diff --git a/.bumpversion.toml b/.bumpversion.toml
index ee405e7d689..25505a1d4b0 100644
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "8.0.0-beta.1"
+current_version = "9.1.0-beta.2"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -55,6 +55,11 @@ filename = "Cargo.toml"
 search = 'lance-datagen = {{ version = "={current_version}"'
 replace = 'lance-datagen = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-derive = {{ version = "={current_version}"'
+replace = 'lance-derive = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-encoding = {{ version = "={current_version}"'
@@ -75,6 +80,11 @@ filename = "Cargo.toml"
 search = 'lance-index = {{ version = "={current_version}"'
 replace = 'lance-index = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-index-core = {{ version = "={current_version}"'
+replace = 'lance-index-core = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-io = {{ version = "={current_version}"'
diff --git a/.cargo/config.toml b/.cargo/config.toml
index 1d9c9ecc9da..c455c4a978d 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -9,6 +9,13 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.release-no-lto]
+inherits = "release"
+debug = true
+lto = false
+# Prioritize compile time when LTO is not relevant to the measurement.
+codegen-units = 16
+
 [profile.bench]
 inherits = "release"
 lto = "thin"
diff --git a/.coderabbit.yaml b/.coderabbit.yaml
new file mode 100644
index 00000000000..9779b0ff8e4
--- /dev/null
+++ b/.coderabbit.yaml
@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: en-US
+early_access: false
+reviews:
+  profile: quiet
+  high_level_summary: false
+  poem: false
+  review_status: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - WIP
+      - Draft
+    drafts: false
+    base_branches:
+      - main
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 00000000000..0e7363697b0
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,79 @@
+name: Bug report
+description: Report incorrect, unexpected, or crashing behavior.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to file a bug report! Please search
+        [existing issues](https://github.com/lance-format/lance/issues) first to
+        avoid filing a duplicate.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: >-
+        A minimal, self-contained code snippet or sequence of steps that
+        reproduces the problem. The more we can copy-paste and run, the faster
+        we can fix it.
+      placeholder: |
+        1. Write a dataset with `...`
+        2. Scan with filter `...`
+        3. Observe `...`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: false
+  - type: input
+    id: lance-version
+    attributes:
+      label: Lance version
+      description: >-
+        Output of `python -c "import lance; print(lance.__version__)"`, or the
+        `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`.
+      placeholder: "e.g. 0.40.0"
+    validations:
+      required: true
+  - type: dropdown
+    id: language
+    attributes:
+      label: Language binding
+      multiple: true
+      options:
+        - Python
+        - Rust
+        - Java
+        - Other / not sure
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...).
+      placeholder: "e.g. Ubuntu 22.04, x86_64, S3"
+    validations:
+      required: false
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs / traceback
+      description: >-
+        Any relevant log output or stack trace. This will be automatically
+        formatted as code, so no need for backticks.
+      render: shell
+    validations:
+      required: false
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 00000000000..32389799971
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,7 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question / usage help
+    url: https://discord.gg/lance
+    about: >-
+      For questions and general discussion, please ask in the Lance Discord
+      rather than opening an issue.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 00000000000..41d6278fc02
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest a new capability or an improvement to an existing one.
+labels: feature
+---
+
+<!--
+Describe what you'd like Lance to do and why. There's no required format —
+mention the problem or use case, and a rough idea of the solution if you have
+one. Please search existing issues first to avoid duplicates.
+-->
diff --git a/.github/ISSUE_TEMPLATE/performance_issue.md b/.github/ISSUE_TEMPLATE/performance_issue.md
new file mode 100644
index 00000000000..65fcd0be6f4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/performance_issue.md
@@ -0,0 +1,12 @@
+---
+name: Performance issue
+about: Report slow operations, high memory use, or a performance regression.
+labels: performance
+---
+
+<!--
+Describe what's slow or memory-hungry. There's no required format, but concrete
+numbers (timings, memory, dataset shape) and a way to reproduce help a lot. If
+it's a regression, note the last version that performed well. Please search
+existing issues first to avoid duplicates.
+-->
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000000..a7b7d11c92b
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,45 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "cargo"
+    directory: "/python"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "cargo"
+    directory: "/java/lance-jni"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      cargo:
+        patterns:
+          - "*"
+
+  - package-ecosystem: "uv"
+    directory: "/python"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      uv:
+        patterns:
+          - "*"
diff --git a/.github/labeler-issues.yml b/.github/labeler-issues.yml
new file mode 100644
index 00000000000..625b321cd6c
--- /dev/null
+++ b/.github/labeler-issues.yml
@@ -0,0 +1,28 @@
+version: 1
+# Never remove labels a template or a human already applied.
+appendOnly: true
+# Content-based labels for issues, applied by srvaroa/labeler via
+# .github/workflows/issue-labeler.yml. This covers issues that bypass the
+# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`,
+# the REST API, or an agent, which never see the web-UI template chooser.
+#
+# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]",
+# "perf -"), which many people type by hand; a modest body keyword match is a
+# secondary fallback. Rules are OR'd (one label per matching entry), so an
+# unmatched issue gets no label and is left for human / LLM triage rather than
+# guessed at. appendOnly means matches stack harmlessly with template labels.
+labels:
+# --- Primary signal: a leading bug/feature/perf marker in the title ---
+- label: bug
+  title: "(?i)^\\s*[\\[(]?\\s*bug\\b"
+- label: feature
+  title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b"
+- label: performance
+  title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b"
+# --- Secondary fallback: keywords anywhere in the body ---
+- label: bug
+  body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)"
+- label: performance
+  body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)"
+- label: feature
+  body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)"
diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml
index fbfcff687ce..d6e6e0f1ada 100644
--- a/.github/workflows/build_linux_wheel/action.yml
+++ b/.github/workflows/build_linux_wheel/action.yml
@@ -3,7 +3,7 @@ name: build-linux-wheel
 description: "Build a manylinux wheel for lance"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"
@@ -41,11 +41,9 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"
     - name: Build x86_64 Manylinux {manylinux} wheel
       if: ${{ inputs.arm-build == 'false' && inputs.manylinux != '2_17' }}
       uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
@@ -60,11 +58,9 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel clang
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"
     - name: Build Arm Manylinux Wheel
       if: ${{ inputs.arm-build == 'true' }}
       uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1
@@ -76,8 +72,6 @@ runs:
         args: ${{ inputs.args }}
         maturin-version: "1.10.2"
         before-script-linux: |
-          set -e
-          yum install -y openssl-devel clang \
-            && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
-            && unzip /tmp/protoc.zip -d /usr/local \
-            && rm /tmp/protoc.zip
+          set -euo pipefail
+          yum install -y openssl-devel clang
+          bash "${GITHUB_WORKSPACE}/.github/workflows/build_linux_wheel/install-protoc.sh"
diff --git a/.github/workflows/build_linux_wheel/install-protoc.sh b/.github/workflows/build_linux_wheel/install-protoc.sh
new file mode 100755
index 00000000000..2d5bf4ced40
--- /dev/null
+++ b/.github/workflows/build_linux_wheel/install-protoc.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version="${PROTOC_VERSION:-24.4}"
+install_dir="${PROTOC_INSTALL_DIR:-/usr/local}"
+machine="${1:-$(uname -m)}"
+
+case "${machine}" in
+  aarch64 | arm64)
+    asset_arch="aarch_64"
+    ;;
+  x86_64)
+    asset_arch="x86_64"
+    ;;
+  *)
+    echo "Unsupported protoc architecture: ${machine}" >&2
+    exit 1
+    ;;
+esac
+
+zip_path="/tmp/protoc-${version}-linux-${asset_arch}.zip"
+url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-linux-${asset_arch}.zip"
+
+for attempt in 1 2 3 4 5; do
+  rm -f "${zip_path}"
+
+  if curl -fsSL --connect-timeout 15 --max-time 120 -o "${zip_path}" "${url}" \
+    && unzip -tq "${zip_path}" >/dev/null; then
+    break
+  fi
+
+  if [[ "${attempt}" == "5" ]]; then
+    echo "Failed to download a valid protoc archive from ${url}" >&2
+    exit 1
+  fi
+
+  sleep "$((attempt * 2))"
+done
+
+unzip -q -o "${zip_path}" -d "${install_dir}"
+rm -f "${zip_path}"
+
+if [[ "$(uname -s)" == "Linux" ]]; then
+  "${install_dir}/bin/protoc" --version
+else
+  test -f "${install_dir}/bin/protoc"
+fi
diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml
index 9d45bde42aa..0cac76c49cf 100644
--- a/.github/workflows/build_mac_wheel/action.yml
+++ b/.github/workflows/build_mac_wheel/action.yml
@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"
diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml
index 03b601db019..94475059c75 100644
--- a/.github/workflows/build_windows_wheel/action.yml
+++ b/.github/workflows/build_windows_wheel/action.yml
@@ -3,7 +3,7 @@ name: build_wheel
 description: "Build a lance wheel"
 inputs:
   python-minor-version:
-    description: "9, 10, 11, 12"
+    description: "10, 11, 12, 13"
     required: true
   args:
     description: "--release"
diff --git a/.github/workflows/compat-pair.yml b/.github/workflows/compat-pair.yml
new file mode 100644
index 00000000000..ba3332d8d7b
--- /dev/null
+++ b/.github/workflows/compat-pair.yml
@@ -0,0 +1,78 @@
+# On-demand cross-version index compatibility run between two arbitrary refs.
+#
+# The scheduled path lives in nightly_run.yml (compat-sequence job), which ages the two
+# previous majors into HEAD nightly. This workflow is the manual escape
+# hatch for any other pairing: each ref (version, sha, branch, or tag) is provisioned by
+# the framework -- a published release installs a wheel, anything else is built from a
+# worktree via maturin -- so two arbitrary refs can be compared even when neither has a
+# published wheel. The suite ages an index under the writer ref and exercises it under the
+# reader, searching maintenance-op sequences for panics or correctness divergence.
+name: Compat (index ref pair)
+
+on:
+  workflow_dispatch:
+    inputs:
+      from_ref:
+        description: "Writer ref (version / sha / branch). Blank = 2nd most recent release."
+        required: false
+        default: ""
+      to_ref:
+        description: "Reader ref (version / sha / branch). Blank = most recent release."
+        required: false
+        default: ""
+      kinds:
+        description: "Comma-separated index kinds (INVERTED,BTREE,...) or 'all'."
+        required: false
+        default: "all"
+      max_length:
+        description: "Max maintenance-op sequence length to search (deeper = slower)."
+        required: false
+        default: "5"
+
+jobs:
+  compat-pair:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    defaults:
+      run:
+        working-directory: python
+    env:
+      COMPAT_FROM_REF: ${{ inputs.from_ref }}
+      COMPAT_TO_REF: ${{ inputs.to_ref }}
+      COMPAT_MAX_LENGTH: ${{ inputs.max_length || '5' }}
+    steps:
+      # Full history so arbitrary refs can be checked out for builds.
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.11"
+      # Toolchain for the build-from-source provisioning path (refs without a wheel).
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Install build deps
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install host deps
+        run: pip install pytest pytest-xdist pyarrow packaging maturin
+      - name: Resolve kinds
+        id: kinds
+        env:
+          KINDS_IN: ${{ inputs.kinds }}
+        run: |
+          if [ -z "$KINDS_IN" ] || [ "$KINDS_IN" = "all" ]; then
+            echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER" >> "$GITHUB_OUTPUT"
+          else
+            echo "value=$KINDS_IN" >> "$GITHUB_OUTPUT"
+          fi
+      # Oversubscribe (4x cores): each scenario writes a small dataset to disk and the
+      # reader spends most of its time in short subprocess round-trips, so a worker waiting
+      # on disk or the sub-venv pipe overlaps another's compute.
+      - name: Run compat suite (${{ inputs.from_ref }} -> ${{ inputs.to_ref }})
+        env:
+          COMPAT_KINDS: ${{ steps.kinds.outputs.value }}
+        run: |
+          python -m pytest python/tests/compat/test_index_sequence.py \
+            --run-compat -n "$(( $(nproc) * 4 ))" -v --no-header
diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml
new file mode 100644
index 00000000000..36f5a9cdbca
--- /dev/null
+++ b/.github/workflows/issue-labeler.yml
@@ -0,0 +1,29 @@
+name: Issue Labeler
+
+# Applies bug / feature / performance labels to issues based on their title and
+# body. This complements the .github/ISSUE_TEMPLATE forms, which only apply
+# labels for issues opened through the web UI; this catches issues opened via
+# `gh issue create`, the REST API, or an agent, which bypass templates.
+#
+# Issues never originate from a fork, so unlike the PR labelers this uses a
+# plain `issues` trigger (no pull_request_target) with a scoped GITHUB_TOKEN.
+on:
+  issues:
+    types: [opened, edited]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+
+jobs:
+  label:
+    name: Apply issue labels
+    permissions:
+      issues: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          config_path: .github/labeler-issues.yml
diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml
index a51cf969a87..6351c5be402 100644
--- a/.github/workflows/java-publish.yml
+++ b/.github/workflows/java-publish.yml
@@ -28,10 +28,24 @@ permissions:
   contents: read
 
 jobs:
-  linux-arm64:
-    name: Build on Linux Arm64
-    runs-on: ubuntu-24.04-arm64-8x
+  build-linux:
+    name: Build on Linux ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86-64
+            runner: ubuntu-24.04
+            docker_platform: linux/amd64
+            protoc_arch: x86_64
+            artifact: liblance_jni_linux_x86_64.zip
+          - arch: arm64
+            runner: ubuntu-24.04-arm64-8x
+            docker_platform: linux/arm64
+            protoc_arch: aarch_64
+            artifact: liblance_jni_linux_arm_64.zip
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -41,9 +55,9 @@ jobs:
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
       - name: Check glibc version outside docker
         run: ldd --version
-      - name: Build and run in Debian 10 Arm64 container
+      - name: Build and run in Debian 10 container
         run: |
-          docker run --platform linux/arm64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
+          docker run --platform ${{ matrix.docker_platform }} -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
 
             set -ex
             # Update sources.list to use archive repositories for Debian 10 (EOL)
@@ -81,7 +95,7 @@ jobs:
               unzip
 
             # https://github.com/databendlabs/databend/issues/8035
-            PROTOC_ZIP=protoc-3.15.0-linux-aarch_64.zip
+            PROTOC_ZIP=protoc-3.15.0-linux-${{ matrix.protoc_arch }}.zip
             curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP
             unzip -o \$PROTOC_ZIP -d /usr/local
             rm -f \$PROTOC_ZIP
@@ -98,105 +112,48 @@ jobs:
             export CXX=clang++
             ldd --version
 
-            cargo build --release
+            cargo build --release --locked
           "
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: liblance_jni_linux_arm_64.zip
+          name: ${{ matrix.artifact }}
           path: java/lance-jni/target/release/liblance_jni.so
           retention-days: 1
           if-no-files-found: error
-  linux-x86:
-    name: Build on Linux x86-64
-    runs-on: ubuntu-24.04
+  build-macos:
+    name: Build on MacOS Arm64
+    runs-on: warp-macos-14-arm64-6x
     timeout-minutes: 60
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-      - name: Check glibc version outside docker
-        run: ldd --version
-      - name: Build and run in Debian 10 X86-64 container
-        run: |
-          docker run --platform linux/amd64 -v ${{ github.workspace }}:/workspace -w /workspace debian:10 bash -c "
-
-            set -ex
-            # Update sources.list to use archive repositories for Debian 10 (EOL)
-            echo 'deb http://archive.debian.org/debian/ buster main' > /etc/apt/sources.list
-            echo 'deb http://archive.debian.org/debian-security buster/updates main' >> /etc/apt/sources.list
-            echo 'deb http://archive.debian.org/debian/ buster-updates main' >> /etc/apt/sources.list
-            apt-get update
-
-            DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --assume-yes \
-              apt-transport-https \
-              ca-certificates \
-              curl \
-              gpg \
-              bash \
-              less \
-              openssl \
-              libssl-dev \
-              pkg-config \
-              libsqlite3-dev \
-              libsqlite3-0 \
-              libreadline-dev \
-              git \
-              cmake \
-              dh-autoreconf \
-              clang \
-              g++ \
-              libc++-dev \
-              libc++abi-dev \
-              libprotobuf-dev \
-              libncurses5-dev \
-              libncursesw5-dev \
-              libudev-dev \
-              libhidapi-dev \
-              zip \
-              unzip
-
-            # https://github.com/databendlabs/databend/issues/8035
-            PROTOC_ZIP=protoc-3.15.0-linux-x86_64.zip
-            curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.15.0/\$PROTOC_ZIP
-            unzip -o \$PROTOC_ZIP -d /usr/local
-            rm -f \$PROTOC_ZIP
-            protoc --version
-
-            curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
-            source \$HOME/.cargo/env
-            cargo --version
-
-            cd java/lance-jni
-
-            # https://github.com/rustls/rustls/issues/1967
-            export CC=clang
-            export CXX=clang++
-            ldd --version
-
-            cargo build --release
-          "
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07
+      - name: Install dependencies
+        run: brew install protobuf
+      - name: Build native lib
+        working-directory: java/lance-jni
+        run: cargo build --release --locked
       - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: liblance_jni_linux_x86_64.zip
-          path: java/lance-jni/target/release/liblance_jni.so
+          name: liblance_jni_darwin_aarch64.zip
+          path: java/lance-jni/target/release/liblance_jni.dylib
           retention-days: 1
           if-no-files-found: error
-  macos-arm64:
-    name: Build on MacOS Arm64 and release
-    runs-on: warp-macos-14-arm64-6x
-    timeout-minutes: 60
+  publish:
+    name: Publish Java packages
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     needs:
-      - linux-arm64
-      - linux-x86
+      - build-linux
+      - build-macos
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Set up Java 11
         uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4
         with:
@@ -208,18 +165,16 @@ jobs:
           server-password: SONATYPE_TOKEN
           gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
           gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
-      - uses: Homebrew/actions/setup-homebrew@50b8c2ab4a835c38897ed2c56c293b07167c0b59 # master 2026-03-07
-      - name: Install dependencies
-        run: |
-          brew install protobuf
-          brew install gpg
-      - name: Download artifact
+      - name: Download artifacts
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
       - name: Copy native libs
         run: |
-          mkdir -p ./java/target/classes/nativelib/linux-x86-64 ./java/target/classes/nativelib/linux-aarch64
+          mkdir -p ./java/target/classes/nativelib/linux-x86-64 \
+            ./java/target/classes/nativelib/linux-aarch64 \
+            ./java/target/classes/nativelib/darwin-aarch64
           cp ./liblance_jni_linux_x86_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-x86-64/liblance_jni.so
           cp ./liblance_jni_linux_arm_64.zip/liblance_jni.so ./java/target/classes/nativelib/linux-aarch64/liblance_jni.so
+          cp ./liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./java/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib
       - name: Set github
         run: |
           git config --global user.email "Lance Github Runner"
@@ -230,7 +185,7 @@ jobs:
           inputs.mode == 'dry_run'
         working-directory: java
         run: |
-          mvn --batch-mode -DskipTests -Drust.release.build=true package
+          mvn --batch-mode -DskipTests -Dskip.build.jni=true package
       - name: Publish with Java 11
         if: |
           github.event_name == 'release' ||
@@ -240,14 +195,14 @@ jobs:
           echo "use-agent" >> ~/.gnupg/gpg.conf
           echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
           export GPG_TTY=$(tty)
-          mvn --batch-mode -DskipTests -Drust.release.build=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh -P shade-jar
+          mvn --batch-mode -DskipTests -Dskip.build.jni=true -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -P deploy-to-ossrh
         env:
           SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
           SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}
   report-failure:
     name: Report Workflow Failure
     runs-on: ubuntu-latest
-    needs: [linux-arm64, linux-x86, macos-arm64]
+    needs: [build-linux, build-macos, publish]
     if: always() && (github.event_name == 'release' || github.event_name == 'workflow_dispatch')
     permissions:
       contents: read
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 83403988244..85b50e10836 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -47,14 +47,14 @@ jobs:
         run: cargo fmt --check
       - name: Rust Clippy
         working-directory: java/lance-jni
-        run: cargo clippy --all-targets -- -D warnings
+        run: cargo clippy --all-targets --locked -- -D warnings
 
   build-and-test-java:
     runs-on: ubuntu-24.04-4x
     timeout-minutes: 60
     strategy:
       matrix:
-        java-version: [11, 17, 21]
+        java-version: [11, 17, 21, 25]
     name: Build and Test with Java ${{ matrix.java-version }}
     steps:
       - name: Checkout repository
@@ -79,7 +79,13 @@ jobs:
           distribution: temurin
           java-version: ${{ matrix.java-version }}
           cache: "maven"
+      # Skip the format gate on Java 25: google-java-format 1.22.0 (pinned via
+      # spotless) reaches into com.sun.tools.javac internals and throws
+      # NoSuchMethodError on JDK 25's javac. No google-java-format release runs
+      # on JDK 25 without also dropping JDK 11/17 support, so we exclude 25 from
+      # the (JDK-independent, already-enforced-on-11/17/21) style check instead.
       - name: Running code style check with Java ${{ matrix.java-version }}
+        if: matrix.java-version != '25'
         working-directory: java
         run: |
           mvn spotless:check
@@ -91,4 +97,7 @@ jobs:
         env:
           LANCE_INTEGRATION_TEST: "1"
         run: |
-          mvn install
+          # spotless is also bound into the build lifecycle, so on Java 25 the
+          # format gate must be skipped here too (-Dspotless.skip=true), not just
+          # in the standalone step above.
+          mvn install ${{ matrix.java-version == '25' && '-Dspotless.skip=true' || '' }}
diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml
index 488ca65e585..2e6e26bbbeb 100644
--- a/.github/workflows/license-header-check.yml
+++ b/.github/workflows/license-header-check.yml
@@ -22,11 +22,25 @@ jobs:
       - name: Check out code
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Install license-header-checker
+        env:
+          # Pin to a specific release and download the asset directly from the
+          # release CDN. This avoids the flaky dynamic "latest" tag lookup in the
+          # upstream install.sh (it fetches the releases page and parses tag_name,
+          # which intermittently returns empty and fails the job).
+          LHC_VERSION: "1.5.0"
         run: |
           set -euo pipefail
-          curl -sfSL https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh -o /tmp/install-lhc.sh
-          bash /tmp/install-lhc.sh
-          rm -f /tmp/install-lhc.sh
+          mkdir -p ./bin
+          base="https://github.com/lluissm/license-header-checker/releases/download/v${LHC_VERSION}"
+          asset="license-header-checker_${LHC_VERSION}_linux_amd64.tar.gz"
+          curl -sfSL "${base}/${asset}" -o "/tmp/${asset}"
+          curl -sfSL "${base}/checksums.txt" -o /tmp/lhc-checksums.txt
+          # Verify the download against the release checksums before extracting so a
+          # corrupted or tampered asset cannot be executed. This restores the
+          # integrity check the upstream install.sh performed via hash_sha256_verify.
+          (cd /tmp && sha256sum -c --ignore-missing lhc-checksums.txt)
+          tar -xzf "/tmp/${asset}" -C ./bin license-header-checker
+          rm -f "/tmp/${asset}" /tmp/lhc-checksums.txt
       - name: Check license headers (rust)
         run: ./bin/license-header-checker -a -v ./rust/license_header.txt rust rs && [[ -z `git status -s` ]]
       - name: Check license headers (python)
diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml
index 955dcec3860..228b0b7de22 100644
--- a/.github/workflows/nightly_run.yml
+++ b/.github/workflows/nightly_run.yml
@@ -44,3 +44,69 @@ jobs:
           # Run only the jumbo tests in release mode with limited parallelism
           # to avoid OOM issues (these tests use >5GiB memory each)
           cargo test --release --package lance-encoding -- --ignored jumbo --test-threads=1
+
+  # Cross-version index maintenance-sequence search (see python/tests/compat/compat_sequence.py).
+  # Ages an index under the latest release of each of the two previous majors and exercises it
+  # under this commit, searching op sequences for panics or correctness divergence. The search is
+  # slow and unbounded in depth, so it runs nightly with a generous timeout rather than blocking
+  # merges; the manual escape hatch for arbitrary ref pairs lives in compat-pair.yml.
+  compat-sequence:
+    if: github.repository == 'lancedb/lance'
+    timeout-minutes: 360
+    runs-on: ubuntu-24.04
+    name: Index Sequence Compat
+    defaults:
+      run:
+        shell: bash
+        working-directory: python
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: 3.13
+      # Toolchain to build the reader (HEAD) wheel from source -- no upstream build job to
+      # borrow a wheel from here, unlike the old post-merge path in python.yml.
+      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+        with:
+          workspaces: python
+      - name: Install build deps
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+      - name: Install host deps
+        run: pip install pytest pytest-xdist pyarrow packaging maturin
+      # Build HEAD once and feed it to the suite as the prebuilt reader, so the two writer
+      # passes below don't each recompile it.
+      - name: Build reader wheel (HEAD)
+        run: python -m maturin build --release -m Cargo.toml --out wheels
+      # Age under each of the two previous majors (writer wheels from PyPI) and read under
+      # this commit's freshly built wheel. Length 5 is the shallowest depth that reaches the
+      # ENT-1662 sequence; -n oversubscribes cores since the work is I/O-bound.
+      - name: Run sequence search (previous two majors -> HEAD)
+        env:
+          COMPAT_MAX_LENGTH: "5"
+          COMPAT_TO_REF: HEAD
+          COMPAT_PREBUILT_REF: HEAD
+        run: |
+          set -euo pipefail
+          wheel=$(ls "$PWD"/wheels/pylance-*.whl | head -1)
+          refs=$(PYTHONPATH=python/tests python -c "
+          from compat.compat_decorator import pylance_stable_versions
+          latest = {}
+          for v in pylance_stable_versions():
+              latest[v.major] = v
+          print(' '.join(str(latest[m]) for m in sorted(latest)[-2:]))
+          ")
+          echo "reader wheel: $wheel"
+          echo "writer refs : $refs"
+          status=0
+          for from_ref in $refs; do
+            echo "::group::$from_ref -> HEAD"
+            COMPAT_FROM_REF="$from_ref" COMPAT_PREBUILT_WHEEL="$wheel" \
+              python -m pytest python/tests/compat/test_index_sequence.py \
+                --run-compat -n "$(( $(nproc) * 4 ))" -v --no-header || status=1
+            echo "::endgroup::"
+          done
+          exit $status
diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml
index b2bfe284fb5..77c76d6fc69 100644
--- a/.github/workflows/pypi-publish.yml
+++ b/.github/workflows/pypi-publish.yml
@@ -35,7 +35,7 @@ jobs:
     name: Python Linux 3.${{ matrix.python-minor-version }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
         config:
           - platform: x86_64
             manylinux: "2_17"
@@ -101,7 +101,7 @@ jobs:
     runs-on: ${{ matrix.config.runner }}
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
         config:
           - target: aarch64-apple-darwin
             runner: warp-macos-14-arm64-6x
@@ -152,7 +152,7 @@ jobs:
     runs-on: windows-latest-4x
     strategy:
       matrix:
-        python-minor-version: ["9"]
+        python-minor-version: ["10"]
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 973d9a632a7..601f2dd9dfc 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -97,7 +97,7 @@ jobs:
     timeout-minutes: 45
     strategy:
       matrix:
-        python-minor-version: ["9", "13"]
+        python-minor-version: ["10", "13"]
     name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
     runs-on: "ubuntu-24.04-4x"
     defaults:
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index e3c17671ce3..6dc8af48a6e 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -96,8 +96,9 @@ jobs:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
       - name: Setup rust toolchain
         run: |
-          rustup toolchain install nightly
-          rustup default nightly
+          # Temporary mitigation for https://github.com/rust-lang/rust/issues/159261.
+          rustup toolchain install nightly-2026-07-13
+          rustup default nightly-2026-07-13
       - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
       - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
       - name: Install dependencies
@@ -111,7 +112,7 @@ jobs:
       - name: Run tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -`
-          cargo +nightly llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
+          cargo +nightly-2026-07-13 llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4
         with:
@@ -253,6 +254,42 @@ jobs:
       - name: Check benchmarks
         run: cargo check --profile ci --benches
 
+  qemu-pre-haswell:
+    # Verifies that lance-linalg's runtime SIMD dispatch still works
+    # correctly when the binary is built with the lower x86-64-v2 baseline
+    # (the legacy build path documented in CONTRIBUTING.md). Emulates a
+    # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA
+    # instructions that leak past the runtime dispatch.
+    #
+    # The published-wheel default baseline (`target-cpu=haswell`) is set in
+    # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to
+    # exercise the legacy path without affecting any other build.
+    name: pre-Haswell SIGILL check (qemu Nehalem)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      CC: clang
+      CXX: clang++
+      RUSTFLAGS: "-C target-cpu=x86-64-v2"
+      CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem"
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - name: Setup rust toolchain
+        run: |
+          rustup toolchain install stable
+          rustup default stable
+      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev qemu-user
+      - name: Build lance-linalg lib tests in release mode
+        run: |
+          cargo test --release -p lance-linalg --lib --no-run
+      - name: Run lance-linalg lib tests under qemu Nehalem
+        run: |
+          cargo test --release -p lance-linalg --lib
+
   msrv:
     # Check the minimum supported Rust version
     name: MSRV Check - Rust v${{ matrix.msrv }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 09c956152fe..502f568908a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,3 +15,16 @@ repos:
     rev: v1.26.0
     hooks:
     - id: typos
+
+  # Catch a Cargo.toml change that wasn't reflected in the matching Cargo.lock.
+  # Runs fully offline (`--frozen` = `--locked` + `--offline`) and does not compile,
+  # so it stays fast. Checks all three lockfiles since a workspace dep change can
+  # touch the excluded python/ and java/ locks too.
+  - repo: local
+    hooks:
+      - id: cargo-lock-sync
+        name: Cargo.lock in sync with Cargo.toml (offline)
+        entry: bash -c 'for m in Cargo.toml python/Cargo.toml java/lance-jni/Cargo.toml; do cargo metadata --frozen --format-version 1 --manifest-path "$m" >/dev/null 2>&1 || { echo "Cargo.lock is out of date for $m. Refresh it (e.g. cargo check --manifest-path $m) and commit the updated lockfile."; exit 1; }; done'
+        language: system
+        files: '(^|/)Cargo\.(toml|lock)$'
+        pass_filenames: false
diff --git a/AGENTS.md b/AGENTS.md
index 2003d6dba10..257ae93ea11 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -30,6 +30,12 @@ Rust workspace with Python and Java bindings:
 
 Key technical traits: async-first (tokio), Arrow-native, versioned writes with manifest tracking, custom ML-optimized encodings, unified object store interface (local/S3/Azure/GCS).
 
+## File Format Stability and Compatibility
+
+- Treat every file format marked stable as a durable compatibility contract. All changes to a stable format must preserve both backward and forward compatibility.
+- Treat every file format marked unstable as disposable. It may change freely; do not add compatibility code, migrations, fallbacks, or tests for files written by earlier unstable revisions.
+- Evaluate compatibility against the latest released stable version while continuing to honor all stable format contracts. Changes that exist only on the current branch or `main` are not compatibility constraints; do not compromise a cleaner or more complete design to preserve those intermediate states.
+
 ## Development Commands
 
 ### Rust
@@ -41,6 +47,9 @@ Key technical traits: async-first (tokio), Arrow-native, versioned writes with m
 * Coverage: `cargo +nightly llvm-cov -q -p <crate> --branch`
 * Coverage HTML: `cargo +nightly llvm-cov -q -p <crate> --branch --html`
 * Coverage for file: `python ci/coverage.py -p <crate> -f <file_path>`
+* Use repository-defined Cargo profiles instead of ad hoc LTO overrides.
+* Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild.
+* Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck.
 
 ### Python / Java
 
@@ -97,6 +106,7 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p
 
 - Prefer implementing functionality with the standard library or existing workspace dependencies before adding new external crates.
 - Keep `Cargo.lock` changes intentional; revert unrelated dependency bumps. Pin broken deps with a comment linking the upstream issue.
+- The repo has three lockfiles: the root `Cargo.lock`, `python/Cargo.lock`, and `java/lance-jni/Cargo.lock` (the latter two are excluded from the workspace). A `workspace.dependencies` change must be reflected in all three — refresh the excluded ones with `cargo check --manifest-path python/Cargo.toml` and `cargo check --manifest-path java/lance-jni/Cargo.toml`, then commit the updated lockfiles. The `cargo-lock-sync` pre-commit hook catches a miss offline.
 - Gate optional/domain-specific deps behind Cargo feature flags. Prefer separate crates for domain functionality (geo, NLP).
 
 ## Testing Standards
@@ -130,6 +140,11 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p
 - Indent content under MkDocs admonition directives (`!!! note`, etc.) with 4 spaces.
 - Proofread comments and docs for typos before committing.
 
+## Filing Issues
+
+- When opening an issue with `gh issue create` or the API, classify it and pass the matching label: `--label bug`, `--label feature`, or `--label performance`. These paths bypass the `.github/ISSUE_TEMPLATE` forms, so the label is not applied automatically.
+- Prefix the title to match, e.g. `bug: ...`, `feature: ...`, or `perf: ...`. A content-based labeler (`.github/workflows/issue-labeler.yml`) uses this as a fallback signal, but an explicit `--label` is the reliable path.
+
 ## Pull Requests
 
 - PR titles must follow the Conventional Commits specification because `.github/workflows/pr-title.yml` validates the PR title and body with commitlint. Use prefixes like `feat:`, `fix:`, `docs:`, `perf:`, `ci:`, `test:`, `build:`, `style:`, or `chore:`; add a scope when useful.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cf332215e49..3940ed7b683 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -19,12 +19,22 @@ If you have any questions, please join our [Discord](https://discord.gg/zMM32dvN
 Currently Lance is implemented in Rust and comes with a Python wrapper. So you'll want to make sure you setup both.
 
 1. Install Rust: https://www.rust-lang.org/tools/install
-2. Install Python 3.9+: https://www.python.org/downloads/
+2. Install Python 3.10+: https://www.python.org/downloads/
 3. Install protoctol buffers: https://grpc.io/docs/protoc-installation/ (make sure you have version 3.20 or higher)
 4. Install commit hooks:
     a. Install pre-commit: https://pre-commit.com/#install
     b. Run `pre-commit install` in the root of the repo
 
+## Building for legacy x86_64 hosts (pre-Haswell)
+
+The default workspace build targets `haswell` (AVX2 + FMA + F16C), matching the published wheels. To build a binary that runs on pre-Haswell silicon (Sandy Bridge / Ivy Bridge / Westmere on Intel, Bulldozer / Piledriver / Steamroller on AMD — i.e. CPUs without AVX2), set the baseline yourself at build time:
+
+```sh
+RUSTFLAGS="-C target-cpu=x86-64-v2" cargo build --release
+```
+
+Runtime SIMD dispatch in `lance-linalg::distance` will then pick the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) based on the host. From Python, use `lance.simd_info()` to verify which tier was selected.
+
 ## Sample Workflow
 
 1. Fork the repo
diff --git a/Cargo.lock b/Cargo.lock
index dc8146a8e01..7873595a4ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -62,24 +62,9 @@ dependencies = [
 
 [[package]]
 name = "all_asserts"
-version = "2.3.3"
+version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514ce16346f9fc96702fd52f2ae7e383b185516ee6f556efd7c3176be8fe7bea"
-
-[[package]]
-name = "alloc-no-stdlib"
-version = "2.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
-
-[[package]]
-name = "alloc-stdlib"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
-dependencies = [
- "alloc-no-stdlib",
-]
+checksum = "62ec8a4dbfbd8acfe181f030be8174b00d3d7dc3447f7dc53ead6cc96e89fe52"
 
 [[package]]
 name = "alloca"
@@ -147,7 +132,7 @@ version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -158,7 +143,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -178,9 +163,9 @@ dependencies = [
 
 [[package]]
 name = "arc-swap"
-version = "1.9.1"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
 dependencies = [
  "rustversion",
 ]
@@ -386,7 +371,7 @@ version = "58.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "serde_core",
  "serde_json",
 ]
@@ -475,7 +460,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -486,7 +471,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -540,7 +525,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "ring",
  "time",
  "tokio",
@@ -602,7 +587,7 @@ dependencies = [
  "bytes-utils",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "percent-encoding",
@@ -630,7 +615,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -660,7 +645,7 @@ dependencies = [
  "hex",
  "hmac 0.12.1",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "lru",
  "percent-encoding",
@@ -689,7 +674,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -713,7 +698,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -738,7 +723,7 @@ dependencies = [
  "aws-types",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -759,7 +744,7 @@ dependencies = [
  "hex",
  "hmac 0.12.1",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "percent-encoding",
  "sha2 0.10.9",
  "time",
@@ -788,7 +773,7 @@ dependencies = [
  "bytes",
  "crc-fast",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "md-5 0.10.6",
@@ -822,7 +807,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "percent-encoding",
@@ -841,7 +826,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-rustls",
  "hyper-util",
@@ -898,7 +883,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -918,7 +903,7 @@ dependencies = [
  "aws-smithy-types",
  "bytes",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "pin-project-lite",
  "tokio",
  "tracing",
@@ -936,7 +921,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -984,7 +969,7 @@ dependencies = [
  "axum-core",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -1017,7 +1002,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "mime",
@@ -1119,9 +1104,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.12.1"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 
 [[package]]
 name = "bitpacking"
@@ -1134,9 +1119,9 @@ dependencies = [
 
 [[package]]
 name = "bitvec"
-version = "1.0.1"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837"
 dependencies = [
  "funty",
  "radium",
@@ -1178,9 +1163,9 @@ dependencies = [
 
 [[package]]
 name = "block-buffer"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
 dependencies = [
  "hybrid-array",
 ]
@@ -1194,27 +1179,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "brotli"
-version = "8.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor",
-]
-
-[[package]]
-name = "brotli-decompressor"
-version = "5.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
-]
-
 [[package]]
 name = "bs58"
 version = "0.5.1"
@@ -1261,7 +1225,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1272,9 +1236,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
 [[package]]
 name = "bytemuck"
-version = "1.25.0"
+version = "1.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
 
 [[package]]
 name = "byteorder"
@@ -1284,9 +1262,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.11.1"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1315,9 +1293,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.63"
+version = "1.2.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
+checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1365,9 +1343,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
  "iana-time-zone",
  "js-sys",
@@ -1426,9 +1404,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.6.1"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
+checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1436,9 +1414,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.6.0"
+version = "4.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
+checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1455,7 +1433,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1491,7 +1469,7 @@ version = "3.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1770,18 +1748,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1944,7 +1922,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1957,7 +1935,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1979,7 +1957,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
  "darling_core 0.20.11",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1990,7 +1968,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
  "darling_core 0.23.0",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2009,14 +1987,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -2026,7 +2003,6 @@ dependencies = [
  "datafusion-datasource-arrow",
  "datafusion-datasource-csv",
  "datafusion-datasource-json",
- "datafusion-datasource-parquet",
  "datafusion-execution",
  "datafusion-expr",
  "datafusion-expr-common",
@@ -2044,13 +2020,11 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
- "parquet",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -2060,9 +2034,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2085,9 +2059,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2108,33 +2082,33 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
- "parquet",
- "paste",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
 dependencies = [
  "futures",
  "log",
@@ -2143,9 +2117,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2165,16 +2139,17 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
- "rand 0.9.4",
+ "parking_lot",
+ "rand 0.9.5",
  "tokio",
  "url",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -2196,9 +2171,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2219,9 +2194,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2236,57 +2211,25 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
-[[package]]
-name = "datafusion-datasource-parquet"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
-dependencies = [
- "arrow",
- "async-trait",
- "bytes",
- "datafusion-common",
- "datafusion-common-runtime",
- "datafusion-datasource",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions-aggregate-common",
- "datafusion-physical-expr",
- "datafusion-physical-expr-adapter",
- "datafusion-physical-expr-common",
- "datafusion-physical-plan",
- "datafusion-pruning",
- "datafusion-session",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "parquet",
- "tokio",
-]
-
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -2295,18 +2238,19 @@ dependencies = [
  "log",
  "object_store",
  "parking_lot",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -2317,29 +2261,27 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2354,26 +2296,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -2383,19 +2324,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -2404,9 +2344,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2420,34 +2360,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2458,14 +2398,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2473,20 +2412,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
 dependencies = [
  "arrow",
  "chrono",
@@ -2503,11 +2442,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2515,20 +2453,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2541,26 +2478,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2576,12 +2513,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2596,7 +2534,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2608,9 +2546,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2619,15 +2557,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2639,9 +2576,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2657,9 +2594,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2702,26 +2639,6 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "deepsize"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
-dependencies = [
- "deepsize_derive",
-]
-
-[[package]]
-name = "deepsize_derive"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -2782,7 +2699,7 @@ dependencies = [
  "darling 0.20.11",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2802,7 +2719,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
 dependencies = [
  "derive_builder_core 0.20.2",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2829,7 +2746,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
 dependencies = [
- "block-buffer 0.12.0",
+ "block-buffer 0.12.1",
  "const-oid 0.10.2",
  "crypto-common 0.2.2",
  "ctutils",
@@ -2853,7 +2770,7 @@ dependencies = [
  "libc",
  "option-ext",
  "redox_users",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2864,7 +2781,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -2949,11 +2866,17 @@ dependencies = [
  "encoding_rs",
 ]
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "env_filter"
-version = "1.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
 dependencies = [
  "log",
  "regex",
@@ -2961,9 +2884,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.10"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2989,7 +2912,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3005,7 +2928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -3086,7 +3009,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "rustc_version",
 ]
 
@@ -3098,7 +3021,6 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
- "zlib-rs",
 ]
 
 [[package]]
@@ -3166,10 +3088,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
- "rand 0.9.4",
+ "rand 0.9.5",
  "test-log",
  "tokio",
 ]
@@ -3245,7 +3167,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3402,12 +3324,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -3468,17 +3391,15 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
 dependencies = [
  "cfg-if 1.0.4",
  "js-sys",
  "libc",
  "r-efi 6.0.0",
  "rand_core 0.10.1",
- "wasip2",
- "wasip3",
  "wasm-bindgen",
 ]
 
@@ -3505,7 +3426,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -3526,18 +3447,42 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "goosefs-sdk"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "dashmap",
+ "hostname",
+ "prost",
+ "prost-types",
+ "rand 0.9.5",
+ "reqwest 0.12.28",
+ "serde",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tonic-prost",
+ "tracing",
+ "uuid",
+]
+
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.4.1",
+ "http 1.4.2",
  "indexmap 2.14.0",
  "slab",
  "tokio",
@@ -3551,6 +3496,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -3603,6 +3549,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3646,13 +3597,13 @@ checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
 dependencies = [
  "dirs",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "indicatif",
  "libc",
  "log",
  "native-tls",
  "num_cpus",
- "rand 0.9.4",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "serde",
  "serde_json",
@@ -3670,7 +3621,7 @@ checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "more-asserts",
  "serde",
  "thiserror 2.0.18",
@@ -3702,6 +3653,17 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "hostname"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
+dependencies = [
+ "cfg-if 1.0.4",
+ "libc",
+ "windows-link",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3715,9 +3677,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.4.1"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -3741,7 +3703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
 ]
 
 [[package]]
@@ -3752,7 +3714,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
 dependencies = [
  "bytes",
  "futures-core",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "pin-project-lite",
 ]
@@ -3771,9 +3733,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
@@ -3795,7 +3757,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "httparse",
  "httpdate",
@@ -3812,7 +3774,7 @@ version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-util",
  "rustls",
@@ -3820,7 +3782,20 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tower-service",
- "webpki-roots 1.0.7",
+ "webpki-roots 1.0.8",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
 ]
 
 [[package]]
@@ -3849,7 +3824,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "hyper",
  "ipnet",
@@ -4067,12 +4042,6 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f"
 
-[[package]]
-name = "id-arena"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -4172,11 +4141,11 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "io-uring"
-version = "0.7.12"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
+checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "cfg-if 1.0.4",
  "libc",
 ]
@@ -4195,7 +4164,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
 dependencies = [
  "hermit-abi",
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -4248,18 +4217,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82"
+checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9"
+checksum = "a813bbf185c8c62eb6fcf54a223177b644824d91612045dfd80bb779acd080eb"
 dependencies = [
  "bytecount",
  "cedarwood",
@@ -4294,7 +4263,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4339,7 +4308,7 @@ dependencies = [
  "quote",
  "rustc_version",
  "simd_cesu8",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4358,7 +4327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
 dependencies = [
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -4373,13 +4342,12 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
 dependencies = [
  "cfg-if 1.0.4",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
@@ -4397,7 +4365,7 @@ dependencies = [
  "nom 8.0.0",
  "num-traits",
  "ordered-float 5.3.0",
- "rand 0.9.4",
+ "rand 0.9.5",
  "serde",
  "serde_json",
  "zmij",
@@ -4431,7 +4399,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "all_asserts",
  "approx",
@@ -4453,7 +4421,6 @@ dependencies = [
  "aws-credential-types",
  "aws-sdk-dynamodb",
  "aws-sdk-s3",
- "bitpacking",
  "byteorder",
  "bytes",
  "chrono",
@@ -4468,7 +4435,6 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-plan",
  "datafusion-substrait",
- "deepsize",
  "either",
  "env_logger",
  "fst",
@@ -4478,8 +4444,9 @@ dependencies = [
  "geoarrow-schema",
  "half",
  "humantime",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-datagen",
@@ -4503,12 +4470,13 @@ dependencies = [
  "parquet",
  "permutation",
  "pin-project",
+ "pprof",
  "pretty_assertions",
  "prost",
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rayon",
  "reqwest 0.12.28",
  "roaring",
@@ -4534,7 +4502,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4543,13 +4511,14 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
  "half",
  "jsonb",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -4582,30 +4551,34 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrayref",
+ "bitpacking",
+ "crunchy",
  "paste",
  "seq-macro",
 ]
 
 [[package]]
 name = "lance-core"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
+ "arrow-data",
  "arrow-schema",
  "async-trait",
  "byteorder",
  "bytes",
  "datafusion-common",
  "datafusion-sql",
- "deepsize",
  "futures",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-derive",
  "libc",
+ "libm",
  "log",
  "moka",
  "num_cpus",
@@ -4613,7 +4586,7 @@ dependencies = [
  "pin-project",
  "proptest",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "roaring",
  "rstest",
  "serde_json",
@@ -4623,12 +4596,13 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "tracing",
+ "twox-hash",
  "url",
 ]
 
 [[package]]
 name = "lance-datafusion"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4661,7 +4635,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4673,15 +4647,23 @@ dependencies = [
  "half",
  "hex",
  "lance-testing",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rand_xoshiro",
- "random_word",
+]
+
+[[package]]
+name = "lance-derive"
+version = "9.1.0-beta.2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4699,7 +4681,7 @@ dependencies = [
  "futures",
  "hex",
  "hyperloglogplus",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
  "lance-bitpacking",
  "lance-core",
@@ -4712,7 +4694,7 @@ dependencies = [
  "prost",
  "prost-build",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_xoshiro",
  "rstest",
  "serial_test",
@@ -4726,7 +4708,7 @@ dependencies = [
 
 [[package]]
 name = "lance-examples"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "all_asserts",
  "arrow",
@@ -4736,7 +4718,7 @@ dependencies = [
  "env_logger",
  "futures",
  "hf-hub",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance",
  "lance-core",
  "lance-datagen",
@@ -4744,7 +4726,7 @@ dependencies = [
  "lance-linalg",
  "object_store",
  "parquet",
- "rand 0.9.4",
+ "rand 0.9.5",
  "tempfile",
  "tokenizers",
  "tokio",
@@ -4752,7 +4734,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4766,7 +4748,6 @@ dependencies = [
  "bytes",
  "criterion",
  "datafusion-common",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -4792,7 +4773,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4806,7 +4787,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "approx",
  "arc-swap",
@@ -4819,7 +4800,6 @@ dependencies = [
  "async-channel",
  "async-recursion",
  "async-trait",
- "bitpacking",
  "bitvec",
  "bytes",
  "chrono",
@@ -4829,7 +4809,6 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "deepsize",
  "dirs",
  "env_logger",
  "fst",
@@ -4839,24 +4818,26 @@ dependencies = [
  "geoarrow-array",
  "geoarrow-schema",
  "half",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "jieba-rs",
  "jsonb",
  "lance-arrow",
  "lance-arrow-stats",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-datagen",
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
  "lance-table",
  "lance-testing",
  "lance-tokenizer",
- "libm",
+ "libsais-rs",
  "log",
  "ndarray",
  "num-traits",
@@ -4865,10 +4846,11 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_distr",
  "rangemap",
  "rayon",
+ "regex-syntax",
  "roaring",
  "rstest",
  "serde",
@@ -4878,13 +4860,34 @@ dependencies = [
  "test-log",
  "tokio",
  "tracing",
- "twox-hash",
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "9.1.0-beta.2"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4902,15 +4905,17 @@ dependencies = [
  "bytes",
  "chrono",
  "criterion",
- "deepsize",
  "futures",
- "http 1.4.1",
+ "goosefs-sdk",
+ "http 1.4.2",
  "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
  "lance-testing",
  "log",
+ "metrics",
+ "metrics-util",
  "mock_instant",
  "mockall",
  "moka",
@@ -4920,7 +4925,7 @@ dependencies = [
  "path_abs",
  "pin-project",
  "prost",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rstest",
  "serde",
  "tempfile",
@@ -4933,7 +4938,7 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "approx",
  "arrow-array",
@@ -4941,19 +4946,20 @@ dependencies = [
  "arrow-schema",
  "cc",
  "criterion",
- "deepsize",
  "half",
  "lance-arrow",
  "lance-core",
  "lance-testing",
  "num-traits",
  "proptest",
- "rand 0.9.4",
+ "rand 0.9.5",
+ "rayon",
+ "rstest",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4965,7 +4971,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-datafusion"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4981,9 +4987,10 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
+ "arrow-array",
  "arrow-ipc",
  "arrow-schema",
  "async-trait",
@@ -4993,9 +5000,12 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "chrono",
+ "datafusion-common",
+ "datafusion-physical-plan",
  "futures",
  "hmac 0.12.1",
  "lance",
+ "lance-arrow",
  "lance-core",
  "lance-index",
  "lance-io",
@@ -5004,28 +5014,32 @@ dependencies = [
  "lance-table",
  "log",
  "object_store",
- "quick-xml 0.38.4",
- "rand 0.9.4",
+ "opendal",
+ "quick-xml 0.40.1",
+ "rand 0.9.5",
  "reqwest 0.12.28",
  "ring",
+ "roaring",
  "rstest",
  "rustls-pki-types",
  "serde",
  "serde_json",
  "sha2 0.10.9",
  "tempfile",
+ "time",
  "tokio",
  "tower",
  "tower-http 0.5.2",
  "url",
+ "uuid",
  "wiremock",
 ]
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.8.0"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3eefb02ded2c3d4b6b60669bb74822d9fa628e144fc748c79ee31f13f566e87b"
+checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -5037,7 +5051,7 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5045,8 +5059,7 @@ dependencies = [
  "byteorder",
  "bytes",
  "criterion",
- "deepsize",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-core",
  "proptest",
  "roaring",
@@ -5056,7 +5069,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5070,7 +5083,6 @@ dependencies = [
  "bytes",
  "chrono",
  "criterion",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -5087,7 +5099,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "protobuf-src",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rangemap",
  "roaring",
  "rstest",
@@ -5103,16 +5115,16 @@ dependencies = [
 
 [[package]]
 name = "lance-test-macros"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "lance-testing"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5120,24 +5132,25 @@ dependencies = [
  "lance-arrow",
  "num-traits",
  "pprof",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
 name = "lance-tokenizer"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",
  "lindera",
  "rust-stemmers",
  "serde",
+ "stop-words",
  "unicode-normalization",
 ]
 
 [[package]]
 name = "lance-tools"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "clap",
  "lance-core",
@@ -5157,12 +5170,6 @@ dependencies = [
  "spin 0.9.8",
 ]
 
-[[package]]
-name = "leb128fmt"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -5241,6 +5248,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libsais-rs"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40fe164dbd47ea0c20e78a121c980ef673326905f1d4fba55e3645a20ef6717f"
+dependencies = [
+ "rayon",
+]
+
 [[package]]
 name = "lindera"
 version = "3.0.7"
@@ -5297,9 +5313,9 @@ dependencies = [
 
 [[package]]
 name = "link-section"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "014e440054ce8170890229eeef5bcda955305e056ec713de40ed366944483f09"
+checksum = "c2b1dd6fe32e55c0fc0ea9493aa57459ca3cf4ff3c857c7d0302290150da6e4f"
 
 [[package]]
 name = "linktime-proc-macro"
@@ -5330,9 +5346,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.31"
+version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
 [[package]]
 name = "loom"
@@ -5476,9 +5492,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.8.1"
+version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "memmap2"
@@ -5489,6 +5505,36 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.24.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
+dependencies = [
+ "portable-atomic",
+ "rapidhash",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
+dependencies = [
+ "aho-corasick",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.5",
+ "indexmap 2.14.0",
+ "metrics",
+ "ordered-float 4.6.0",
+ "quanta",
+ "radix_trie",
+ "rand 0.9.5",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -5534,9 +5580,9 @@ dependencies = [
 
 [[package]]
 name = "mock_instant"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6"
+checksum = "9bb517913cfcfb9eeda59f36020269075a152701a01606c612f547e4890be399"
 
 [[package]]
 name = "mockall"
@@ -5561,7 +5607,7 @@ dependencies = [
  "cfg-if 1.0.4",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5603,7 +5649,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5635,7 +5681,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5670,6 +5716,15 @@ dependencies = [
  "rawpointer",
 ]
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "nix"
 version = "0.26.4"
@@ -5715,7 +5770,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -5828,7 +5883,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -5843,7 +5898,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
 ]
 
 [[package]]
@@ -5888,7 +5943,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body-util",
  "httparse",
  "humantime",
@@ -5955,7 +6010,7 @@ version = "6.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "libc",
  "once_cell",
  "onig_sys",
@@ -5993,9 +6048,11 @@ dependencies = [
  "opendal-service-azdls",
  "opendal-service-cos",
  "opendal-service-gcs",
+ "opendal-service-goosefs",
  "opendal-service-hf",
  "opendal-service-oss",
  "opendal-service-s3",
+ "opendal-service-tos",
 ]
 
 [[package]]
@@ -6008,7 +6065,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "jiff",
  "log",
@@ -6033,7 +6090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
 dependencies = [
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "mea",
  "opendal-core",
 ]
@@ -6077,7 +6134,7 @@ checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
 dependencies = [
  "base64 0.22.1",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -6098,7 +6155,7 @@ checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
 dependencies = [
  "base64 0.22.1",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -6116,7 +6173,7 @@ version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "opendal-core",
 ]
 
@@ -6127,7 +6184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -6145,7 +6202,7 @@ checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -6158,6 +6215,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "opendal-service-goosefs"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
+dependencies = [
+ "bytes",
+ "goosefs-sdk",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+]
+
 [[package]]
 name = "opendal-service-hf"
 version = "0.57.0"
@@ -6166,7 +6237,7 @@ checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
 dependencies = [
  "bytes",
  "hf-xet",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -6182,7 +6253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -6201,7 +6272,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "crc32c",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "md-5 0.11.0",
  "opendal-core",
@@ -6213,13 +6284,30 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "opendal-service-tos"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1"
+dependencies = [
+ "bytes",
+ "http 1.4.2",
+ "opendal-core",
+ "quick-xml 0.39.4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-volcengine-tos",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "openssl"
-version = "0.10.80"
+version = "0.10.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "cfg-if 1.0.4",
  "foreign-types",
  "libc",
@@ -6235,7 +6323,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6246,9 +6334,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.116"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
 dependencies = [
  "cc",
  "libc",
@@ -6271,6 +6359,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-float"
 version = "5.3.0"
@@ -6358,26 +6455,19 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "base64 0.22.1",
- "brotli",
  "bytes",
  "chrono",
- "flate2",
  "futures",
  "half",
  "hashbrown 0.17.1",
- "lz4_flex",
  "num-bigint",
  "num-integer",
  "num-traits",
- "object_store",
  "paste",
  "seq-macro",
- "simdutf8",
- "snap",
  "thrift",
  "tokio",
  "twox-hash",
- "zstd",
 ]
 
 [[package]]
@@ -6562,7 +6652,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6755,7 +6845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -6784,9 +6874,9 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744"
 dependencies = [
  "bit-set",
  "bit-vec",
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
  "rand_chacha 0.9.0",
  "rand_xorshift",
  "regex-syntax",
@@ -6797,9 +6887,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -6807,9 +6897,9 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
+checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
  "itertools 0.14.0",
@@ -6820,28 +6910,28 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn 2.0.117",
+ "syn 2.0.119",
  "tempfile",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
 dependencies = [
  "prost",
 ]
@@ -6872,7 +6962,22 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
 ]
 
 [[package]]
@@ -6890,15 +6995,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -6949,7 +7045,7 @@ dependencies = [
  "bytes",
  "getrandom 0.3.4",
  "lru-slab",
- "rand 0.9.4",
+ "rand 0.9.5",
  "ring",
  "rustc-hash",
  "rustls",
@@ -6972,14 +7068,14 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.45"
+version = "1.0.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
 dependencies = [
  "proc-macro2",
 ]
@@ -7002,6 +7098,16 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rancor"
 version = "0.1.1"
@@ -7024,9 +7130,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
 dependencies = [
  "rand_chacha 0.9.0",
  "rand_core 0.9.5",
@@ -7039,7 +7145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
 dependencies = [
  "chacha20",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "rand_core 0.10.1",
 ]
 
@@ -7094,7 +7200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
 dependencies = [
  "num-traits",
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -7116,23 +7222,28 @@ dependencies = [
 ]
 
 [[package]]
-name = "random_word"
-version = "0.5.2"
+name = "rangemap"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81"
+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+
+[[package]]
+name = "rapidhash"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
 dependencies = [
- "ahash",
- "brotli",
- "paste",
- "rand 0.9.4",
- "unicase",
+ "rustversion",
 ]
 
 [[package]]
-name = "rangemap"
-version = "1.7.1"
+name = "raw-cpuid"
+version = "11.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+dependencies = [
+ "bitflags 2.13.0",
+]
 
 [[package]]
 name = "rawpointer"
@@ -7186,7 +7297,7 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
 ]
 
 [[package]]
@@ -7217,14 +7328,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -7251,9 +7362,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "regress"
@@ -7288,7 +7399,7 @@ checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d"
 dependencies = [
  "anyhow",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -7307,7 +7418,7 @@ dependencies = [
  "bytes",
  "form_urlencoded",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "quick-xml 0.40.1",
@@ -7329,7 +7440,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "pem",
  "percent-encoding",
@@ -7353,7 +7464,7 @@ dependencies = [
  "futures",
  "hex",
  "hmac 0.13.0",
- "http 1.4.1",
+ "http 1.4.2",
  "jiff",
  "log",
  "percent-encoding",
@@ -7383,7 +7494,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2"
 dependencies = [
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-aws-v4",
@@ -7401,7 +7512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb"
 dependencies = [
  "anyhow",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -7409,6 +7520,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "reqsign-volcengine-tos"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3"
+dependencies = [
+ "anyhow",
+ "http 1.4.2",
+ "log",
+ "percent-encoding",
+ "reqsign-core",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.12.28"
@@ -7421,7 +7545,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -7455,7 +7579,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams 0.4.2",
  "web-sys",
- "webpki-roots 1.0.7",
+ "webpki-roots 1.0.8",
 ]
 
 [[package]]
@@ -7468,7 +7592,7 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -7506,7 +7630,7 @@ checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58"
 dependencies = [
  "anyhow",
  "async-trait",
- "http 1.4.1",
+ "http 1.4.2",
  "reqwest 0.13.4",
  "thiserror 2.0.18",
  "tower-service",
@@ -7562,7 +7686,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7638,7 +7762,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.117",
+ "syn 2.0.119",
  "unicode-ident",
 ]
 
@@ -7670,9 +7794,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -7689,11 +7813,11 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -7726,9 +7850,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
 dependencies = [
  "web-time",
  "zeroize",
@@ -7752,7 +7876,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -7875,7 +7999,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7907,7 +8031,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -7967,7 +8091,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7978,7 +8102,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -7987,6 +8111,7 @@ version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -8013,7 +8138,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8025,7 +8150,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8042,9 +8167,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"
+checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
 dependencies = [
  "base64 0.22.1",
  "bs58",
@@ -8062,14 +8187,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac"
+checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8120,7 +8245,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8251,6 +8376,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -8259,9 +8390,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "smallvec"
-version = "1.15.1"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
@@ -8281,15 +8412,9 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
-[[package]]
-name = "snap"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
-
 [[package]]
 name = "socket2"
 version = "0.6.4"
@@ -8297,7 +8422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -8362,9 +8487,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -8378,7 +8503,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8415,6 +8540,15 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
 
+[[package]]
+name = "stop-words"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d68df56303396bcfb639455b3c166804aeb7994005010aab5e9e8a1277b8871d"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "str_stack"
 version = "0.1.1"
@@ -8461,7 +8595,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8473,16 +8607,17 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -8496,7 +8631,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.117",
+ "syn 2.0.119",
  "typify",
  "walkdir",
 ]
@@ -8549,9 +8684,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.117"
+version = "2.0.119"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -8575,7 +8710,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8598,7 +8733,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -8632,10 +8767,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -8663,7 +8798,7 @@ checksum = "c26ef8b00e4d382e59f6a8ddb3cd790b3a5bb29f21a358a9a69ea2f29f13f27b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8672,7 +8807,7 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "944ad38adcbb71eaa682c56bceeb079e4ca82b4b3edc2a0fde5cb297b77dac8d"
 dependencies = [
- "syn 2.0.117",
+ "syn 2.0.119",
  "test-log-core",
 ]
 
@@ -8702,7 +8837,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8713,7 +8848,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8879,7 +9014,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -8894,9 +9029,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-retry"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640"
+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"
 dependencies = [
  "pin-project-lite",
  "rand 0.10.1",
@@ -8968,6 +9103,45 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
+dependencies = [
+ "async-trait",
+ "base64 0.22.1",
+ "bytes",
+ "h2",
+ "http 1.4.2",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "socket2",
+ "sync_wrapper",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tonic-prost"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
+dependencies = [
+ "bytes",
+ "prost",
+ "tonic",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.3"
@@ -8976,9 +9150,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap 2.14.0",
  "pin-project-lite",
+ "slab",
  "sync_wrapper",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -8990,9 +9167,9 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -9008,11 +9185,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
 dependencies = [
  "async-compression",
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -9069,7 +9246,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -9167,7 +9344,7 @@ version = "2.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
 dependencies = [
- "rand 0.9.4",
+ "rand 0.9.5",
 ]
 
 [[package]]
@@ -9207,7 +9384,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.117",
+ "syn 2.0.119",
  "thiserror 2.0.18",
  "unicode-ident",
 ]
@@ -9225,7 +9402,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.117",
+ "syn 2.0.119",
  "typify-impl",
 ]
 
@@ -9283,12 +9460,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "unicode_categories"
 version = "0.1.1"
@@ -9365,11 +9536,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.2"
+version = "1.24.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
+checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
 dependencies = [
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "js-sys",
  "serde_core",
  "wasm-bindgen",
@@ -9444,20 +9615,11 @@ dependencies = [
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
- "wit-bindgen 0.57.1",
-]
-
-[[package]]
-name = "wasip3"
-version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
-dependencies = [
- "wit-bindgen 0.51.0",
+ "wit-bindgen",
 ]
 
 [[package]]
@@ -9471,9 +9633,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
 dependencies = [
  "cfg-if 1.0.4",
  "once_cell",
@@ -9484,9 +9646,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.72"
+version = "0.4.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -9494,9 +9656,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -9504,48 +9666,26 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
 dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "wasm-encoder"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
-dependencies = [
- "leb128fmt",
- "wasmparser",
-]
-
-[[package]]
-name = "wasm-metadata"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
-dependencies = [
- "anyhow",
- "indexmap 2.14.0",
- "wasm-encoder",
- "wasmparser",
-]
-
 [[package]]
 name = "wasm-streams"
 version = "0.4.2"
@@ -9572,23 +9712,11 @@ dependencies = [
  "web-sys",
 ]
 
-[[package]]
-name = "wasmparser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
-dependencies = [
- "bitflags 2.12.1",
- "hashbrown 0.15.5",
- "indexmap 2.14.0",
- "semver",
-]
-
 [[package]]
 name = "web-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -9606,9 +9734,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -9619,14 +9747,14 @@ version = "0.26.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
 dependencies = [
- "webpki-roots 1.0.7",
+ "webpki-roots 1.0.8",
 ]
 
 [[package]]
 name = "webpki-roots"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
+checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -9666,7 +9794,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -9728,7 +9856,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -9739,7 +9867,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -9980,7 +10108,7 @@ dependencies = [
  "base64 0.22.1",
  "deadpool",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body-util",
  "hyper",
  "hyper-util",
@@ -9993,100 +10121,12 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
-dependencies = [
- "wit-bindgen-rust-macro",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.57.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
-[[package]]
-name = "wit-bindgen-core"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
-dependencies = [
- "anyhow",
- "heck",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-bindgen-rust"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
-dependencies = [
- "anyhow",
- "heck",
- "indexmap 2.14.0",
- "prettyplease",
- "syn 2.0.117",
- "wasm-metadata",
- "wit-bindgen-core",
- "wit-component",
-]
-
-[[package]]
-name = "wit-bindgen-rust-macro"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
-dependencies = [
- "anyhow",
- "prettyplease",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "wit-bindgen-core",
- "wit-bindgen-rust",
-]
-
-[[package]]
-name = "wit-component"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
-dependencies = [
- "anyhow",
- "bitflags 2.12.1",
- "indexmap 2.14.0",
- "log",
- "serde",
- "serde_derive",
- "serde_json",
- "wasm-encoder",
- "wasm-metadata",
- "wasmparser",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-parser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
-dependencies = [
- "anyhow",
- "id-arena",
- "indexmap 2.14.0",
- "log",
- "semver",
- "serde",
- "serde_derive",
- "serde_json",
- "unicode-xid",
- "wasmparser",
-]
-
 [[package]]
 name = "wkb"
 version = "0.9.2"
@@ -10140,7 +10180,7 @@ dependencies = [
  "clap",
  "crc32fast",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "lazy_static",
  "more-asserts",
@@ -10181,7 +10221,7 @@ dependencies = [
  "csv",
  "futures",
  "futures-util",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "heapify",
  "itertools 0.14.0",
  "lazy_static",
@@ -10214,7 +10254,7 @@ dependencies = [
  "chrono",
  "clap",
  "gearhash",
- "http 1.4.1",
+ "http 1.4.2",
  "itertools 0.14.0",
  "lazy_static",
  "more-asserts",
@@ -10282,9 +10322,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.15"
+version = "0.8.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72"
 
 [[package]]
 name = "yansi"
@@ -10294,9 +10334,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -10311,28 +10351,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "synstructure",
 ]
 
 [[package]]
 name = "zerocopy"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -10352,15 +10392,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
  "synstructure",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"
@@ -10394,15 +10434,9 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn 2.0.119",
 ]
 
-[[package]]
-name = "zlib-rs"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
-
 [[package]]
 name = "zmij"
 version = "1.0.21"
diff --git a/Cargo.toml b/Cargo.toml
index 5044498dd41..8133dd306db 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
     "rust/lance-file",
     "rust/lance-geo",
     "rust/lance-index",
+    "rust/lance-index-core",
     "rust/lance-io",
     "rust/lance-linalg",
     "rust/lance-namespace",
@@ -17,6 +18,7 @@ members = [
     "rust/lance-select",
     "rust/lance-tokenizer",
     "rust/lance-table",
+    "rust/lance-derive",
     "rust/lance-test-macros",
     "rust/lance-testing",
     "rust/lance-tools",
@@ -31,7 +33,7 @@ resolver = "3"
 
 
 [workspace.package]
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -56,26 +58,28 @@ rust-version = "1.91.0"
 [workspace.dependencies]
 arc-swap = "1.7"
 libc = "0.2.176"
-lance = { version = "=8.0.0-beta.1", path = "./rust/lance", default-features = false }
-lance-arrow = { version = "=8.0.0-beta.1", path = "./rust/lance-arrow" }
-lance-core = { version = "=8.0.0-beta.1", path = "./rust/lance-core" }
-lance-datafusion = { version = "=8.0.0-beta.1", path = "./rust/lance-datafusion" }
-lance-datagen = { version = "=8.0.0-beta.1", path = "./rust/lance-datagen" }
-lance-encoding = { version = "=8.0.0-beta.1", path = "./rust/lance-encoding" }
-lance-file = { version = "=8.0.0-beta.1", path = "./rust/lance-file" }
-lance-geo = { version = "=8.0.0-beta.1", path = "./rust/lance-geo" }
-lance-index = { version = "=8.0.0-beta.1", path = "./rust/lance-index" }
-lance-io = { version = "=8.0.0-beta.1", path = "./rust/lance-io", default-features = false }
-lance-linalg = { version = "=8.0.0-beta.1", path = "./rust/lance-linalg" }
-lance-namespace = { version = "=8.0.0-beta.1", path = "./rust/lance-namespace" }
-lance-namespace-impls = { version = "=8.0.0-beta.1", path = "./rust/lance-namespace-impls" }
+lance = { version = "=9.1.0-beta.2", path = "./rust/lance", default-features = false }
+lance-arrow = { version = "=9.1.0-beta.2", path = "./rust/lance-arrow" }
+lance-core = { version = "=9.1.0-beta.2", path = "./rust/lance-core" }
+lance-datafusion = { version = "=9.1.0-beta.2", path = "./rust/lance-datafusion" }
+lance-datagen = { version = "=9.1.0-beta.2", path = "./rust/lance-datagen" }
+lance-derive = { version = "=9.1.0-beta.2", path = "./rust/lance-derive" }
+lance-encoding = { version = "=9.1.0-beta.2", path = "./rust/lance-encoding" }
+lance-file = { version = "=9.1.0-beta.2", path = "./rust/lance-file" }
+lance-geo = { version = "=9.1.0-beta.2", path = "./rust/lance-geo" }
+lance-index = { version = "=9.1.0-beta.2", path = "./rust/lance-index" }
+lance-index-core = { version = "=9.1.0-beta.2", path = "./rust/lance-index-core" }
+lance-io = { version = "=9.1.0-beta.2", path = "./rust/lance-io", default-features = false }
+lance-linalg = { version = "=9.1.0-beta.2", path = "./rust/lance-linalg" }
+lance-namespace = { version = "=9.1.0-beta.2", path = "./rust/lance-namespace" }
+lance-namespace-impls = { version = "=9.1.0-beta.2", path = "./rust/lance-namespace-impls" }
 lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" }
-lance-namespace-reqwest-client = "0.8.0"
-lance-select = { version = "=8.0.0-beta.1", path = "./rust/lance-select" }
-lance-tokenizer = { version = "=8.0.0-beta.1", path = "./rust/lance-tokenizer" }
-lance-table = { version = "=8.0.0-beta.1", path = "./rust/lance-table" }
-lance-test-macros = { version = "=8.0.0-beta.1", path = "./rust/lance-test-macros" }
-lance-testing = { version = "=8.0.0-beta.1", path = "./rust/lance-testing" }
+lance-namespace-reqwest-client = "0.8.6"
+lance-select = { version = "=9.1.0-beta.2", path = "./rust/lance-select" }
+lance-tokenizer = { version = "=9.1.0-beta.2", path = "./rust/lance-tokenizer" }
+lance-table = { version = "=9.1.0-beta.2", path = "./rust/lance-table" }
+lance-test-macros = { version = "=9.1.0-beta.2", path = "./rust/lance-test-macros" }
+lance-testing = { version = "=9.1.0-beta.2", path = "./rust/lance-testing" }
 approx = "0.5.1"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false, features = ["prettyprint"] }
@@ -101,10 +105,14 @@ aws-sdk-s3 = { version = "1.38.0", default-features = false }
 half = { "version" = "2.1", default-features = false, features = [
     "num-traits",
     "std",
+    "bytemuck",
 ] }
-lance-bitpacking = { version = "=8.0.0-beta.1", path = "./rust/compression/bitpacking" }
+lance-bitpacking = { version = "=9.1.0-beta.2", path = "./rust/compression/bitpacking" }
 bitpacking = "0.9"
 bitvec = "1"
+bytemuck = { version = "1", default-features = false, features = [
+    "extern_crate_alloc",
+] }
 bytes = "1.11.1"
 byteorder = "1.5"
 clap = { version = "4", features = ["derive"] }
@@ -120,7 +128,7 @@ criterion = { version = "0.8.2", features = [
 ] }
 crossbeam-queue = "0.3"
 crossbeam-skiplist = "0.1"
-datafusion = { version = "53.0.0", default-features = false, features = [
+datafusion = { version = "54.0.0", default-features = false, features = [
     "crypto_expressions",
     "datetime_expressions",
     "encoding_expressions",
@@ -130,35 +138,37 @@ datafusion = { version = "53.0.0", default-features = false, features = [
     "string_expressions",
     "unicode_expressions",
 ] }
-datafusion-common = "53.0.0"
-datafusion-functions = { version = "53.0.0", features = ["regex_expressions"] }
-datafusion-sql = "53.0.0"
-datafusion-expr = "53.0.0"
-datafusion-ffi = "53.0.0"
-datafusion-physical-expr = "53.0.0"
-datafusion-physical-plan = "53.0.0"
-datafusion-substrait = "53.0.0"
-deepsize = "0.2.0"
+datafusion-common = "54.0.0"
+datafusion-functions = { version = "54.0.0", default-features = false, features = ["regex_expressions"] }
+datafusion-sql = "54.0.0"
+datafusion-expr = "54.0.0"
+datafusion-ffi = "54.0.0"
+datafusion-physical-expr = "54.0.0"
+datafusion-physical-plan = "54.0.0"
+datafusion-substrait = { version = "54.0.0", default-features = false }
 dirs = "6.0.0"
 either = "1.0"
 fst = { version = "0.4.7", features = ["levenshtein"] }
-fsst = { version = "=8.0.0-beta.1", path = "./rust/compression/fsst" }
+fsst = { version = "=9.1.0-beta.2", path = "./rust/compression/fsst" }
 futures = "0.3"
 geoarrow-array = "0.8"
 geoarrow-schema = "0.8"
-geodatafusion = "0.4.0"
+geodatafusion = "0.5.0"
 geo-traits = "0.3.0"
 geo-types = "0.7.16"
+goosefs-sdk = "=0.1.5"
 http = "1.1.0"
 humantime = "2.2.0"
 hyperloglogplus = { version = "0.4.1", features = ["const-loop"] }
 icu_segmenter = { version = "2.2", default-features = false, features = ["compiled_data"] }
 io-uring = "0.7"
-itertools = "0.13"
+itertools = "0.14"
 jieba-rs = { version = "0.10.0", default-features = false }
 jsonb = { version = "0.5.3", default-features = false, features = ["databend"] }
 libm = "0.2.15"
 log = "0.4"
+metrics = { version = "0.24" }
+metrics-util = { version = "0.19" }
 mockall = { version = "0.14.0" }
 mock_instant = { version = "0.6.0" }
 moka = { version = "0.12", features = ["future", "sync"] }
@@ -179,6 +189,7 @@ rand_distr = { version = "0.5.1" }
 rand_xoshiro = "0.7.0"
 rangemap = { version = "1.0" }
 rayon = "1.10"
+regex-syntax = "0.8.10"
 roaring = "0.11.4"
 rstest = "0.26.1"
 serde = { version = "^1" }
@@ -202,6 +213,7 @@ tower = "0.5"
 tower-http = "0.5"
 tracing = "0.1"
 tracing-mock = { version = "=0.1.0-beta.3" }
+twox-hash = "2.0"
 url = "2.5.7"
 uuid = { version = "1.2", features = ["v4", "serde"] }
 wiremock = "0.6"
diff --git a/LICENSE b/LICENSE
index 79de57d6670..cd4c38213a9 100644
--- a/LICENSE
+++ b/LICENSE
@@ -226,3 +226,30 @@ under the MIT license:
    SOFTWARE.
 
 https://github.com/pola-rs/polars/blob/main/LICENSE
+
+--------------------------------------------------------------------------------
+
+This project includes code adapted from the quickwit-oss/bitpacking crate, which
+is licensed under the MIT license:
+
+   Copyright (c) 2016 Paul Masurel
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+
+https://github.com/quickwit-oss/bitpacking/blob/main/LICENSE
diff --git a/README.md b/README.md
index 2f0b2bca18f..08716c84ead 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,17 @@ For more details, see the full [Lance format specification](https://lance.org/fo
 > [!TIP]
 > Lance is in active development and we welcome contributions. Please see our [contributing guide](https://lance.org/community/contributing/) for more information.
 
+## File format stability
+
+Lance releases frequently because the SDKs, integrations, and performance work are moving quickly. This does not mean the Lance file format changes incompatibly in every release. The Lance file format is identified by the `data_storage_version` stored in each dataset, and stable storage versions are a long-term compatibility contract.
+
+* Once a dataset is written with a stable `data_storage_version`, future Lance releases will continue to support reading that storage version.
+* SDK and API compatibility is separate from file format compatibility. SDK/API changes follow semantic versioning and are documented in the [migration guide](https://lance.org/guide/migration/).
+* Older Lance releases may not understand file format versions introduced later. If you run mixed Lance versions, pin `data_storage_version` for deterministic writes.
+* The `next` file format alias is unstable and should only be used for experimentation, never for production data.
+
+For production, write data with a stable `data_storage_version`. See the [format versioning guide](https://lance.org/format/file/versioning/) for the current compatibility matrix.
+
 ## Quick Start
 
 **Installation**
@@ -67,7 +78,7 @@ pip install pylance
 To install a preview release:
 
 ```shell
-pip install --pre --extra-index-url https://pypi.fury.io/lance-format/pylance
+pip install --pre --extra-index-url https://pypi.fury.io/lance-format pylance
 ```
 
 > [!TIP]
diff --git a/ci/create_release_branch.sh b/ci/create_release_branch.sh
index 9c7d9d3e58a..db88f5b6b24 100755
--- a/ci/create_release_branch.sh
+++ b/ci/create_release_branch.sh
@@ -229,9 +229,9 @@ else
     bump-my-version bump -vv --new-version "${RC_VERSION}" --no-tag patch
 
     # Update Cargo.lock files after version bump
-    cargo update
-    (cd python && cargo update)
-    (cd java/lance-jni && cargo update)
+    cargo update --workspace
+    (cd python && cargo update --workspace)
+    (cd java/lance-jni && cargo update --workspace)
 
     # Commit the RC version
     git add -A
@@ -259,9 +259,9 @@ else
     bump-my-version bump -vv --new-version "${NEXT_VERSION}" --no-tag patch
 
     # Update Cargo.lock files after version bump
-    cargo update
-    (cd python && cargo update)
-    (cd java/lance-jni && cargo update)
+    cargo update --workspace
+    (cd python && cargo update --workspace)
+    (cd java/lance-jni && cargo update --workspace)
 
     git add -A
     git commit -m "chore: bump main to ${NEXT_VERSION}
diff --git a/ci/publish_beta.sh b/ci/publish_beta.sh
index f50798a52e0..06fa5c16a91 100644
--- a/ci/publish_beta.sh
+++ b/ci/publish_beta.sh
@@ -93,9 +93,9 @@ if [[ "${BRANCH}" == "main" ]] && [[ "${CURRENT_VERSION}" =~ -beta\.[0-9]+$ ]];
                 bump-my-version bump -vv --new-version "${NEXT_VERSION}" --no-tag patch
 
                 # Update Cargo.lock files after version bump
-                cargo update
-                (cd python && cargo update)
-                (cd java/lance-jni && cargo update)
+                cargo update --workspace
+                (cd python && cargo update --workspace)
+                (cd java/lance-jni && cargo update --workspace)
 
                 git add -A
                 git commit -m "chore: bump to ${NEXT_VERSION} based on breaking change detection"
@@ -133,9 +133,9 @@ echo "Bumping beta version"
 bump-my-version bump -vv prerelease_num
 
 # Update Cargo.lock files after version bump
-cargo update
-(cd python && cargo update)
-(cd java/lance-jni && cargo update)
+cargo update --workspace
+(cd python && cargo update --workspace)
+(cd java/lance-jni && cargo update --workspace)
 
 # Get new version
 NEW_VERSION=$(grep '^version = ' Cargo.toml | head -n1 | cut -d'"' -f2)
diff --git a/ci/release_common.sh b/ci/release_common.sh
index cd653212aae..573202d1689 100644
--- a/ci/release_common.sh
+++ b/ci/release_common.sh
@@ -29,9 +29,9 @@ bump_and_commit_version() {
     bump-my-version bump -vv --new-version "${NEW_VERSION}" --no-tag patch
 
     # Update Cargo.lock files after version bump
-    cargo update
-    (cd python && cargo update)
-    (cd java/lance-jni && cargo update)
+    cargo update --workspace
+    (cd python && cargo update --workspace)
+    (cd java/lance-jni && cargo update --workspace)
 
     git add -A
     git commit -m "${COMMIT_MESSAGE}"
diff --git a/deny.toml b/deny.toml
index 3a65cdd9ed6..75b92e53447 100644
--- a/deny.toml
+++ b/deny.toml
@@ -83,6 +83,8 @@ ignore = [
     { id = "RUSTSEC-2024-0436", reason = "`paste` is used by datafusion" },
     { id = "RUSTSEC-2023-0071", reason = "`rsa` is used by opendal via reqsign" },
     { id = "RUSTSEC-2025-0119", reason = "`number_prefix` used by hf-hub in examples" },
+    { id = "RUSTSEC-2026-0194", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" },
+    { id = "RUSTSEC-2026-0195", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" },
 ]
 # If this is true, then cargo deny will use the git executable to fetch advisory database.
 # If this is false, then it uses a built-in git library.
diff --git a/docs/README.md b/docs/README.md
index 80092b157c7..8cb33f86387 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,6 +1,7 @@
 # Lance Documentation
 
-This directory contains the documentation for Lance, built with MkDocs and Material theme.
+This directory contains the documentation for Lance, built with MkDocs and a
+custom theme (`theme/`) implementing the Lance Docs design.
 
 ## Getting Started with uv
 
@@ -62,5 +63,18 @@ uv sync --upgrade
 ## Project Structure
 
 - `src/` - Source markdown files for documentation
+- `theme/` - Custom MkDocs theme implementing the Lance Docs design
+  (templates in `*.html`, styles/behaviour in `theme/assets/`)
 - `mkdocs.yml` - MkDocs configuration
 - `pyproject.toml` - Python project configuration (uv compatible)
+
+## Theme Notes
+
+- Light/dark mode follows `prefers-color-scheme`, is toggleable from the
+  header, and persists in `localStorage` (`ld-theme`).
+- The GitHub star count is fetched from the public GitHub API and cached in
+  `localStorage` for an hour; the button degrades to a plain link offline.
+- Search is a lightweight overlay (`/` or `Cmd/Ctrl+K`) over the standard
+  `search` plugin index — no external search dependencies.
+- Mermaid diagrams render client-side; the library is loaded from a CDN only
+  on pages that contain a diagram.
diff --git a/docs/clean-full-website.sh b/docs/clean-full-website.sh
index db8013cd744..2ebe0b31b87 100755
--- a/docs/clean-full-website.sh
+++ b/docs/clean-full-website.sh
@@ -39,7 +39,7 @@ nav:
   - Apache DataFusion: datafusion.md
   - PostgreSQL: https://github.com/lancedb/pglance
   - PyTorch: pytorch.md
-  - Tensorflow: tensorflow.md
+  - TensorFlow: tensorflow.md
 EOF
 
 mkdir -p "$docs_src/format/catalog/dir"
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 8144a42e5f5..de75e65f5d9 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -6,37 +6,13 @@ docs_dir: src
 repo_name: lance-format/lance
 repo_url: https://github.com/lance-format/lance
 
+# Custom theme implementing the "Lance Docs" design (see theme/).
 theme:
-  name: material
-  custom_dir: overrides
-  logo: logo/white.png
-  favicon: logo/logo.png
-  palette:
-    - scheme: default
-      primary: custom
-      accent: custom
-      toggle:
-        icon: material/brightness-7
-        name: Switch to dark mode
-    - scheme: slate
-      primary: custom
-      accent: custom
-      toggle:
-        icon: material/brightness-4
-        name: Switch to light mode
-  features:
-    - navigation.tabs
-    - navigation.sections
-    - navigation.instant
-    - navigation.indexes
-    - navigation.tracking
-    - navigation.top
-    - search.highlight
-    - search.share
-    - content.code.copy
-    - content.code.annotate
-  icon:
-    repo: fontawesome/brands/github
+  name: null
+  custom_dir: theme
+  locale: en
+  static_templates:
+    - 404.html
 
 markdown_extensions:
   - admonition
@@ -51,9 +27,16 @@ markdown_extensions:
       line_spans: __span
       pygments_lang_class: true
   - pymdownx.inlinehilite
-  - pymdownx.snippets
+  - pymdownx.snippets:
+      # Allow snippets to pull in files from the repo root (e.g. shared docs
+      # kept alongside Rust source). Paths are relative to the docs/ dir.
+      base_path:
+        - .
+        - ..
   - pymdownx.tabbed:
       alternate_style: true
+  # Autolink bare URLs (e.g. repository tables on the community pages).
+  - pymdownx.magiclink
   - attr_list
   - md_in_html
   - tables
@@ -66,18 +49,4 @@ plugins:
   - mkdocs_protobuf:
       proto_dir: ../protos
 
-extra:
-  generator: false
-  social:
-    - icon: fontawesome/brands/github
-      link: https://github.com/lance-format/lance
-    - icon: fontawesome/brands/discord
-      link: https://discord.gg/lance
-
 copyright: © 2025 Lance Format. All rights reserved.
-
-extra_css:
-  - assets/stylesheets/home.css
-extra_javascript:
-  - assets/javascripts/nav-expand.js
-
diff --git a/docs/overrides/home.html b/docs/overrides/home.html
deleted file mode 100644
index 26eb288aee6..00000000000
--- a/docs/overrides/home.html
+++ /dev/null
@@ -1,295 +0,0 @@
-{% extends "main.html" %}
-
-{% block tabs %}
-  {{ super() }}
-
-  <style>
-    /* Prevent horizontal overflow */
-    body {
-      overflow-x: hidden;
-    }
-
-    /* Hide main content for home page */
-    .md-content {
-      display: none;
-    }
-
-    /* Hide table of contents */
-    @media screen and (min-width: 60em) {
-      .md-sidebar--secondary {
-        display: none;
-      }
-    }
-
-    /* Hide navigation */
-    @media screen and (min-width: 76.25em) {
-      .md-sidebar--primary {
-        display: none;
-      }
-    }
-
-    /* Make header static */
-    .md-header {
-      position: initial;
-    }
-
-    .md-main__inner {
-      margin: 0;
-    }
-
-    /* Style Learn More and View Integrations links */
-    .lance-feature-section .md-button,
-    .lance-intro-section .md-button:not(.md-button--primary) {
-      background-color: transparent;
-      color: inherit;
-      text-decoration: none;
-      border: none;
-      box-shadow: none;
-    }
-
-    .lance-feature-section .md-button:hover,
-    .lance-intro-section .md-button:not(.md-button--primary):hover {
-      background-color: transparent;
-      color: #625EFF;
-      text-decoration: none;
-    }
-  </style>
-
-  <script>
-    // Fix palette toggle jumping to bottom of page
-    (function() {
-      // Store scroll position
-      let savedScrollY = 0;
-      let isTogglingPalette = false;
-
-      // Prevent hash-based scrolling for palette changes
-      if ('scrollRestoration' in history) {
-        history.scrollRestoration = 'manual';
-      }
-
-      // Intercept scroll attempts during palette toggle
-      const preventScroll = function() {
-        if (isTogglingPalette) {
-          window.scrollTo(0, savedScrollY);
-        }
-      };
-
-      // Save scroll position and intercept clicks on palette toggle
-      document.addEventListener('click', function(e) {
-        const paletteLabel = e.target.closest('[for^="__palette"]');
-        if (paletteLabel) {
-          savedScrollY = window.scrollY;
-          isTogglingPalette = true;
-
-          // Prevent any scrolling for a brief moment
-          window.addEventListener('scroll', preventScroll, { passive: false });
-
-          setTimeout(function() {
-            isTogglingPalette = false;
-            window.removeEventListener('scroll', preventScroll);
-          }, 100);
-        }
-      }, true);
-
-      // Listen for hash changes from palette toggle
-      window.addEventListener('hashchange', function(e) {
-        if (window.location.hash.startsWith('#__palette')) {
-          e.preventDefault();
-          // Immediately restore scroll position
-          window.scrollTo(0, savedScrollY);
-          // Clean up hash without triggering another hashchange
-          history.replaceState(null, null, window.location.pathname);
-        }
-      }, false);
-
-      // Also handle palette input changes directly
-      document.addEventListener('DOMContentLoaded', function() {
-        const paletteInputs = document.querySelectorAll('input[name="__palette"]');
-        paletteInputs.forEach(function(input) {
-          input.addEventListener('change', function() {
-            window.scrollTo(0, savedScrollY);
-          });
-        });
-      });
-    })();
-  </script>
-
-  <!-- Hero Section -->
-  <section class="mdx-container">
-    <div class="container">
-      <div class="intro-message">
-        <div class="hero-logo">
-          <img src="logo/white.png" alt="Lance Logo">
-          <h1>Lance<sup>™</sup></h1>
-        </div>
-        <h3>The Open Lakehouse Format for Multimodal AI</h3>
-        <hr class="intro-divider" />
-        <ul class="list-inline">
-          <li>
-            <a href="quickstart" class="md-button md-button--primary">Get Started</a>
-          </li>
-          <li>
-            <a href="format" class="md-button">Read the Spec</a>
-          </li>
-          <li>
-            <a href="examples/python/llm_training" class="md-button">Train an LLM</a>
-          </li>
-          <li>
-            <a href="https://discord.gg/lance" class="md-button" target="_blank" rel="noopener">Join Discord</a>
-          </li>
-        </ul>
-      </div>
-    </div>
-  </section>
-
-  <!-- What is Lance Section -->
-  <section class="lance-intro-section">
-    <div class="container">
-      <div class="lance-intro-content">
-        <h2>What is Lance?</h2>
-        <p>
-          Lance is a modern, open source lakehouse format for multimodal AI. It contains a file format, table format, and catalog spec,
-          allowing you to build a complete open lakehouse on top of object storage to power your AI workflows.
-          Lance brings high-performance vector search, full-text search, random access, and feature
-          engineering capabilities to the lakehouse, while you can still get all the existing lakehouse benefits
-          like SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB, etc.)
-          and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore, etc.)
-        </p>
-        <p>
-          Learn more about Lance's technical details by reading our 
-          <a href="https://arxiv.org/abs/2504.15247" class="lance-paper-link" target="_blank" rel="noopener">research paper</a>
-          published at <em>VLDB 2025</em>.
-        </p>
-        <a href="quickstart" class="md-button md-button--primary">Read the Docs</a>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 1: Expressive Hybrid Search -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Expressive Hybrid Search</h2>
-          <p>
-            Lance enables powerful hybrid search combining vector similarity, full-text search,
-            and SQL analytics on the same dataset. All query types are accelerated by corresponding
-            secondary indexes as part of the Lance specification.
-          </p>
-          <p>
-            Run semantic search on embeddings, BM25 search on keywords, and apply complex SQL predicates -
-            all using a single table with a unified interface.
-          </p>
-          <a href="quickstart/vector-search" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/hybrid-search.png" alt="Hybrid Search Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 2: Lightning-fast Random Access -->
-  <section class="lance-feature-section reverse">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Lightning-fast Random Access</h2>
-          <p>
-            Lance delivers 100x faster random access compared to Parquet or Iceberg.
-            Unlike traditional formats, Lance maintains high performance even when
-            randomly accessing scattered rows across your entire dataset.
-          </p>
-          <p>
-            With a highly optimized file format plus efficient row-addressing and secondary indexes at table level,
-            you can access individual records across multiple files instantly,
-            making it perfect for real-time ML serving, random sampling, and interactive applications.
-          </p>
-          <a href="guide/read_and_write#random-access" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/random-access.png" alt="Random Access Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 3: Native Multimodal Data Support -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Native Multimodal Data Support</h2>
-          <p>
-            Store images, videos, audio, text, and embeddings alongside your traditional tabular data in a single unified format.
-            Lance's blob encoding efficiently handles large binary objects with lazy loading,
-            while optimized vector storage accelerates similarity search.
-          </p>
-          <p>
-            Perfect for AI/ML workloads where you need to store raw data, ML features, generated captions and embeddings
-            all together for multimodal retrieval and genAI workflows.
-          </p>
-          <a href="guide/blob" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/multimodal-data.png" alt="Multimodal Data Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 4: Data Evolution -->
-  <section class="lance-feature-section reverse">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Data Evolution > Schema Evolution</h2>
-          <p>
-            Schema evolution in most open table formats are metadata only and fast.
-            But when trying to backfill column values in existing rows, a full table rewrite is typically required.
-            Lance supports data evolution (efficient schema evolution with backfill), making it perfect for ML
-            feature engineering, embedding and media content management.
-          </p>
-          <p>
-            Adding a new column with data is as simple as writing new Lance files to the Lance table -
-            no need to rewrite your entire dataset.
-          </p>
-          <a href="guide/data_evolution" class="md-button">Learn More</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/data-evolution.png" alt="Data Evolution Example" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-  <!-- Feature 5: Rich Ecosystem Integration -->
-  <section class="lance-feature-section">
-    <div class="container">
-      <div class="lance-feature-content">
-        <div class="lance-feature-text">
-          <h2>Rich Ecosystem Integrations</h2>
-          <p>
-            As an open format, Lance integrates seamlessly with the Python data ecosystem and modern data platforms.
-            Work with your favorite tools including Pandas, Polars, Ray and PyTorch for data processing and machine learning.
-          </p>
-          <p>
-            Connect with leading query engines like Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink/Fluss
-            to run SQL analytics and distributed processing on your Lance datasets.
-          </p>
-          <a href="integrations/" class="md-button">View Integrations</a>
-        </div>
-        <div class="lance-feature-demo">
-          <img src="assets/images/ecosystem-integrations.png" alt="Lance Ecosystem Integrations" style="max-width: 500px; width: 100%; height: auto; border-radius: 8px;">
-        </div>
-      </div>
-    </div>
-  </section>
-
-
-{% endblock %}
-
-{% block content %}{% endblock %}
-{% block footer %}
-  {{ super() }}
-{% endblock %}
diff --git a/docs/pyproject.toml b/docs/pyproject.toml
index 4112230aec5..3aa6847e4f3 100644
--- a/docs/pyproject.toml
+++ b/docs/pyproject.toml
@@ -6,10 +6,12 @@ readme = "README.md"
 requires-python = ">=3.10,<3.11"
 dependencies = [
     "mkdocs>=1.5.0",
-    "mkdocs-material>=9.4.0",
+    "pymdown-extensions>=10.0",
+    "pygments>=2.16",
     "mkdocs-protobuf>=0.1.0",
     "mkdocs-linkcheck>=1.0.0",
-    "mkdocs-awesome-pages-plugin>=2.10.1"
+    "mkdocs-awesome-pages-plugin>=2.10.1",
+    "requests>=2.31.0"  # mkdocs-linkcheck imports it but doesn't declare it
 ]
 
 [tool.uv]
diff --git a/docs/src/assets/javascripts/nav-expand.js b/docs/src/assets/javascripts/nav-expand.js
deleted file mode 100644
index 17171f5a295..00000000000
--- a/docs/src/assets/javascripts/nav-expand.js
+++ /dev/null
@@ -1,16 +0,0 @@
-// Auto-expand sidebar navigation to 2 levels on page load.
-// Level 1 sections are already expanded by navigation.sections.
-// This expands level 2 (e.g. Operations, Models become visible)
-// but leaves level 3+ collapsed.
-document.addEventListener("DOMContentLoaded", function () {
-  // In mkdocs-material with navigation.sections, the top-level items
-  // are rendered as non-collapsible sections. The collapsible items
-  // start at the next level. We want to expand one more level.
-  //
-  // Selector: inside the primary nav, find toggle checkboxes that are
-  // exactly 2 nesting levels deep (the second-level sections).
-  var toggles = document.querySelectorAll(
-    ".md-sidebar--primary .md-nav--primary > .md-nav__list > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item > .md-nav__toggle"
-  );
-  toggles.forEach(function (t) { t.checked = true; });
-});
diff --git a/docs/src/assets/stylesheets/home.css b/docs/src/assets/stylesheets/home.css
deleted file mode 100644
index eafdd5afbf9..00000000000
--- a/docs/src/assets/stylesheets/home.css
+++ /dev/null
@@ -1,244 +0,0 @@
-/* Lance Homepage Styles */
-
-/* Override with custom color #625EFF site-wide */
-:root > * {
-  --md-primary-fg-color: #625EFF;
-  --md-primary-fg-color--light: #8481FF;
-  --md-primary-fg-color--dark: #4A46CC;
-  --md-accent-fg-color: #625EFF;
-  --md-accent-fg-color--transparent: rgba(98, 94, 255, 0.1);
-}
-
-* {
-  box-sizing: border-box;
-}
-
-.container {
-  width: 100%;
-  max-width: 1140px;
-  margin-right: auto;
-  margin-left: auto;
-  padding-right: 15px;
-  padding-left: 15px;
-}
-
-/* Hero Section - Fullscreen with Background Image */
-.mdx-container {
-  text-align: center;
-  color: #f8f8f8;
-  background: url("../images/lance-mj.png") no-repeat center center;
-  background-size: cover;
-  min-height: 100vh;
-  height: 100vh;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-}
-
-.intro-message {
-  position: relative;
-  padding: 40px 20px;
-  font-family: "Lato", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
-  max-width: 1000px;
-  margin: 0 auto;
-}
-
-.hero-logo {
-  display: inline-flex;
-  align-items: center;
-  margin-bottom: 16px;
-}
-
-.hero-logo img {
-  height: 120px;
-  width: auto;
-  margin-right: 24px;
-  margin-top: 12px;
-  filter: drop-shadow(3px 3px 8px rgba(0, 0, 0, 0.9));
-}
-
-.intro-message h1 {
-  font-weight: 400;
-  margin: 0;
-  display: inline-block;
-  text-shadow: 3px 3px 8px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1);
-  font-size: 8em;
-  line-height: 1.2;
-  color: #ffffff;
-  vertical-align: middle;
-}
-
-.intro-message h1 sup {
-  font-size: 2rem;
-  text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9);
-}
-
-.intro-message h3 {
-  font-size: 1.1rem;
-  text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1);
-  font-weight: 600;
-  margin-bottom: 32px;
-  color: #ffffff;
-}
-
-.intro-divider {
-  width: 400px;
-  max-width: 80%;
-  border-top: 1px solid rgba(255, 255, 255, 0.8);
-  border-bottom: 1px solid rgba(0, 0, 0, 0.2);
-  margin: 24px auto;
-}
-
-.list-inline {
-  padding-left: 0;
-  margin-left: -5px;
-  list-style: none;
-  margin-bottom: 0;
-}
-
-.list-inline li {
-  display: inline-block;
-  padding-right: 5px;
-  padding-left: 5px;
-}
-
-.intro-message .md-button {
-  margin: 8px;
-  padding: 14px 36px;
-  font-size: 1.1rem;
-  font-weight: 600;
-  text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.8);
-  transition: all 0.3s ease;
-}
-
-.intro-message .md-button:hover {
-  transform: translateY(-2px);
-  box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3);
-}
-
-.intro-message .md-button--primary:hover {
-  box-shadow: 0 8px 20px rgba(98, 94, 255, 0.5);
-}
-
-.intro-message .md-button:not(.md-button--primary):hover {
-  box-shadow: 0 8px 20px rgba(255, 255, 255, 0.4);
-}
-
-/* What is Lance Section */
-.lance-intro-section {
-  padding: 80px 0;
-  background-color: rgba(128, 128, 128, 0.03);
-  border-bottom: 1px solid rgba(128, 128, 128, 0.1);
-}
-
-.lance-intro-content {
-  max-width: 900px;
-  margin: 0 auto;
-  text-align: center;
-}
-
-.lance-intro-content h2 {
-  font-size: 36px;
-  font-weight: 500;
-  margin-bottom: 32px;
-  color: var(--md-primary-fg-color);
-}
-
-.lance-intro-content p {
-  font-size: 16px;
-  line-height: 1.8;
-  margin-bottom: 32px;
-  opacity: 0.9;
-  text-align: left;
-}
-
-.lance-paper-link {
-  color: var(--md-primary-fg-color);
-  text-decoration: none;
-}
-
-.lance-paper-link:hover {
-  color: var(--md-primary-fg-color);
-  text-decoration: none;
-}
-
-.lance-intro-content a:hover {
-  color: #757575;
-  text-decoration: none;
-}
-
-.lance-intro-content .md-button {
-  margin-top: 16px;
-  padding: 10px 28px;
-  font-size: 14px;
-  border: 2px solid currentColor;
-  background-color: transparent;
-  transition: all 0.3s ease;
-}
-
-.lance-intro-content .md-button:hover {
-  color: var(--md-primary-fg-color);
-  background-color: transparent;
-}
-
-/* Feature Sections */
-.lance-feature-section {
-  padding: 80px 0;
-  border-bottom: 1px solid rgba(128, 128, 128, 0.1);
-}
-
-.lance-feature-section:last-child {
-  border-bottom: none;
-}
-
-.lance-feature-content {
-  display: flex;
-  flex-wrap: wrap;
-  align-items: center;
-  gap: 60px;
-}
-
-.lance-feature-text {
-  flex: 1;
-  min-width: 300px;
-}
-
-.lance-feature-text h2 {
-  font-size: 30px;
-  font-weight: 500;
-  margin-bottom: 16px;
-  color: var(--md-primary-fg-color);
-}
-
-.lance-feature-text p {
-  font-size: 15px;
-  line-height: 1.6;
-  opacity: 0.85;
-  margin-bottom: 16px;
-}
-
-.lance-feature-text .md-button {
-  font-size: 0.6rem;
-  padding: 0;
-  transition: all 0.3s ease;
-}
-
-.lance-feature-text .md-button:hover {
-  transform: translateX(4px);
-  color: var(--md-primary-fg-color);
-}
-
-.lance-feature-demo {
-  flex: 1;
-  min-width: 400px;
-  display: flex;
-  justify-content: center;
-  overflow: hidden;
-}
-
-/* Alternating layout */
-.lance-feature-section.reverse .lance-feature-content {
-  flex-direction: row-reverse;
-}
-
-
diff --git a/docs/src/community/maintainers.md b/docs/src/community/maintainers.md
index 755201f069e..f3ba6e70304 100644
--- a/docs/src/community/maintainers.md
+++ b/docs/src/community/maintainers.md
@@ -40,38 +40,38 @@ Maintainers with GitHub write access are additionally encouraged to:
 
 ## Roster
 
-| Name                   | GitHub Handle        | Affiliation       | GitHub Write Access | Ecosystem Roles                                 |
-|------------------------|----------------------|-------------------|---------------------|-------------------------------------------------|
-| Wyatt Alt              | wkalt                | LanceDB           | ✓                   |                                                 |
-| Matt Basta             | mattbasta            | Runway AI         |                     |                                                 |
-| Giuseppe Battista      | giusedroid           | AWS               |                     |                                                 |
-| Timothy Carambat       | timothycarambat      | Anything LLM      |                     |                                                 |
-| Ayush Chaurasia        | AyushExel            | LanceDB           |                     |
-| Chongchen Chen         | chenkovsky           | MiraclePlus       |                     |                                                 |
-| Akela Drissner-Schmid  | akelad               | dltHub            |                     |                                                 |
-| Ty Dunn                | TyDunn               | Continue          |                     |                                                 |
-| Enwei Jiao             | jiaoew1991           | Luma.ai           | ✓                   | Milvus Maintainer                               |
-| Bryan Keller           | bryanck              | Netflix           |                     | Apache Iceberg Committer                        |
-| Aman Kishore           | AmanKishore          | Harvey.ai         |                     |                                                 |
-| Sangwu Lee             | RE-N-Y               | Krea.ai           |                     |                                                 |
-| Jeremy Leibs           | jleibs               | Rerun.io          |                     |                                                 |
-| Haocheng Liu           | HaochengLIU          | Seven Research    | ✓                   |                                                 |
-| Nathan Ma              | majin1102            | ByteDance         | ✓                   | Apache Amoro (incubating) PPMC Member           |
-| ChanChan Mao           | ccmao1130            | LanceDB           |                     |                                                 |
-| Lu Qiu                 | LuQQiu               | LanceDB           | ✓                   | Alluxio PMC Member                              |
-| Dan Rammer             | hamersaw             | LanceDB           | ✓                   |                                                 |
-| Rong Rong              | walterddr            | Google DeepMind   |                     | Apache Pinot PMC Member, Apache Flink Committer |
-| Nat Roth               | nrothGIT             | Meta AI           |                     |                                                 |
-| Kevin Shaffer-Morrison | kevinshaffermorrison | AWS               |                     |                                                 |
-| Noah Shpak             | noahshpak            | Thinking Machines |                     |                                                 |
-| Chunxu Tang            | ChunxuTang           | Google            |                     | PrestoDB Committer                              |
-| Ankit Vij              | ankitvij-db          | Databricks        |                     |                                                 |
-| Beinan Wang            | beinan               | Microsoft AI      |                     | Alluxio PMC Member, Presto TSC Member           |
-| Jiacheng Yang          | jiachengdb           | Google AI         |                     |                                                 |
-| Yang Jie               | LuciferYang          | Baidu Inc.        |                     | Apache Spark PMC Member, Apache Uniffle PMC Member |
-| Jianjian Xie           | jja725               | Uber              |                     |                                                 |
-| Zhang Yue              | zhangyue19921010     | ByteDance         |                     |                                                 |
-| Jinglun                | wojiaodoubao         | ByteDance         |                     | Apache Hadoop Committer                         |
+| Name                   | GitHub Handle        | Affiliation       | GitHub Write Access | Ecosystem Roles                                    |
+|------------------------|----------------------|-------------------|---------------------|----------------------------------------------------|
+| Wyatt Alt              | wkalt                | LanceDB           | ✓                   |                                                    |
+| Matt Basta             | mattbasta            | Runway AI         |                     |                                                    |
+| Giuseppe Battista      | giusedroid           | AWS               |                     |                                                    |
+| Timothy Carambat       | timothycarambat      | Anything LLM      |                     |                                                    |
+| Ayush Chaurasia        | AyushExel            | LanceDB           |                     |                                                    |
+| Chongchen Chen         | chenkovsky           | MiraclePlus       |                     |                                                    |
+| Akela Drissner-Schmid  | akelad               | dltHub            |                     |                                                    |
+| Ty Dunn                | TyDunn               | Continue          |                     |                                                    |
+| Enwei Jiao             | jiaoew1991           | Luma.ai           | ✓                   | Milvus Maintainer                                  |
+| Bryan Keller           | bryanck              | Netflix           |                     | Apache Iceberg Committer                           |
+| Aman Kishore           | AmanKishore          | Harvey.ai         |                     |                                                    |
+| Sangwu Lee             | RE-N-Y               | Krea.ai           |                     |                                                    |
+| Jeremy Leibs           | jleibs               | Rerun.io          |                     |                                                    |
+| Haocheng Liu           | HaochengLIU          | Seven Research    | ✓                   |                                                    |
+| Nathan Ma              | majin1102            | ByteDance         | ✓                   | Apache Amoro (incubating) PPMC Member              |
+| ChanChan Mao           | ccmao1130            | LanceDB           |                     |                                                    |
+| Lu Qiu                 | LuQQiu               | LanceDB           | ✓                   | Alluxio PMC Member                                 |
+| Dan Rammer             | hamersaw             | LanceDB           | ✓                   |                                                    |
+| Rong Rong              | walterddr            | Google DeepMind   |                     | Apache Pinot PMC Member, Apache Flink Committer    |
+| Nat Roth               | nrothGIT             | Meta AI           |                     |                                                    |
+| Kevin Shaffer-Morrison | kevinshaffermorrison | AWS               |                     |                                                    |
+| Noah Shpak             | noahshpak            | Thinking Machines |                     |                                                    |
+| Chunxu Tang            | ChunxuTang           | Google            |                     | PrestoDB Committer                                 |
+| Ankit Vij              | ankitvij-db          | Databricks        |                     |                                                    |
+| Beinan Wang            | beinan               | Microsoft AI      |                     | Alluxio PMC Member, Presto TSC Member              |
+| Jiacheng Yang          | jiachengdb           | Google AI         |                     |                                                    |
+| Yang Jie               | LuciferYang          | Baidu Inc.        |                     | Apache Spark PMC Member, Apache Uniffle PMC Member | 
+| Jianjian Xie           | jja725               | Uber              |                     |                                                    |
+| Zhang Yue              | zhangyue19921010     | ByteDance         |                     | Apache Hudi PMC, Apache Druid Committer, Kafka Contributor |
+| Jinglun                | wojiaodoubao         | ByteDance         |                     | Apache Hadoop Committer                            |
 
 ## Becoming a Maintainer
 
diff --git a/docs/src/format/file/encoding.md b/docs/src/format/file/encoding.md
index a3d99ef39cb..4ca053d4fa6 100644
--- a/docs/src/format/file/encoding.md
+++ b/docs/src/format/file/encoding.md
@@ -683,9 +683,10 @@ the default mini-block size is negligible. You should only consider changing thi
 confirmed — through profiling — that mini-block read amplification is saturating your available bandwidth
 (for example, accessing a remote object store over a constrained network link).
 
-The maximum number of values per mini-block can be lowered via an environment variable:
+The maximum number of values per mini-block can be tuned via an environment variable:
 
-- `LANCE_MINIBLOCK_MAX_VALUES` (default `4096`): upper bound on the number of values in a single mini-block chunk.
+- `LANCE_MINIBLOCK_MAX_VALUES` (default `4096`, maximum `32768`): upper bound on the number of values in a single mini-block chunk.
 
 Reducing this value produces smaller mini-blocks, which reduces the amount of data fetched per read at the
-cost of more mini-blocks and slightly more metadata overhead.
+cost of more mini-blocks and slightly more metadata overhead. Increasing it can reduce metadata overhead and
+improve throughput for highly compressible data, but it may increase random-read amplification.
diff --git a/docs/src/format/file/versioning.md b/docs/src/format/file/versioning.md
index 67684c95be9..2add82ef2fa 100644
--- a/docs/src/format/file/versioning.md
+++ b/docs/src/format/file/versioning.md
@@ -11,7 +11,7 @@ no longer readable by any newer versions of Lance. The `next` version should onl
 benchmarking upcoming features.
 
 The `stable` and `next` aliases are resolved by the specific Lance release you are using. During a format rollout
-(for example, 2.2), prefer explicit version pinning for deterministic behavior across environments.
+(for example, 2.3), prefer explicit version pinning for deterministic behavior across environments.
 
 The following values are supported:
 
@@ -20,7 +20,8 @@ The following values are supported:
 | 0.1            | Any                   | 0.34 (write)          | This is the initial Lance format. It is no longer writable. |
 | 2.0            | 0.16.0                | Any                   | Rework of the Lance file format that removed row groups and introduced null support for lists, fixed size lists, and primitives |
 | 2.1            | 0.38.1                | Any                   | Enhances integer and string compression, adds support for nulls in struct fields, and improves random access performance with nested fields. |
-| 2.2 (unstable) | None                  | Any                   | Adds support for newer nested type/encoding capabilities (including map support) and 2.2-era storage features. |
+| 2.2            | None                  | Any                   | Adds support for newer nested type/encoding capabilities (including map support) and 2.2-era storage features. |
+| 2.3 (unstable) | None                  | Any                   | Adds experimental encodings for upcoming features. |
 | legacy         | N/A                   | N/A                   | Alias for 0.1 |
-| stable         | N/A                   | N/A                   | Alias for the latest stable version in the Lance release you are running. |
+| stable         | N/A                   | N/A                   | Alias for the default version for new datasets in the Lance release you are running. |
 | next           | N/A                   | N/A                   | Alias for the latest unstable version in the Lance release you are running.|
diff --git a/docs/src/format/index.md b/docs/src/format/index.md
index c6956453b05..055c9129c5f 100644
--- a/docs/src/format/index.md
+++ b/docs/src/format/index.md
@@ -1,44 +1,44 @@
-# Lance Lakehouse Format Specification
+# Lance Lakehouse Format Specifications
 
-Lance is a lakehouse format designed as a stack of interoperating specifications instead of a single file or metadata layout. The storage-facing layers are the file format, table format, index formats, and catalog specifications, with a unified namespace interface sitting above them.
+Lance is a lakehouse format defined as a stack of interoperating specifications, rather than as a single file format or metadata layout. The storage-facing layers cover files, tables, indices, and catalogs. A unified namespace interface sits above those layers and gives engines a consistent way to work with Lance tables across catalog implementations.
 
 ## Architecture Overview
 
-Modern lakehouses are built from cooperating layers. Lance keeps those layers intentionally decoupled so that the file format, table metadata, indices, and catalogs can evolve independently without forcing lock-in across the stack.
+Modern lakehouses are built from complementary layers. Lance keeps those layers intentionally decoupled so that the file format, table metadata, indices, and catalogs can evolve independently without forcing lock-in across the stack.
 
 ![Lakehouse Stack](../images/lakehouse_stack.png)
 
 At a high level:
 
-- The **file format** stores column data in large random-access-friendly pages and avoids row groups.
+- The **file format** stores column data in large pages optimized for random access and avoids row groups.
 - The **table format** manages fragments, manifests, deletions, schema evolution, and ACID commits.
 - The **index formats** define redundant search structures such as scalar, vector, full-text, and system indices.
 - The **catalog specs** define how tables are discovered, registered, and coordinated across engines and services.
-- The **namespace client spec** provides a unified interface for engines to interact with any catalog implementation.
+- The **namespace client spec** provides a unified interface for engines to interact with any catalog implementations.
 
-The layers are designed so that only table readers, table writers, and index readers or writers need to know the on-disk Lance file layout.
+The layers are designed so that only table readers, table writers, and index readers or writers need to understand the on-disk Lance file layout.
 
 ## Design Themes
 
 ### File Format
 
-The Lance file format is optimized for cloud object storage and highly selective reads. It avoids Parquet-style row groups, uses structural encodings that support efficient random access, and keeps statistics and search structures out of the file format so those concerns can evolve as independent indices.
+The Lance file format is optimized for cloud object storage and highly selective reads. It avoids Parquet-style row groups, uses structural encodings for efficient random access, and keeps statistics and search structures out of the file format so those concerns can evolve independently as indices.
 
 ### Table Format
 
-The Lance table format stores data in two dimensions: rows are grouped into fragments, and each fragment can contain multiple data files that each contribute a subset of columns. This makes column additions and backfills metadata-heavy instead of rewrite-heavy, which is especially useful for feature engineering and embedding workflows.
+The Lance table format organizes data in two dimensions: rows are grouped into fragments, and each fragment can contain multiple data files, each contributing a subset of columns. This makes column additions and backfills primarily metadata operations instead of data rewrites, which is especially useful for feature engineering and embedding workflows.
 
 ### Index Formats
 
-Indices are first-class table objects. Lance tables define how indices are discovered, versioned, and coordinated transactionally, while the index formats themselves remain decoupled from both the file encoding and the table manifest structure.
+Indices are first-class table objects. Lance tables define how indices are discovered, versioned, and coordinated transactionally. The index formats themselves remain decoupled from both the file encoding and the table manifest structure.
 
 ### Catalog Specs
 
-Lance provides storage-native and service-oriented catalog options. The [Directory Catalog](catalog/dir/index.md) supports zero-infrastructure deployments directly on object stores, while the [REST Catalog](catalog/rest/index.md) standardizes enterprise-facing APIs and can act as an external manifest store.
+Lance provides both storage-native and service-oriented catalog options. The [Directory Catalog](catalog/dir/index.md) supports zero-infrastructure deployments directly on object stores, while the [REST Catalog](catalog/rest/index.md) standardizes enterprise-facing APIs and can act as an external manifest store.
 
 ### Namespace Client Spec
 
-The [Namespace Client Spec](namespace/index.md) provides a unified interface for engines to interact with any catalog implementation, across both Lance native catalog specs and third-party catalog systems, in any programming language. This abstraction allows applications to switch between directory-based, REST-based, or third-party catalogs without changing their code.
+The [Namespace Client Spec](namespace/index.md) provides a language-agnostic interface for engines to interact with any catalog implementation, including Lance-native catalogs and third-party catalog systems. This abstraction allows applications to switch between directory-based, REST-based, and third-party catalogs without changing their code.
 
 ## Specifications
 
diff --git a/docs/src/format/index/index.md b/docs/src/format/index/index.md
index 4a2e33c60a4..af8fdf595b4 100644
--- a/docs/src/format/index/index.md
+++ b/docs/src/format/index/index.md
@@ -177,7 +177,7 @@ or updated. These should be filtered out during query execution.
   </figcaption>
 </figure>
 
-There are three situations to consider:
+There are four situations to consider:
 
 1. **A fragment has some deleted rows.** A few of the rows in the fragment have been marked
    as deleted, but some of the rows are still present. The row addresses from the deletion
@@ -188,6 +188,17 @@ There are three situations to consider:
 3. **A fragment has had the indexed column updated in place.** This cannot be detected just
    by examining metadata. To prevent reading invalid data, the engine should filter out any
    row addresses that are not in the index's current `fragment_bitmap`.
+4. **A fragment has an updated value in an [overlay file](../table/data_overlay_file.md).**
+   This can be detected by checking if any of the fragments in the index's `fragment_bitmap`
+   have overlay files. For each overlay whose `committed_version` is greater than the index
+   segment's `dataset_version`, the overlay carries updated values not reflected in the index,
+   so its covered rows must be excluded from index results. Excluded rows are re-evaluated
+   against their current (overlaid) values on the flat path — dropping them without
+   re-evaluation would silently lose rows that match under the new value. Exclusion is
+   field-aware: only overlays covering the indexed field matter. You may exclude just the
+   affected rows or the whole fragment; the latter is simpler and safer but re-evaluates more
+   rows than necessary. See [Data Overlay Files](../table/data_overlay_file.md#index-integration)
+   for the exclusion set, re-evaluation, and correctness invariant.
 
 ## Compaction and remapping
 
diff --git a/docs/src/format/index/scalar/bloom_filter.md b/docs/src/format/index/scalar/bloom_filter.md
index 5a0e08e4228..6d924f8705d 100644
--- a/docs/src/format/index/scalar/bloom_filter.md
+++ b/docs/src/format/index/scalar/bloom_filter.md
@@ -4,6 +4,9 @@ Bloom filters are probabilistic data structures that allow for fast membership t
 They are space-efficient and can test whether an element is a member of a set.
 It's an inexact filter - they may include false positives but never false negatives.
 
+In addition, since finding NULLs is a common query pattern, the index also maintains a
+bitmap of null rows which allows it to return exact results for IS NULL queries.
+
 ## Index Details
 
 ```protobuf
@@ -32,6 +35,13 @@ The bloom filter index stores zone-based bloom filters in a single file:
 |---------------------------|--------|-------------------------------------------------------------|
 | `bloomfilter_item`        | String | Expected number of items per zone (default: "8192")         |
 | `bloomfilter_probability` | String | False positive probability (default: "0.00057", ~1 in 1754) |
+| `null_bitmap`             | UInt32 | Index of null bitmap global buffer                          |
+
+### Global Buffers
+
+| Metadata Key        | Description                                                |
+|---------------------|------------------------------------------------------------|
+| `null_bitmap`       | A serialized RowAddrTreeMap specifying which rows are null |
 
 ## Bloom Filter Spec
 
@@ -122,10 +132,11 @@ Offset 60-63:  Block 1, Word 7 (32-bit LE)
 
 ## Accelerated Queries
 
-The bloom filter index provides inexact results for the following query types:
+The bloom filter index provides inexact results for the following query types (nullability queries
+return exact results):
 
 | Query Type | Description               | Operation                                 | Result Type |
 |------------|---------------------------|-------------------------------------------|-------------|
 | **Equals** | `column = value`          | Tests if value exists in bloom filter     | AtMost      |
 | **IsIn**   | `column IN (v1, v2, ...)` | Tests if any value exists in bloom filter | AtMost      |
-| **IsNull** | `column IS NULL`          | Returns zones where has_null is true      | AtMost      |
\ No newline at end of file
+| **IsNull** | `column IS NULL`          | Returns zones where has_null is true      | Exact       |
diff --git a/docs/src/format/index/scalar/fmindex.md b/docs/src/format/index/scalar/fmindex.md
new file mode 100644
index 00000000000..1138024896f
--- /dev/null
+++ b/docs/src/format/index/scalar/fmindex.md
@@ -0,0 +1,64 @@
+# FM-Index (Full-text / Substring / Regex Search)
+
+The FM-Index (Ferragina-Manzini Index) is a compressed substring index based on the Burrows-Wheeler Transform (BWT). Unlike traditional inverted indexes (Full-Text Search) which index distinct words, the FM-Index enables efficient **arbitrary substring search**, **prefix match**, and **suffix/regular-expression search** directly on raw bytes.
+
+In Lance, the FM-Index is designed to scale dynamically across millions of documents or large-scale datasets, and is partitioned using Lance's **Segmented Index** architecture to support incremental appends, disjoint fragment tracking, and segment merging.
+
+## High-Level Architecture
+
+The FM-Index indexes raw text by treating columns of strings or binary payloads as raw byte arrays. 
+
+```
+                     +----------------------------------------+
+                     |            Lance Dataset               |
+                     |   (Disjoint groups of Fragments 0..N)   |
+                     +----------------------------------------+
+                                         |
+                       Divide fragments into num_segments
+                                         |
+                                         v
+                     +----------------------------------------+
+                     |            Segmented Index             |
+                     |  +-----------+ +-----------+ +-------+ |
+                     |  | Segment 1 | | Segment 2 | | ...   | |
+                     |  |  (FM-Idx) | |  (FM-Idx) | |       | |
+                     |  +-----------+ +-----------+ +-------+ |
+                     +----------------------------------------+
+```
+
+Each segment contains its own self-contained physical FM-Index mapping byte sub-sequences to Lance global row IDs.
+
+## Data Normalization & Sanitization
+
+The FM-Index is **normalization-independent by design** because it operates entirely on raw bytes. 
+
+### Byte Sanitization vs. Text Normalization
+
+1. **Byte Sanitization (Core Index Layer)**:
+   The physical FM-Index uses specific sentinel bytes internally to mark boundaries:
+   - `\x00` is reserved as the global Burrows-Wheeler Transform (BWT) terminator character.
+   - `\xFF` is reserved as the document/row separator character.
+   
+   To avoid breaking the indexing structures, any incoming occurrences of `\x00` or `\xFF` are sanitized by remapping them to space (`\x20`) characters at index-build time. No other bytes are changed in this layer.
+
+2. **Text Normalization (User/Application Layer)**:
+   Because the index faithfully maps raw bytes, any semantic normalization (such as case folding `Hello` -> `hello`, Unicode NFKC normalization, stemming, or whitespace collapsing) is fully decoupled from the core index engine:
+   - To build a case-insensitive search index, users apply a lowercase transform to the column *prior* to indexing.
+   - When querying, the user's query text must undergo the exact same normalization pipeline.
+
+## Configurable Segment Partitioning
+
+Merging or appending to BWT-based indexes cannot be done via simple concatenation; the BWT suffix array must be reconstructed by re-reading the text and rebuilding. To balance build cost and search performance, Lance allows configuring how fragments map to index segments.
+
+- **`num_segments` parameter**: Configured at index-creation time. If `num_segments` is specified (e.g. `num_segments = 4`), Lance splits the target dataset fragments into disjoint subsets and builds independent FM-Index segments over each chunk.
+- **Unindexed Appends**: When new fragments are appended to the dataset, a subsequent `create_index` execution with unindexed fragment coverage will construct a new separate segment representing only those new fragments, keeping existing segments fully intact.
+- **Segment Merging**: Multiple existing index segments can be merged into a single segment under Lance's `merge_segments` protocol. Lance unions the fragment coverage bitmaps of the selected segments, re-reads the raw text from those covered fragments, and constructs a fresh unified FM-Index.
+
+## Query Evaluation
+
+When a substring query is submitted (e.g., `CONTAINS(column, "query_string")`):
+1. The search string is sanitized (remapping any `\x00` or `\xFF` to spaces) and optionally normalized if the target index is normalized.
+2. The query is dispatched across all active segments in the logical index in parallel.
+3. Each segment performs a BWT backward-search to locate occurrences of the pattern.
+4. Matching offsets are mapped back to absolute dataset Row IDs.
+5. Results from all segments are unioned to produce the final selection.
diff --git a/docs/src/format/index/scalar/fts.md b/docs/src/format/index/scalar/fts.md
index 792702aca08..d5c75158011 100644
--- a/docs/src/format/index/scalar/fts.md
+++ b/docs/src/format/index/scalar/fts.md
@@ -43,6 +43,8 @@ An FTS index may contain multiple partitions. Each partition has its own set of
 | `_length`              | UInt32                  | false    | Number of documents containing the token                         |
 | `_compressed_position` | List<List<LargeBinary>> | true     | Optional compressed position lists for phrase queries            |
 
+The posting-list file schema metadata includes `posting_block_size`, the number of documents encoded per compressed posting block. Older indexes that do not have this metadata use the legacy block size `128`.
+
 ### Metadata File Schema
 
 The metadata file contains JSON-serialized configuration and partition information:
@@ -67,6 +69,7 @@ The metadata file contains JSON-serialized configuration and partition informati
 | `min_gram`          | UInt32  | 2         | Minimum n-gram length (only for ngram tokenizer)               |
 | `max_gram`          | UInt32  | 15        | Maximum n-gram length (only for ngram tokenizer)               |
 | `prefix_only`       | Boolean | false     | Generate only prefix n-grams (only for ngram tokenizer)        |
+| `block_size`        | UInt32  | 128       | Documents per compressed posting block. Must be 128 or 256. Missing values from older indexes read as 128. `256` is experimental and may introduce breaking changes. |
 
 ## Tokenizers
 
@@ -81,6 +84,7 @@ The full text search index supports multiple tokenizer types for different text
 | **raw**        | No tokenization, treats entire text as single token                       | Exact matching         |
 | **ngram**      | Breaks text into overlapping character sequences                          | Substring/fuzzy search |
 | **icu**        | ICU dictionary-based Unicode word segmentation                            | Mixed-language text    |
+| **icu/split**  | ICU segmentation with simple-style delimiter splitting                    | Mixed-language identifiers |
 | **jieba/***    | Chinese text tokenizer with word segmentation                             | Chinese text           |
 | **lindera/***  | Japanese text tokenizer with morphological analysis                       | Japanese text          |
 
@@ -88,8 +92,10 @@ The full text search index supports multiple tokenizer types for different text
 
 The ICU tokenizer uses Unicode word boundary rules and dictionary-based segmentation for complex scripts. It is useful for mixed-language text where the default `simple` tokenizer would keep an unspaced CJK span as one large token.
 
+By default, Lance preserves ICU word segments as returned by ICU. Use `base_tokenizer: "icu/split"` to split ICU word segments again on non-alphanumeric delimiters such as underscores and punctuation. For example, `hello_world こんにちは世界` is tokenized as `hello`, `world`, `こんにちは`, and `世界`.
+
 - **Models**: Uses compiled ICU4X segmenter data bundled with Lance
-- **Usage**: Specify as `icu`
+- **Usage**: Specify as `icu`, or `icu/split` to split punctuation-delimited identifiers
 - **Features**:
   - Unicode-aware word boundary detection
   - Dictionary-based segmentation for Chinese, Japanese, Khmer, Lao, Myanmar, and Thai
diff --git a/docs/src/format/index/scalar/ngram.md b/docs/src/format/index/scalar/ngram.md
index bdf78474d50..d437363d264 100644
--- a/docs/src/format/index/scalar/ngram.md
+++ b/docs/src/format/index/scalar/ngram.md
@@ -29,4 +29,10 @@ The N-gram index provides inexact results for the following query types:
 
 | Query Type     | Description              | Operation                                             | Result Type |
 |----------------|--------------------------|-------------------------------------------------------|-------------|
-| **contains**   | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost      |
\ No newline at end of file
+| **contains**   | Substring search in text | Finds all trigrams in query, intersects posting lists | AtMost      |
+| **regexp_like** / **regexp_match** | Regular-expression match | Derives a necessary trigram condition from the pattern (AND of intersections, OR of unions), then rechecks the true regex | AtMost |
+| **LIKE** (infix) | Wildcard match such as `%foo%bar%` | Uses the literal segments of the pattern as a trigram condition, then rechecks the LIKE | AtMost |
+
+Patterns from which no trigram can be derived - for example `a.b`, `.*`,
+case-insensitive matches, or literal runs shorter than three characters - fall
+back to rechecking every row. This is always correct, just not accelerated.
diff --git a/docs/src/format/index/scalar/zonemap.md b/docs/src/format/index/scalar/zonemap.md
index 256edc2671d..552d476cc86 100644
--- a/docs/src/format/index/scalar/zonemap.md
+++ b/docs/src/format/index/scalar/zonemap.md
@@ -8,6 +8,9 @@ zones that cannot contain matching values.
 Zone maps are "inexact" filters - they can definitively exclude zones but may include
 false positives that require rechecking.
 
+In addition, since finding NULLs is a common query pattern, the index also maintains a
+bitmap of null rows which allows it to return exact results for IS NULL queries.
+
 ## Index Details
 
 ```protobuf
@@ -34,17 +37,25 @@ The zone map index stores zone statistics in a single file:
 
 ### Schema Metadata
 
-| Key             | Type   | Description                               |
-|-----------------|--------|-------------------------------------------|
-| `rows_per_zone` | String | Number of rows per zone (default: "8192") |
+| Key                 | Type   | Description                               |
+|---------------------|--------|-------------------------------------------|
+| `rows_per_zone`     | String | Number of rows per zone (default: "8192") |
+| `null_bitmap`       | UInt32 | Index of null bitmap global buffer        |
+
+### Global Buffers
+
+| Metadata Key        | Description                                                |
+|---------------------|------------------------------------------------------------|
+| `null_bitmap`       | A serialized RowAddrTreeMap specifying which rows are null |
 
 ## Accelerated Queries
 
-The zone map index provides inexact results for the following query types:
+The zone map index provides inexact results for the following query types (nullability queries
+return exact results):
 
 | Query Type | Description               | Operation                                   | Result Type |
 |------------|---------------------------|---------------------------------------------|-------------|
 | **Equals** | `column = value`          | Includes zones where min ≤ value ≤ max      | AtMost      |
 | **Range**  | `column BETWEEN a AND b`  | Includes zones where ranges overlap         | AtMost      |
 | **IsIn**   | `column IN (v1, v2, ...)` | Includes zones that could contain any value | AtMost      |
-| **IsNull** | `column IS NULL`          | Includes zones where null_count > 0         | AtMost      |
\ No newline at end of file
+| **IsNull** | `column IS NULL`          | Includes zones where null_count > 0         | Exact       |
diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md
index e565bd737f0..7aaf9b55996 100644
--- a/docs/src/format/index/vector/index.md
+++ b/docs/src/format/index/vector/index.md
@@ -192,12 +192,16 @@ Compresses vectors using scalar quantization for moderate memory savings:
 
 Compresses vectors using RabitQ with random rotation and binary quantization for extreme compression:
 
-| Column            | Type                       | Nullable | Description                                                     |
-| ----------------- | -------------------------- | -------- | --------------------------------------------------------------- |
-| `_rowid`          | uint64                     | false    | Row identifier                                                  |
-| `_rabit_codes`    | list<uint8>[dimension / 8] | false    | Binary quantized codes (1 bit per dimension, packed into bytes) |
-| `__add_factors`   | float32                    | false    | Additive correction factors for distance computation            |
-| `__scale_factors` | float32                    | false    | Scale correction factors for distance computation               |
+| Column               | Type                                             | Nullable                 | Description                                                     |
+| -------------------- | ------------------------------------------------ | ------------------------ | --------------------------------------------------------------- |
+| `_rowid`             | uint64                                           | false                    | Row identifier                                                  |
+| `_rabit_codes`       | list<uint8>[dimension / 8]                       | false                    | Binary quantized codes (1 bit per dimension, packed into bytes) |
+| `__add_factors`      | float32                                          | false                    | Additive correction factors for distance computation            |
+| `__scale_factors`    | float32                                          | false                    | Scale correction factors for distance computation               |
+| `__error_factors`    | float32                                          | false for `raw_query`    | Error factors for raw-query lower-bound pruning                 |
+| `__ex_codes`         | list<uint8>[ceil(dimension * (num_bits - 1) / 8)] | false for `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ                         |
+| `__add_factors_ex`   | float32                                          | false for `num_bits > 1` | Additive correction factors for ex-code distance computation    |
+| `__scale_factors_ex` | float32                                          | false for `num_bits > 1` | Scale correction factors for ex-code distance computation       |
 
 #### Arrow Schema Metadata
 
@@ -248,9 +252,10 @@ For **RabitQ (RQ)**:
 | JSON Key              | Type | Description                                          |
 | --------------------- | ---- | ---------------------------------------------------- |
 | `rotate_mat_position` | u32  | Position of the rotation matrix in the global buffer |
-| `num_bits`            | u8   | Number of bits per dimension (currently always 1)    |
+| `num_bits`            | u8   | Number of bits per dimension, in the range 1..=9     |
 | `code_dim`            | u32  | Rotated vector dimension for the 1-bit binary code   |
 | `packed`              | bool | Whether codes are packed for optimized computation   |
+| `query_estimator`     | string | Distance estimator layout: `residual_query` or `raw_query`. Missing values are read as `residual_query` for compatibility with released 1-bit IVF_RQ indexes. |
 
 #### Lance File Global Buffer
 
@@ -274,9 +279,11 @@ to rotate vectors before binary quantization:
 ```
 
 The rotation matrix has shape `[code_dim, code_dim]` where `code_dim` is the rotated vector
-dimension. Current IVF_RQ stores the 1-bit binary code in `_rabit_codes`; future multi-bit support
-will store the remaining `num_bits - 1` ex-code bits separately instead of widening this binary
-code path.
+dimension. IVF_RQ always stores the 1-bit binary sign code in `_rabit_codes`; for `num_bits > 1`,
+the remaining `num_bits - 1` ex-code bits are stored in `__ex_codes` instead of widening the
+binary code path. New IVF_RQ indexes store raw-query estimator factors. `num_bits=1` indexes only
+store the binary-code factor columns; multi-bit indexes also store separate ex-code additive and
+scale factors.
 
 ## Appendices
 
@@ -319,7 +326,8 @@ pa.schema([
 ### Appendix 2: Example IVF_RQ Format
 
 This example shows how an `IVF_RQ` index is physically laid out. Assume vectors have dimension 128,
-RQ uses 1 bit per dimension (num_bits=1), and distance type is "l2".
+RQ uses 1 bit per dimension (`num_bits=1`), and distance type is "l2". For `num_bits > 1`, the
+auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_factors_ex`.
 
 #### Index File
 
@@ -340,7 +348,7 @@ RQ uses 1 bit per dimension (num_bits=1), and distance type is "l2".
 - Arrow Schema Metadata:
   - `"distance_type"` → `"l2"`
   - `"lance:ivf"` → tracks per-partition `offsets` and `lengths` (no centroids here)
-  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true}"`
+  - `"lance:rabit"` → `"{"rotate_mat_position":1,"num_bits":1,"packed":true,"query_estimator":"raw_query"}"`
 - Lance File Global buffer:
   - `Tensor` rotation matrix with shape `[code_dim, code_dim]` = `[128, 128]` (float32)
 - Rows with Arrow schema:
@@ -351,6 +359,7 @@ pa.schema([
     pa.field("_rabit_codes", pa.list(pa.uint8(), list_size=16)), # dimension/8 = 128/8 = 16 bytes
     pa.field("__add_factors", pa.float32()),
     pa.field("__scale_factors", pa.float32()),
+    pa.field("__error_factors", pa.float32()),
 ])
 ```
 
diff --git a/docs/src/format/table/.pages b/docs/src/format/table/.pages
index 16c20058608..5b0cb0e95e6 100644
--- a/docs/src/format/table/.pages
+++ b/docs/src/format/table/.pages
@@ -6,4 +6,5 @@ nav:
   - Layout: layout.md
   - Branch & Tag: branch_tag.md
   - Row ID & Lineage: row_id_lineage.md
+  - Data Overlay Files: data_overlay_file.md
   - MemTable & WAL: mem_wal.md
diff --git a/docs/src/format/table/data_overlay_file.md b/docs/src/format/table/data_overlay_file.md
new file mode 100644
index 00000000000..9da739a29e3
--- /dev/null
+++ b/docs/src/format/table/data_overlay_file.md
@@ -0,0 +1,388 @@
+# Data Overlay Files
+
+!!! warning "Experimental"
+
+    This feature is currently experimental and not yet supported in any library.
+
+<!-- TODO: When overlay file support is implemented, update this note to state
+     the released version that first supports the feature (and drop the
+     "experimental" framing once it is stable). -->
+
+!!! note "Overlay files require feature flag 64 (data overlay files)"
+
+    A reader or writer that does not understand overlay files must refuse a
+    dataset that uses them. Silently ignoring an overlay would return stale base
+    values, which is a correctness bug rather than a degraded experience.
+
+Overlay files supply new values for a subset of `(row offset, field)` cells
+within a fragment **without rewriting the fragment's base data files**. They make
+updates cheap when only a small fraction of rows and/or columns change: instead
+of rewriting whole columns or moving rows to a new fragment, a writer appends a
+small file carrying just the changed cells.
+
+This is Lance's third mechanism for changing data in place, alongside
+[deletion files](index.md#deletion-files) (which remove rows) and
+[data evolution](index.md#data-evolution) (which adds or rewrites whole columns).
+An overlay changes individual cells.
+
+## Concepts
+
+### Coverage and resolution
+
+Each overlay declares which cells it provides through a **coverage** bitmap (or,
+for sparse overlays, one bitmap per field). The bitmaps index **physical row
+offsets**. They include deleted rows and are stable even as deletion vectors change.
+
+To resolve a cell `(offset, field)` on read, walk the fragment's overlays from
+**newest to oldest**. The first overlay that covers `(offset, field)` wins; its
+value is used. If no overlay covers the cell, the value falls through to the base
+data file (or is `NULL` if no base data file holds that field).
+
+Precedence among overlays is determined by:
+
+1. `committed_version` — higher wins (see [Versioning](#versioning-and-ordering)).
+2. Position in `DataFragment.overlays` as a tiebreaker — a later entry is newer.
+
+A covered offset whose value is `NULL` overrides the cell **to** `NULL`. This is
+distinct from an offset that is simply absent from the bitmap, which falls
+through to the base. Coverage, not value-nullness, decides whether an overlay
+applies.
+
+### Interaction with deletions
+
+Deletions take precedence over overlays. If a row offset is marked deleted in the
+fragment's deletion file, any overlay value for that offset is dead and is
+ignored, regardless of commit order.
+
+### Physical layout
+
+An overlay's data file stores **one value column per field**, in the order of
+`data_file.fields`. It does **not** store a row-offset key column. The position of
+a covered offset's value within its column is the **rank** of that offset in the
+field's coverage bitmap — the number of set bits below it. Resolving a cell is a
+rank lookup plus one value fetch, with no separate offset column to store or
+search.
+
+Because different fields may cover different offset sets, the value columns of a
+single sparse overlay may have **different lengths**. The Lance file format
+permits columns of differing item counts within one file, so a sparse overlay is
+representable as a single file. (See [Writer support](#writer-support) for the
+current implementation status.)
+
+### Dense vs. sparse overlays
+
+A single overlay is one of two shapes:
+
+- **Dense (rectangular).** One `shared_offset_bitmap` applies to every field. Every
+  covered offset has a value for every field. This is the common case for a plain
+  `UPDATE`, where one `SET` list is applied to one set of rows.
+- **Sparse.** A `FieldCoverage` carries one bitmap per field, used when different
+  fields cover different offset sets — for example a `MERGE` with multiple
+  `WHEN MATCHED` branches, where different rows update different columns. A dense
+  overlay would have to widen to the bounding rectangle and fill the untouched
+  cells with their current values (post-images), which for wide columns such as
+  embeddings means re-storing data that did not change. A sparse overlay stores
+  exactly the changed cells.
+
+## Protobuf
+
+<details>
+<summary>DataOverlayFile protobuf message</summary>
+
+```protobuf
+%%% proto.message.DataOverlayFile %%%
+```
+
+</details>
+
+<details>
+<summary>FieldCoverage protobuf message</summary>
+
+```protobuf
+%%% proto.message.FieldCoverage %%%
+```
+
+</details>
+
+## Versioning and ordering
+
+Overlays reuse the dataset version as their ordering clock rather than
+introducing a separate generation counter.
+
+`committed_version` is the dataset version at which an overlay **became
+effective** — the version of the commit that introduced it, **not** the version
+it was read from. It is stamped at commit time and re-stamped if the commit is
+retried, in the same way as the created-at / last-updated-at version sequences.
+
+This single value drives every ordering decision:
+
+- **Overlay vs. overlay** (read precedence): higher `committed_version` wins.
+- **Overlay vs. index** (query correctness): an index records the
+  `dataset_version` it was built from. An index whose `dataset_version >=
+  committed_version` already incorporates the overlay. An overlay whose
+  `committed_version > index.dataset_version` is newer than the index and its
+  cells must be excluded from index results and re-evaluated.
+- **Scheduler signal**: the gap between an overlay's `committed_version` and an
+  index's `dataset_version`, or between an overlay and the base, is a staleness
+  measure the compaction scheduler can use.
+
+!!! note "Why effective version, not read version"
+
+    Suppose an overlay reads version 5 and commits at version 6, while an index
+    is built reading version 5 (before the overlay) and commits at version 7 with
+    `dataset_version = 5`. If the overlay stored its *read* version (5), the test
+    `5 > 5` is false, the row would not be excluded, and the index — which never
+    saw the overlay — would return a stale result. Storing the *effective*
+    version (6) makes `6 > 5` true, the cell is excluded and re-evaluated, and the
+    result is correct.
+
+## Index integration
+
+Building an index over a fragment that has overlays does **not** require dropping
+the fragment from the index's coverage. The fragment stays indexed, and the query
+path reconciles overlays at query time using an **exclusion set**.
+
+The exclusion set for an index on field `F` is the union of the coverage bitmaps,
+restricted to field `F`, of every overlay whose `committed_version >
+index.dataset_version`. The exclusion is **field-aware**: an overlay that touches
+only unrelated columns does not exclude anything from the index on `F`.
+
+The query then proceeds as:
+
+1. Run the index search as usual, producing candidate rows.
+2. Remove any candidate in the exclusion set. (Its indexed value may be stale.)
+3. **Re-evaluate** the excluded rows against their current values — the same flat
+   path already used for the unindexed tail of fragments. For a scalar predicate
+   this re-applies the filter; for a vector query it re-scores the row's current
+   vector. Rows that still match are added back to the result.
+
+Step 3 is what makes exclusion correct rather than merely safe: removing a row
+from index candidates without re-evaluating it would silently drop a row that
+should match under its new value.
+
+Exclusion is always *sufficient* because a write changes a cell only by adding an
+overlay, and that overlay's `committed_version` — the version of the commit that
+adds it — necessarily exceeds the `dataset_version` of any pre-existing index. So
+every cell a write changes is guaranteed to fall in that index's exclusion set.
+Compaction may remove an overlay only if no index still relies on it for exclusion
+(see [Compaction](#compaction)).
+
+## Compaction
+
+Overlays accumulate read cost — every overlay is a bitmap to test, a possible
+file to open, and additional work to interleave values. Compaction bounds that cost in two modes:
+
+- **Overlay → overlay.** Merge several overlays into fewer, computing the
+  post-image per `(offset, field)` by walking the merged overlays newest-first.
+  The merged overlay takes the **maximum** `committed_version` of its inputs, so
+  the exclusion semantics are preserved. The merged overlays must be **contiguous
+  in `committed_version`** — with overlays at v10, v30, and v50 you cannot merge
+  just v10 and v50, because stamping the result v50 would incorrectly promote
+  v10's values above the intervening v30 for any cell v30 also covers. Indexes can
+  still be re-used, but they may now need to exclude more rows. This is cheap to
+  write and does not touch the base.
+- **Overlay → base.** Fold overlays into a fresh base data file, computing the
+  post-image for every covered cell, then clear the overlays. The base is
+  complete, so every post-image is well defined. Overlay offsets are physical, so
+  they cannot survive a rewrite that reorders rows; folding therefore materializes
+  values rather than carrying overlays forward.
+
+!!! warning "Folding an indexed field must update its index"
+
+    An overlay→base fold removes the overlay, which removes the exclusion signal
+    that kept an index correct. Folding an overlay that covers an indexed field
+    `F` is therefore equivalent to a column rewrite of `F` and must, in the same
+    commit, either rebuild the index to a `dataset_version` at least the folded
+    overlay's `committed_version`, or remove the fragment from the index's
+    coverage so the rows fall to the flat path. Otherwise the index would serve
+    stale values with no overlay to exclude them. This is the same rule that
+    already governs rewriting a column that an index is built on.
+
+When a fragment with overlays is compacted by a row-rewriting operation
+(`RewriteRows`, which produces new fragments with new row addresses), the
+overlays are folded into the new base as part of the rewrite, and existing
+[fragment-reuse remapping](row_id_lineage.md) handles the row-address changes as
+it does today.
+
+## Row lineage
+
+An overlay write updates the `last_updated_at_version` of every covered row, so
+change-data-feed and time-travel queries observe the update. Because overlays are
+addressed by physical offset, they do **not** require stable row IDs to be
+enabled; lineage updates apply only when those features are on.
+
+## Worked example
+
+The following example illustrates how overlays function across their lifecycle, to make the rules above concrete.
+
+A table `users` with stable row IDs enabled and these fields:
+
+| field id | name      | type                    |
+|----------|-----------|-------------------------|
+| 1        | id        | `int32` (primary key)   |
+| 2        | name      | `utf8`                  |
+| 3        | age       | `int32`                 |
+| 4        | embedding | `fixed_size_list<f32,4>`|
+
+Created at version 1 as a single fragment `0` with one base data file
+`data/file0.lance` holding all four columns. `physical_rows = 4`:
+
+| offset | id | name  | age | embedding        |
+|--------|----|-------|-----|------------------|
+| 0      | 1  | Alice | 30  | …                |
+| 1      | 2  | Bob   | 25  | …                |
+| 2      | 3  | Carol | 40  | …                |
+| 3      | 4  | Dave  | 22  | …                |
+
+A BTree scalar index on `age` is built at version 1, covering fragment `0`
+(`dataset_version = 1`).
+
+### Step 1 — write an overlay
+
+```sql
+UPDATE users SET age = age + 1 WHERE id IN (2, 4);   -- Bob (offset 1), Dave (offset 3)
+```
+
+This touches one field (`age`) for two rows, so the writer emits a dense overlay —
+one shared bitmap covering both offsets — and commits it as version 2. Fragment
+`0` gains:
+
+```text
+DataOverlayFile {
+  data_file: { path: "data/overlay-<uuid>.lance", fields: [3], column_indices: [0] }
+  coverage:  shared_offset_bitmap = {1, 3}
+  committed_version: 2
+}
+```
+
+The overlay file stores a single `age` column with two values, `[26, 23]`, at
+ranks `{1,3}.rank(1) = 0` and `{1,3}.rank(3) = 1`. `last_updated_at_version` is
+set to 2 for offsets 1 and 3.
+
+### Step 2 — read
+
+`SELECT id, age FROM users` reads base ages `[30, 25, 40, 22]`. For `age`
+(field 3), the overlay covers offsets 1 and 3, so `age[1]` is replaced with the
+overlay value at rank `{1,3}.rank(1) = 0` → `26`, and `age[3]` with the value at
+rank `{1,3}.rank(3) = 1` → `23`. Result ages: `[30, 26, 40, 23]`.
+
+### Step 3 — index query
+
+```sql
+SELECT * FROM users WHERE age = 26;
+```
+
+The `age` index was built at `dataset_version = 1`; the overlay's
+`committed_version` is 2. Since `2 > 1`, the overlay's coverage for `age`, `{1, 3}`,
+is the exclusion set for this query.
+
+- The index (built at v1) holds Bob's *old* `age = 25`, so a lookup for `26`
+  returns nothing from the index.
+- The whole exclusion set is re-evaluated on the flat path, not just the rows the
+  index returned. Offset 1's current `age` (26, via the overlay) matches, so Bob
+  is returned; offset 3's current `age` (23) does not match and is dropped.
+
+The mirror case `WHERE age = 25` shows exclusion preventing a stale hit: the index
+returns offset 1 (stale `25`), but offset 1 is excluded, re-evaluated to `26`, and
+correctly dropped.
+
+### Step 4 — a second, non-rectangular write
+
+```sql
+MERGE INTO users USING staged ON users.id = staged.id
+WHEN MATCHED AND staged.kind = 'rename'  THEN UPDATE SET name = staged.name        -- Carol(2), Dave(3)
+WHEN MATCHED AND staged.kind = 'embed'   THEN UPDATE SET embedding = staged.embedding -- Bob(1)
+```
+
+`name` is updated for offsets `{2, 3}` and `embedding` for offset `{1}` — different
+fields over different rows. This is a sparse overlay, committed as version 3:
+
+```text
+DataOverlayFile {
+  data_file: { path: "data/overlay-<uuid2>.lance", fields: [2, 4], column_indices: [0, 1] }
+  coverage:  field_coverage { offset_bitmaps: [ {2,3},   {1} ] }
+  //                                     name (field 2)  ^    ^ embedding (field 4)
+  committed_version: 3
+}
+```
+
+The file's `name` column has **two** values (`["Caroline", "David"]`, at
+ranks 0 and 1 of `{2,3}`) and its `embedding` column has **one** value (at rank 0
+of `{1}`) — columns of different lengths in one file.
+
+### Step 5 — read after the second write
+
+`SELECT name, age, embedding FROM users` resolves each field independently,
+newest overlay first:
+
+- `name`: the v3 overlay covers `{2,3}` → `["Alice", "Bob", "Caroline", "David"]`.
+- `age`: the v3 overlay does not cover `age`; the v2 overlay still applies at
+  offsets 1 and 3 → `[30, 26, 40, 23]`.
+- `embedding`: the v3 overlay covers `{1}` → Bob's vector is the new one, others
+  from base.
+
+Overlays from different versions coexist and apply per field.
+
+### Step 6 — compaction (overlay → base)
+
+The scheduler folds both overlays into fragment `0` at version 4, computing
+post-images for `age`, `name`, and `embedding`, and writing a new base data file
+`data/file1.lance` with those columns. In the old file, fields 2, 3, and 4 are
+marked with a tombstone (`-2`); field 1 (`id`) remains. The fragment's `overlays` list is
+cleared. Row addresses are preserved (a column rewrite, not a row rewrite), so
+stable row IDs and the deletion vector are untouched.
+
+Because the fold removed the overlay that was excluding offsets 1 and 3 from the
+`age` index, the commit must drop fragment `0` from its coverage so `age` queries
+fall to the flat path.
+
+## Guidance
+
+!!! note "This section is a stub."
+
+    The following are implementation considerations, not part of the on-disk
+    specification.
+
+### When to overlay vs. rewrite a column vs. move rows
+
+<!-- TODO: Replace the rough heuristic below with concrete thresholds once we
+     have benchmarked the crossover points between overlays, column rewrites,
+     and row moves. -->
+
+*(To be expanded.)* The choice between appending an overlay, rewriting a full
+column (data evolution), and moving updated rows to a new fragment depends on the
+fraction of rows changed, the fraction of columns changed, column width, the
+presence of indexes on the changed columns, and the accumulated overlay read
+cost. Roughly: few rows changed favors overlays; most rows in a few columns
+favors a column rewrite; most columns changed favors moving rows to a new
+fragment.
+
+### Writer support
+
+<!-- TODO: Fill in as writer implementation progresses, including the status of
+     single-file sparse overlays (independent-length columns). -->
+
+*(To be expanded.)* Dense (rectangular) overlays write with the existing
+equal-length file writer today. Sparse overlays stored as a **single** file
+require the writer to emit columns of independent lengths, which the current v2
+writer does not yet do (it advances all columns from one global row counter).
+Until that support lands, a writer can express a sparse update as multiple dense
+overlays in one transaction.
+
+### Scheduling compaction
+
+<!-- TODO: Fill in with a concrete cost/benefit policy once compaction is
+     implemented and benchmarked. -->
+
+*(To be expanded.)* The overlay→overlay and overlay→base modes have very
+different costs; a cost/benefit scheduler decides when each is worthwhile, using
+the version gap as a staleness signal.
+
+## Related specifications
+
+- [Table format overview](index.md)
+- [Transactions: DataOverlay operation](transaction.md#dataoverlay) — write path
+  and conflict semantics
+- [Row ID & Lineage](row_id_lineage.md)
+- [Index Formats: handling overlay rows](../index/index.md#handling-deleted-and-invalidated-rows)
+- [Format Versioning](versioning.md)
diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md
index 94ea4b90dc9..f9da132cf3b 100644
--- a/docs/src/format/table/index.md
+++ b/docs/src/format/table/index.md
@@ -168,6 +168,29 @@ However, this invalidates row addresses and requires rebuilding indices, which c
 
 </details>
 
+## Data Overlay Files
+
+!!! warning "Experimental"
+
+    This feature is currently experimental and not yet supported in any library.
+
+<!-- TODO: When overlay file support is implemented, update this note to state
+     the released version that first supports the feature. -->
+
+!!! note "Overlay files require feature flag 64 (data overlay files)"
+
+Overlay files supply new values for a subset of cells within
+a fragment without rewriting the base data files. They make updates cheap when only
+a small percentage of rows and/or columns change: a writer appends a small file
+carrying just the changed cells instead of rewriting whole columns or moving rows
+to a new fragment.
+
+For the full specification — coverage and resolution rules, dense vs. sparse layout,
+versioning, index integration, compaction, and a worked example — see the
+[Data Overlay Files Specification](data_overlay_file.md).
+
+
+
 ## Related Specifications
 
 ### Storage Layout
diff --git a/docs/src/format/table/mem_wal.md b/docs/src/format/table/mem_wal.md
index 8a228721123..92a5c5fce4a 100644
--- a/docs/src/format/table/mem_wal.md
+++ b/docs/src/format/table/mem_wal.md
@@ -7,684 +7,674 @@ scan, point lookup, vector search and full-text search.
 
 ![MemWAL Overview](../../images/mem_wal_overview.png)
 
-A Lance table is called a **base table** under the context of the MemWAL spec.
-It may have an [unenforced primary key](index.md#unenforced-primary-key) defined in the table schema.
-Primary keys are required for primary-key lookups and last-write-wins upsert semantics,
-but append-only MemWAL tables may omit them.
+A Lance table is called the **base table** in this document.
+The base table may have an [unenforced primary key](index.md#unenforced-primary-key) in its schema.
+Primary keys are required for primary-key lookups and last-write-wins upsert semantics.
+Append-only MemWAL tables may omit a primary key.
 
-On top of the base table, the MemWAL spec defines a set of shards.
-Writers write to shards, and data in each shard is merged into the base table asynchronously.
-An index is kept in the base table for readers to quickly discover the state of all shards at a point of time.
+MemWAL adds a set of shards on top of the base table.
+Writers append to shards.
+Each shard keeps recent data in an in-memory MemTable, persists writes to a per-shard WAL, flushes MemTables as small Lance datasets, and later merges those flushed generations into the base table.
 
-### MemWAL Shard
-
-A **MemWAL Shard** is the main unit to horizontally scale out writes.
-
-Each shard has exactly one active writer at any time.
-Writers claim a shard and then write data to that shard.
-Data in each shard is expected to be merged into the base table asynchronously.
+The base table manifest contains one MemWAL system index entry named `__lance_mem_wal`.
+This index stores MemWAL configuration and global progress metadata inline in `IndexMetadata.index_details`.
+Each shard's own manifest remains authoritative for shard-local mutable state.
 
-For tables with a primary key, rows of the same primary key must be written to one and only one shard.
-If two shards contain rows with the same primary key, the following scenario can cause data corruption:
-
-1. Shard A receives a write with primary key `pk=1` at time T1
-2. Shard B receives a write with primary key `pk=1` at time T2 (T2 > T1)
-3. The row in shard B is merged into the base table first
-4. The row in shard A is merged into the base table second
-5. The row from Shard A (older) now overwrites the row from Shard B (newer)
+### MemWAL Shard
 
-This violates the expected "last write wins" semantics.
-By ensuring each primary key is assigned to exactly one shard via the sharding spec,
-merge order between shards becomes irrelevant for correctness.
-Append-only tables without a primary key do not rely on last-write-wins conflict resolution
-and may shard by any deterministic append key or partitioning column.
+A **MemWAL shard** is the unit of horizontal write scaling.
+Each shard has exactly one active writer epoch at a time.
+Writers claim a shard, append WAL entries, update the in-memory MemTable, and publish flushed MemTable generations by updating the shard manifest.
 
-See [MemWAL Shard Architecture](#shard-architecture) for the complete shard architecture.
+For primary-key tables, all rows for the same primary key must map to the same shard.
+If one primary key can appear in multiple shards, asynchronous merge order between shards can make an older row overwrite a newer row.
+Append-only tables without a primary key do not rely on last-write-wins conflict resolution and may use any deterministic shard assignment suitable for the workload.
 
 ### MemWAL Index
 
-A **MemWAL Index** is the centralized structure for all MemWAL metadata on top of a base table.
-A table has at most one MemWAL index. It stores:
+The MemWAL index is a system index entry on the base table.
+It has `name = "__lance_mem_wal"`, no indexed fields, and no index files.
+`IndexMetadata.files` is `None`.
+All MemWAL index data is stored in the `MemWalIndexDetails` protobuf message in `IndexMetadata.index_details`.
 
-- **Configuration**: Sharding specs defining how rows map to shards, and which indexes to maintain
-- **Merge progress**: Last generation merged to base table for each shard
-- **Index catchup progress**: Which merged generation each base table index has been rebuilt to cover
-- **Shard snapshots**: Point-in-time snapshot of shard states for read optimization
+The index stores:
 
-The index is the source of truth for **configuration**, **merge progress** and **index catchup progress**
-Writers and mergers read the MemWAL index to get these configurations before writing.
+- **Configuration**: `sharding_specs`, `maintained_indexes`, and `writer_config_defaults`.
+- **Merge progress**: `merged_generations`, the last generation merged into the base table for each shard.
+- **Index catchup progress**: `index_catchup`, the merged generation covered by each base-table index.
+- **Shard snapshots**: optional point-in-time snapshot fields for read optimization.
 
-Each [shard's manifest](#shard-manifest) is authoritative for its own state.
-Readers may use **shard snapshots** as a read-only optimization to see a point-in-time view of shards without opening each shard manifest.
-Readers that need the latest shard set must discover shard directories in storage and read each shard's latest manifest.
-
-See [MemWAL Index Details](#memwal-index-details) for the complete structure.
+Shard snapshots are not authoritative.
+Readers that need the latest shard set list `_mem_wal/` and read each shard's latest manifest.
 
 ## Shard Architecture
 
 ![Shard Architecture](../../images/mem_wal_regional.png)
 
-Within a shard, writes are stored in an **in-memory table (MemTable)**.
-It is also written to the shard's **Write-Ahead Log (WAL)** for durability guarantee.
-The MemTable is periodically **flushed** to storage based on memory pressure and other conditions.
-**Flushed MemTables** in storage are then asynchronously **merged** into the base table.
+Within a shard, writes first enter an in-memory **MemTable** and are durably appended to the shard **write-ahead log (WAL)**.
+The MemTable is periodically **flushed** to storage as a Lance dataset.
+Flushed MemTables are asynchronously **merged** into the base table.
 
 ### MemTable
 
-A MemTable holds rows inserted into the shard before flushing to storage.
-It serves 2 purposes:
+A MemTable holds rows inserted into a shard before those rows are flushed to storage.
+It serves two purposes:
 
-1. build up data and related indexes to be flushed to storage as a flushed MemTable
-2. allow a reader to potentially access data that is not flushed to storage yet
+1. It buffers data and per-MemTable indexes before a flushed generation is written.
+2. It lets readers access data that has not been flushed yet when strong consistency is required.
 
-#### MemTable Format
+The storage format does not prescribe the in-memory MemTable layout.
+Conceptually, a MemTable is an append log of Arrow record batches.
+Later appends have larger in-memory row positions.
+For primary-key tables, in-memory reads use the largest visible row position as the newest row for a key.
 
-The complete in-memory format of a MemTable is implementation-specific and out of the scope of this spec.
-The Lance core Rust SDK maintains one default implementation and is available through all its language binding SDKs,
-but integrations are free to build their own MemTable format depending on the specific use cases,
-as long as it follows the MemWAL storage layout, reader and writer requirements when flushing MemTable.
+### MemTable Generation
 
-Conceptually, because Lance uses [Arrow as its in-memory data exchange format](https://arrow.apache.org/docs/format/index.html),
-for the ease of explanation in this spec, we will treat MemTable as a list of Arrow record batches,
-and each write into the MemTable is a new Arrow record batch.
+Each MemTable has a monotonically increasing generation number starting from 1.
+When generation `N` is flushed and discarded, the next MemTable uses generation `N + 1`.
 
-#### MemTable Generation
+Generation numbers order data freshness within one shard:
 
-Based on conditions like memory limit and durability requirements,
-a MemTable needs to be **flushed** to storage and discarded.
-When that happens, new writes go to a new MemTable and the cycle repeats.
-Each MemTable is assigned a monotonically increasing generation number starting from 1.
-When MemTable of generation `N` is discarded, the next MemTable gets assigned generation `N+1`.
+- Base table data has generation 0.
+- Higher MemWAL generations are newer.
+- Within the active in-memory generation, higher row positions are newer.
+- Within a flushed generation, flush-time deletion vectors hide older duplicate primary-key rows, so readers see at most the newest row for each primary key.
 
-### WAL
+## WAL
 
-WAL serves as the durable storage of all MemTables in a shard.
-It consists of data in MemTables ordered by generation.
-Every time we write to the WAL, we call it a **WAL Flush**.
+The WAL is the durable append log for a shard.
+Every durable WAL append creates one **WAL entry**.
 
-#### WAL Durability
+### WAL Entry Positions
 
-When a write is flushed to WAL, the specific write becomes durable.
-Otherwise, if the MemTable is lost, data is also lost.
+WAL entry positions are 1-based.
+The first data entry is position 1.
+Position 0 is reserved as the sentinel value meaning no WAL entry has been covered.
 
-Multiple writes can be batched together in a single WAL flush to reduce WAL flush frequency and improve throughput.
-The more writes a single WAL flush batches, the longer it takes for a write to be durable.
+Writers append WAL entries in increasing position order.
+If entry `N` is not fully written, entry `N + 1` must not exist.
+Recovery replays from `replay_after_wal_entry_position + 1`.
 
-The whole LSM tree's durability is determined by the durability of the WAL.
-For example, if WAL is stored in Amazon S3, it has 99.999999999% durability.
-If it is stored in local disk, the data will be lost if the local disk is damaged.
+### WAL Entry Format
 
-#### WAL Entry
+Each WAL entry is an Apache Arrow IPC stream file.
+The Arrow schema metadata includes:
 
-Each time a WAL flush happens, it adds a new **WAL Entry** to the WAL.
-In other words, a WAL consists of an ordered list of WAL entries starting from position 0.
-Writer must flush WAL entries in sequential order from lower to higher position.
-If WAL entry `N` is not flushed fully, WAL entry `N+1` must not exist in storage.
+- `writer_epoch`: decimal string containing the writer epoch that created the entry.
+- `fence_sentinel`: optional marker for a data-less fence sentinel entry.
 
-#### WAL Replay
+A normal WAL entry contains one or more record batches.
+A fence sentinel entry contains no batches and is skipped during replay.
+Sentinels are used so an older writer collides on the next WAL position and discovers that it has been fenced.
 
-**Replaying** a WAL means to read data in the WAL from a lower to a higher position.
-This is commonly used to recover the latest MemTable after it is lost,
-by reading from the start position of the latest MemTable generation till the highest position in the WAL,
-assuming proper fencing to guard against multiple writers to the same shard.
+### WAL Storage Layout
 
-See [Writer Fencing](#writer-fencing) for the full fencing mechanism.
+WAL entries live under `_mem_wal/{shard_id}/wal/`.
+Filenames use bit-reversed 64-bit binary names with the `.arrow` suffix:
 
-#### WAL Entry Format
+```text
+_mem_wal/{shard_id}/wal/{bit_reversed_position}.arrow
+```
 
-Each WAL entry is a file in storage following the [Apache Arrow IPC stream format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) to store the batch of writes in the MemTable.
-The writer epoch is stored in the stream's Arrow schema metadata with key `writer_epoch` for fencing validation during replay.
+The bit-reversal spreads sequential positions across object-store keyspace.
+For example, position 5 is encoded as:
 
-#### WAL Storage Layout
+```text
+1010000000000000000000000000000000000000000000000000000000000000.arrow
+```
 
-Each WAL entry is stored within the WAL directory of the shard located at `_mem_wal/{shard_id}/wal`.
+## Flushed MemTable
 
-WAL files use bit-reversed 64-bit binary naming to distribute files evenly across the directory keyspace.
-This optimizes S3 throughput by spreading sequential writes across S3's internal partitions, minimizing throttling.
-The filename is the bit-reversed binary representation of the entry ID with suffix `.arrow`.
-For example, entry ID 5 (binary `000...101`) becomes `1010000000000000000000000000000000000000000000000000000000000000.arrow`.
+A flushed MemTable is a persisted MemTable generation.
+It is stored as a Lance dataset under its shard directory.
 
-### Flushed MemTable
+!!! note
+    This structure is similar to a sorted string table in other LSM implementations, but MemWAL flushed generations are not sorted by key.
 
-A flushed MemTable is created by flushing the MemTable to storage.
-In Lance MemWAL spec, a flushed MemTable must be a Lance table following the Lance table format spec.
+### Flushed MemTable Storage Layout
 
-!!!note
-This is called Sorted String Table (SSTable) or Sorted Run in many LSM-tree literatures and implementations.
-However, since our MemTable is not sorted, we just use the term flushed MemTable to avoid confusion.
+Generation `i` is flushed to:
+
+```text
+_mem_wal/{shard_id}/{random8}_gen_{i}/
+```
 
-#### Flushed MemTable Storage Layout
+`{random8}` is an 8-character random hex value generated for each flush attempt.
+If a flush attempt fails, a retry writes a different directory instead of reusing a partially written one.
+The shard manifest records the successful directory name in `flushed_generations.path`.
+
+The generation directory is a standard Lance dataset written with the base table's data storage version.
+Each flushed generation is written as one fragment.
+Additional MemWAL sidecars may be present:
+
+```text
+{random8}_gen_{i}/
+├── _versions/
+│   └── {version}.manifest
+├── _deletions/                         # Present when within-generation dedup deletes rows
+├── _indices/                           # Present when maintained user indexes are built
+│   └── {index_uuid}/
+├── _pk_index/                          # Primary-key sidecar BTree, not a manifest index
+└── bloom_filter.bin                    # Primary-key bloom filter
+```
 
-The MemTable of generation `i` is flushed to `_mem_wal/{shard_id}/{random_hex}_gen_{i}/` directory,
-where `{random_hex}` is a random 8-character hex value generated at flush time.
-The random hex value is necessary to ensure if one MemTable flush attempt fails,
-The retry can use another directory.
-The content within the generation directory follows the [Lance table storage layout](layout.md).
+The exact Lance dataset internals follow the [Lance table storage layout](layout.md).
 
-#### Merging MemTable to Base Table
+### Flushed Row Order
 
-Generation numbers determine merge order of flushed MemTable into base table:
-lower numbers represent older data and must be merged to the base table first to preserve correct upsert semantics.
+Flushed MemTable rows are written in forward insert order.
+Physical row offsets increase with write time.
+For a duplicate primary key within one flushed generation, the newest row has the largest physical offset.
 
-Within a single flushed MemTable for a primary-key table,
-if there are multiple rows of the same primary key, the row that is last inserted wins.
-Append-only tables without a primary key retain all inserted rows.
+Primary-key flushed generations use a deletion vector to expose last-write-wins semantics.
+During flush, the writer scans rows in forward order, keeps the last occurrence of each primary key, and marks all earlier duplicate offsets deleted.
+The deletion vector is attached to fragment 0 in the generation manifest.
 
-### Shard Manifest
+Append-only flushed generations without a primary key do not perform primary-key deduplication and retain every row.
 
-Each shard has a manifest file. This is the source of truth for the state of a shard.
+### Tombstone Rows
 
-#### Shard Manifest Contents
+Delete operations are represented as rows with the internal `_tombstone` column.
+Tombstone rows follow the same forward row ordering and deletion-vector rules as ordinary rows.
+If the newest row for a primary key is a tombstone, the deletion vector keeps that tombstone row and hides older rows for the key.
+Read planning then filters `_tombstone = false`, so the key is absent from query results.
 
-The manifest contains:
+### Flushed Primary-Key Sidecars
 
-- **Fencing state**: `writer_epoch` as the latest writer fencing token, see [Writer Fencing](#writer-fencing) for more details.
-- **Shard assignment**: `shard_spec_id` and `shard_field_values` record how this shard maps to its sharding spec. `shard_field_values` is a map from shard field id to the raw Arrow scalar bytes of the computed value; the matching `ShardingField.result_type` in the `ShardingSpec` determines how to interpret each entry (e.g., 4 little-endian bytes for int32, raw UTF-8 bytes for utf8).
-- **WAL pointers**: `replay_after_wal_entry_position` (last entry position flushed to MemTable, 0-based), `wal_entry_position_last_seen` (last entry position seen at manifest update, 0-based)
-- **Generation trackers**: `current_generation` (next generation to flush), `flushed_generations` list of generation number and directory path pairs (e.g., generation 1 at `a1b2c3d4_gen_1`)
+Primary-key MemTables maintain an implicit BTree for primary-key deduplication, independent of `maintained_indexes`.
+When a primary-key MemTable is flushed, the flushed generation writes two primary-key sidecars:
 
-Note: `wal_entry_position_last_seen` is a hint that may be stale since it's not updated on WAL write.
-It is updated opportunistically by any reader that can update the shard manifest.
-The manifest itself is atomically written, but recovery must try to get newer WAL files to find the actual state beyond this hint.
+- `bloom_filter.bin` stores the generation's primary-key bloom filter and lets point lookups skip generations that cannot contain the queried key.
+- `_pk_index/` stores a standalone BTree over primary-key values to forward row ids.
 
-The manifest is serialized as a protobuf binary file using the `ShardManifest` message.
+The `_pk_index/` sidecar is not a maintained user index, is not registered in the generation manifest, and has no manifest UUID.
+Its identity is its immutable generation path.
+Readers open it directly from `{generation_path}/_pk_index`.
 
-<details>
-<summary>ShardManifest protobuf message</summary>
+The `_pk_index/` directory is a Lance scalar BTree index store:
 
-```protobuf
-%%% mem_wal.message.ShardManifest %%%
+```text
+_pk_index/
+├── page_data.lance
+└── page_lookup.lance
 ```
 
-</details>
+Readers load this directory as a BTree index using `BTreeIndexDetails` with default parameters.
+The primary-key index type is the Arrow type of the primary-key column for a single-column primary key, or `Binary` for a composite primary key.
 
-#### Shard Manifest Versioning
+The `page_lookup.lance` file has the following schema:
 
-Manifests are versioned starting from 1 and immutable.
-Each update creates a new manifest file at the next version number.
-Updates use put-if-not-exists or file rename to ensure atomicity depending on the storage system.
-If two processes compete, one wins and the other retries.
+| Column       | Type                    | Nullable | Description                                    |
+|--------------|-------------------------|----------|------------------------------------------------|
+| `min`        | {PrimaryKeyIndexType}   | true     | Minimum primary-key index value in the page    |
+| `max`        | {PrimaryKeyIndexType}   | true     | Maximum primary-key index value in the page    |
+| `null_count` | UInt32                  | false    | Number of null values in the page              |
+| `page_idx`   | UInt32                  | false    | Page number pointing into `page_data.lance`    |
 
-To commit a manifest version:
+The `page_data.lance` file has the following schema:
 
-1. Compute the next version number
-2. Write the manifest to `{bit_reversed_version}.binpb` using put-if-not-exists
-3. In parallel best-effort write to `version_hint.json` with `{"version": <new_version>}` (failure is acceptable)
+| Column   | Type                  | Nullable | Description                                                       |
+|----------|-----------------------|----------|-------------------------------------------------------------------|
+| `values` | {PrimaryKeyIndexType} | true     | Sorted primary-key index values                                   |
+| `ids`    | UInt64                | false    | Forward row ids corresponding to each primary-key index value     |
 
-To read the latest manifest version:
+For a single-column primary key, the indexed value stores the primary-key scalar directly.
+For a composite primary key, the indexed value stores an order-preserving binary tuple encoding of all primary-key columns in primary-key column order.
+Each tuple column is encoded as:
 
-1. Read `version_hint.json` to get the latest version hint. If not found, start from version 1
-2. Check existence for subsequent versions from the starting version
-3. Continue until a version is not found
-4. The latest version is the last found version
+- `0x00` for null.
+- `0x01` followed by the non-null value encoding otherwise.
 
-!!!note
-This works because the write rate to shard manifests is significantly lower than read rates. Shard manifests are only updated when shard metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version.
+Supported non-null value encodings are:
 
-#### Shard Manifest Storage Layout
+- Signed integers and date values: sign-flipped 8-byte big-endian integer bytes.
+- Unsigned integers: 8-byte big-endian unsigned integer bytes.
+- Boolean: one byte, `0x00` for false and `0x01` for true.
+- UTF-8 and binary values: raw bytes, with each `0x00` byte escaped as `0x00 0xff`, followed by a `0x00 0x00` terminator.
 
-All shard manifest versions are stored in `_mem_wal/{shard_id}/manifest` directory.
+This encoding is injective and preserves primary-key tuple ordering under lexicographic byte comparison.
+Composite primary-key columns must use one of the supported encodings above.
 
-Each shard manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files.
-For example, version 5 becomes `1010000000000000000000000000000000000000000000000000000000000000.binpb`.
+The sidecar row ids are in the same forward row-position space as the data files, deletion vector, and maintained user indexes.
+The sidecar is used for cross-generation membership and block-list checks.
+It is not used to choose the newest row inside the same flushed generation; the deletion vector has already hidden older same-generation duplicates.
 
-## MemWAL Index Details
+### Maintained User Indexes
 
-The MemWAL Index uses the [standard index storage](../index/index.md#index-storage) at `_indices/{UUID}/`.
+When the MemWAL index lists `maintained_indexes`, flush may build matching indexes inside the flushed generation.
+These index files live in the generation's `_indices/{index_uuid}/` directory and are recorded in the generation manifest.
+The implicit primary-key BTree sidecar is not included in `maintained_indexes` and does not live under `_indices/`.
 
-The index stores its data in two parts:
+These indexes use the same row-position space as the forward-written data files.
+If the generation has a primary key, the generation deletion vector masks stale duplicate rows for indexed reads as well.
 
-1. **Index details** (`index_details` in `IndexMetadata`): Contains configuration, merge progress, and snapshot metadata
-2. **Shard snapshots**: Stored as a Lance file or inline, depending on shard count
+### Merging Flushed Generations
 
-### Index Details
+Flushed generations are merged into the base table in ascending generation order within each shard.
+Lower generation numbers are older and must merge before higher generation numbers.
+The base table merge uses merge-insert semantics so newer rows overwrite older rows for the same primary key.
 
-The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` protobuf message with the following key fields:
+## Shard Manifest
 
-- **Configuration fields** (`sharding_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration.
-  Writers read these fields to determine how to partition data and which indexes to maintain.
-- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each shard.
-  This field is updated atomically with merge-insert data commits, enabling conflict resolution when multiple mergers operate concurrently.
-  Each entry contains the shard UUID and generation number.
-- **Index catchup progress** (`index_catchup`) tracks which merged generation each base table index has been rebuilt to cover.
-  When data is merged from a flushed MemTable to the base table, the base table's indexes may be rebuilt asynchronously.
-  During this window, queries should use the flushed MemTable's pre-built indexes instead of scanning unindexed data in the base table.
-  See [Indexed Read Plan](#indexed-read-plan) for details.
-- **Shard snapshot fields** (`snapshot_ts_millis`, `num_shards`, `inline_snapshots`) provide a snapshot of shard states.
-  The actual shard manifests remain authoritative for shard state.
-  When `num_shards` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema.
+Each shard has a versioned manifest.
+The latest shard manifest is the source of truth for shard-local state.
+
+### Shard Manifest Contents
+
+The manifest contains:
+
+- **Identity**: `shard_id`, `shard_spec_id`, and `shard_field_entries`.
+- **Fencing state**: `writer_epoch`.
+- **WAL pointers**: `replay_after_wal_entry_position` and `wal_entry_position_last_seen`.
+- **Generation state**: `current_generation` and `flushed_generations`.
+- **Lifecycle state**: `status`, either `ACTIVE` or `SEALED`.
+
+`shard_field_entries` stores computed shard field values as raw Arrow scalar bytes keyed by `ShardingField.field_id`.
+The matching `ShardingField.result_type` determines how to decode each value.
+For example, `int32` values are four little-endian bytes and `utf8` values are raw UTF-8 bytes.
+
+`replay_after_wal_entry_position` is the most recent 1-based WAL position covered by a flushed generation.
+The default value 0 means no WAL entry has been covered and recovery starts at position 1.
+
+`wal_entry_position_last_seen` is a best-effort hint for the most recent WAL position observed at manifest update time.
+It is not authoritative because it is not updated on every WAL write.
+Recovery must still probe or list WAL files to find the actual tail.
+
+`status = SEALED` marks a reversible in-flight drop-table operation.
+Sealed shards refuse new writer claims.
+
+The manifest is serialized as the `ShardManifest` protobuf message.
 
 <details>
-<summary>MemWalIndexDetails protobuf message</summary>
+<summary>ShardManifest protobuf message</summary>
 
 ```protobuf
-%%% mem_wal.message.MemWalIndexDetails %%%
+%%% mem_wal.message.ShardManifest %%%
 ```
 
 </details>
 
-### Shard Identifier
-
-Each shard has a unique UUID identifier within the table.
-When a new shard is created, implementations may assign either a random UUID or
-a deterministic UUID derived from the shard assignment when deterministic
-writer fencing is required.
-
-### Shard Discovery
+### Shard Manifest Versioning
 
-The MemWAL index can store shard snapshots for read optimization, but those snapshots may lag the latest shard set.
-Implementations that need to discover the current shard set should list `_mem_wal/` shard directories and read each shard's latest [shard manifest](#shard-manifest).
+Manifest versions start at 1.
+Each update writes a new immutable protobuf file:
 
-Each shard manifest records the shard UUID, sharding spec ID, and computed shard field values needed to map the shard back to a sharding spec assignment.
-
-### Sharding Spec
+```text
+_mem_wal/{shard_id}/manifest/{bit_reversed_version}.binpb
+```
 
-A **Sharding Spec** defines how all rows in a table are logically divided into different shards,
-enabling automatic shard assignment and query-time shard pruning.
+Writers use put-if-not-exists or atomic rename, depending on storage support.
+If two processes race to write the same next version, one wins and the other reloads and retries.
 
-Each sharding spec has:
+After a successful version write, the writer best-effort updates:
 
-- **Spec ID**: A positive integer that uniquely identifies this spec within the MemWAL index. IDs are never reused.
-- **Sharding fields**: An array of field definitions that determine how to compute shard values.
+```json
+{"version": <new_version>}
+```
 
-Each shard is bound to a specific sharding spec ID, recorded in its [manifest](#shard-manifest).
-Shards without a spec ID (`spec_id = 0`) are manually-created shards not governed by any spec.
+in:
 
-A sharding spec's field array consists of **sharding field** definitions.
-Each sharding field has the following properties:
+```text
+_mem_wal/{shard_id}/manifest/version_hint.json
+```
 
-| Property      | Description                                                               |
-| ------------- | ------------------------------------------------------------------------- |
-| `field_id`    | Unique string identifier for this sharding field                         |
-| `source_ids`  | Array of field IDs referencing source columns in the schema               |
-| `transform`   | A well-known shard expression, specify this or `expression`              |
-| `expression`  | A DataFusion SQL expression for custom logic, specify this or `transform` |
-| `result_type` | The output type of the shard value                                       |
+Readers use `version_hint.json` as a starting point and then probe subsequent versions until a version is missing.
+The latest manifest is the last existing version.
 
-#### Shard Expression
+## MemWAL Index Details
 
-A **Shard Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a shard value from source column(s).
-Source columns are referenced as `col0`, `col1`, etc., corresponding to the order of field IDs in `source_ids`.
+The MemWAL index is stored inline in the base table's `IndexMetadata`.
+It is a system index with no file directory.
+The `index_details` field contains a `MemWalIndexDetails` protobuf message.
 
-Shard expressions must satisfy the following requirements:
+Important fields:
 
-1. **Deterministic**: The same input value must always produce the same output value.
-2. **Stateless**: The expression must not depend on external state (e.g., current time, random values, session variables).
-3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same shard value).
-4. **Column removal resistant**: If a source field ID is not found in the schema, the column should be interpreted as NULL.
-5. **NULL-safe**: The expression should properly handle NULL inputs and have defined behavior (e.g., return NULL if input is NULL for single-column expressions).
-6. **Consistent with result type**: The expression's return type must be consistent with `result_type` in non-NULL cases.
+- `sharding_specs`: sharding configuration used by writers and shard pruning.
+- `maintained_indexes`: names of base-table indexes to maintain in MemTables and flushed generations.
+- `writer_config_defaults`: string map of default writer configuration values persisted for all writers.
+- `merged_generations`: per-shard merge progress, updated atomically with base-table merge commits.
+- `index_catchup`: per-index coverage progress after data has merged to the base table.
+- `snapshot_ts_millis`, `num_shards`, and `inline_snapshots`: optional shard snapshot fields for read optimization.
 
-#### Shard Transform
+If a shard is absent from `index_catchup` for an index, that index is assumed to be fully caught up for the shard.
 
-A **Shard Transform** is a well-known shard expression with a predefined name.
-When a transform is specified, the expression is derived automatically.
+Shard snapshots, when present, use the following Lance file schema:
 
-| Transform      | Parameters    | Shard Expression                                         | Result Type    |
-| -------------- | ------------- | --------------------------------------------------------- | -------------- |
-| `identity`     | (none)        | `col0`                                                    | same as source |
-| `year`         | (none)        | `date_part('year', col0)`                                 | `int32`        |
-| `month`        | (none)        | `date_part('month', col0)`                                | `int32`        |
-| `day`          | (none)        | `date_part('day', col0)`                                  | `int32`        |
-| `hour`         | (none)        | `date_part('hour', col0)`                                 | `int32`        |
-| `bucket`       | `num_buckets` | `abs(murmur3(col0)) % N`                                  | `int32`        |
-| `multi_bucket` | `num_buckets` | `abs(murmur3_multi(col0, col1, ...)) % N`                 | `int32`        |
-| `truncate`     | `width`       | `left(col0, W)` (string) or `col0 - (col0 % W)` (numeric) | same as source |
+| Column                     | Type                         | Nullable | Description                                            |
+|----------------------------|------------------------------|----------|--------------------------------------------------------|
+| `shard_id`                 | Utf8                         | false    | Shard UUID string                                      |
+| `shard_spec_id`            | UInt32                       | false    | Sharding spec that produced the shard                  |
+| `shard_field_{field_id}`   | `ShardingField.result_type`  | false    | Computed shard field value for the given sharding field |
 
-The `bucket` and `multi_bucket` transforms use Murmur3 hash functions:
+The MemWAL index data is stored inline.
+Readers discover the latest shard set by listing `_mem_wal/` shard directories and reading shard manifests.
 
-- **`murmur3(col)`**: Computes the 32-bit Murmur3 hash (x86 variant, seed 0) of a single column. Returns a signed 32-bit integer. Returns NULL if input is NULL.
-- **`murmur3_multi(col0, col1, ...)`**: Computes the Murmur3 hash across multiple columns. Returns a signed 32-bit integer. NULL fields are ignored during hashing; returns NULL only if all inputs are NULL.
+<details>
+<summary>MemWalIndexDetails protobuf message</summary>
 
-The hash result is wrapped with `abs()` and modulo `N` to produce a non-negative bucket number in the range `[0, N)`.
+```protobuf
+%%% mem_wal.message.MemWalIndexDetails %%%
+```
 
-### Shard Snapshot Storage
+</details>
 
-Shard snapshots are stored using one of two strategies based on the number of shards:
+## Sharding
 
-| Shard Count       | Storage Strategy    | Location                                  |
-| ------------------ | ------------------- | ----------------------------------------- |
-| <= 100 (threshold) | Inline              | `inline_snapshots` field in index details |
-| > 100              | External Lance file | `_indices/{UUID}/index.lance`             |
+A **ShardingSpec** defines how rows map to shards.
+Each spec has a positive `spec_id` and one or more `ShardingField` entries.
+Each shard manifest records the `shard_spec_id` and the computed shard field values for that shard.
+`spec_id = 0` means the shard was manually created and is not governed by a sharding spec.
 
-The threshold (100 shards) is implementation-defined and may vary.
+Each `ShardingField` contains:
 
-**Inline storage**: For small shard counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field.
-This keeps the index metadata compact while avoiding an additional file read for common cases.
+- `field_id`: stable identifier for the computed shard field.
+- `source_ids`: field IDs of source columns in the Lance schema.
+- `transform`: well-known transform name, when using built-in transform evaluation.
+- `expression`: reserved custom expression text, mutually exclusive with `transform`.
+- `result_type`: Arrow type name for the computed value.
+- `parameters`: transform-specific string parameters.
 
-**External Lance file**: For large shard counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`.
-This file uses standard Lance format with the shard snapshot schema, enabling efficient columnar access and compression.
+The supported built-in transforms are:
 
-### Shard Snapshot Arrow Schema
+- `unsharded`: takes no source columns, always returns `int32` value 0, and creates one shard.
+- `bucket`: takes one source column and `num_buckets`, hashes the value, and returns an `int32` bucket id in `[0, num_buckets)`.
+- `identity`: takes one source column and returns the raw scalar value as the shard value.
 
-Shard snapshots are stored as a Lance file with one row per shard.
-The snapshot schema is optimized for shard discovery. Full mutable shard state
-remains in the authoritative shard manifest files.
+`bucket` computes a deterministic 32-bit hash with seed 0 and then computes:
 
-| Column                   | Type          | Description                                                                                                |
-| ------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------- |
-| `shard_id`               | `utf8`        | Shard UUID string                                                                                          |
-| `shard_spec_id`          | `uint32`      | Sharding spec ID (0 if manual)                                                                             |
-| `shard_field_{field_id}` | varies        | One column per sharding field defined in the sharding spec, typed to match the field's `ShardingField.result_type`. |
+```text
+(hash & i32::MAX) % num_buckets
+```
 
-For example, with a sharding spec containing a field `user_bucket` of type `int32`:
+`num_buckets` must be in `[1, 1024]`.
+Null bucket values hash to 0 and therefore map to bucket 0.
+See [Appendix 3: Bucket Hashing](#appendix-3-bucket-hashing) for the exact hash algorithm and test vectors.
 
-| Column                     | Type    | Description                  |
-| -------------------------- | ------- | ---------------------------- |
-| ...                        | ...     | (base columns above)         |
-| `shard_field_user_bucket`  | `int32` | Bucket value for this shard |
+The `bucket` transform supports scalar boolean, integer, floating-point, date32, time, timestamp, utf8, and large_utf8 source types.
+The `identity` transform supports scalar boolean, integer, utf8, and large_utf8 source types.
 
-This schema records the fields needed to map each shard back to its sharding spec
-assignment. Readers that need fencing epochs, WAL positions, or flushed
-generation state must read the latest shard manifests directly.
+The `year`, `month`, `day`, `hour`, `multi_bucket`, and `truncate` transform names are not supported MemWAL sharding transforms and must not be used in `ShardingSpec.transform`.
 
 ## Storage Layout
 
-Here is a recap of the storage layout with all the files and concepts defined so far:
+The MemWAL storage layout is:
 
-```
+```text
 {table_path}/
+├── _versions/
+│   └── ...                              # Base table manifests, including __lance_mem_wal index metadata
 ├── _indices/
-│   └── {index_uuid}/                    # MemWAL Index (uses standard index storage)
-│       └── index.lance                  # Serialized shard snapshots (Lance file)
-│
+│   └── ...                              # Ordinary base table index files; MemWAL index has no files
 └── _mem_wal/
-    └── {shard_id}/                   # Shard directory (UUID v4)
+    └── {shard_id}/
         ├── manifest/
-        │   ├── {bit_reversed_version}.binpb     # Serialized shard manifest (bit-reversed naming)
-        │   └── version_hint.json                # Version hint file
+        │   ├── {bit_reversed_version}.binpb
+        │   └── version_hint.json
         ├── wal/
-        │   ├── {bit_reversed_entry_id}.arrow    # WAL data files (bit-reversed naming)
+        │   ├── {bit_reversed_position}.arrow
         │   └── ...
-        └── {random_hash}_gen_{i}/        # Flushed MemTable (generation i, random prefix)
+        └── {random8}_gen_{generation}/
             ├── _versions/
-            │   └── {version}.manifest    # Table manifest (V2 naming scheme)
-            ├── _indices/                 # Indexes
-            │   ├── {vector_index}/
-            │   └── {scalar_index}/
-            └── bloom_filter.bin          # Primary key bloom filter
+            │   └── {version}.manifest
+            ├── _deletions/
+            ├── _indices/
+            │   └── {index_uuid}/
+            ├── _pk_index/
+            └── bloom_filter.bin
 ```
 
-## Implementation Expectation
-
-This specification describes the storage layout for the LSM tree architecture. Implementations are free to use any approach to fulfill the storage layout requirements. Once data is written to the expected storage layout, the reader and writer expectations apply.
-
-The specification defines:
+Some flushed-generation subdirectories are conditional.
+For example, `_deletions/` is present only when the generation manifest references a deletion vector, `_indices/` is present only when maintained user indexes are built, and `_pk_index/` plus `bloom_filter.bin` are meaningful for primary-key tables.
 
-- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, shard manifests, and the MemWAL index
-- **Durability guarantees**: How data is persisted through WAL entries and flushed MemTables
-- **Consistency model**: How readers and writers coordinate through manifests and epoch-based fencing
+## Implementation Expectation
 
-Implementations may choose different approaches for:
+This document specifies the storage layout and observable reader and writer invariants.
+Implementations may choose different in-memory structures, buffering policies, background scheduling, and query execution plans.
 
-- In-memory data structures and indexing
-- Buffering strategies before WAL flush
-- Background task scheduling and concurrency
-- Query execution strategies
+An implementation is compatible when it:
 
-As long as the storage layout is correct and the documented invariants are maintained, implementations can optimize for their specific use cases.
+1. Writes WAL entries, shard manifests, flushed generations, and MemWAL index metadata using the documented layout.
+2. Preserves WAL position, writer fencing, and manifest versioning invariants.
+3. Exposes last-write-wins semantics for primary-key tables.
+4. Preserves append-only semantics for tables without primary keys.
+5. Maintains generation ordering when merging flushed MemTables into the base table.
 
 ## Writer Expectations
 
-A writer operates on a single shard and is responsible for:
+A writer operates on one shard and is responsible for:
 
-1. Claiming the shard using epoch-based fencing
-2. Writing data to WAL entries and flushed MemTables following the [storage layout](#storage-layout)
-3. Maintaining the shard manifest to track WAL and generation progress
+1. Claiming the shard with epoch-based fencing.
+2. Appending WAL entries in sequential 1-based positions.
+3. Maintaining in-memory MemTable state.
+4. Flushing MemTable generations to Lance datasets.
+5. Updating the shard manifest after a generation is durably flushed.
 
 ### Writer Fencing
 
-Writers use epoch-based fencing to ensure single-writer semantics per shard.
+Writers use `writer_epoch` to enforce single-writer semantics per shard.
 
 To claim a shard:
 
-1. Load the latest shard manifest
-2. Increment `writer_epoch` by one
-3. Atomically write a new manifest version
-4. If the write fails (another writer claimed the epoch), reload and retry with a higher epoch
+1. Load the latest shard manifest.
+2. Verify the shard is `ACTIVE`.
+3. Increment `writer_epoch`.
+4. Atomically write a new manifest version.
+5. If the manifest write loses a race, reload and retry.
 
-Before any manifest update, a writer must verify its `writer_epoch` remains valid:
+Before a manifest update, a writer verifies its local epoch is still current:
 
-- If `local_writer_epoch == stored_writer_epoch`: The writer is still active and may proceed
-- If `local_writer_epoch < stored_writer_epoch`: The writer has been fenced and must abort
+- If `local_writer_epoch == stored_writer_epoch`, the writer may proceed.
+- If `local_writer_epoch < stored_writer_epoch`, the writer has been fenced and must abort.
 
-For a concrete example, see [Appendix 1: Writer Fencing Example](#appendix-1-writer-fencing-example).
+WAL append conflicts also detect fencing.
+If an older writer collides with a newer writer's WAL entry at the same position, it reloads the manifest and observes the higher epoch.
+Fence sentinel entries make this collision path explicit without storing data batches.
 
 ## Background Job Expectations
 
-Background jobs handle merging flushed MemTables to the base table and garbage collection.
+Background jobs merge flushed generations into the base table and remove obsolete shard data.
 
 ### MemTable Merger
 
-Flushed MemTables must be merged to the base table in **ascending generation order** within each shard. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones.
-
-The merge uses Lance's merge-insert operation with atomic transaction semantics:
+Flushed MemTables must merge into the base table in ascending generation order within each shard.
+The merge uses Lance merge-insert semantics and updates `merged_generations[shard_id]` atomically with the base-table commit.
 
-- `merged_generations[shard_id]` is updated atomically with the data commit
-- On commit conflict, check the conflicting commit's `merged_generations` to determine if the generation was already merged
+On commit conflict, a merger reloads the conflicting base-table version:
 
-For a concrete example, see [Appendix 2: Concurrent Merger Example](#appendix-2-concurrent-merger-example).
+- If the committed `merged_generations[shard_id]` is already greater than or equal to the generation being merged, the merger skips that generation.
+- Otherwise, the merger retries from the latest base-table version.
 
 ### Garbage Collector
 
-The garbage collector removes obsolete data from shard directories. Flushed MemTables and their referenced WAL files may be deleted after:
-
-1. The generation has been merged to the base table (`generation <= merged_generations[shard_id]`)
-2. All maintained indexes have caught up (`generation <= min(index_catchup[I].caught_up_generation)`)
-3. No retained base table version references the generation for time travel
+The garbage collector may remove obsolete flushed generations after:
 
-!!!warning
-    Deleting WAL files weakens [writer fencing](#writer-fencing) and can lead to silent acknowledgement of lost writes.
+1. The generation has been merged to the base table.
+2. Every maintained index has caught up to cover the merged generation, or the generation is no longer needed for indexed reads.
+3. No retained base-table version needs the generation for time travel or consistency.
 
-    Fencing detects a stalled writer when its `put-if-not-exists` for the next WAL entry collides with a newer writer's entry at the same position — only that collision triggers the epoch check. If GC has already removed the WAL file at that position, the stalled writer's PUT lands on empty space and succeeds against its old `writer_epoch`. The entry is acknowledged to the client, but the new manifest's `replay_after_wal_entry_position` has already advanced past it, so the data is never replayed.
+!!! warning
+    Deleting WAL files can weaken writer fencing.
 
-    Implementations that GC WAL files must compensate, for example by re-checking fence state after each successful WAL write, encoding the writer epoch into the WAL filename so positions are partitioned by epoch, or otherwise guaranteeing a stalled writer cannot land at a position that has been or will be GC'd.
+    Fencing detects a stalled writer when its put-if-not-exists for the next WAL entry collides with a newer writer's entry at the same position.
+    If garbage collection has removed that WAL file, the stalled writer may write into empty space with an old `writer_epoch`.
+    Implementations that garbage collect WAL files must compensate by re-checking fence state after WAL writes, partitioning WAL positions by epoch, or otherwise preventing stale writers from landing at positions that have been garbage collected.
 
 ## Reader Expectations
 
 ### LSM Tree Merging Read
 
-For tables with a primary key, readers **MUST** merge results from multiple data sources
-(base table, flushed MemTables, in-memory MemTables) by primary key to ensure correctness.
-
-When the same primary key exists in multiple sources, the reader must keep only the newest version based on:
-
-1. **Generation number** (`_gen`): Higher generation wins. The base table has generation 0, MemTables have positive integers starting from 1.
-2. **Row address** (`_rowaddr`): Within the same generation, higher row address wins (later writes within a batch overwrite earlier ones).
+For primary-key tables, readers merge rows from the base table, flushed MemTables, and optionally in-memory MemTables by primary key.
+The newest row wins.
 
-The ordering for "newest" is: highest `_gen` first, then highest `_rowaddr`.
+Freshness ordering within one shard is:
 
-This deduplication is essential because:
+1. Higher generation wins.
+2. Within the active in-memory generation, higher row position wins.
+3. Within a flushed generation, the generation's deletion vector has already hidden older duplicate primary-key rows.
 
-- A row updated in a MemTable also exists (with older data) in the base table
-- A flushed MemTable that has been merged to the base table may not yet be garbage collected, causing the same row to appear in both
-- A single write batch may contain multiple updates to the same primary key
-
-Without proper merging, queries would return duplicate or stale rows.
+The base table has generation 0.
+MemWAL generations are positive.
+This ordering applies only to sources selected for the same read plan.
+Readers must not include a flushed generation that is already covered by the base table according to `merged_generations[shard_id]`, because otherwise the positive MemWAL generation would incorrectly outrank base-table rows during deduplication.
+Rows from different shards do not need primary-key deduplication if the sharding spec guarantees that each primary key maps to exactly one shard.
 
 Append-only tables without a primary key do not perform primary-key deduplication.
-Readers should include the relevant base table, flushed MemTables, and in-memory MemTables
-according to the requested consistency level; duplicate values are treated as distinct appended rows.
-
-### Reader Consistency
+Rows from all selected sources are distinct appended rows.
 
-Reader consistency depends on two factors:
+### Tombstones
 
-1. access to in-memory MemTables
-2. the source of shard metadata (either through MemWAL index or shard manifests)
+Readers must treat `_tombstone = true` rows as delete markers.
+In flushed generations, deletion vectors first resolve same-generation duplicate primary keys.
+Then query planning filters tombstone rows from user-visible results.
+In active in-memory MemTables, the newest visible row position for a primary key wins; if that row is a tombstone, the key is absent.
 
-Strong consistency requires access to in-memory MemTables for all shards involved in the query and reading shard manifests directly.
-Otherwise, the query is eventually consistent due to missing unflushed data or stale MemWAL Index snapshots.
+### Reader Consistency
 
-!!!note
-Reading a stale MemWAL Index does not impact correctness, only freshness:
+Reader consistency depends on:
 
-    - **Merged MemTable still in index**: If a flushed MemTable has been merged to the base table but still shows in the MemWAL index, readers query both. This results in some inefficiency for querying the same data twice, but [LSM-tree merging](#lsm-tree-merging-read) ensures correct results since both contain the same data. The inefficiency is also compensated by the fact that the data is covered by index and we rarely end up scanning both data.
-    - **Garbage collected MemTable still in index**: If a flushed MemTable has been garbage collected, but is still in the MemWAL index, readers would fail to open it and skip it. This is also safe because if it is garbage collected, the data must already exist in the base table.
-    - **Newly flushed MemTable not in index**: If a newly flushed MemTable is added after the snapshot was built, it is not queried. The result is eventually consistent but correct for the snapshot's point in time.
+1. Whether the reader can access active in-memory MemTables.
+2. Whether shard metadata comes from latest shard manifests or from an older MemWAL index snapshot.
 
-### Query Planning
+Strong consistency requires active in-memory MemTable access for relevant shards and direct reads of latest shard manifests.
+Otherwise, reads are eventually consistent because unflushed data or newly-created shards may be absent from the read plan.
 
-#### MemTable Collection
+Reading a stale MemWAL index snapshot does not corrupt last-write-wins ordering, but it can reduce freshness:
 
-The query planner collects datasets from multiple sources and assembles them for unified query execution.
-Datasets come from:
+- If a merged flushed generation is still listed, readers must skip it when `generation <= merged_generations[shard_id]`.
+  For primary-key tables, including it would let an older flushed row outrank newer base-table contents because MemWAL generations are positive and the base table is modeled as generation 0.
+  For append-only tables, including it would return the same append twice.
+- If a garbage-collected flushed generation is still listed, readers may skip it after failing to open it because its data must already be in the base table or be filtered out by `merged_generations`.
+- If a newly flushed generation is not listed, the read is consistent with the older snapshot but may miss fresher data.
 
-1. base table (representing already-merged data)
-2. flushed MemTables (persisted but not yet merged)
-3. optionally in-memory MemTables (if accessible).
+Readers that require latest shard membership should list `_mem_wal/` and read shard manifests instead of relying only on snapshots.
 
-Each dataset is tagged with a generation number: 0 for the base table, and positive integers for MemTable generations.
-Within a shard, the generation number determines data freshness, with higher numbers representing newer data.
-For primary-key tables, rows from different shards do not need deduplication
-since each primary key maps to exactly one shard.
-Append-only tables without a primary key do not require cross-shard primary-key deduplication.
-
-The planner also collects bloom filters from each generation for staleness detection during search queries.
+### Query Planning
 
-#### Shard Pruning
+A query planner collects sources from:
 
-Before executing queries, if sharding spec is available,
-the planner evaluates filter predicates against sharding specs to determine which shards may contain matching data.
-This pruning step reduces the number of shards to scan.
+1. The base table.
+2. Flushed MemTables that are not yet safely replaceable by base-table indexed reads.
+3. Active in-memory MemTables, when available and required by the requested consistency level.
 
-For each filter predicate:
+Each source is tagged with its shard and generation.
+For primary-key reads, the planner applies LSM deduplication across selected sources.
+For append-only reads, the planner concatenates selected sources without primary-key deduplication.
 
-1. Extract predicates on columns used in sharding specs
-2. Evaluate which shard values can satisfy the predicate
-3. Prune shards whose values cannot match
+Bloom filters and `_pk_index/` sidecars help prune flushed generations during point lookups and cross-generation deduplication.
 
-For example, with a sharding spec using `bucket(user_id, 10)` and a filter `user_id = 123`:
+### Shard Pruning
 
-1. Compute `bucket(123, 10) = 3`
-2. Only scan shards with bucket value 3
-3. Skip all other shards
+When sharding specs are available, the planner evaluates query predicates against shard fields and skips shards whose computed shard values cannot match.
 
-Shard pruning applies to both scan queries and prefilters in search queries.
+For example, with `bucket(user_id, 10)` and predicate `user_id = 123`:
 
-#### Indexed Read Plan
+1. Compute the bucket id for `123`.
+2. Scan only shards whose manifest has the same computed bucket value.
+3. Skip all other bucket shards.
 
-When data is merged from a flushed MemTable to the base table, the base table's indexes are rebuilt asynchronously by the base table index builders.
-During this window, the merged data exists in the base table but is not yet covered by the base table's indexes.
+### Indexed Read Plan
 
-Without special handling, indexed queries would fall back to expensive full scans for the unindexed part of the base table.
-To maintain indexed read performance, the query planner should use `index_catchup` progress to determine the optimal data source for each query.
+When data is merged from a flushed MemTable into the base table, base-table indexes may lag behind the data commit.
+`index_catchup` records which merged generation each base-table index covers.
 
-The key insight is that flushed MemTables serve as a bridge between the base table's index catchup and the current merged state.
-For a query that requires a specific index for acceleration, when `index_gen < merged_gen`,
-the generations in the gap `(index_gen, merged_gen]` have data already merged in the base table but are not covered by the base table's index.
-Since flushed MemTables contain pre-built indexes (created during [MemTable flush](#flushed-memtable)), queries can use these indexes instead of scanning unindexed data in the base table.
-This ensures all reads remain indexed regardless of how far behind the async index builder is.
+If an indexed query needs index `I` and `I` has only caught up to generation `G` while `merged_generations[shard_id]` is higher, the planner should read the gap from flushed-generation indexes instead of scanning unindexed base-table rows.
+Once index `I` catches up, the planner can use the base-table index for those merged rows.
 
 ## Appendices
 
 ### Appendix 1: Writer Fencing Example
 
-This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same shard.
-
-#### Initial State
+Initial shard manifest:
 
+```text
+version: 1
+writer_epoch: 5
+replay_after_wal_entry_position: 10
+wal_entry_position_last_seen: 12
+status: ACTIVE
 ```
-Shard manifest (version 1):
-  writer_epoch: 5
-  replay_after_wal_entry_position: 10
-  wal_entry_position_last_seen: 12
-```
-
-#### Scenario
-
-| Step | Writer A                                      | Writer B                                  | Manifest State     |
-| ---- | --------------------------------------------- | ----------------------------------------- | ------------------ |
-| 1    | Loads manifest, sees epoch=5                  |                                           | epoch=5, version=1 |
-| 2    | Increments to epoch=6, writes manifest v2     |                                           | epoch=6, version=2 |
-| 3    | Starts writing WAL entries 13, 14, 15         |                                           |                    |
-| 4    |                                               | Loads manifest v2, sees epoch=6           | epoch=6, version=2 |
-| 5    |                                               | Increments to epoch=7, writes manifest v3 | epoch=7, version=3 |
-| 6    |                                               | Starts writing WAL entries 16, 17         |                    |
-| 7    | Tries to flush MemTable, loads manifest       |                                           |                    |
-| 8    | Sees epoch=7, but local epoch=6               |                                           |                    |
-| 9    | **Writer A is fenced!** Aborts all operations |                                           |                    |
-| 10   |                                               | Continues writing normally                | epoch=7, version=3 |
-
-#### What Happens to Writer A's WAL Entries?
-
-Writer A wrote WAL entries 13, 14, 15 with `writer_epoch=6` in their schema metadata.
-
-When Writer B performs crash recovery or MemTable flush:
-
-1. Reads WAL entries sequentially starting from `replay_after_wal_entry_position + 1` (entry 11, since positions are 0-based)
-2. For each entry, checks existence using HEAD request on the bit-reversed filename
-3. Continues until an entry is not found (e.g., entry 18 doesn't exist)
-4. Finds entries 13, 14, 15, 16, 17
-5. Reads each file's `writer_epoch` from schema metadata
-6. Entries 13, 14, 15 have `writer_epoch=6` which is <= current epoch (7) -> **valid, will be replayed**
-7. Entries 16, 17 have `writer_epoch=7` -> **valid, will be replayed**
 
-#### Key Points
+Writer A loads version 1, claims epoch 6, and writes manifest version 2.
+It appends WAL entries 13, 14, and 15 with `writer_epoch = 6`.
 
-1. **No data loss**: Writer A's entries are not discarded. They were written with a valid epoch at the time and will be included in recovery.
+Writer B then loads version 2, claims epoch 7, and writes manifest version 3.
+It appends WAL entries 16 and 17 with `writer_epoch = 7`.
 
-2. **Consistency preserved**: Writer A is prevented from making further writes that could conflict with Writer B.
+When Writer A later tries to flush or update the shard manifest, it reloads the manifest and sees stored epoch 7 while its local epoch is 6.
+Writer A is fenced and must abort.
 
-3. **Orphaned files are safe**: WAL files from fenced writers remain on storage and are replayed by the new writer. They are only garbage collected after being included in a flushed MemTable that has been merged.
-
-4. **Epoch validation timing**: Writers check their epoch before manifest updates (MemTable flush), not on every WAL write. This keeps the hot path fast while ensuring consistency at commit boundaries.
+Recovery starts from `replay_after_wal_entry_position + 1`, which is entry 11.
+Entries 13, 14, 15, 16, and 17 are valid replay inputs because they were written by epochs that were valid at write time and are not greater than the current shard epoch.
 
 ### Appendix 2: Concurrent Merger Example
 
-This example demonstrates how MemWAL Index and conflict resolution handle concurrent mergers safely.
-
-#### Initial State
+Initial state:
 
-```
-MemWAL Index:
+```text
+MemWAL index:
   merged_generations: {shard: 5}
 
-Shard manifest (version 1):
+Shard manifest:
   current_generation: 8
-  flushed_generations: [(6, "abc123_gen_6"), (7, "def456_gen_7")]
+  flushed_generations:
+    - generation: 6, path: "abc12345_gen_6"
+    - generation: 7, path: "def67890_gen_7"
 ```
 
-#### Scenario 1: Racing on the Same Generation
-
-Two mergers both try to merge generation 6 concurrently.
-
-| Step | Merger A                  | Merger B                       | MemWAL Index     |
-| ---- | ------------------------- | ------------------------------ | ---------------- |
-| 1    | Reads index: merged_gen=5 |                                | merged_gen=5     |
-| 2    | Reads shard manifest     |                                |                  |
-| 3    | Starts merging gen 6      |                                |                  |
-| 4    |                           | Reads index: merged_gen=5      | merged_gen=5     |
-| 5    |                           | Reads shard manifest          |                  |
-| 6    |                           | Starts merging gen 6           |                  |
-| 7    | Commits (merged_gen=6)    |                                | **merged_gen=6** |
-| 8    |                           | Tries to commit                |                  |
-| 9    |                           | **Conflict**: reads new index  |                  |
-| 10   |                           | Sees merged_gen=6 >= 6, aborts |                  |
-| 11   |                           | Reloads, continues to gen 7    |                  |
+Two mergers both try to merge generation 6.
+Merger A commits first and updates `merged_generations[shard]` to 6 in the same base-table commit as the data.
+Merger B then hits a commit conflict, reloads the latest MemWAL index, sees `merged_generations[shard] >= 6`, skips generation 6, and continues with generation 7.
 
-Merger B's conflict resolution detected that generation 6 was already merged by checking the MemWAL Index in the conflicting commit.
+The MemWAL index is the authoritative merge-progress record because it is committed atomically with the base-table data changes.
 
-#### Scenario 2: Crash After Table Commit
+### Appendix 3: Bucket Hashing
 
-Merger A crashes after committing to the table.
+The bucket transform hash uses 32-bit wrapping arithmetic with these mixing functions.
+Right shifts in `fmix` are logical shifts of the `u32` bit pattern.
 
-| Step | Merger A                  | Merger B                         | MemWAL Index     |
-| ---- | ------------------------- | -------------------------------- | ---------------- |
-| 1    | Reads index: merged_gen=5 |                                  | merged_gen=5     |
-| 2    | Merges gen 6, commits     |                                  | **merged_gen=6** |
-| 3    | **CRASH**                 |                                  | merged_gen=6     |
-| 4    |                           | Reads index: merged_gen=6        | merged_gen=6     |
-| 5    |                           | Reads shard manifest            |                  |
-| 6    |                           | **Skips gen 6** (already merged) |                  |
-| 7    |                           | Merges gen 7, commits            | **merged_gen=7** |
-
-The MemWAL Index is the single source of truth. Merger B correctly used it to determine that generation 6 was already merged.
-
-#### Key Points
+```text
+mix_k1(k) = rotl32(k * 0xcc9e2d51, 15) * 0x1b873593
+mix_h1(h, k) = rotl32(h ^ k, 13) * 5 + 0xe6546b64
+fmix(h, len) =
+    h = h ^ len
+    h = (h ^ (h >> 16)) * 0x85ebca6b
+    h = (h ^ (h >> 13)) * 0xc2b2ae35
+    h ^ (h >> 16)
+```
 
-1. **Single source of truth**: `merged_generations` is the authoritative source for merge progress, updated atomically with data.
+Signed and unsigned casts use two's-complement wrapping.
+Values are normalized and hashed as follows:
+
+- `bool`: `false` as `0`, `true` as `1`, then `hash_i32`.
+- `int8`, `int16`, `int32`, `uint8`, `uint16`, `uint32`, `date32`, `time32`: cast to `i32`, then `hash_i32`.
+- `int64`, `uint64`, `timestamp`, `time64`: cast to `i64`, then `hash_i64`.
+- `float32`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7fc00000`; other values use IEEE 754 bits cast to `i32`, then `hash_i32`.
+- `float64`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7ff8000000000000`; other values use IEEE 754 bits cast to `i64`, then `hash_i64`.
+- `utf8` and `large_utf8`: hash the UTF-8 bytes with `hash_bytes`.
+
+The helper hashes are:
+
+```text
+hash_i32(v) = fmix(mix_h1(0, mix_k1(v)), 4)
+
+hash_i64(v) =
+    low = low 32 bits of v as i32
+    high = high 32 bits of v as i32
+    fmix(mix_h1(mix_h1(0, mix_k1(low)), mix_k1(high)), 8)
+
+hash_bytes(bytes) =
+    h = 0
+    for each complete 4-byte little-endian chunk:
+        h = mix_h1(h, mix_k1(chunk_as_i32))
+    for each remaining byte:
+        h = mix_h1(h, mix_k1(sign_extend_i8(byte)))
+    fmix(h, byte_length)
+```
 
-2. **Conflict resolution uses MemWAL Index**: When a commit conflicts, the merger checks the conflicting commit's MemWAL Index.
+Test vectors for `num_buckets = 8`:
 
-3. **No progress regression**: Because MemWAL Index is updated atomically with data, concurrent mergers cannot regress the merge progress.
+- `int32` or `date32`: `1 -> 2`, `2 -> 7`, `null -> 0`, `3 -> 1`.
+- `utf8`: `"a" -> 1`, `"b" -> 5`, `null -> 0`.
+- `bool`: `true -> 2`.
+- `float32`: `1.25 -> 0`.
+- `float64`: `1.25 -> 0`.
diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md
index d1a5191bf54..ef3384c6254 100644
--- a/docs/src/format/table/transaction.md
+++ b/docs/src/format/table/transaction.md
@@ -445,20 +445,85 @@ Replaces data in specific column regions with new data files.
 #### DataReplacement Compatibility
 
 A DataReplacement operation only replaces a single column's worth of data. As a result, it can be safer and simpler than Merge
-or Update operations. Here are the operations that conflict with DataReplacement:
+or Update operations. It rewrites a column file positionally against the fragments it targets, so a concurrent operation only
+conflicts when it removes one of those fragments or invalidates the rows the column file covers. Here are the operations that
+conflict with DataReplacement (non-retryable):
 
 - Overwrite
 - Restore
 - UpdateMemWalState
+- Delete (only if it removes a target fragment outright)
+- Update (only if it removes a target fragment outright)
 
 The following operations are retryable conflicts with DataReplacement:
 
 - DataReplacement (only if same field and overlapping fragments)
 - CreateIndex (only if the field being replaced is being indexed)
 - Rewrite (only if overlapping fragments)
-- Update (only if overlapping fragments)
+- Update (only if it rewrites rows out of a target fragment, or rewrites one of the replaced fields in place)
 - Merge (always)
 
+A concurrent Delete or Update that only adds a deletion vector to a target fragment (without removing it) is compatible: the
+positional column file stays aligned and the rebase preserves the deletion vector.
+
+### DataOverlay
+
+Attaches [overlay files](data_overlay_file.md) to fragments, supplying new values
+for a subset of `(row offset, field)` cells without rewriting the fragments' base
+data files. The overlays are appended to each fragment's existing `overlays` list,
+so overlays written by concurrent commits are preserved. Each overlay's
+`committed_version` is stamped to the new dataset version at commit time (and
+re-stamped on retry), like the created-at / last-updated-at version sequences.
+
+<details>
+<summary>DataOverlay protobuf message</summary>
+
+```protobuf
+%%% proto.message.DataOverlay %%%
+
+%%% proto.message.DataOverlayGroup %%%
+```
+
+</details>
+
+#### DataOverlay Compatibility
+
+A DataOverlay operation only changes cells within existing fragments and preserves
+physical row addresses, so — like DataReplacement — it is intentionally permissive.
+Because overlays stack and the higher `committed_version` wins each covered cell,
+independent backfills never conflict, and a concurrent Delete simply makes the
+overlay value for a deleted offset inert. Here are the operations that conflict
+with DataOverlay:
+
+- Overwrite
+- Restore
+- UpdateMemWalState
+
+The following operations are retryable conflicts with DataOverlay:
+
+- Rewrite (only if overlapping fragments) — row-rewriting compaction or an
+  overlay→base fold changes physical row addresses or consumes the overlays, so
+  the overlay's offsets are no longer valid; the writer must re-read the new
+  fragment, recompute, and retry.
+- Merge (always).
+- A row-moving Update that touches an overlaid fragment — a delete-and-reinsert
+  update (any update that is not a `REWRITE_COLUMNS` column rewrite) relocates the
+  updated rows into new fragments, so the overlay's physical offsets no longer
+  address them; the writer must re-read and retry.
+
+DataOverlay is compatible with another DataOverlay (any fields), Append, Delete, a
+`REWRITE_COLUMNS` column rewrite, and DataReplacement, because all of these
+preserve physical row addresses: overlay offsets stay valid, the overlay is newer
+and wins its covered cells, and the version gate excludes those cells from any
+rebuilt index.
+
+When a DataReplacement or a `REWRITE_COLUMNS` update writes new base values for a
+field, it supersedes any older overlay on that field: the writer tombstones the
+overlay's entry for the rewritten field — replacing the field id with the obsolete
+sentinel, as with obsolete base columns — so the fresh base values are not silently
+shadowed. Overlay entries for other fields are preserved, and an overlay left with
+no live fields is dropped.
+
 ### UpdateMemWalState
 
 Updates the state of MemWal indices (write-ahead log based indices).
diff --git a/docs/src/guide/.pages b/docs/src/guide/.pages
index 46ddd475799..7a0fd817b1c 100644
--- a/docs/src/guide/.pages
+++ b/docs/src/guide/.pages
@@ -6,6 +6,7 @@ nav:
   - JSON Support: json.md
   - Tags and Branches: tags_and_branches.md
   - Object Store Configuration: object_store.md
+  - Observability: observability.md
   - Distributed Write: distributed_write.md
   - Distributed Indexing: distributed_indexing.md
   - Migration Guide: migration.md
diff --git a/docs/src/guide/arrays.md b/docs/src/guide/arrays.md
index f824dd641c1..5765ca7675b 100644
--- a/docs/src/guide/arrays.md
+++ b/docs/src/guide/arrays.md
@@ -119,11 +119,11 @@ calling `lance.arrow.ImageTensorArray.to_encoded`.
 
 A `lance.arrow.EncodedImageArray.to_tensor` method is provided to decode
 encoded images and return them as `lance.arrow.FixedShapeImageTensorArray`, from
-which they can be converted to numpy arrays or TensorFlow tensors.
+which they can be converted to numpy arrays.
 For decoding images, it will first attempt to use a decoder provided via the optional
 function parameter. If decoder is not provided it will attempt to use
-[Pillow](https://pillow.readthedocs.io/en/stable/) and [tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) in that
-order. If neither library or custom decoder is available an exception will be raised.
+[Pillow](https://pillow.readthedocs.io/en/stable/). If neither Pillow nor a custom
+decoder is available an exception will be raised.
 
 ```python
 from lance.arrow import ImageURIArray
@@ -132,13 +132,16 @@ uris = [os.path.join(os.path.dirname(__file__), "images/1.png")]
 encoded_images = ImageURIArray.from_uris(uris).read_uris()
 print(encoded_images.to_tensor())
 
-def tensorflow_decoder(images):
-    import tensorflow as tf
+def pillow_decoder(images):
+    import io
     import numpy as np
+    from PIL import Image
 
-    return np.stack(tf.io.decode_png(img.as_py(), channels=3) for img in images.storage)
+    return np.stack(
+        np.asarray(Image.open(io.BytesIO(img.as_py()))) for img in images.storage
+    )
 
-print(encoded_images.to_tensor(tensorflow_decoder))
+print(encoded_images.to_tensor(pillow_decoder))
 # <lance.arrow.FixedShapeImageTensorArray object at 0x...>
 # [[42, 42, 42, 255]]
 # <lance.arrow.FixedShapeImageTensorArray object at 0x...>
@@ -164,8 +167,8 @@ created by calling `lance.arrow.ImageArray.from_array` and passing in a
 It can be encoded into to `lance.arrow.EncodedImageArray` by calling
 `lance.arrow.FixedShapeImageTensorArray.to_encoded` and passing custom encoder
 If encoder is not provided it will attempt to use
-[tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) and [Pillow](https://pillow.readthedocs.io/en/stable/) in that order. Default encoders will
-encode to PNG. If neither library is available it will raise an exception.
+[Pillow](https://pillow.readthedocs.io/en/stable/). The default encoder will encode
+to PNG. If neither Pillow nor a custom encoder is available it will raise an exception.
 
 ```python
 from lance.arrow import ImageURIArray
@@ -176,4 +179,4 @@ tensor_images.to_encoded()
 # <lance.arrow.EncodedImageArray object at 0x...>
 # [...
 # b'\x89PNG\r\n\x1a...'
-``` 
\ No newline at end of file
+```
diff --git a/docs/src/guide/blob.md b/docs/src/guide/blob.md
index b1f956a19e7..00bd5d086e3 100644
--- a/docs/src/guide/blob.md
+++ b/docs/src/guide/blob.md
@@ -1,20 +1,26 @@
 # Blob Columns
 
-Lance supports large binary objects (images, videos, audio, model artifacts) through blob columns.
-Blob access is lazy: reads return `BlobFile` handles so callers can stream bytes on demand.
+Lance can store large binary objects (images, videos, audio, model artifacts) in blob columns, where they are treated like any other column payload in the dataset.
+Blob columns support both planned full-payload reads and lazy file-like access.
 
-![Blob](../images/blob.png)
+!!! tip "Choosing between `read_blobs` and `take_blobs`"
+    - For data loaders and batch processing that need complete byte payloads, use `read_blobs`.
+    - Use `take_blobs` when you need a `BlobFile` handle for streaming, seeking, or partial reads.
 
-## What This Page Covers
 
-This page focuses on Python blob workflows and uses Lance file format terminology.
+![Blob v2 overview](../images/blob-v2-overview-light.png#only-light)
+![Blob v2 overview](../images/blob-v2-overview-dark.png#only-dark)
+
+If you're unsure about whether you need a blob column in the first place (and why it's useful), read the "[when to use blob column vs. inline binary](#when-to-use-a-blob-column-vs-inline-binary)" section below.
+
+## Quick Start: Blob v2
+
+This page focuses on blob workflows in Python and uses Lance file format terminology.
 
 - `data_storage_version` means the Lance **file format version** of a dataset.
 - A dataset's `data_storage_version` is fixed once the dataset is created.
 - If you need a different file format version, write a **new dataset**.
 
-## Quick Start (Blob v2)
-
 ```python
 import lance
 import pyarrow as pa
@@ -35,23 +41,27 @@ table = pa.table(
 
 ds = lance.write_dataset(table, "./blobs_v22.lance", data_storage_version="2.2")
 
-blob = ds.take_blobs("blob", indices=[0])[0]
-with blob as f:
-    assert f.read() == b"hello blob v2"
+_row_address, payload = ds.read_blobs("blob", indices=[0])[0]
+assert payload == b"hello blob v2"
 ```
 
-## Version Compatibility (Single Source of Truth)
+## Version Compatibility
+
+Blob support is tied to the dataset's file format version. Earlier file format versions
+(`< 2.2`) stored blobs using the `lance-encoding:blob` metadata field, while Blob
+v2 introduces a new storage layout that requires file format `>= 2.2`.
+
+The two
+schemes are mutually exclusive: for file format `>= 2.2`, legacy blob metadata
+(`lance-encoding:blob`) is rejected on write. The table below is the single
+source of truth for which scheme is supported at each `data_storage_version`.
 
 | Dataset `data_storage_version` | Legacy blob metadata (`lance-encoding:blob`) | Blob v2 (`lance.blob.v2`) |
 |---|---|---|
 | `0.1`, `2.0`, `2.1` | Supported for write/read | Not supported |
 | `2.2+` | Not supported for write | Supported for write/read (recommended) |
 
-Important:
-
-- For file format `>= 2.2`, legacy blob metadata (`lance-encoding:blob`) is rejected on write.
-
-## Blob v2 Write Patterns
+## Blob v2: Write Patterns
 
 Use `blob_field` and `blob_array` to build blob v2 columns.
 
@@ -95,6 +105,16 @@ Note:
 - By default, external blob URIs must map to a registered non-dataset-root base path.
 - If you need to reference external objects outside those bases, set
   `allow_external_blob_outside_bases=True` when writing.
+- Blob v2 storage layout thresholds can be configured per column with
+  `blob_field(..., inline_size_threshold=..., dedicated_size_threshold=...)`.
+  The inline threshold controls when values move from the data file to packed
+  `.blob` sidecar storage. The dedicated threshold controls when values move
+  from packed sidecar storage to a dedicated `.blob` file. The dedicated
+  threshold is checked first. For existing columns, these thresholds are stored
+  in the dataset schema; appends that explicitly provide different threshold
+  metadata for the same column are rejected.
+- `blob_pack_file_size_threshold` is a write option for rolling packed `.blob`
+  sidecar files. It does not control inline-vs-packed placement.
 
 ### Example: packed external blobs (single container file)
 
@@ -148,10 +168,22 @@ ds = lance.write_dataset(
 )
 ```
 
-## Blob v2 Read Patterns
+## Blob v2: Read Patterns
+
+Choose the read API based on the payload shape you want:
+
+| API | Returns | Use When |
+|---|---|---|
+| `read_blobs` | `List[Tuple[int, bytes]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. |
+| `take_blobs` | `List[BlobFile]` | You need file-like objects for streaming, seeking, or partial reads. |
+| `scanner(..., blob_handling="all_binary")` | Arrow binary columns | You want blob columns in a scan result or `pyarrow.Table`. |
 
-Use `take_blobs` to fetch file-like handles.
-Exactly one selector must be provided: `ids`, `indices`, or `addresses`.
+Do not wrap `take_blobs` in your own thread pool just to call `read()` or
+`readall()` on every blob. Use `read_blobs` instead; it plans and executes
+batched blob reads through Lance's scheduler.
+
+Exactly one selector must be provided to `read_blobs` or `take_blobs`: `ids`,
+`indices`, or `addresses`.
 
 | Selector | Typical Use | Stability |
 |---|---|---|
@@ -159,19 +191,17 @@ Exactly one selector must be provided: `ids`, `indices`, or `addresses`.
 | `ids` | Logical row-id based reads | Stable logical identity (when row ids are available) |
 | `addresses` | Low-level physical reads and debugging | Unstable physical location |
 
-### Read by row indices
+### Read complete payloads by row indices
 
 ```python
 import lance
 
 ds = lance.dataset("./blobs_v22.lance")
-blobs = ds.take_blobs("blob", indices=[0, 1])
-
-with blobs[0] as f:
-    data = f.read()
+rows = ds.read_blobs("blob", indices=[0, 1])
+payloads = [payload for _row_address, payload in rows]
 ```
 
-### Read by row ids
+### Read complete payloads by row ids
 
 ```python
 import lance
@@ -179,10 +209,10 @@ import lance
 ds = lance.dataset("./blobs_v22.lance")
 row_ids = ds.to_table(columns=[], with_row_id=True).column("_rowid").to_pylist()
 
-blobs = ds.take_blobs("blob", ids=row_ids[:2])
+rows = ds.read_blobs("blob", ids=row_ids[:2])
 ```
 
-### Read by row addresses
+### Read complete payloads by row addresses
 
 ```python
 import lance
@@ -190,7 +220,29 @@ import lance
 ds = lance.dataset("./blobs_v22.lance")
 row_addrs = ds.to_table(columns=[], with_row_address=True).column("_rowaddr").to_pylist()
 
-blobs = ds.take_blobs("blob", addresses=row_addrs[:2])
+rows = ds.read_blobs("blob", addresses=row_addrs[:2])
+```
+
+### Read blob columns as Arrow binary
+
+```python
+import lance
+
+ds = lance.dataset("./blobs_v22.lance")
+table = ds.scanner(columns=["blob"], blob_handling="all_binary").to_table()
+payloads = table.column("blob").to_pylist()
+```
+
+### Open file-like blob handles lazily
+
+```python
+import lance
+
+ds = lance.dataset("./blobs_v22.lance")
+blobs = ds.take_blobs("blob", indices=[0, 1])
+
+with blobs[0] as f:
+    header = f.read(1024)
 ```
 
 ### Example: decode video frames lazily
@@ -219,10 +271,10 @@ with av.open(blob) as container:
         pass
 ```
 
-## Legacy Compatibility Appendix (`data_storage_version` <= `2.1`)
+## Legacy Compatibility (`data_storage_version` <= `2.1`)
 
 If you need to keep writing legacy blob columns, use file format `0.1`, `2.0`, or `2.1`
-and mark `LargeBinary` fields with `lance-encoding:blob = true`.
+and mark `LargeBinary` fields with a metadata kwarg `"lance-encoding:blob": true`.
 
 ```python
 import lance
@@ -252,12 +304,12 @@ ds = lance.write_dataset(
 )
 ```
 
-This write pattern is invalid for `data_storage_version >= 2.2`.
-For new datasets, prefer blob v2.
+As mentioned above, this write pattern is invalid for `data_storage_version >= 2.2`.
+For new datasets, it's recommended to use Lance file format 2.2, which uses blob v2 by default.
 
 ## Rewrite to a New Blob v2 Dataset
 
-If your current dataset is legacy blob and you want blob v2, rewrite into a new dataset with `data_storage_version="2.2"`.
+If your current dataset consists of legacy blobs (stored in file formats <2.2) and you want to opt in to blob v2, you must rewrite it as a new dataset with `data_storage_version="2.2"`.
 
 ```python
 import lance
@@ -287,39 +339,33 @@ lance.write_dataset(
 )
 ```
 
-Warning:
-
-- The example above materializes binary payloads in memory (`blob_handling="all_binary"` and `to_pylist()`).
-- For large datasets, prefer chunked/batched rewrite pipelines.
-
-## Troubleshooting
-
-### "Blob v2 requires file version >= 2.2"
+!!! warning
+    - The example above materializes binary payloads in memory (`blob_handling="all_binary"` and `to_pylist()`).
+    - For large datasets, prefer chunked/batched rewrite pipelines.
 
-Cause:
+## When to Use a Blob Column vs. Inline Binary
 
-- You are writing blob v2 values into a dataset/file format below `2.2`.
+Not every binary column needs to be a blob column. Plain Arrow `binary`/`large_binary` stores bytes *inline*, interleaved with your other columns, which is simplest and fastest for really small blobs (e.g., thumbnail images). Using a blob column to store the binary payload makes sense when either of these holds:
 
-Fix:
+- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `take_blobs` → `BlobFile` handles that seek and range-read, so you pay only for the bytes you touch.
+- **Your values are large (roughly 1 MB or more on average).** Operations that rewrite entire rows, such as compaction or some updates, must copy the large inline payloads forward into the new version — even when those bytes never changed. The bigger the payload, the more bytes you rewrite per logical change (write amplification). A blob column keeps large payloads in separate `.blob` files that are referenced rather than re-copied, so these operations don't rewrite the heavy bytes.
 
-- Write to a dataset created with `data_storage_version="2.2"` (or newer).
+!!! tip
+    As a rule of thumb, if average payload size is below a few tens of KB and you only ever read whole values, plain inline binary is fine. Above ~1 MB, or any time you want file-like access, prefer a blob column. Blob v2 also tunes this automatically: by default it keeps payloads under 16 KiB inline, packs mid-sized payloads into shared `.blob` sidecars, and gives payloads over 2 MiB their own dedicated `.blob` file.
 
-### "Legacy blob columns ... are not supported for file version >= 2.2"
-
-Cause:
-
-- You are using legacy blob metadata (`lance-encoding:blob`) while writing `2.2+` data.
-
-Fix:
-
-- Replace legacy metadata-based columns with blob v2 columns (`blob_field` / `blob_array`).
+## Troubleshooting
 
-### "Exactly one of ids, indices, or addresses must be specified"
+This section contains commonly noticed issues or errors, and explains how to address them.
 
-Cause:
+### Blob v2 requires file version >= 2.2
+**Cause**: You are writing blob v2 values into a dataset/file format below `2.2`.  
+**Fix**: Write to a dataset created with `data_storage_version="2.2"` (or newer).
 
-- `take_blobs` received none or multiple selectors.
+### Legacy blob columns ... are not supported for file version >= 2.2
+**Cause**: You are using legacy blob metadata (`lance-encoding:blob`) while writing `2.2+` data.  
+**Fix**: Replace legacy metadata-based columns with blob v2 columns (`blob_field` / `blob_array`).
 
-Fix:
 
-- Provide exactly one of `ids`, `indices`, or `addresses`.
+### Exactly one of ids, indices, or addresses must be specified
+**Cause**: `read_blobs` or `take_blobs` received none or multiple selectors.  
+**Fix**: Provide exactly one of `ids`, `indices`, or `addresses`.
diff --git a/docs/src/guide/data_types.md b/docs/src/guide/data_types.md
index 178b155485b..06f26da1db3 100644
--- a/docs/src/guide/data_types.md
+++ b/docs/src/guide/data_types.md
@@ -32,7 +32,7 @@ Lance supports the full Apache Arrow type system. When writing data through Pyth
 
 ### Blob Type for Large Binary Objects
 
-Lance provides a specialized **Blob** type for efficiently storing and retrieving very large binary objects such as videos, images, audio files, or other multimedia content. Unlike regular binary columns, blobs support lazy loading, which means you can read portions of the data without loading everything into memory.
+Lance provides a specialized **Blob** type for efficiently storing and retrieving very large binary objects such as videos, images, audio files, or other multimedia content. Blob columns support planned full-payload reads as well as lazy file-like access for streaming, seeking, and partial reads.
 
 For new datasets, use blob v2 (`lance.blob.v2`) via `blob_field` and `blob_array`.
 
@@ -62,9 +62,7 @@ table = pa.table(
 )
 
 ds = lance.write_dataset(table, "./videos_v22.lance", data_storage_version="2.2")
-blob = ds.take_blobs("video", indices=[0])[0]
-with blob as f:
-    payload = f.read()
+_row_address, payload = ds.read_blobs("video", indices=[0])[0]
 ```
 
 For legacy compatibility (`data_storage_version <= 2.1`), you can still write blob columns using `LargeBinary` with `lance-encoding:blob=true`.
@@ -101,11 +99,18 @@ ds = lance.write_dataset(
 )
 ```
 
-To read blob data, use `take_blobs()` which returns file-like objects for lazy reading:
+To read complete blob payloads into memory, use `read_blobs()`:
+
+```python
+rows = ds.read_blobs("video", indices=[0])
+_row_address, payload = rows[0]
+```
+
+Use `take_blobs()` only when you need file-like objects for lazy, partial, or seek-based reading:
 
 ```python
 # Retrieve blob as a file-like object (lazy loading)
-blobs = ds.take_blobs("video", ids=[0])
+blobs = ds.take_blobs("video", indices=[0])
 
 # Use with libraries that accept file-like objects
 import av  # pip install av
diff --git a/docs/src/guide/distributed_indexing.md b/docs/src/guide/distributed_indexing.md
index ae17b9bb0f2..389e5a1bc09 100644
--- a/docs/src/guide/distributed_indexing.md
+++ b/docs/src/guide/distributed_indexing.md
@@ -18,9 +18,10 @@ write:
 3. Lance plans and builds index artifacts from the worker outputs supplied by the caller
 4. the built artifacts are committed into the dataset manifest
 
-For vector indices, the worker outputs are segments stored directly
-under `indices/<segment_uuid>/`. Lance can turn these outputs into one or more
-physical segments and then commit them as one logical index.
+For vector indices and segment-native scalar indices, the worker outputs are
+segments stored directly under `indices/<segment_uuid>/`. Lance can turn these
+outputs into one or more physical segments and then commit them as one logical
+index.
 
 ![Distributed Vector Segment Build](../images/distributed_vector_segment_build.svg)
 
@@ -81,7 +82,7 @@ launching workers and driving the overall workflow.
 
 ## Current Model
 
-The current model for distributed vector indexing has two layers of parallelism.
+The current model for distributed indexing has two layers of parallelism.
 
 ### Worker Build
 
@@ -105,9 +106,46 @@ or merged into larger segments:
 
 Within a single commit, built segments must have disjoint fragment coverage.
 
+`merge_existing_index_segments(...)` currently supports vector, inverted,
+bitmap, BTree, and zone map segments. Other scalar index families can still
+commit multiple compatible segments directly when their build path supports
+fragment-scoped segments, but cannot be merged into a larger physical segment
+until they add a merge implementation.
+
+### Vector Model Scope
+
+Distributed vector builds support two model scopes.
+
+**Shared model artifacts**: the caller trains or provides IVF centroids once and
+passes the same artifacts to every worker. For IVF-PQ segments that should be
+physically mergeable, workers should also use the same PQ codebook. This makes
+partition ids and quantizer state have the same meaning across segments.
+
+**Independent segment models**: each worker trains the IVF/PQ model for its own
+`fragment_ids`. The resulting segments can be committed together as one logical
+index without sharing centroids or codebooks.
+
+At query time, Lance searches each physical segment independently:
+
+1. Lance opens each segment by index UUID
+2. each segment ranks IVF partitions using its own centroids
+3. each segment searches the selected partitions using its own quantizer storage
+4. Lance merges the candidate rows from all segments by `_distance`
+
+Because partition ids are interpreted only within a segment during this fanout
+query path, independently trained committed segments can return valid results.
+For L2 and cosine IVF-PQ, each segment computes residuals against its own IVF
+centroid during both build and query, so distances remain estimates of the
+original query-to-vector metric.
+
+Physical merge is a separate operation. It rewrites several segment artifacts
+into one artifact with one model metadata scope. Use shared compatible model
+artifacts for segments you plan to merge physically, or keep independently
+trained segments as separate physical segments.
+
 ## Internal Finalize Model
 
-Internally, Lance models distributed vector segment build as:
+Internally, Lance models distributed segment build as:
 
 1. **build** one uncommitted segment per worker
 2. **optionally merge** caller-defined groups of existing segments
diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md
index 32d2d89ee7f..5efd7b26b7f 100644
--- a/docs/src/guide/migration.md
+++ b/docs/src/guide/migration.md
@@ -6,6 +6,27 @@ stable and breaking changes should generally be communicated (via warnings) for
 give users a chance to migrate.  This page documents the breaking changes between releases and gives advice on how to
 migrate.
 
+## 9.0.0
+
+* Newly created FTS / inverted indexes now default to format v2 instead of v1.
+  The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the
+  format used for newly created indexes. Users who need a specific index layout
+  should pass the index creation parameter `format_version` explicitly.
+
+* This affects users who create FTS / inverted indexes and need those indexes to
+  be readable by older Lance versions, or who depend on the v1 index layout. In
+  those cases, pass `format_version=1` when creating the index. Otherwise, newly
+  created indexes will use v2 by default, and older Lance readers may not be able
+  to read them.
+
+  ```python
+  dataset.create_scalar_index("text", "INVERTED", format_version=1)
+  ```
+
+* Existing v1 FTS indexes remain queryable. Operations that maintain an existing
+  v1 index, including append, incremental indexing, optimize, and mem-wal
+  maintained-index flush, should continue preserving the v1 format.
+
 ## 7.2.0
 
 * The `IndexSegmentBuilder` API has been removed from Rust, Python, and Java.
diff --git a/docs/src/guide/object_store.md b/docs/src/guide/object_store.md
index 1710e3b5100..9ce4800b800 100644
--- a/docs/src/guide/object_store.md
+++ b/docs/src/guide/object_store.md
@@ -41,6 +41,35 @@ These options apply to all object stores.
 | `client_max_retries`         | Number of times for the object store client to retry the request. Default, `3`.                                                                                                                                                                                                                         |
 | `client_retry_timeout`       | Timeout for the object store client to retry the request in seconds. Default, `180`.                                                                                                                                                                                                                    |
 
+## Per-Base Configuration
+
+A dataset can register additional base paths that store part of its data, and each
+base may live in a different bucket, account, or storage provider. A storage option
+key of the form `base_<id>.<key>` applies `<key>` only to the base path with that
+manifest id. Every base inherits the unscoped options; base-scoped entries add to or
+override them for that base only.
+
+```python
+import lance
+ds = lance.dataset(
+    "az://account-a/path",
+    storage_options={
+        # Shared defaults, used by the primary dataset and inherited by bases
+        "account_name": "account-a",
+        "account_key": "key-a",
+        # Overrides for the base path with id 1
+        "base_1.account_name": "account-b",
+        "base_1.account_key": "key-b",
+    },
+)
+```
+
+Base ids are assigned when bases are registered (`initial_bases` ids are assigned
+sequentially starting at 1, in order) and can be inspected through the manifest base
+paths. Keys that do not match `base_<id>.<key>` exactly (e.g. `base_url`) are treated
+as regular storage options. Exact per-base parameter maps (`base_store_params`,
+keyed by base path URI) take precedence over base-scoped keys for that base.
+
 ## S3 Configuration
 
 S3 (and S3-compatible stores) have additional configuration options that configure
@@ -218,3 +247,238 @@ ds = lance.dataset(
 | `oss_secret_access_key` | Access key secret used for OSS authentication. Optional if credentials are provided by environment. |
 | `oss_region` | OSS region (for example, `cn-hangzhou`). Optional. |
 | `oss_security_token` | Security token for temporary credentials (STS). Optional. |
+
+## Volcengine TOS Configuration
+
+TOS credentials can be set in the environment variables `TOS_ACCESS_KEY_ID`,
+`TOS_SECRET_ACCESS_KEY`, `TOS_ENDPOINT`, `TOS_REGION`, and `TOS_SECURITY_TOKEN`.
+Lance also accepts the corresponding `VOLCENGINE_` environment variable prefix.
+Alternatively, credentials can be passed as parameters to the `storage_options`
+parameter; explicit `storage_options` override environment variables:
+
+```python
+import lance
+ds = lance.dataset(
+    "tos://bucket/path",
+    storage_options={
+        "tos_endpoint": "https://tos-cn-beijing.volces.com",
+        "tos_region": "cn-beijing",
+        "tos_access_key_id": "my-access-key",
+        "tos_secret_access_key": "my-secret-key",
+        "tos_security_token": "my-session-token",
+    }
+)
+```
+
+| Key | Description |
+|-----|-------------|
+| `tos_endpoint` | TOS endpoint. Required (for example, `https://tos-cn-beijing.volces.com`). |
+| `tos_region` | TOS signing region (for example, `cn-beijing`). Optional. |
+| `tos_access_key_id` | Access key ID used for TOS authentication. Optional if credentials are provided by environment. |
+| `tos_secret_access_key` | Secret access key used for TOS authentication. Optional if credentials are provided by environment. |
+| `tos_security_token` | Security token for temporary credentials. Optional. |
+
+## Tencent Cloud COS Configuration
+
+[COS (Cloud Object Storage)](https://cloud.tencent.com/product/cos) credentials can be set in environment variables prefixed
+with `COS_` or `TENCENTCLOUD_` (for example, `COS_ENDPOINT`, `COS_SECRET_ID`,
+`COS_SECRET_KEY`, `TENCENTCLOUD_REGION`, `TENCENTCLOUD_SECURITY_TOKEN`).
+Alternatively, credentials can be passed as parameters to the `storage_options`
+parameter; explicit `storage_options` override environment variables:
+
+=== "Python"
+
+    ```python
+    import lance
+    ds = lance.dataset(
+        "cos://bucket/path",
+        storage_options={
+            "cos_endpoint": "https://cos.ap-guangzhou.myqcloud.com",
+            "cos_secret_id": "my-secret-id",
+            "cos_secret_key": "my-secret-key",
+        }
+    )
+    ```
+
+=== "Rust"
+
+    In this Lance distribution, `tencent` is already part of the **default
+    features** of the `lance` crate, so simply depending on `lance` is enough:
+
+    ```toml
+    [dependencies]
+    lance = "*"
+    ```
+
+    You only need to enable the `tencent` feature explicitly in the following
+    cases:
+
+    - You opted out of default features, e.g.
+      `lance = { version = "*", default-features = false, features = ["tencent", ...] }`.
+    - You depend on `lance-io` directly (without `lance`); `tencent` is **not**
+      a default feature of `lance-io`:
+      `lance-io = { version = "*", features = ["tencent"] }`.
+
+| Key | Description |
+|-----|-------------|
+| `cos_endpoint` | COS endpoint. Required (for example, `https://cos.ap-guangzhou.myqcloud.com`). Can also be set via the `COS_ENDPOINT` environment variable. |
+| `cos_secret_id` | Secret ID used for COS authentication. Optional if credentials are provided by environment. |
+| `cos_secret_key` | Secret key used for COS authentication. Optional if credentials are provided by environment. |
+| `cos_enable_versioning` | Whether to enable object versioning on the bucket. Optional. |
+
+!!! note
+
+    The OpenDAL `CosConfig` currently exposes a limited set of options. Additional
+    settings such as the security token (`TENCENTCLOUD_SECURITY_TOKEN`) and region
+    (`TENCENTCLOUD_REGION`) must be configured via environment variables.
+
+## GooseFS Configuration
+
+[GooseFS](https://cloud.tencent.com/product/goosefs) is a distributed caching
+filesystem. Lance accesses GooseFS through its Master gRPC service. The URL format
+is `goosefs://host:port/path`, where `host:port` is the GooseFS Master address
+(default port: `9200`, may be omitted, e.g. `goosefs://10.0.0.1/path`) and
+`/path` is the filesystem path within GooseFS.
+
+!!! note "About the dataset path"
+
+    `/path` is just an arbitrary directory inside GooseFS — Lance does **not**
+    require the path to end with a `.lance` suffix. Any valid GooseFS directory
+    works, for example:
+
+    - `goosefs://10.0.0.1:9200/data/my-dataset`
+    - `goosefs://10.0.0.1:9200/data/my-dataset.lance`
+    - `goosefs://10.0.0.1:9200/lance-test/lance-io`
+
+    The `.lance` suffix used in the examples below is only a naming convention
+    that makes it easy to recognize a Lance dataset directory at a glance; it
+    has no special meaning to Lance itself. The only requirement is that the
+    same path is used consistently for reads and writes of a given dataset.
+
+=== "Python"
+
+    ```python
+    import lance
+
+    ds = lance.dataset(
+        "goosefs://10.0.0.1:9200/data/my-dataset.lance",
+        storage_options={
+            "goosefs_auth_type": "simple",
+            "goosefs_auth_username": "lance",
+        },
+    )
+    ```
+
+=== "Rust"
+
+    In this Lance distribution, `goosefs` is already part of the **default
+    features** of the `lance` crate, so simply depending on `lance` is enough:
+
+    ```toml
+    [dependencies]
+    lance = "*"
+    ```
+
+    You only need to enable the `goosefs` feature explicitly in the following
+    cases:
+
+    - You opted out of default features, e.g.
+      `lance = { version = "*", default-features = false, features = ["goosefs", ...] }`.
+    - You depend on `lance-io` directly (without `lance`); `goosefs` is **not**
+      a default feature of `lance-io`:
+      `lance-io = { version = "*", features = ["goosefs"] }`.
+
+    Open the underlying `lance_io::object_store::ObjectStore` directly (mirrors
+    the integration test in `rust/lance-io/tests/goosefs_integration.rs`):
+
+    ```rust
+    use lance_io::object_store::ObjectStore;
+
+    let uri = "goosefs://10.0.0.1:9200/lance-test/lance-io";
+    let (store, path) = ObjectStore::from_uri(uri).await?;
+
+    // Read / write through the underlying `object_store::ObjectStore` API
+    store.inner.put(&path, (&b"hello"[..]).into()).await?;
+    let result = store.inner.get(&path).await?;
+    let bytes = result.bytes().await?;
+    ```
+
+    Open a Lance dataset with custom storage options:
+
+    ```rust
+    use std::collections::HashMap;
+    use lance::dataset::DatasetBuilder;
+
+    let mut storage_options = HashMap::new();
+    storage_options.insert("goosefs_master_addr".to_string(), "10.0.0.1:9200".to_string());
+    storage_options.insert("goosefs_auth_type".to_string(), "simple".to_string());
+    storage_options.insert("goosefs_auth_username".to_string(), "lance".to_string());
+
+    let dataset = DatasetBuilder::from_uri("goosefs://10.0.0.1:9200/data/my-dataset.lance")
+        .with_storage_options(storage_options)
+        .load()
+        .await?;
+    ```
+
+=== "Java"
+
+    Pass the GooseFS configuration through `ReadOptions.setStorageOptions`
+    when opening the dataset:
+
+    ```java
+    import org.lance.Dataset;
+    import org.lance.ReadOptions;
+
+    import java.util.HashMap;
+    import java.util.Map;
+
+    Map<String, String> storageOptions = new HashMap<>();
+    storageOptions.put("goosefs_master_addr", "10.0.0.1:9200");
+    storageOptions.put("goosefs_auth_type", "simple");
+    storageOptions.put("goosefs_auth_username", "lance");
+
+    ReadOptions options = new ReadOptions.Builder()
+        .setStorageOptions(storageOptions)
+        .build();
+
+    try (Dataset dataset = Dataset.open()
+            .uri("goosefs://10.0.0.1:9200/data/my-dataset.lance")
+            .readOptions(options)
+            .build()) {
+        // ... use the dataset
+    }
+    ```
+
+    For writes, the same `storageOptions(...)` setter is available on
+    `WriteDatasetBuilder` and `WriteFragmentBuilder`.
+
+The Master address can be resolved from (in priority order):
+
+1. The `goosefs_master_addr` storage option (supports HA: `"addr1:port,addr2:port"`).
+2. The `GOOSEFS_MASTER_ADDR` environment variable.
+3. The host and port from the URL authority.
+
+The following keys can be used as both environment variables or keys in the
+`storage_options` parameter:
+
+| Key | Description |
+|-----|-------------|
+| `goosefs_master_addr` / `GOOSEFS_MASTER_ADDR` | GooseFS Master address. Supports a single address (`host:port`) or comma-separated HA addresses (`addr1:port,addr2:port`). Optional if the address is provided in the URL. |
+| `goosefs_write_type` / `GOOSEFS_WRITE_TYPE` | Write type, e.g. `MUST_CACHE`, `CACHE_THROUGH`, `THROUGH`, `ASYNC_THROUGH`. Optional. |
+| `goosefs_block_size` / `GOOSEFS_BLOCK_SIZE` | GooseFS block size in bytes (this is the GooseFS-side block size, not Lance's I/O block size). Optional. |
+| `goosefs_chunk_size` / `GOOSEFS_CHUNK_SIZE` | Chunk size in bytes used when reading or writing files. Optional. |
+| `goosefs_auth_type` / `GOOSEFS_AUTH_TYPE` | Authentication type. Either `nosasl` or `simple` (case-insensitive; the value is passed through to OpenDAL). Optional. |
+| `goosefs_auth_username` / `GOOSEFS_AUTH_USERNAME` | Username used in `simple` authentication mode. Optional. |
+
+!!! note "Running the GooseFS integration tests"
+
+    The Rust integration tests for GooseFS live at
+    `rust/lance-io/tests/goosefs_integration.rs` and are gated behind feature
+    flags. They require a reachable GooseFS cluster (configured via the
+    `GOOSEFS_MASTER_ADDR` and `GOOSEFS_AUTH_TYPE` environment variables) and
+    can be run with:
+
+    ```bash
+    cargo test -p lance-io --features "goosefs goosefs-test" \
+        --test goosefs_integration -- --ignored --nocapture --test-threads=1
+    ```
diff --git a/docs/src/guide/observability.md b/docs/src/guide/observability.md
new file mode 100644
index 00000000000..f2053b59447
--- /dev/null
+++ b/docs/src/guide/observability.md
@@ -0,0 +1,69 @@
+# Observability
+
+Lance can publish operational metrics to your monitoring stack. The table below
+is the authoritative catalogue of the metrics Lance emits, shared verbatim with
+the Rust [`lance::metrics`](https://docs.rs/lance/latest/lance/metrics/) module
+documentation.
+
+--8<-- "rust/lance/src/metrics.md"
+
+## Collecting metrics
+
+Lance emits through the [`metrics`](https://docs.rs/metrics) crate facade, so it
+is not tied to a specific backend — you install a recorder/exporter and route
+the metrics wherever you like. Metrics are available from **both** the Rust and
+Python APIs.
+
+### Rust
+
+Enable the `metrics` feature on the `lance` crate:
+
+```toml
+lance = { version = "...", features = ["metrics"] }
+```
+
+Then install any `metrics`-compatible recorder once at startup, before opening
+datasets. For example, with
+[`metrics-exporter-prometheus`](https://docs.rs/metrics-exporter-prometheus):
+
+```rust
+metrics_exporter_prometheus::PrometheusBuilder::new()
+    .install()
+    .expect("install Prometheus recorder");
+```
+
+Any recorder works — Prometheus, StatsD, an OpenTelemetry bridge, and so on.
+When no recorder is installed, emission is a cheap no-op.
+
+### Python
+
+Unlike Rust, the Python bindings do not let you plug in an arbitrary recorder:
+bridging one across the FFI boundary into the Rust `metrics` facade would be
+complicated and inefficient. Instead `pylance` standardizes on OpenTelemetry,
+which has good Python support, as its recorder.
+
+The `pylance` wheels are built with the `metrics` feature enabled. Install the
+OpenTelemetry extra and call `instrument_lance_metrics`, which registers Lance's
+metrics as observable instruments on your OpenTelemetry `MeterProvider`:
+
+```bash
+pip install "pylance[otel]"
+```
+
+```python
+from lance.otel import instrument_lance_metrics
+
+# Uses the global MeterProvider; pass meter_provider=... to target a specific one.
+instrument_lance_metrics()
+```
+
+From there the metrics flow through whatever OpenTelemetry pipeline you have
+configured (OTLP, Prometheus, console, …). Because OpenTelemetry has no
+asynchronous histogram instrument, histograms are exported Prometheus-style as
+three observable counters: `<name>_bucket`, `<name>_count`, and `<name>_sum`.
+Each `<name>_bucket` sample carries an `le` ("less than or equal") attribute
+giving that bucket's inclusive upper bound in the metric's unit; the bucket
+count is cumulative, covering every observation at or below `le`. For example, a
+`lance_object_store_request_duration_seconds_bucket` sample with `le="0.5"`
+counts all requests that completed in 0.5 seconds or less, while `le="+Inf"` is
+the total count.
diff --git a/docs/src/guide/performance.md b/docs/src/guide/performance.md
index 14181eb69af..acccdf14776 100644
--- a/docs/src/guide/performance.md
+++ b/docs/src/guide/performance.md
@@ -219,6 +219,37 @@ use cases. For example, S3 can typically get up to 5000
 req/s and with these settings we should get there in about
 10 seconds.
 
+## Fragment Sizing
+
+A Lance table is a collection of fragments tracked by a manifest. How you size those fragments
+trades off two classes of work:
+
+- **Manifest-level operations** scale with the *number* of fragments. Every dataset mutation
+  (appends, metadata updates, schema changes, compactions, etc.) rewrites the manifest, so a
+  larger fragment list makes every write slower. Reads pay a similar cost up front: opening a
+  dataset, listing fragments, planning a scan, and resolving transaction conflicts at the
+  dataset level all walk the manifest.
+- **Fragment-level operations** scale with the *size* of a fragment. These include scans
+  against a matching fragment, compaction, updates, deletes, and `merge_insert`. Conflict
+  detection for these operations is also done at the fragment level.
+
+Fewer, larger fragments make manifest-level operations cheap but make each fragment-level
+operation heavier and increase the chance of conflicts when many writers target the same
+fragment. More, smaller fragments do the reverse.
+
+Practical guidance:
+
+- The default of 1M rows per fragment works well up to ~1B rows. Past that, bumping toward
+  ~100M rows per fragment is reasonable, though fragment-count limits are rarely the bottleneck
+  in practice.
+- Tens of thousands of fragments per table is generally fine.
+- Keep individual fragments well under object-store object-size limits (S3 caps at 5 TB, and
+  stores tend to misbehave well before that). 10 GB–100 GB per fragment is a reasonable upper
+  range; 1 TB is a hard ceiling.
+- If you run many concurrent updates, deletes, or `merge_insert` operations, err toward more
+  fragments — conflict detection is per-fragment, so too few fragments leads to excess
+  retries.
+
 ## Conflict Handling
 
 Lance supports concurrent operations on the same table using optimistic concurrency control. When two
diff --git a/docs/src/guide/read_and_write.md b/docs/src/guide/read_and_write.md
index ec6cde5173a..c7feb69144b 100644
--- a/docs/src/guide/read_and_write.md
+++ b/docs/src/guide/read_and_write.md
@@ -456,3 +456,146 @@ affected files are no longer part of any ANN index if they were before. Because
 of this, it's recommended to rewrite files before re-building indices.
 
 <!-- TODO: remove this last comment once stable row ids are default. -->
+
+### Cleanup old versions
+
+Lance is an immutable format — every write creates a new version. The new version
+only writes the data that changed, so an insert writes the new rows and an update
+rewrites the affected columns for the affected rows. Even a delete creates a
+small deletion file. However, old versions still reference the previous data
+files, so those files are kept on disk until explicitly removed. Over time this
+means storage grows with each operation — inserts, updates, and deletes alike.
+
+Keeping old versions has important benefits: readers that opened an older version
+can continue reading it without interference from concurrent writers, providing
+snapshot isolation. Old versions also enable time travel queries, letting you
+read the dataset as it existed at any prior point in time.
+
+`cleanup_old_versions` deletes old version metadata and any data files that are
+no longer referenced by any version, reclaiming the accumulated storage.
+
+!!! warning
+
+    Once old versions are cleaned up, time travel queries to those versions are
+    no longer possible. Choose your retention window (`older_than`) accordingly —
+    any version removed by cleanup cannot be recovered.
+
+```python
+import lance
+
+dataset = lance.dataset("./my_dataset.lance")
+dataset.cleanup_old_versions()
+```
+
+By default, versions older than 7 days are removed. You can override this with
+the `older_than` parameter (a `timedelta`):
+
+```python
+from datetime import timedelta
+
+dataset.cleanup_old_versions(older_than=timedelta(days=1))
+```
+
+!!! note
+
+    Tagged versions are exempt from cleanup. See [Tags and Branches](tags_and_branches.md)
+    for details.
+
+By default, Lance only removes files that it can **verify** are no longer needed.
+A file is verified when Lance can see that it was referenced by an older version
+and is no longer referenced by any newer version. However, some orphaned files
+cannot be verified this way — for example, files left behind by aborted or failed
+commits that were never recorded in any version. These files are
+indistinguishable from files being written by an in-progress operation.
+
+Cleanup will never delete the current (active) version. This means passing
+`older_than=timedelta(0)` is safe and will delete all versions except the current
+one.
+
+The `delete_unverified` flag enables a more aggressive strategy that will also
+delete these unverified files:
+
+```python
+dataset.cleanup_old_versions(
+    older_than=timedelta(hours=2),
+    delete_unverified=True,
+)
+```
+
+!!! danger
+
+    Only use `delete_unverified=True` when you are confident that no other
+    concurrent operation has been in-progress for longer than the `older_than`
+    duration. Lance uses the file's age to decide whether an unverified file is
+    safe to remove, so any operation that is still running past the `older_than`
+    window risks having its files deleted out from under it.
+
+    In particular, combining `delete_unverified=True` with `older_than=timedelta(0)`
+    is **extremely dangerous** — if any other operation is in-progress at all,
+    its data files may be deleted, leading to dataset corruption.
+
+### Automatic cleanup
+
+Instead of calling `cleanup_old_versions` manually, you can configure Lance to
+clean up old versions automatically during writes. When auto cleanup is enabled,
+Lance will run cleanup every *N* commits (the **interval**), removing versions
+older than a specified duration.
+
+Auto cleanup can be enabled when creating a new dataset:
+
+```python
+import lance
+import pyarrow as pa
+from lance.dataset import AutoCleanupConfig
+
+table = pa.table({"id": range(100)})
+ds = lance.write_dataset(
+    table,
+    "./my_dataset.lance",
+    auto_cleanup_options=AutoCleanupConfig(
+        interval=20,             # run cleanup every 20 commits
+        older_than_seconds=3600, # remove versions older than 1 hour
+    ),
+)
+```
+
+Or enabled on an existing dataset:
+
+```python
+ds = lance.dataset("./my_dataset.lance")
+ds.optimize.enable_auto_cleanup(
+    AutoCleanupConfig(
+        interval=20,
+        older_than_seconds=3600,
+    )
+)
+```
+
+And disabled again:
+
+```python
+ds.optimize.disable_auto_cleanup()
+```
+
+Auto cleanup parameters can also be set directly via dataset config keys:
+
+```python
+ds.update_config({
+    "lance.auto_cleanup.interval": "20",
+    "lance.auto_cleanup.older_than": "3600s",
+})
+```
+
+!!! warning
+
+    Auto cleanup runs as part of the commit path. If your writer does not have
+    delete permissions, or you are doing high-frequency writes where the extra
+    latency matters, pass `skip_auto_cleanup=True` to `write_dataset` to skip it
+    on a per-write basis.
+
+### Other cleanup strategies
+
+It is common to run cleanup as a periodic background task on a dedicated server
+(for example, via a cron job or scheduled workflow). This keeps cleanup off the
+write path entirely, avoiding any impact to write latency, but requires setting
+up and maintaining additional infrastructure.
diff --git a/docs/src/guide/tokenizer.md b/docs/src/guide/tokenizer.md
index 192574656f5..eedac585fe9 100644
--- a/docs/src/guide/tokenizer.md
+++ b/docs/src/guide/tokenizer.md
@@ -20,6 +20,12 @@ ICU uses Unicode word boundary rules and bundled dictionary data for complex scr
 ds.create_scalar_index("text", "INVERTED", base_tokenizer="icu")
 ```
 
+Use `icu/split` when mixed-language text also contains punctuation-delimited identifiers that should be searchable by part.
+
+```python
+ds.create_scalar_index("text", "INVERTED", base_tokenizer="icu/split")
+```
+
 ## Language Models of Jieba
 
 ### Downloading the Model
diff --git a/docs/src/images/blob-v2-overview-dark.png b/docs/src/images/blob-v2-overview-dark.png
new file mode 100644
index 00000000000..04853012405
Binary files /dev/null and b/docs/src/images/blob-v2-overview-dark.png differ
diff --git a/docs/src/images/blob-v2-overview-light.png b/docs/src/images/blob-v2-overview-light.png
new file mode 100644
index 00000000000..7dae4f70cc3
Binary files /dev/null and b/docs/src/images/blob-v2-overview-light.png differ
diff --git a/docs/src/images/lakehouse_stack.png b/docs/src/images/lakehouse_stack.png
index e34c999efd8..b3e4db709d6 100644
Binary files a/docs/src/images/lakehouse_stack.png and b/docs/src/images/lakehouse_stack.png differ
diff --git a/docs/src/integrations/.pages b/docs/src/integrations/.pages
index 62feffae067..ba764bf40f0 100644
--- a/docs/src/integrations/.pages
+++ b/docs/src/integrations/.pages
@@ -3,7 +3,7 @@ nav:
   - Apache DataFusion: datafusion.md
   - PostgreSQL: https://github.com/lancedb/pglance
   - PyTorch: pytorch.md
-  - Tensorflow: tensorflow.md
+  - TensorFlow: tensorflow.md
   - Apache Spark: spark
   - Ray: ray
   - Trino: trino
diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md
index 0304f8f5277..eb3b25051cc 100644
--- a/docs/src/integrations/index.md
+++ b/docs/src/integrations/index.md
@@ -27,7 +27,7 @@ GitHub organization.
 | Integration | Description | Source |
 |---|---|---|
 | [PyTorch](pytorch.md) | Use `lance.torch.data.LanceDataset` as a `torch.utils.data.IterableDataset` for training and inference. | Built-in |
-| [TensorFlow](tensorflow.md) | Use `lance.tf.data.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | Built-in |
+| [TensorFlow](tensorflow.md) | Use `lance_tensorflow.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | [lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) |
 | [Ray](ray) | Distributed read/write of Lance datasets with Ray Data. | [lance-format/lance-ray](https://github.com/lance-format/lance-ray) |
 | [Hugging Face](huggingface) | Convert and load Hugging Face datasets to and from Lance in a single call. | [lance-format/lance-huggingface](https://github.com/lance-format/lance-huggingface) |
 
diff --git a/docs/src/integrations/tensorflow.md b/docs/src/integrations/tensorflow.md
index 1c5d6b87157..03a5c5c5eca 100644
--- a/docs/src/integrations/tensorflow.md
+++ b/docs/src/integrations/tensorflow.md
@@ -1,92 +1,71 @@
-# Tensorflow Integration
+---
+title: TensorFlow
+description: Stream Lance datasets into TensorFlow tf.data pipelines with lance-tensorflow.
+---
 
-Lance can be used as a regular [tf.data.Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
-in [Tensorflow](https://www.tensorflow.org/).
+# TensorFlow Integration
 
-!!! warning
+The TensorFlow integration is maintained in the
+[lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow)
+project.
 
-    This feature is experimental and the APIs may change in the future.
+The main Lance Python package no longer includes `lance.tf`. Install
+`lance-tensorflow` and import `lance_tensorflow` instead.
+
+```bash
+pip install lance-tensorflow
+```
 
 ## Reading from Lance
 
-Using `lance.tf.data.from_lance`, you can create an `tf.data.Dataset` easily.
+Use `lance_tensorflow.from_lance` to create a `tf.data.Dataset` from a Lance
+dataset.
 
 ```python
-import tensorflow as tf
-import lance
+from lance_tensorflow import from_lance
 
-# Create tf dataset
-ds = lance.tf.data.from_lance("s3://my-bucket/my-dataset")
-
-# Chain tf dataset with other tf primitives
+ds = from_lance(
+    "s3://my-bucket/my-dataset",
+    columns=["image", "label"],
+    filter="split = 'train'",
+    batch_size=256,
+)
 
-for batch in ds.shuffling(32).map(lambda x: tf.io.decode_png(x["image"])):
-    print(batch)
+for batch in ds:
+    print(batch["label"])
 ```
 
-Backed by the Lance [columnar format](../format/index.md), using `lance.tf.data.from_lance` supports
-efficient column selection, filtering, and more.
+## Dataset Convenience Methods
+
+If you want `tf.data.Dataset.from_lance`, register the convenience methods
+explicitly after importing `lance_tensorflow`.
 
 ```python
-ds = lance.tf.data.from_lance(
-    "s3://my-bucket/my-dataset",
-    columns=["image", "label"],
-    filter="split = 'train' AND collected_time > timestamp '2020-01-01'",
-    batch_size=256)
-```
+import tensorflow as tf
+import lance_tensorflow
 
-By default, Lance will infer the Tensor spec from the projected columns. You can also specify `tf.TensorSpec` manually.
+lance_tensorflow.register_tensorflow_dataset()
 
-```python
-batch_size = 256
-ds = lance.tf.data.from_lance(
-    "s3://my-bucket/my-dataset",
-    columns=["image", "labels"],
-    batch_size=batch_size,
-    output_signature={
-        "image": tf.TensorSpec(shape=(), dtype=tf.string),
-        "labels": tf.RaggedTensorSpec(
-            dtype=tf.int32, shape=(batch_size, None), ragged_rank=1),
-    },
+ds = tf.data.Dataset.from_lance("s3://my-bucket/my-dataset")
 ```
 
-## Distributed Training and Shuffling
+## Migration
 
-Since [a Lance Dataset is a set of Fragments](../format/index.md), we can distribute and shuffle Fragments to different
-workers.
+Replace old imports:
 
 ```python
-import tensorflow as tf
-from lance.tf.data import from_lance, lance_fragments
+import lance.tf.data
 
-world_size = 32
-rank = 10
-seed = 123  #
-epoch = 100
+ds = lance.tf.data.from_lance(uri)
+```
 
-dataset_uri = "s3://my-bucket/my-dataset"
+with:
 
-# Shuffle fragments distributedly.
-fragments =
-    lance_fragments("s3://my-bucket/my-dataset")
-    .shuffling(32, seed=seed)
-    .repeat(epoch)
-    .enumerate()
-    .filter(lambda i, _: i % world_size == rank)
-    .map(lambda _, fid: fid)
+```python
+from lance_tensorflow import from_lance
 
-ds = from_lance(
-    uri,
-    columns=["image", "label"],
-    fragments=fragments,
-    batch_size=32
-    )
-for batch in ds:
-    print(batch)
+ds = from_lance(uri)
 ```
 
-!!! warning
-
-    For multiprocessing you should probably not use fork as lance is
-    multi-threaded internally and fork and multi-thread do not work well.
-    Refer to [this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555). 
\ No newline at end of file
+See the [lance-tensorflow README](https://github.com/lance-format/lance-tensorflow)
+for the current installation and compatibility details.
diff --git a/docs/src/quickstart/full-text-search.md b/docs/src/quickstart/full-text-search.md
index 17327e40bc5..7f09c4325b7 100644
--- a/docs/src/quickstart/full-text-search.md
+++ b/docs/src/quickstart/full-text-search.md
@@ -90,7 +90,7 @@ ds.create_scalar_index(
     index_type="INVERTED",
     name="text_idx",              # Optional index name (if omitted, default is "text_idx")
     with_position=False,          # Set True to enable phrase queries (stores token positions)
-    base_tokenizer="simple",      # Tokenizer: "simple" (whitespace+punct), "icu", "whitespace", or "raw" (no tokenization)
+    base_tokenizer="simple",      # Tokenizer: "simple", "icu", "icu/split", "whitespace", "raw", or "ngram"
     language="English",           # Language used for stemming + stop words (only used if `stem` or `remove_stop_words` is True)
     max_token_length=40,          # Drop tokens longer than this length
     lower_case=True,              # Lowercase text before tokenization
@@ -98,6 +98,7 @@ ds.create_scalar_index(
     remove_stop_words=True,       # Remove stop words (language-dependent)
     custom_stop_words=None,       # Optional additional stop words (only used if remove_stop_words=True)
     ascii_folding=True,           # Fold accents to ASCII when possible (e.g., "é" -> "e")
+    block_size=128,               # Posting block size: 128 or 256; 256 is experimental
 )
 ```
 
diff --git a/docs/src/quickstart/index.md b/docs/src/quickstart/index.md
index 606948263c4..34367c7177f 100644
--- a/docs/src/quickstart/index.md
+++ b/docs/src/quickstart/index.md
@@ -22,17 +22,17 @@ For the latest features and bug fixes, you can install the preview version:
 === "pip"
 
     ```bash
-    pip install --pre --extra-index-url https://pypi.fury.io/lance-format/ pylance
+    pip install --pre --extra-index-url https://pypi.fury.io/lance-format pylance
     ```
 
 === "uv"
 
     ```bash
     uv venv
-    uv pip install --prerelease allow --index https://pypi.fury.io/lance-format/ pylance
+    uv pip install --prerelease allow --index https://pypi.fury.io/lance-format pylance
 
     # To add to pyproject.toml, just do:
-    uv add --prerelease allow --index https://pypi.fury.io/lance-format/ pylance
+    uv add --prerelease allow --index https://pypi.fury.io/lance-format pylance
     ```
 
 !!! note
diff --git a/docs/theme/404.html b/docs/theme/404.html
new file mode 100644
index 00000000000..51897bd0805
--- /dev/null
+++ b/docs/theme/404.html
@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+
+{% block container %}
+  <main class="ld-notfound">
+    <div class="ld-kicker">404 · Page not found</div>
+    <h1>This page doesn't exist.</h1>
+    <p>The page you're looking for may have moved.</p>
+    <div class="ld-hero__cta">
+      <a class="ld-btn ld-btn--primary ld-btn--lg" href="{{ 'index.html' | url }}">Back to home</a>
+      <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'quickstart/' | url }}">Getting started</a>
+    </div>
+  </main>
+{% endblock %}
diff --git a/docs/theme/assets/site.css b/docs/theme/assets/site.css
new file mode 100644
index 00000000000..3d565d027d2
--- /dev/null
+++ b/docs/theme/assets/site.css
@@ -0,0 +1,734 @@
+/* Lance docs site chrome — faithful port of the "Lance Docs" design prototype.
+   Layout/nav classes (ld-*) come from the theme templates; article content
+   styles target python-markdown + pymdownx output. */
+
+[hidden] { display: none !important; }
+
+.ld-shell {
+  min-height: 100vh;
+  display: flex;
+  flex-direction: column;
+  background: var(--surface-page);
+  font-family: var(--font-body);
+  color: var(--text-body);
+}
+
+/* ---------- header ---------- */
+.ld-header {
+  position: sticky;
+  top: 0;
+  z-index: 60;
+  display: flex;
+  align-items: center;
+  gap: 20px;
+  height: 60px;
+  padding: 0 clamp(16px, 3vw, 32px);
+  min-width: 0;
+  background: var(--surface-header);
+  backdrop-filter: blur(12px);
+  border-bottom: 1px solid var(--line-1);
+}
+.ld-brand {
+  display: inline-flex;
+  align-items: center;
+  text-decoration: none;
+  user-select: none;
+  flex-shrink: 0;
+}
+.ld-brand img { display: block; height: 24px; width: auto; }
+.ld-topnav {
+  display: flex;
+  gap: 2px;
+  margin-right: auto;
+  overflow-x: auto;
+  scrollbar-width: none;
+  min-width: 0;
+  flex: 1;
+}
+.ld-topnav::-webkit-scrollbar { display: none; }
+.ld-toptab {
+  font-family: var(--font-body);
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--text-muted);
+  text-decoration: none;
+  padding: 7px 12px;
+  cursor: pointer;
+  transition: color var(--dur-fast) var(--ease-out);
+  position: relative;
+  white-space: nowrap;
+}
+.ld-toptab:hover { color: var(--text-body); }
+.ld-toptab.active { color: var(--fg-1); }
+.ld-toptab.active::after {
+  content: "";
+  position: absolute;
+  left: 12px;
+  right: 12px;
+  bottom: -10px;
+  height: 2px;
+  background: var(--beam-400);
+}
+.ld-header__actions { display: flex; gap: 10px; align-items: center; flex-shrink: 0; }
+/* All header controls share one 34px height regardless of content (icon/text). */
+.ld-header__actions .ld-btn { height: 34px; padding-top: 0; padding-bottom: 0; }
+.ld-header__actions .ld-btn--icon { width: 34px; padding: 0; justify-content: center; }
+
+/* ---------- buttons ---------- */
+.ld-btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 8px;
+  text-decoration: none;
+  white-space: nowrap;
+  flex-shrink: 0;
+}
+.ld-btn--primary {
+  font-family: var(--font-body);
+  font-size: 13.5px;
+  font-weight: 500;
+  color: #ffffff;
+  background: var(--beam-400);
+  padding: 8px 16px;
+}
+.ld-btn--primary:hover { background: var(--beam-600); color: #ffffff; }
+.ld-btn--ghost {
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  color: var(--text-secondary);
+  border: 1px solid var(--line-1);
+  padding: 7px 14px;
+}
+.ld-btn--ghost:hover { border-color: var(--beam-400); color: var(--fg-1); }
+.ld-btn--icon { padding: 7px 9px; background: none; cursor: pointer; }
+.ld-btn__count {
+  border-left: 1px solid var(--line-2);
+  padding-left: 8px;
+  color: var(--fg-1);
+  font-weight: 600;
+}
+:root[data-theme="light"] .ld-icon-sun { display: none; }
+:root[data-theme="dark"] .ld-icon-moon { display: none; }
+.ld-btn--outline {
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--fg-1);
+  border: 1px solid var(--line-1);
+  padding: 11px 24px;
+}
+.ld-btn--outline:hover { border-color: var(--beam-400); color: var(--fg-1); }
+.ld-btn--text {
+  font-size: 14px;
+  font-weight: 500;
+  color: var(--text-secondary);
+  padding: 11px 12px;
+}
+.ld-btn--text:hover { color: var(--fg-1); }
+.ld-btn--lg.ld-btn--primary { font-size: 14px; padding: 12px 24px; }
+
+/* ---------- home: hero ---------- */
+.ld-hero { max-width: var(--container-max); margin: 0 auto; padding: 96px 32px 72px; width: 100%; }
+.ld-kicker {
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--beam-600);
+  margin-bottom: 24px;
+}
+.ld-kicker a { color: inherit; text-decoration: underline; text-underline-offset: 3px; }
+.ld-kicker a:hover { color: var(--beam-700); }
+.ld-hero h1 {
+  font-family: var(--font-display);
+  font-size: 72px;
+  font-weight: 600;
+  letter-spacing: var(--tracking-display);
+  line-height: 0.97;
+  color: var(--fg-1);
+  margin: 0 0 28px;
+  max-width: 900px;
+  text-wrap: balance;
+}
+.ld-hero__lead {
+  font-size: 17px;
+  line-height: 1.6;
+  color: var(--text-secondary);
+  max-width: 640px;
+  margin: 0 0 36px;
+  text-wrap: pretty;
+}
+.ld-hero__cta { display: flex; gap: 12px; flex-wrap: wrap; }
+
+/* ---------- home: stats band ---------- */
+.ld-band { max-width: var(--container-max); margin: 0 auto; padding: 0 32px; width: 100%; }
+.ld-stats {
+  display: grid;
+  grid-template-columns: 1fr 1fr 1fr;
+  border-top: 1px solid var(--line-1);
+  border-bottom: 1px solid var(--line-1);
+}
+.ld-stat { padding: 28px 32px; }
+.ld-stat:first-child { padding-left: 0; }
+.ld-stat:last-child { padding-right: 0; }
+.ld-stat + .ld-stat { border-left: 1px solid var(--line-2); }
+.ld-stat--link { display: block; text-decoration: none; }
+.ld-stat--link .ld-stat__label { transition: color var(--dur-fast) var(--ease-out); }
+.ld-stat--link:hover .ld-stat__label { color: var(--beam-600); }
+.ld-stat__value {
+  font-family: var(--font-display);
+  font-size: 40px;
+  font-weight: 600;
+  letter-spacing: -0.03em;
+  color: var(--fg-1);
+}
+.ld-stat__value--beam { color: var(--beam-400); }
+.ld-stat__label {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  letter-spacing: 0.1em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin-top: 6px;
+}
+
+/* ---------- home: what is lance ---------- */
+.ld-what { max-width: var(--container-max); margin: 0 auto; padding: 72px 32px 40px; width: 100%; }
+.ld-what h2 {
+  font-family: var(--font-display);
+  font-size: 34px;
+  font-weight: 600;
+  letter-spacing: -0.03em;
+  color: var(--fg-1);
+  margin: 0 0 16px;
+}
+.ld-what__body {
+  font-size: 16px;
+  line-height: 1.65;
+  color: var(--text-secondary);
+  max-width: 760px;
+  margin: 0 0 12px;
+  text-wrap: pretty;
+}
+.ld-what__more { font-size: 15px; line-height: 1.6; color: var(--text-secondary); margin: 0; }
+.ld-inline-link {
+  color: var(--beam-600);
+  text-decoration: underline;
+  text-decoration-thickness: 2px;
+  text-underline-offset: 3px;
+}
+
+/* ---------- home: features ---------- */
+.ld-features { max-width: var(--container-max); margin: 0 auto; padding: 24px 32px 96px; width: 100%; }
+.ld-feature {
+  display: grid;
+  grid-template-columns: 90px minmax(0, 1fr) minmax(0, 460px);
+  gap: 40px;
+  align-items: center;
+  border-top: 1px solid var(--line-1);
+  padding: 40px 0;
+}
+.ld-feature__num {
+  font-family: var(--font-mono);
+  font-size: 15px;
+  color: var(--beam-600);
+  align-self: start;
+  padding-top: 6px;
+}
+.ld-feature__body h3 {
+  font-family: var(--font-display);
+  font-size: 24px;
+  font-weight: 600;
+  letter-spacing: -0.02em;
+  color: var(--fg-1);
+  margin: 0 0 12px;
+}
+.ld-feature__body p {
+  font-size: 15px;
+  line-height: 1.65;
+  color: var(--text-secondary);
+  margin: 0 0 16px;
+  text-wrap: pretty;
+}
+.ld-more {
+  font-family: var(--font-mono);
+  font-size: 13px;
+  color: var(--beam-600);
+  text-decoration: none;
+}
+.ld-more:hover { text-decoration: underline; }
+/* Logo grids stay on white in both themes — the artwork assumes a light background. */
+.ld-feature__img { border: 1px solid var(--line-1); padding: 16px; background: #ffffff; }
+.ld-feature__img img { display: block; width: 100%; height: auto; }
+
+/* homepage code windows (always ink-dark, like docs code blocks) */
+.ld-feature__code { border: 1px solid var(--line-2); background: var(--ink-900); min-width: 0; border-radius: var(--radius-box); overflow: hidden; }
+.ld-win { display: flex; gap: 6px; padding: 12px 16px 0; }
+.ld-win i { width: 10px; height: 10px; border-radius: 999px; background: var(--ink-600); }
+.ld-feature__code pre {
+  margin: 0;
+  padding: 14px 20px 18px;
+  overflow-x: auto;
+  font-family: var(--font-mono);
+  font-size: 12.5px;
+  line-height: 1.7;
+  color: #EDEBF5;
+}
+.ld-feature__code code { font-family: inherit; }
+.tok-kw { color: #B8A8FF; }
+.tok-str { color: #7CC0FF; }
+.tok-com { color: #6F6A85; font-style: italic; }
+.tok-num { color: #F2CE6B; }
+.tok-arg { color: #FF95A8; }
+.tok-fn { color: #8FE8CE; }
+
+/* ---------- docs layout ---------- */
+.ld-docs {
+  flex: 1;
+  display: grid;
+  grid-template-columns: 240px minmax(0, 1fr) 190px;
+  max-width: 1280px;
+  width: 100%;
+  margin: 0 auto;
+  gap: 44px;
+  padding: 0 32px;
+}
+.ld-sidenav {
+  border-right: 1px solid var(--line-2);
+  padding: 36px 24px 48px 0;
+  position: sticky;
+  top: 60px;
+  align-self: start;
+  height: calc(100vh - 60px);
+  overflow-y: auto;
+}
+.ld-sidenav__group { margin-bottom: 6px; }
+.ld-sidenav__label {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 22px 0 8px;
+}
+.ld-sidenav a {
+  display: block;
+  font-size: 13.5px;
+  color: var(--text-secondary);
+  text-decoration: none;
+  padding: 5px 12px;
+  cursor: pointer;
+  border-left: 1px solid var(--line-2);
+  line-height: 1.45;
+}
+.ld-sidenav a:hover { color: var(--text-body); background: var(--surface-overlay); }
+.ld-sidenav a.active {
+  color: var(--beam-600);
+  border-left: 2px solid var(--beam-400);
+  padding-left: 11px;
+  font-weight: 500;
+}
+.ld-sidenav a .ext { color: var(--text-muted); font-size: 11px; margin-left: 4px; }
+
+/* ---------- right-hand page TOC ---------- */
+.ld-toc {
+  position: sticky;
+  top: 60px;
+  align-self: start;
+  max-height: calc(100vh - 60px);
+  overflow-y: auto;
+  padding: 44px 0 48px;
+}
+.ld-toc__label {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 0 0 10px;
+}
+.ld-toc a {
+  display: block;
+  font-size: 12.5px;
+  line-height: 1.45;
+  color: var(--text-muted);
+  text-decoration: none;
+  border-left: 1px solid var(--line-2);
+  padding: 4px 0 4px 12px;
+}
+.ld-toc a.ld-toc__h3 { padding-left: 24px; }
+.ld-toc a:hover { color: var(--text-body); }
+.ld-toc a.active {
+  color: var(--beam-600);
+  border-left: 2px solid var(--beam-400);
+  padding-left: 11px;
+  font-weight: 500;
+}
+.ld-toc a.ld-toc__h3.active { padding-left: 23px; }
+
+.ld-crumbs {
+  font-family: var(--font-mono);
+  font-size: 11.5px;
+  color: var(--text-muted);
+  display: flex;
+  gap: 8px;
+  flex-wrap: wrap;
+  margin-bottom: 24px;
+}
+.ld-crumbs b { color: var(--beam-600); font-weight: 500; }
+
+.ld-pagenav {
+  display: flex;
+  justify-content: space-between;
+  gap: 16px;
+  border-top: 1px solid var(--line-1);
+  margin-top: 56px;
+  padding-top: 20px;
+}
+.ld-pagenav a { text-decoration: none; color: var(--fg-1); }
+.ld-pagenav a:hover { color: var(--beam-600); }
+.ld-pagenav a.next { text-align: right; }
+.ld-pagenav__k {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin-bottom: 4px;
+}
+.ld-pagenav__t { font-size: 14.5px; font-weight: 500; }
+
+/* ---------- article prose (python-markdown output) ---------- */
+.ld-article-col { max-width: 760px; min-width: 0; padding: 40px 0 96px; }
+.ld-article { font-family: var(--font-body); color: var(--text-secondary); }
+.ld-article h1 { font-family: var(--font-display); font-size: 42px; font-weight: 600; letter-spacing: -0.03em; line-height: 1.05; color: var(--fg-1); margin: 0 0 20px; text-wrap: balance; }
+.ld-article h2 { font-family: var(--font-display); font-size: 26px; font-weight: 600; letter-spacing: -0.02em; color: var(--fg-1); margin: 48px 0 14px; padding-top: 28px; border-top: 1px solid var(--line-2); }
+.ld-article h3 { font-family: var(--font-display); font-size: 19px; font-weight: 600; letter-spacing: -0.01em; color: var(--fg-1); margin: 32px 0 10px; }
+.ld-article h4, .ld-article h5, .ld-article h6 { font-family: var(--font-mono); font-size: 12.5px; font-weight: 600; letter-spacing: .14em; text-transform: uppercase; color: var(--fg-1); margin: 28px 0 8px; }
+.ld-article p { font-size: 15.5px; line-height: 1.65; margin: 0 0 16px; text-wrap: pretty; }
+.ld-article a { color: var(--beam-600); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 3px; text-decoration-color: rgba(98, 94, 255, .35); }
+.ld-article a:hover { text-decoration-color: var(--beam-600); }
+.ld-article code { font-family: var(--font-mono); font-size: 13px; background: var(--surface-inline-code); border-radius: var(--radius-chip); padding: 1px 5px; color: var(--fg-1); }
+.ld-article ul, .ld-article ol { margin: 0 0 16px; padding-left: 22px; font-size: 15.5px; line-height: 1.65; }
+.ld-article li { margin-bottom: 6px; }
+.ld-article li p { margin-bottom: 8px; }
+.ld-article hr { border: 0; border-top: 1px solid var(--line-1); margin: 40px 0; }
+.ld-article blockquote { margin: 0 0 16px; padding: 4px 0 4px 20px; border-left: 2px solid var(--beam-400); }
+.ld-article blockquote p:last-child { margin-bottom: 0; }
+
+/* heading permalinks (toc: permalink) */
+.ld-article .headerlink {
+  margin-left: 8px;
+  color: var(--beam-400);
+  text-decoration: none;
+  opacity: 0;
+  transition: opacity var(--dur-fast) var(--ease-out);
+}
+.ld-article :is(h1, h2, h3, h4, h5, h6):hover .headerlink { opacity: 1; }
+
+/* figures: lone images sit on a white plate in both themes */
+.ld-article img { max-width: 100%; height: auto; }
+.ld-article p > img:only-child {
+  display: block;
+  margin: 24px auto;
+  border: 1px solid var(--line-1);
+  padding: 16px;
+  background: #ffffff;
+}
+
+/* tables (JS wraps them in .ld-tablewrap for overflow) */
+.ld-tablewrap { overflow-x: auto; margin: 0 0 20px; border: 1px solid var(--line-1); }
+.ld-article table { border-collapse: collapse; width: 100%; font-size: 14px; }
+.ld-article th { font-family: var(--font-mono); font-size: 11px; font-weight: 600; letter-spacing: .12em; text-transform: uppercase; text-align: left; color: var(--fg-1); background: var(--surface-overlay); padding: 10px 14px; border-bottom: 1px solid var(--line-1); }
+.ld-article td { padding: 10px 14px; border-bottom: 1px solid var(--line-2); vertical-align: top; line-height: 1.55; color: var(--text-secondary); }
+.ld-article tr:last-child td { border-bottom: 0; }
+
+/* ---------- code blocks (pymdownx.highlight output, JS adds the bar) ---------- */
+.ld-code { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; }
+.ld-code__bar { display: flex; justify-content: space-between; align-items: center; padding: 7px 14px; border-bottom: 1px solid var(--line-2); background: var(--surface-card); }
+.ld-code__bar span { display: inline-flex; align-items: center; gap: 7px; font-family: var(--font-mono); font-size: 11px; letter-spacing: .12em; text-transform: uppercase; color: var(--text-muted); }
+.ld-code__bar span svg { width: 13px; height: 13px; display: block; flex-shrink: 0; }
+.ld-copy { font-family: var(--font-mono); font-size: 11px; letter-spacing: .08em; text-transform: uppercase; background: none; border: none; color: var(--beam-600); cursor: pointer; padding: 2px 0; }
+.ld-copy:hover { color: var(--beam-700); }
+.ld-code .highlight { margin: 0; }
+.ld-code pre { margin: 0; background: var(--surface-code); padding: 16px 18px; overflow-x: auto; }
+.ld-code pre code { background: none; border: none; padding: 0; font-size: 13px; line-height: 1.6; color: var(--code-fg); }
+/* Ink code surfaces (dark-theme docs blocks + the always-dark homepage windows)
+   need an explicit light selection color; light-theme docs blocks use the default. */
+:root[data-theme="dark"] .ld-code pre ::selection, :root[data-theme="dark"] .ld-code pre::selection,
+.ld-feature__code pre ::selection, .ld-feature__code pre::selection {
+  background: rgba(98, 94, 255, 0.55);
+  color: #ffffff;
+}
+
+/* Pygments tokens — colors resolve per theme via the --code-* palette */
+.highlight .k, .highlight .kn, .highlight .kd, .highlight .kt, .highlight .kr, .highlight .kp, .highlight .ow { color: var(--code-kw); }
+.highlight .kc { color: var(--code-kw); }
+.highlight .s, .highlight .s1, .highlight .s2, .highlight .sb, .highlight .sd, .highlight .sa, .highlight .se, .highlight .si, .highlight .sx, .highlight .sr, .highlight .ss, .highlight .sh { color: var(--code-str); }
+.highlight .c, .highlight .c1, .highlight .cm, .highlight .ch, .highlight .cs, .highlight .cp, .highlight .cpf { color: var(--code-com); font-style: italic; }
+.highlight .m, .highlight .mi, .highlight .mf, .highlight .mh, .highlight .mo, .highlight .mb, .highlight .il { color: var(--code-num); }
+.highlight .nf, .highlight .fm, .highlight .nd, .highlight .ne { color: var(--code-fn); }
+.highlight .nc, .highlight .nn { color: var(--code-name); font-weight: 500; }
+.highlight .nb, .highlight .bp { color: var(--code-kw); }
+.highlight .nt { color: var(--code-tag); }
+.highlight .na { color: var(--code-tag); }
+.highlight .o { color: var(--code-punct); }
+.highlight .p { color: var(--code-punct); }
+.highlight .gp { color: var(--code-prompt); }
+.highlight .go { color: var(--code-punct); }
+.highlight .gh, .highlight .gu { color: var(--code-name); font-weight: 600; }
+.highlight .hll { background: var(--code-hll); display: block; }
+
+/* mermaid diagrams: white plate (rendered by JS) */
+.ld-article pre.mermaid, .ld-article div.mermaid {
+  border: 1px solid var(--line-1);
+  padding: 16px;
+  background: #ffffff;
+  margin: 0 0 20px;
+  text-align: center;
+  overflow-x: auto;
+}
+
+/* ---------- admonitions ---------- */
+.ld-article .admonition {
+  border: 1px solid var(--line-2);
+  border-radius: var(--radius-box);
+  overflow: hidden;
+  margin: 0 0 20px;
+  background: var(--surface-card);
+  padding: 0;
+  font-size: 14.5px;
+}
+.ld-article .admonition > .admonition-title {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-family: var(--font-mono);
+  font-size: 11px;
+  font-weight: 600;
+  letter-spacing: .14em;
+  text-transform: uppercase;
+  color: var(--beam-600);
+  padding: 9px 16px;
+  margin: 0;
+  border-bottom: 1px solid var(--line-2);
+  background: none;
+}
+/* Material-style type icon, drawn in the title color via a CSS mask */
+.ld-article .admonition > .admonition-title::before {
+  content: "";
+  width: 14px;
+  height: 14px;
+  flex-shrink: 0;
+  background-color: currentColor;
+  -webkit-mask: var(--ld-adm-icon) no-repeat center / contain;
+  mask: var(--ld-adm-icon) no-repeat center / contain;
+}
+/* Icon shapes are Material Design Icons paths (Apache 2.0), matching mkdocs-material's defaults. */
+.ld-article .admonition { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.abstract, .admonition.summary, .admonition.tldr) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.info, .admonition.todo) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,9H11V7H13M13,17H11V11H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.tip, .admonition.hint, .admonition.important) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17.66,11.2C17.43,10.9 17.15,10.64 16.89,10.38C16.22,9.78 15.46,9.35 14.82,8.72C13.33,7.26 13,4.85 13.95,3C13,3.23 12.17,3.75 11.46,4.32C8.87,6.4 7.85,10.07 9.07,13.22C9.11,13.32 9.15,13.42 9.15,13.55C9.15,13.77 9,13.97 8.8,14.05C8.57,14.15 8.33,14.09 8.14,13.93C8.08,13.88 8.04,13.83 8,13.76C6.87,12.33 6.69,10.28 7.45,8.64C5.78,10 4.87,12.3 5,14.47C5.06,14.97 5.12,15.47 5.29,15.97C5.43,16.57 5.7,17.17 6,17.7C7.08,19.43 8.95,20.67 10.96,20.92C13.1,21.19 15.39,20.8 17.03,19.32C18.86,17.66 19.5,15 18.56,12.72L18.43,12.46C18.22,12 17.66,11.2 17.66,11.2M14.5,17.5C14.22,17.74 13.76,18 13.4,18.1C12.28,18.5 11.16,17.94 10.5,17.28C11.69,17 12.4,16.12 12.61,15.23C12.78,14.43 12.46,13.77 12.33,13C12.21,12.26 12.23,11.63 12.5,10.94C12.69,11.32 12.89,11.7 13.13,12C13.9,13 15.11,13.44 15.37,14.8C15.41,14.94 15.43,15.08 15.43,15.23C15.46,16.05 15.1,16.95 14.5,17.5H14.5Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.success, .admonition.check, .admonition.done) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M9,20.42L2.79,14.21L5.62,11.38L9,14.77L18.88,4.88L21.71,7.71L9,20.42Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.question, .admonition.help, .admonition.faq) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M15.07,11.25L14.17,12.17C13.45,12.89 13,13.5 13,15H11V14.5C11,13.39 11.45,12.39 12.17,11.67L13.41,10.41C13.78,10.05 14,9.55 14,9C14,7.89 13.1,7 12,7A2,2 0 0,0 10,9H8A4,4 0 0,1 12,5A4,4 0 0,1 16,9C16,9.88 15.64,10.67 15.07,11.25M13,19H11V17H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12C22,6.47 17.5,2 12,2Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.warning, .admonition.caution, .admonition.attention) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,14H11V9H13M13,18H11V16H13M1,21H23L12,2L1,21Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.failure, .admonition.fail, .admonition.missing) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20 6.91L17.09 4L12 9.09L6.91 4L4 6.91L9.09 12L4 17.09L6.91 20L12 14.91L17.09 20L20 17.09L14.91 12L20 6.91Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.danger, .admonition.error) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M11,15H6L13,1V9H18L11,23V15Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition.bug { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,12H10V10H14M14,16H10V14H14M20,8H17.19C16.74,7.22 16.12,6.55 15.37,6.04L17,4.41L15.59,3L13.42,5.17C12.96,5.06 12.5,5 12,5C11.5,5 11.04,5.06 10.59,5.17L8.41,3L7,4.41L8.62,6.04C7.88,6.55 7.26,7.22 6.81,8H4V10H6.09C6.04,10.33 6,10.66 6,11V12H4V14H6V15C6,15.34 6.04,15.67 6.09,16H4V18H6.81C7.85,19.79 9.78,21 12,21C14.22,21 16.15,19.79 17.19,18H20V16H17.91C17.96,15.67 18,15.34 18,15V14H20V12H18V11C18,10.66 17.96,10.33 17.91,10H20V8Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition.example { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M7,13V11H21V13H7M7,19V17H21V19H7M7,7V5H21V7H7M3,8V5H2V4H4V8H3M2,17V16H5V20H2V19H4V18.5H3V17.5H4V17H2M4.25,10A0.75,0.75 0 0,1 5,10.75C5,10.95 4.92,11.14 4.79,11.27L3.12,13H5V14H2V13.08L4,11H2V10H4.25Z"/%3E%3C/svg%3E'); }
+.ld-article :is(.admonition.quote, .admonition.cite) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,17H17L19,13V7H13V13H16M6,17H9L11,13V7H5V13H8L6,17Z"/%3E%3C/svg%3E'); }
+.ld-article .admonition > :not(.admonition-title) { margin: 14px 16px; font-size: 14.5px; }
+.ld-article .admonition > .highlight, .ld-article .admonition > .ld-code { margin: 14px 16px; }
+.ld-article .admonition.warning > .admonition-title,
+.ld-article .admonition.caution > .admonition-title,
+.ld-article .admonition.attention > .admonition-title { color: var(--warn-500); }
+.ld-article .admonition.danger > .admonition-title,
+.ld-article .admonition.error > .admonition-title,
+.ld-article .admonition.bug > .admonition-title,
+.ld-article .admonition.failure > .admonition-title { color: var(--danger-500); }
+.ld-article .admonition.success > .admonition-title,
+.ld-article .admonition.check > .admonition-title { color: var(--ok-500); }
+
+/* ---------- details / summary (pymdownx.details) ---------- */
+.ld-article details {
+  border: 1px solid var(--line-2);
+  border-radius: var(--radius-box);
+  overflow: hidden;
+  margin: 0 0 20px;
+  background: var(--surface-card);
+}
+.ld-article details > summary {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  font-weight: 600;
+  letter-spacing: .1em;
+  text-transform: uppercase;
+  color: var(--beam-600);
+  padding: 10px 16px;
+  cursor: pointer;
+  user-select: none;
+  list-style: none;
+}
+.ld-article details > summary::-webkit-details-marker { display: none; }
+.ld-article details > summary:hover { background: var(--surface-overlay); }
+.ld-article details[open] > summary { border-bottom: 1px solid var(--line-2); }
+.ld-article details > :not(summary) { margin: 14px 16px; }
+
+/* ---------- content tabs (pymdownx.tabbed, alternate style) ---------- */
+.ld-article .tabbed-set { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; position: relative; }
+.ld-article .tabbed-set > input { position: absolute; opacity: 0; pointer-events: none; }
+.ld-article .tabbed-labels {
+  display: flex;
+  border-bottom: 1px solid var(--line-2);
+  background: var(--surface-card);
+  overflow-x: auto;
+  scrollbar-width: none;
+}
+.ld-article .tabbed-labels > label {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  letter-spacing: .06em;
+  padding: 9px 18px;
+  border-right: 1px solid var(--line-2);
+  color: var(--text-muted);
+  cursor: pointer;
+  white-space: nowrap;
+}
+.ld-article .tabbed-labels > label:hover { color: var(--fg-1); }
+.ld-article .tabbed-content { display: block; }
+.ld-article .tabbed-block { display: none; padding: 16px 16px 0; }
+.ld-article .tabbed-block > :last-child { margin-bottom: 16px; }
+.ld-article .tabbed-block > .ld-code:last-child { margin-bottom: 16px; }
+/* nth-input → nth-label/nth-block mapping (supports up to 8 tabs) */
+.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-labels > label:nth-child(1),
+.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > label:nth-child(2),
+.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > label:nth-child(3),
+.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > label:nth-child(4),
+.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > label:nth-child(5),
+.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > label:nth-child(6),
+.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > label:nth-child(7),
+.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > label:nth-child(8) {
+  color: var(--fg-1);
+  box-shadow: inset 0 -2px 0 var(--beam-400);
+}
+.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-content > .tabbed-block:nth-child(1),
+.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-content > .tabbed-block:nth-child(2),
+.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-content > .tabbed-block:nth-child(3),
+.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-content > .tabbed-block:nth-child(4),
+.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-content > .tabbed-block:nth-child(5),
+.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-content > .tabbed-block:nth-child(6),
+.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-content > .tabbed-block:nth-child(7),
+.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-content > .tabbed-block:nth-child(8) {
+  display: block;
+}
+
+/* ---------- search overlay ---------- */
+.ld-search { position: fixed; inset: 0; z-index: 100; }
+.ld-search__scrim { position: absolute; inset: 0; background: rgba(14, 12, 20, 0.5); }
+.ld-search__panel {
+  position: relative;
+  max-width: 640px;
+  margin: 96px auto 0;
+  background: var(--surface-page);
+  border: 1px solid var(--line-1);
+}
+.ld-search__bar { display: flex; align-items: center; border-bottom: 1px solid var(--line-1); }
+.ld-search__input {
+  flex: 1;
+  font-family: var(--font-body);
+  font-size: 15px;
+  color: var(--fg-1);
+  background: none;
+  border: none;
+  outline: none;
+  padding: 14px 16px;
+}
+.ld-search__input::placeholder { color: var(--text-muted); }
+.ld-search__close {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: .1em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+  background: none;
+  border: 1px solid var(--line-2);
+  margin-right: 12px;
+  padding: 3px 8px;
+  cursor: pointer;
+}
+.ld-search__close:hover { color: var(--fg-1); border-color: var(--beam-400); }
+.ld-search__results { max-height: 55vh; overflow-y: auto; }
+.ld-search__hit { display: block; padding: 12px 16px; text-decoration: none; border-top: 1px solid var(--line-2); }
+.ld-search__hit:first-child { border-top: 0; }
+.ld-search__hit:hover, .ld-search__hit.active { background: var(--surface-overlay); }
+.ld-search__hit-title { font-size: 14px; font-weight: 500; color: var(--fg-1); }
+.ld-search__hit-title mark, .ld-search__hit-text mark { background: none; color: var(--beam-600); }
+.ld-search__hit-crumb { font-family: var(--font-mono); font-size: 10.5px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); margin-bottom: 2px; }
+.ld-search__hit-text { font-size: 12.5px; color: var(--text-muted); line-height: 1.5; margin-top: 2px; }
+.ld-search__empty { font-family: var(--font-mono); font-size: 12px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); padding: 20px 16px; }
+
+/* ---------- 404 ---------- */
+.ld-notfound { max-width: var(--container-max); margin: 0 auto; padding: 120px 32px 160px; width: 100%; flex: 1; }
+.ld-notfound h1 {
+  font-family: var(--font-display);
+  font-size: 56px;
+  font-weight: 600;
+  letter-spacing: var(--tracking-display);
+  color: var(--fg-1);
+  margin: 0 0 16px;
+}
+.ld-notfound p { color: var(--text-secondary); margin: 0 0 32px; }
+
+/* ---------- footer ---------- */
+.ld-footer {
+  border-top: 1px solid var(--line-1);
+  padding: 40px 32px;
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 32px;
+  flex-wrap: wrap;
+}
+.ld-footer__brand img { display: block; height: 24px; width: auto; }
+.ld-footer__brand p {
+  font-size: 12.5px;
+  color: var(--text-muted);
+  max-width: 300px;
+  line-height: 1.6;
+  margin: 12px 0 0;
+}
+.ld-footer__cols { display: flex; gap: 64px; flex-wrap: wrap; }
+.ld-footer__col { display: flex; flex-direction: column; gap: 8px; }
+.ld-footer__col h5 {
+  font-family: var(--font-mono);
+  font-size: 10.5px;
+  letter-spacing: var(--tracking-caps);
+  text-transform: uppercase;
+  color: var(--text-muted);
+  margin: 0 0 4px;
+}
+.ld-footer__col a { font-size: 13.5px; color: var(--text-secondary); text-decoration: none; }
+.ld-footer__col a:hover { color: var(--fg-1); }
+
+/* ---------- responsive ---------- */
+@media (max-width: 1180px) {
+  .ld-docs { grid-template-columns: 240px minmax(0, 1fr); }
+  .ld-toc { display: none; }
+}
+@media (max-width: 960px) {
+  .ld-hero h1 { font-size: clamp(38px, 9vw, 56px); }
+  .ld-stats { grid-template-columns: 1fr; }
+  .ld-stat { padding: 20px 0; }
+  .ld-stat + .ld-stat { border-left: 0; border-top: 1px solid var(--line-2); }
+  .ld-feature { grid-template-columns: 1fr; gap: 16px; }
+  .ld-feature__num { padding-top: 0; }
+  .ld-docs { display: block; padding: 0 20px; }
+  .ld-sidenav {
+    position: static;
+    height: auto;
+    border-right: 0;
+    border-bottom: 1px solid var(--line-2);
+    padding: 20px 0;
+  }
+  .ld-hero, .ld-band, .ld-what, .ld-features { padding-left: 20px; padding-right: 20px; }
+  .ld-hero { padding-top: 56px; padding-bottom: 48px; }
+}
diff --git a/docs/theme/assets/site.js b/docs/theme/assets/site.js
new file mode 100644
index 00000000000..cdb36c0ecfe
--- /dev/null
+++ b/docs/theme/assets/site.js
@@ -0,0 +1,252 @@
+/* Lance docs theme behaviours: theme toggle, GitHub stars, code block chrome,
+   TOC scroll-spy, search overlay, mermaid rendering. */
+(function () {
+  "use strict";
+
+  var BASE = (window.LD_BASE || ".").replace(/\/$/, "");
+
+  /* ---------- theme toggle ---------- */
+  var themeBtn = document.getElementById("theme-toggle");
+  if (themeBtn) {
+    themeBtn.addEventListener("click", function () {
+      var next = document.documentElement.dataset.theme === "dark" ? "light" : "dark";
+      document.documentElement.dataset.theme = next;
+      try { localStorage.setItem("ld-theme", next); } catch (e) { /* private mode */ }
+    });
+  }
+
+  /* ---------- GitHub stars ---------- */
+  (function loadStars() {
+    var el = document.getElementById("gh-stars");
+    if (!el) return;
+    var TTL = 3600e3;
+    function show(count) {
+      if (!(count > 0)) return;
+      var label = count >= 1000 ? (count / 1000).toFixed(1).replace(/\.0$/, "") + "k" : String(count);
+      el.textContent = "★ " + label;
+      el.hidden = false;
+    }
+    try {
+      var cached = JSON.parse(localStorage.getItem("ld-gh-stars") || "null");
+      if (cached && Date.now() - cached.t < TTL) { show(cached.v); return; }
+    } catch (e) { /* fall through to fetch */ }
+    fetch("https://api.github.com/repos/lance-format/lance")
+      .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+      .then(function (d) {
+        show(d.stargazers_count);
+        try { localStorage.setItem("ld-gh-stars", JSON.stringify({ v: d.stargazers_count, t: Date.now() })); } catch (e) { /* ignore */ }
+      })
+      .catch(function () { /* rate-limited or offline: button still works without the count */ });
+  })();
+
+  /* ---------- article enhancements ---------- */
+  var article = document.querySelector(".ld-article");
+
+  if (article) {
+    // Language logos for code-bar labels (Simple Icons + Devicon paths, drawn in currentColor).
+    var LANG_ICONS = {
+      python: ["0 0 24 24", "M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z"],
+      rust: ["0 0 24 24", "M23.8346 11.7033l-1.0073-.6236a13.7268 13.7268 0 00-.0283-.2936l.8656-.8069a.3483.3483 0 00-.1154-.578l-1.1066-.414a8.4958 8.4958 0 00-.087-.2856l.6904-.9587a.3462.3462 0 00-.2257-.5446l-1.1663-.1894a9.3574 9.3574 0 00-.1407-.2622l.49-1.0761a.3437.3437 0 00-.0274-.3361.3486.3486 0 00-.3006-.154l-1.1845.0416a6.7444 6.7444 0 00-.1873-.2268l.2723-1.153a.3472.3472 0 00-.417-.4172l-1.1532.2724a14.0183 14.0183 0 00-.2278-.1873l.0415-1.1845a.3442.3442 0 00-.49-.328l-1.076.491c-.0872-.0476-.1742-.0952-.2623-.1407l-.1903-1.1673A.3483.3483 0 0016.256.955l-.9597.6905a8.4867 8.4867 0 00-.2855-.086l-.414-1.1066a.3483.3483 0 00-.5781-.1154l-.8069.8666a9.2936 9.2936 0 00-.2936-.0284L12.2946.1683a.3462.3462 0 00-.5892 0l-.6236 1.0073a13.7383 13.7383 0 00-.2936.0284L9.9803.3374a.3462.3462 0 00-.578.1154l-.4141 1.1065c-.0962.0274-.1903.0567-.2855.086L7.744.955a.3483.3483 0 00-.5447.2258L7.009 2.348a9.3574 9.3574 0 00-.2622.1407l-1.0762-.491a.3462.3462 0 00-.49.328l.0416 1.1845a7.9826 7.9826 0 00-.2278.1873L3.8413 3.425a.3472.3472 0 00-.4171.4171l.2713 1.1531c-.0628.075-.1255.1509-.1863.2268l-1.1845-.0415a.3462.3462 0 00-.328.49l.491 1.0761a9.167 9.167 0 00-.1407.2622l-1.1662.1894a.3483.3483 0 00-.2258.5446l.6904.9587a13.303 13.303 0 00-.087.2855l-1.1065.414a.3483.3483 0 00-.1155.5781l.8656.807a9.2936 9.2936 0 00-.0283.2935l-1.0073.6236a.3442.3442 0 000 .5892l1.0073.6236c.008.0982.0182.1964.0283.2936l-.8656.8079a.3462.3462 0 00.1155.578l1.1065.4141c.0273.0962.0567.1914.087.2855l-.6904.9587a.3452.3452 0 00.2268.5447l1.1662.1893c.0456.088.0922.1751.1408.2622l-.491 1.0762a.3462.3462 0 00.328.49l1.1834-.0415c.0618.0769.1235.1528.1873.2277l-.2713 1.1541a.3462.3462 0 00.4171.4161l1.153-.2713c.075.0638.151.1255.2279.1863l-.0415 1.1845a.3442.3442 0 00.49.327l1.0761-.49c.087.0486.1741.0951.2622.1407l.1903 1.1662a.3483.3483 0 00.5447.2268l.9587-.6904a9.299 9.299 0 00.2855.087l.414 1.1066a.3452.3452 0 00.5781.1154l.8079-.8656c.0972.0111.1954.0203.2936.0294l.6236 1.0073a.3472.3472 0 00.5892 0l.6236-1.0073c.0982-.0091.1964-.0183.2936-.0294l.8069.8656a.3483.3483 0 00.578-.1154l.4141-1.1066a8.4626 8.4626 0 00.2855-.087l.9587.6904a.3452.3452 0 00.5447-.2268l.1903-1.1662c.088-.0456.1751-.0931.2622-.1407l1.0762.49a.3472.3472 0 00.49-.327l-.0415-1.1845a6.7267 6.7267 0 00.2267-.1863l1.1531.2713a.3472.3472 0 00.4171-.416l-.2713-1.1542c.0628-.0749.1255-.1508.1863-.2278l1.1845.0415a.3442.3442 0 00.328-.49l-.49-1.076c.0475-.0872.0951-.1742.1407-.2623l1.1662-.1893a.3483.3483 0 00.2258-.5447l-.6904-.9587.087-.2855 1.1066-.414a.3462.3462 0 00.1154-.5781l-.8656-.8079c.0101-.0972.0202-.1954.0283-.2936l1.0073-.6236a.3442.3442 0 000-.5892zm-6.7413 8.3551a.7138.7138 0 01.2986-1.396.714.714 0 11-.2997 1.396zm-.3422-2.3142a.649.649 0 00-.7715.5l-.3573 1.6685c-1.1035.501-2.3285.7795-3.6193.7795a8.7368 8.7368 0 01-3.6951-.814l-.3574-1.6684a.648.648 0 00-.7714-.499l-1.473.3158a8.7216 8.7216 0 01-.7613-.898h7.1676c.081 0 .1356-.0141.1356-.088v-2.536c0-.074-.0536-.0881-.1356-.0881h-2.0966v-1.6077h2.2677c.2065 0 1.1065.0587 1.394 1.2088.0901.3533.2875 1.5044.4232 1.8729.1346.413.6833 1.2381 1.2685 1.2381h3.5716a.7492.7492 0 00.1296-.0131 8.7874 8.7874 0 01-.8119.9526zM6.8369 20.024a.714.714 0 11-.2997-1.396.714.714 0 01.2997 1.396zM4.1177 8.9972a.7137.7137 0 11-1.304.5791.7137.7137 0 011.304-.579zm-.8352 1.9813l1.5347-.6824a.65.65 0 00.33-.8585l-.3158-.7147h1.2432v5.6025H3.5669a8.7753 8.7753 0 01-.2834-3.348zm6.7343-.5437V8.7836h2.9601c.153 0 1.0792.1772 1.0792.8697 0 .575-.7107.7815-1.2948.7815zm10.7574 1.4862c0 .2187-.008.4363-.0243.651h-.9c-.09 0-.1265.0586-.1265.1477v.413c0 .973-.5487 1.1846-1.0296 1.2382-.4576.0517-.9648-.1913-1.0275-.4717-.2704-1.5186-.7198-1.8436-1.4305-2.4034.8817-.5599 1.799-1.386 1.799-2.4915 0-1.1936-.819-1.9458-1.3769-2.3153-.7825-.5163-1.6491-.6195-1.883-.6195H5.4682a8.7651 8.7651 0 014.907-2.7699l1.0974 1.151a.648.648 0 00.9182.0213l1.227-1.1743a8.7753 8.7753 0 016.0044 4.2762l-.8403 1.8982a.652.652 0 00.33.8585l1.6178.7188c.0283.2875.0425.577.0425.8717zm-9.3006-9.5993a.7128.7128 0 11.984 1.0316.7137.7137 0 01-.984-1.0316zm8.3389 6.71a.7107.7107 0 01.9395-.3625.7137.7137 0 11-.9405.3635z"],
+      java: ["0 0 128 128", "M47.617 98.12c-19.192 5.362 11.677 16.439 36.115 5.969-4.003-1.556-6.874-3.351-6.874-3.351-10.897 2.06-15.952 2.222-25.844 1.092-8.164-.935-3.397-3.71-3.397-3.71zm33.189-10.46c-14.444 2.779-22.787 2.69-33.354 1.6-8.171-.845-2.822-4.805-2.822-4.805-21.137 7.016 11.767 14.977 41.309 6.336-3.14-1.106-5.133-3.131-5.133-3.131zm11.319-60.575c.001 0-42.731 10.669-22.323 34.187 6.024 6.935-1.58 13.17-1.58 13.17s15.289-7.891 8.269-17.777c-6.559-9.215-11.587-13.793 15.634-29.58zm9.998 81.144s3.529 2.91-3.888 5.159c-14.102 4.272-58.706 5.56-71.095.171-4.45-1.938 3.899-4.625 6.526-5.192 2.739-.593 4.303-.485 4.303-.485-4.952-3.487-32.013 6.85-13.742 9.815 49.821 8.076 90.817-3.637 77.896-9.468zM85 77.896c2.395-1.634 5.703-3.053 5.703-3.053s-9.424 1.685-18.813 2.474c-11.494.964-23.823 1.154-30.012.326-14.652-1.959 8.033-7.348 8.033-7.348s-8.812-.596-19.644 4.644C17.455 81.134 61.958 83.958 85 77.896zm5.609 15.145c-.108.29-.468.616-.468.616 31.273-8.221 19.775-28.979 4.822-23.725-1.312.464-2 1.543-2 1.543s.829-.334 2.678-.72c7.559-1.575 18.389 10.119-5.032 22.286zM64.181 70.069c-4.614-10.429-20.26-19.553.007-35.559C89.459 14.563 76.492 1.587 76.492 1.587c5.23 20.608-18.451 26.833-26.999 39.667-5.821 8.745 2.857 18.142 14.688 28.815zm27.274 51.748c-19.187 3.612-42.854 3.191-56.887.874 0 0 2.874 2.38 17.646 3.331 22.476 1.437 57-.8 57.816-11.436.001 0-1.57 4.032-18.575 7.231z"],
+      bash: ["0 0 24 24", "M21.038,4.9l-7.577-4.498C13.009,0.134,12.505,0,12,0c-0.505,0-1.009,0.134-1.462,0.403L2.961,4.9 C2.057,5.437,1.5,6.429,1.5,7.503v8.995c0,1.073,0.557,2.066,1.462,2.603l7.577,4.497C10.991,23.866,11.495,24,12,24 c0.505,0,1.009-0.134,1.461-0.402l7.577-4.497c0.904-0.537,1.462-1.529,1.462-2.603V7.503C22.5,6.429,21.943,5.437,21.038,4.9z M15.17,18.946l0.013,0.646c0.001,0.078-0.05,0.167-0.111,0.198l-0.383,0.22c-0.061,0.031-0.111-0.007-0.112-0.085L14.57,19.29 c-0.328,0.136-0.66,0.169-0.872,0.084c-0.04-0.016-0.057-0.075-0.041-0.142l0.139-0.584c0.011-0.046,0.036-0.092,0.069-0.121 c0.012-0.011,0.024-0.02,0.036-0.026c0.022-0.011,0.043-0.014,0.062-0.006c0.229,0.077,0.521,0.041,0.802-0.101 c0.357-0.181,0.596-0.545,0.592-0.907c-0.003-0.328-0.181-0.465-0.613-0.468c-0.55,0.001-1.064-0.107-1.072-0.917 c-0.007-0.667,0.34-1.361,0.889-1.8l-0.007-0.652c-0.001-0.08,0.048-0.168,0.111-0.2l0.37-0.236 c0.061-0.031,0.111,0.007,0.112,0.087l0.006,0.653c0.273-0.109,0.511-0.138,0.726-0.088c0.047,0.012,0.067,0.076,0.048,0.151 l-0.144,0.578c-0.011,0.044-0.036,0.088-0.065,0.116c-0.012,0.012-0.025,0.021-0.038,0.028c-0.019,0.01-0.038,0.013-0.057,0.009 c-0.098-0.022-0.332-0.073-0.699,0.113c-0.385,0.195-0.52,0.53-0.517,0.778c0.003,0.297,0.155,0.387,0.681,0.396 c0.7,0.012,1.003,0.318,1.01,1.023C16.105,17.747,15.736,18.491,15.17,18.946z M19.143,17.859c0,0.06-0.008,0.116-0.058,0.145 l-1.916,1.164c-0.05,0.029-0.09,0.004-0.09-0.056v-0.494c0-0.06,0.037-0.093,0.087-0.122l1.887-1.129 c0.05-0.029,0.09-0.004,0.09,0.056V17.859z M20.459,6.797l-7.168,4.427c-0.894,0.523-1.553,1.109-1.553,2.187v8.833 c0,0.645,0.26,1.063,0.66,1.184c-0.131,0.023-0.264,0.039-0.398,0.039c-0.42,0-0.833-0.114-1.197-0.33L3.226,18.64 c-0.741-0.44-1.201-1.261-1.201-2.142V7.503c0-0.881,0.46-1.702,1.201-2.142l7.577-4.498c0.363-0.216,0.777-0.33,1.197-0.33 c0.419,0,0.833,0.114,1.197,0.33l7.577,4.498c0.624,0.371,1.046,1.013,1.164,1.732C21.686,6.557,21.12,6.411,20.459,6.797z"],
+    };
+    var LANG_ALIASES = { py: "python", python3: "python", rs: "rust", sh: "bash", shell: "bash", console: "bash", zsh: "bash" };
+
+    // Code blocks: wrap .highlight in a window with a language bar + copy button.
+    article.querySelectorAll("div.highlight").forEach(function (hl) {
+      if (hl.closest(".ld-code")) return;
+      var code = hl.querySelector("code");
+      var m = (hl.className + " " + (code ? code.className : "")).match(/language-([\w+-]+)/);
+      var lang = m ? m[1] : "text";
+      var wrap = document.createElement("div");
+      wrap.className = "ld-code";
+      var bar = document.createElement("div");
+      bar.className = "ld-code__bar";
+      bar.innerHTML = "<span></span><button class=\"ld-copy\" type=\"button\">Copy</button>";
+      var span = bar.querySelector("span");
+      var icon = LANG_ICONS[LANG_ALIASES[lang] || lang];
+      if (icon) span.innerHTML = '<svg viewBox="' + icon[0] + '" fill="currentColor" aria-hidden="true"><path d="' + icon[1] + '"/></svg>';
+      span.appendChild(document.createTextNode(lang));
+      hl.parentNode.insertBefore(wrap, hl);
+      wrap.appendChild(bar);
+      wrap.appendChild(hl);
+    });
+
+    document.addEventListener("click", function (e) {
+      var copy = e.target.closest(".ld-copy");
+      if (!copy) return;
+      var pre = copy.closest(".ld-code").querySelector("pre");
+      if (navigator.clipboard && pre) navigator.clipboard.writeText(pre.textContent);
+      copy.textContent = "Copied";
+      setTimeout(function () { copy.textContent = "Copy"; }, 1400);
+    });
+
+    // Tables: wrap for horizontal overflow.
+    article.querySelectorAll("table").forEach(function (t) {
+      if (t.closest(".ld-tablewrap")) return;
+      var wrap = document.createElement("div");
+      wrap.className = "ld-tablewrap";
+      t.parentNode.insertBefore(wrap, t);
+      wrap.appendChild(t);
+    });
+
+    // Mermaid: render fenced diagrams on demand.
+    var mermaidNodes = article.querySelectorAll("pre.mermaid, div.mermaid");
+    if (mermaidNodes.length) {
+      import("https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs").then(function (mod) {
+        var mermaid = mod.default;
+        mermaidNodes.forEach(function (node) {
+          if (node.tagName === "PRE") {
+            var div = document.createElement("div");
+            div.className = "mermaid";
+            div.textContent = node.textContent;
+            node.replaceWith(div);
+          }
+        });
+        mermaid.initialize({ startOnLoad: false, securityLevel: "loose", theme: "neutral" });
+        mermaid.run({ querySelector: ".ld-article div.mermaid" });
+      }).catch(function () { /* offline: leave the diagram source visible */ });
+    }
+  }
+
+  /* ---------- TOC scroll-spy ---------- */
+  var tocLinks = Array.prototype.slice.call(document.querySelectorAll(".ld-toc a"));
+  if (tocLinks.length && article) {
+    var targets = tocLinks.map(function (a) {
+      var id = decodeURIComponent((a.getAttribute("href") || "").replace(/^#/, ""));
+      return document.getElementById(id);
+    });
+    var update = function () {
+      var active = 0;
+      for (var i = 0; i < targets.length; i++) {
+        if (targets[i] && targets[i].getBoundingClientRect().top <= 90) active = i;
+      }
+      tocLinks.forEach(function (a, i) { a.classList.toggle("active", i === active); });
+    };
+    window.addEventListener("scroll", update, { passive: true });
+    update();
+  }
+
+  /* ---------- search overlay ---------- */
+  var overlay = document.getElementById("search-overlay");
+  var input = document.getElementById("search-input");
+  var results = document.getElementById("search-results");
+  var indexPromise = null;
+
+  function loadIndex() {
+    if (!indexPromise) {
+      indexPromise = fetch(BASE + "/search/search_index.json")
+        .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); })
+        .then(function (d) {
+          return d.docs.map(function (doc) {
+            return {
+              location: doc.location,
+              title: doc.title || "",
+              text: (doc.text || "").replace(/\s+/g, " "),
+            };
+          });
+        });
+    }
+    return indexPromise;
+  }
+
+  function esc(s) {
+    return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+  }
+
+  function highlight(text, terms) {
+    var out = esc(text);
+    terms.forEach(function (t) {
+      out = out.replace(new RegExp("(" + t.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ")", "ig"), "<mark>$1</mark>");
+    });
+    return out;
+  }
+
+  function search(docs, query) {
+    var terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    if (!terms.length) return [];
+    var scored = [];
+    docs.forEach(function (doc) {
+      var title = doc.title.toLowerCase();
+      var text = doc.text.toLowerCase();
+      var score = 0;
+      for (var i = 0; i < terms.length; i++) {
+        var t = terms[i];
+        var inTitle = title.indexOf(t) !== -1;
+        var inText = text.indexOf(t) !== -1;
+        if (!inTitle && !inText) { score = 0; break; }
+        score += (inTitle ? 10 : 0) + (inText ? 1 : 0);
+      }
+      // Prefer page-level entries slightly over deep anchors.
+      if (score > 0) scored.push({ doc: doc, score: score + (doc.location.indexOf("#") === -1 ? 2 : 0) });
+    });
+    scored.sort(function (a, b) { return b.score - a.score; });
+    return scored.slice(0, 12).map(function (s) { return s.doc; });
+  }
+
+  function snippet(text, terms) {
+    var lower = text.toLowerCase();
+    var pos = -1;
+    for (var i = 0; i < terms.length; i++) {
+      pos = lower.indexOf(terms[i]);
+      if (pos !== -1) break;
+    }
+    if (pos === -1) pos = 0;
+    var start = Math.max(0, pos - 60);
+    var s = (start > 0 ? "…" : "") + text.slice(start, start + 160) + (start + 160 < text.length ? "…" : "");
+    return s;
+  }
+
+  function render(docs, query) {
+    var terms = query.toLowerCase().split(/\s+/).filter(Boolean);
+    if (!docs.length) {
+      results.innerHTML = "<div class=\"ld-search__empty\">No results</div>";
+      return;
+    }
+    results.innerHTML = docs.map(function (doc) {
+      var crumb = doc.location.split("#")[0].replace(/\/$/, "").replace(/\//g, " / ") || "home";
+      return "<a class=\"ld-search__hit\" href=\"" + BASE + "/" + doc.location + "\">" +
+        "<div class=\"ld-search__hit-crumb\">" + esc(crumb) + "</div>" +
+        "<div class=\"ld-search__hit-title\">" + highlight(doc.title, terms) + "</div>" +
+        "<div class=\"ld-search__hit-text\">" + highlight(snippet(doc.text, terms), terms) + "</div>" +
+        "</a>";
+    }).join("");
+  }
+
+  function openSearch() {
+    overlay.hidden = false;
+    input.value = "";
+    results.innerHTML = "";
+    input.focus();
+    loadIndex();
+  }
+  function closeSearch() { overlay.hidden = true; }
+
+  if (overlay && input && results) {
+    var openBtn = document.getElementById("search-open");
+    if (openBtn) openBtn.addEventListener("click", openSearch);
+    overlay.addEventListener("click", function (e) {
+      if (e.target.closest("[data-search-close]")) closeSearch();
+    });
+    document.addEventListener("keydown", function (e) {
+      if (e.key === "Escape" && !overlay.hidden) { closeSearch(); return; }
+      var typing = /^(INPUT|TEXTAREA|SELECT)$/.test((document.activeElement || {}).tagName || "");
+      if ((e.key === "/" && !typing) || ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k")) {
+        e.preventDefault();
+        if (overlay.hidden) openSearch(); else closeSearch();
+      }
+    });
+    var pending = 0;
+    input.addEventListener("input", function () {
+      var q = input.value.trim();
+      var seq = ++pending;
+      if (q.length < 2) { results.innerHTML = ""; return; }
+      loadIndex().then(function (docs) {
+        if (seq !== pending) return;
+        render(search(docs, q), q);
+      }).catch(function () {
+        results.innerHTML = "<div class=\"ld-search__empty\">Search index unavailable</div>";
+      });
+    });
+  }
+})();
diff --git a/docs/theme/assets/tokens.css b/docs/theme/assets/tokens.css
new file mode 100644
index 00000000000..29c08f2711e
--- /dev/null
+++ b/docs/theme/assets/tokens.css
@@ -0,0 +1,142 @@
+/* Lance Design System — tokens (Swiss editorial: white, ink, brand purple #625EFF).
+   Merged from the design project's ds/tokens set; dark theme mirrors the same
+   structure with paper rules on ink. */
+@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600&family=Roboto+Mono:wght@400;500;600&display=swap");
+
+:root {
+  color-scheme: light;
+
+  /* ---- Typography ---- */
+  --font-display: "Space Grotesk", "Segoe UI", sans-serif;
+  --font-body: "IBM Plex Sans", "Segoe UI", sans-serif;
+  --font-mono: "Roboto Mono", "SF Mono", monospace;
+  --text-base: 16px;
+  --leading-body: 1.6;
+  --tracking-display: -0.045em;
+  --tracking-caps: 0.14em;
+
+  /* ---- Ink scale (dark values — code surfaces & tooltips only) ---- */
+  --ink-950: #0E0C14;
+  --ink-900: #14111D;
+  --ink-850: #1A1626;
+  --ink-800: #221D30;
+  --ink-700: #2C2640;
+  --ink-600: #3A3352;
+
+  /* ---- Foreground scale (on white) ---- */
+  --fg-1: #0E0C14;
+  --fg-2: #4A4459;
+  --fg-3: #8A8499;
+  --fg-inverse: #ffffff;
+
+  /* ---- Brand purple ramp (official #625EFF) ---- */
+  --beam-300: #B3B1FF;
+  --beam-400: #625EFF;
+  --beam-500: #625EFF;
+  --beam-600: #4B47E0;
+  --beam-700: #3936B4;
+  --beam-glow: rgba(98, 94, 255, 0.25);
+  --beam-dim: rgba(98, 94, 255, 0.09);
+
+  /* ---- Semantic status ---- */
+  --ok-500: #0E8F63;
+  --warn-500: #B26205;
+  --danger-500: #D92D20;
+
+  /* ---- Lines: ink rules carry the structure ---- */
+  --line-1: #14111D;
+  --line-2: rgba(20, 17, 29, 0.16);
+
+  /* ---- Surfaces ---- */
+  --surface-page: #FFFFFF;
+  --surface-card: #FFFFFF;
+  --surface-overlay: #F5F4F8;
+  --surface-header: rgba(255, 255, 255, 0.88);
+  --surface-code: #F5F4F8; /* light code surface */
+  --surface-inline-code: rgba(20, 17, 29, 0.05);
+
+  --text-body: var(--fg-1);
+  --text-secondary: var(--fg-2);
+  --text-muted: var(--fg-3);
+
+  /* ---- Syntax highlighting (Pygments), light theme ---- */
+  --code-fg: #2A2635;
+  --code-kw: #7C3AED;     /* keywords, builtins */
+  --code-str: #0B62C4;    /* strings */
+  --code-com: #6E6A80;    /* comments */
+  --code-num: #9A6700;    /* numbers */
+  --code-fn: #0F766E;     /* functions, decorators */
+  --code-name: #2A2635;   /* class/module names, headings */
+  --code-tag: #BE123C;    /* HTML tags, attributes */
+  --code-punct: #57534E;  /* operators, punctuation, output */
+  --code-prompt: #8A8499; /* REPL prompts */
+  --code-hll: rgba(98, 94, 255, 0.12); /* highlighted line */
+
+  /* ---- Layout ---- */
+  --container-max: 1200px;
+  --radius-box: 6px;   /* content boxes: code, admonitions, tabs */
+  --radius-chip: 4px;  /* inline code chips */
+
+  /* ---- Motion — fast, precise, no bounce ---- */
+  --ease-out: cubic-bezier(0.2, 0.8, 0.2, 1);
+  --dur-fast: 120ms;
+}
+
+/* ---- Dark theme: same structure, paper rules on ink ---- */
+:root[data-theme="dark"] {
+  color-scheme: dark;
+
+  --fg-1: #F2F0FA;
+  --fg-2: #B7B1C6;
+  --fg-3: #837D96;
+  --fg-inverse: #0E0C14;
+
+  /* text accents need a lighter purple to stay readable on ink */
+  --beam-600: #8E8BFF;
+  --beam-700: #A9A6FF;
+
+  --ok-500: #34C08E;
+  --warn-500: #E8A13C;
+  --danger-500: #FF6B5E;
+
+  --line-1: rgba(237, 235, 245, 0.7);
+  --line-2: rgba(237, 235, 245, 0.15);
+
+  --surface-page: #0E0C14;
+  --surface-card: #14111D;
+  --surface-overlay: #1A1626;
+  --surface-header: rgba(14, 12, 20, 0.85);
+  --surface-code: #14111D;
+  --surface-inline-code: rgba(237, 235, 245, 0.08);
+
+  /* ---- Syntax highlighting (Pygments), dark theme (on ink) ---- */
+  --code-fg: #EDEBF5;
+  --code-kw: #B8A8FF;
+  --code-str: #7CC0FF;
+  --code-com: #6F6A85;
+  --code-num: #F2CE6B;
+  --code-fn: #8FE8CE;
+  --code-name: #EDEBF5;
+  --code-tag: #FF95A8;
+  --code-punct: #B7B1C6;
+  --code-prompt: #6F6A85;
+  --code-hll: rgba(98, 94, 255, 0.22);
+}
+
+/* ---- Minimal base ---- */
+* { box-sizing: border-box; }
+
+body {
+  margin: 0;
+  background: var(--surface-page);
+  color: var(--text-body);
+  font-family: var(--font-body);
+  font-size: var(--text-base);
+  line-height: var(--leading-body);
+  -webkit-font-smoothing: antialiased;
+}
+
+/* No `color` here: forcing ink text breaks selection inside dark code blocks. */
+::selection { background: rgba(98, 94, 255, 0.3); }
+
+code, kbd, pre { font-family: var(--font-mono); }
diff --git a/docs/theme/base.html b/docs/theme/base.html
new file mode 100644
index 00000000000..54312685410
--- /dev/null
+++ b/docs/theme/base.html
@@ -0,0 +1,118 @@
+{#- Lance Docs custom theme — shared skeleton: head, header, footer, scripts. -#}
+{%- macro first_url(item) -%}
+  {%- if item.is_link or item.is_page -%}
+    {{- item.url -}}
+  {%- elif item.children -%}
+    {{- first_url(item.children[0]) -}}
+  {%- endif -%}
+{%- endmacro -%}
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+{% if page and page.title and not page.is_homepage %}
+<title>{{ page.title }} — {{ config.site_name }}</title>
+{% else %}
+<title>{{ config.site_name }} — The open lakehouse format for multimodal AI</title>
+{% endif %}
+{% if page and page.meta and page.meta.description %}
+<meta name="description" content="{{ page.meta.description }}">
+{% elif config.site_description %}
+<meta name="description" content="{{ config.site_description }}">
+{% endif %}
+{% if page and page.canonical_url %}<link rel="canonical" href="{{ page.canonical_url }}">{% endif %}
+<link rel="icon" href="{{ 'logo/logo.png' | url }}">
+<link rel="stylesheet" href="{{ 'assets/tokens.css' | url }}">
+<link rel="stylesheet" href="{{ 'assets/site.css' | url }}">
+<script>
+/* Set the theme before first paint to avoid a light-mode flash. */
+(function () {
+  try {
+    var t = localStorage.getItem("ld-theme");
+    if (t !== "light" && t !== "dark") {
+      t = window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light";
+    }
+    document.documentElement.dataset.theme = t;
+  } catch (e) { document.documentElement.dataset.theme = "light"; }
+})();
+</script>
+</head>
+<body>
+<div class="ld-shell">
+  <header class="ld-header">
+    <a class="ld-brand" href="{{ nav.homepage.url | url }}" aria-label="{{ config.site_name }}">
+      <img src="{{ 'logo/wide.png' | url }}" alt="{{ config.site_name }}">
+    </a>
+    <nav class="ld-topnav" aria-label="Sections">
+      {%- for item in nav %}
+      <a class="ld-toptab{% if item.active %} active{% endif %}" href="{{ first_url(item) | url }}">{{ item.title }}</a>
+      {%- endfor %}
+    </nav>
+    <div class="ld-header__actions">
+      <button class="ld-btn ld-btn--ghost ld-btn--icon" id="search-open" type="button" aria-label="Search docs" title="Search (/)">
+        <svg viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><circle cx="11" cy="11" r="7"/><path d="m21 21-4.3-4.3"/></svg>
+      </button>
+      <a class="ld-btn ld-btn--ghost" href="{{ config.repo_url }}" target="_blank" rel="noopener" aria-label="Lance on GitHub">
+        <svg viewBox="0 0 16 16" width="14" height="14" fill="currentColor" aria-hidden="true"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82a7.4 7.4 0 0 1 2-.27c.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58-8-8-8Z"/></svg>
+        GitHub
+        <span class="ld-btn__count" id="gh-stars" hidden></span>
+      </a>
+      <a class="ld-btn ld-btn--ghost ld-btn--icon" href="https://discord.gg/lance" target="_blank" rel="noopener" aria-label="Join the Lance Discord" title="Discord">
+        <svg viewBox="0 0 24 24" width="15" height="15" fill="currentColor" aria-hidden="true"><path d="M20.32 4.37a19.8 19.8 0 0 0-4.89-1.52.07.07 0 0 0-.08.04c-.21.38-.44.87-.61 1.25a18.27 18.27 0 0 0-5.48 0 12.64 12.64 0 0 0-.62-1.25.08.08 0 0 0-.08-.04c-1.71.3-3.35.81-4.88 1.52a.07.07 0 0 0-.04.03C.53 9.05-.32 13.58.1 18.06c0 .02.01.04.03.05a19.9 19.9 0 0 0 6 3.03c.03.01.06 0 .08-.02.46-.63.87-1.3 1.22-2a.08.08 0 0 0-.04-.11 13.1 13.1 0 0 1-1.87-.89.08.08 0 0 1-.01-.13c.13-.09.25-.19.37-.29a.07.07 0 0 1 .08-.01c3.93 1.79 8.18 1.79 12.06 0a.07.07 0 0 1 .08.01c.12.1.25.2.37.29a.08.08 0 0 1-.01.13c-.6.35-1.22.64-1.87.89a.08.08 0 0 0-.04.11c.36.7.77 1.36 1.22 2 .02.02.05.03.08.02a19.84 19.84 0 0 0 6.02-3.03.08.08 0 0 0 .03-.05c.5-5.18-.84-9.68-3.55-13.66a.06.06 0 0 0-.03-.03ZM8.02 15.33c-1.18 0-2.16-1.08-2.16-2.42 0-1.33.96-2.42 2.16-2.42 1.21 0 2.18 1.1 2.16 2.42 0 1.34-.96 2.42-2.16 2.42Zm7.97 0c-1.18 0-2.15-1.08-2.15-2.42 0-1.33.95-2.42 2.15-2.42 1.21 0 2.18 1.1 2.16 2.42 0 1.34-.95 2.42-2.16 2.42Z"/></svg>
+      </a>
+      <button class="ld-btn ld-btn--ghost ld-btn--icon" id="theme-toggle" type="button" aria-label="Toggle light / dark theme" title="Toggle theme">
+        <svg class="ld-icon-sun" viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><circle cx="12" cy="12" r="4"/><path d="M12 2v2m0 16v2M4.93 4.93l1.41 1.41m11.32 11.32 1.41 1.41M2 12h2m16 0h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"/></svg>
+        <svg class="ld-icon-moon" viewBox="0 0 24 24" width="15" height="15" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79Z"/></svg>
+      </button>
+      <a class="ld-btn ld-btn--primary" href="{{ 'quickstart/' | url }}">Get started</a>
+    </div>
+  </header>
+
+  {% block container %}{% endblock %}
+
+  <footer class="ld-footer">
+    <div class="ld-footer__brand">
+      <img src="{{ 'logo/wide.png' | url }}" alt="{{ config.site_name }}">
+      <p>Lance is an open-source columnar format for multimodal AI. Apache-2.0. {{ config.copyright }}</p>
+    </div>
+    <div class="ld-footer__cols">
+      <div class="ld-footer__col">
+        <h5>Format</h5>
+        <a href="{{ 'format/' | url }}">Specification</a>
+        <a href="{{ 'format/file/' | url }}">File format</a>
+        <a href="{{ 'format/table/' | url }}">Table format</a>
+      </div>
+      <div class="ld-footer__col">
+        <h5>Develop</h5>
+        <a href="{{ 'quickstart/' | url }}">Getting started</a>
+        <a href="{{ 'guide/read_and_write/' | url }}">User guide</a>
+        <a href="{{ 'sdk_docs/' | url }}">SDK docs</a>
+      </div>
+      <div class="ld-footer__col">
+        <h5>Community</h5>
+        <a href="{{ config.repo_url }}" target="_blank" rel="noopener">GitHub</a>
+        <a href="https://discord.gg/lance" target="_blank" rel="noopener">Discord</a>
+        <a href="https://groups.google.com/a/lance.org/g/dev" target="_blank" rel="noopener">Mailing list</a>
+        <a href="{{ 'community/contributing/' | url }}">Contributing</a>
+      </div>
+    </div>
+  </footer>
+</div>
+
+<!-- Search overlay -->
+<div class="ld-search" id="search-overlay" hidden>
+  <div class="ld-search__scrim" data-search-close></div>
+  <div class="ld-search__panel" role="dialog" aria-label="Search documentation">
+    <div class="ld-search__bar">
+      <input class="ld-search__input" id="search-input" type="search" placeholder="Search documentation…" autocomplete="off" spellcheck="false">
+      <button class="ld-search__close" type="button" data-search-close aria-label="Close search">Esc</button>
+    </div>
+    <div class="ld-search__results" id="search-results"></div>
+  </div>
+</div>
+
+<script>window.LD_BASE = "{{ base_url }}";</script>
+<script src="{{ 'assets/site.js' | url }}" defer></script>
+</body>
+</html>
diff --git a/docs/theme/home.html b/docs/theme/home.html
new file mode 100644
index 00000000000..e42d6f70fee
--- /dev/null
+++ b/docs/theme/home.html
@@ -0,0 +1,147 @@
+{% extends "base.html" %}
+
+{% block container %}
+  <main class="ld-home">
+    <section class="ld-hero">
+      <div class="ld-kicker">Lance format · Open source · Apache-2.0 · <a href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">VLDB '25 paper ↗</a></div>
+      <h1>The open lakehouse format for multimodal AI.</h1>
+      <p class="ld-hero__lead">A file format, table format, and catalog spec for building a complete lakehouse on object storage — powering vector and full-text search, feature engineering, and model training with the fast random access and scans that AI workloads need.</p>
+      <div class="ld-hero__cta">
+        <a class="ld-btn ld-btn--primary ld-btn--lg" href="{{ 'quickstart/' | url }}">Get started</a>
+        <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'format/' | url }}">Read the spec</a>
+        <a class="ld-btn ld-btn--outline ld-btn--lg" href="{{ 'examples/python/llm_training/' | url }}">Train an LLM</a>
+        <a class="ld-btn ld-btn--text ld-btn--lg" href="https://discord.gg/lance" target="_blank" rel="noopener">Join Discord →</a>
+      </div>
+    </section>
+
+    <section class="ld-band">
+      <div class="ld-stats">
+        <div class="ld-stat">
+          <div class="ld-stat__value">100×</div>
+          <div class="ld-stat__label">Faster random access than Parquet</div>
+        </div>
+        <div class="ld-stat">
+          <div class="ld-stat__value">1 line</div>
+          <div class="ld-stat__label">To convert Parquet to Lance</div>
+        </div>
+        <a class="ld-stat ld-stat--link" href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">
+          <div class="ld-stat__value ld-stat__value--beam">VLDB '25</div>
+          <div class="ld-stat__label">Peer-reviewed research paper →</div>
+        </a>
+      </div>
+    </section>
+
+    <section class="ld-what">
+      <h2>What is Lance?</h2>
+      <p class="ld-what__body">Lance is a modern, open source lakehouse format for multimodal AI. It brings high-performance vector and full-text search, feature engineering, and model training to the lakehouse, powered by fast random access and scans — while keeping SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB) and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore).</p>
+      <p class="ld-what__more">Learn more in the <a class="ld-inline-link" href="https://arxiv.org/abs/2504.15247" target="_blank" rel="noopener">research paper</a> published at VLDB 2025.</p>
+    </section>
+
+    <section class="ld-features">
+      <div class="ld-feature">
+        <div class="ld-feature__num">01</div>
+        <div class="ld-feature__body">
+          <h3>Expressive hybrid search</h3>
+          <p>Combine vector similarity, full-text search (BM25), and SQL analytics on the same dataset. All query types are accelerated by secondary indexes that are part of the Lance specification.</p>
+          <a class="ld-more" href="{{ 'quickstart/vector-search/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/docs"</span>)
+
+<span class="tok-com"># Full text search</span>
+ds.to_table(<span class="tok-arg">full_text_query=</span><span class="tok-str">"machine learning"</span>)
+
+<span class="tok-com"># Hybrid search</span>
+ds.to_table(
+    <span class="tok-arg">nearest=</span>{
+        <span class="tok-str">"column"</span>: <span class="tok-str">"embedding"</span>, <span class="tok-str">"q"</span>: query_vec, <span class="tok-str">"k"</span>: <span class="tok-num">10</span>
+    },
+    <span class="tok-arg">filter=</span><span class="tok-str">"year &gt; 2020"</span>,
+)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">02</div>
+        <div class="ld-feature__body">
+          <h3>Lightning-fast random access</h3>
+          <p>100x faster random access than Parquet or Iceberg. An optimized file format plus row addressing and secondary indexes let you fetch individual records across files instantly — for ML serving, sampling, and interactive apps.</p>
+          <a class="ld-more" href="{{ 'guide/read_and_write/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/embeddings.lance"</span>)
+
+<span class="tok-com"># Access the 2nd &amp; 51st rows</span>
+ds.take([<span class="tok-num">2</span>, <span class="tok-num">51</span>], <span class="tok-arg">columns=</span>[<span class="tok-str">"id"</span>, <span class="tok-str">"vec_gemma3"</span>])
+
+<span class="tok-com"># Take 1000 random samples</span>
+ds.sample(<span class="tok-num">1000</span>, <span class="tok-arg">columns=</span>[<span class="tok-str">"id"</span>, <span class="tok-str">"vec_llama"</span>])</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">03</div>
+        <div class="ld-feature__body">
+          <h3>Native multimodal data</h3>
+          <p>Store images, videos, audio, text, and embeddings alongside tabular data in one format. Blob encoding handles large binary objects with lazy loading; optimized vector storage accelerates similarity search.</p>
+          <a class="ld-more" href="{{ 'guide/blob/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+<span class="tok-kw">import</span> av
+
+ds = lance.dataset(<span class="tok-str">"s3://my-bucket/videos.lance"</span>)
+
+<span class="tok-com"># Get blobs from the 2nd and 51st rows</span>
+blobs = ds.take_blobs(<span class="tok-str">"video"</span>, <span class="tok-arg">ids=</span>[<span class="tok-num">2</span>, <span class="tok-num">51</span>])
+
+<span class="tok-kw">for</span> blob <span class="tok-kw">in</span> blobs:
+    <span class="tok-kw">with</span> av.open(blob) <span class="tok-kw">as</span> container:
+        stream = container.streams.video[<span class="tok-num">0</span>]
+        container.seek(<span class="tok-arg">start_time=</span><span class="tok-num">500</span>, <span class="tok-arg">stream=</span>stream)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">04</div>
+        <div class="ld-feature__body">
+          <h3>Data evolution &gt; schema evolution</h3>
+          <p>Backfilling column values normally forces a full table rewrite. Lance supports efficient schema evolution with backfill — adding a column with data is just writing new Lance files to the table.</p>
+          <a class="ld-more" href="{{ 'guide/data_evolution/' | url }}">Learn more →</a>
+        </div>
+        <div class="ld-feature__code">
+          <div class="ld-win" aria-hidden="true"><i></i><i></i><i></i></div>
+          <pre><code><span class="tok-kw">import</span> lance
+
+dataset = lance.dataset(<span class="tok-str">"my_data.lance"</span>)
+
+<span class="tok-fn">@lance.batch_udf</span>()
+<span class="tok-kw">def</span> <span class="tok-fn">add_embeddings</span>(batch):
+    vectors = model.encode(batch[<span class="tok-str">"text"</span>])
+    <span class="tok-kw">return</span> {<span class="tok-str">"embedding"</span>: vectors}
+
+dataset.add_columns(add_embeddings)</code></pre>
+        </div>
+      </div>
+
+      <div class="ld-feature">
+        <div class="ld-feature__num">05</div>
+        <div class="ld-feature__body">
+          <h3>Rich ecosystem integrations</h3>
+          <p>Works with Pandas, Polars, Ray, and PyTorch for processing and ML. Connects to Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink for SQL analytics and distributed processing.</p>
+          <a class="ld-more" href="{{ 'integrations/' | url }}">View integrations →</a>
+        </div>
+        <div class="ld-feature__img">
+          <img src="{{ 'assets/images/ecosystem-integrations.png' | url }}" alt="Lance ecosystem integrations" loading="lazy">
+        </div>
+      </div>
+    </section>
+  </main>
+{% endblock %}
diff --git a/docs/theme/main.html b/docs/theme/main.html
new file mode 100644
index 00000000000..3a4f4adf04f
--- /dev/null
+++ b/docs/theme/main.html
@@ -0,0 +1,88 @@
+{% extends "base.html" %}
+
+{#- Render one nav section as sidenav groups: its direct pages/links under the
+    section's own label, then each child section as a further group. -#}
+{%- macro sidenav_section(sec, label) -%}
+  {%- set direct = sec.children | selectattr("is_section", "ne", true) | list -%}
+  {%- if direct -%}
+  <div class="ld-sidenav__group">
+    <div class="ld-sidenav__label">{{ label }}</div>
+    {%- for child in direct %}
+    {%- if child.is_link %}
+    <a href="{{ child.url }}" target="_blank" rel="noopener">{{ child.title }} <span class="ext">↗</span></a>
+    {%- else %}
+    <a{% if child == page %} class="active"{% endif %} href="{{ child.url | url }}">{{ child.title }}</a>
+    {%- endif %}
+    {%- endfor %}
+  </div>
+  {%- endif -%}
+  {%- for child in sec.children if child.is_section -%}
+    {{ sidenav_section(child, child.title) }}
+  {%- endfor -%}
+{%- endmacro -%}
+
+{% block container %}
+  {%- set active_section = namespace(item=none) -%}
+  {%- for item in nav %}{% if item.active %}{% set active_section.item = item %}{% endif %}{% endfor %}
+
+  <main class="ld-docs">
+    <aside class="ld-sidenav" aria-label="Section pages">
+      {%- if active_section.item and active_section.item.is_section %}
+        {{ sidenav_section(active_section.item, active_section.item.title) }}
+      {%- elif active_section.item %}
+      <div class="ld-sidenav__group">
+        <div class="ld-sidenav__label">{{ active_section.item.title }}</div>
+        <a class="active" href="{{ active_section.item.url | url }}">{{ active_section.item.title }}</a>
+      </div>
+      {%- endif %}
+    </aside>
+
+    <div class="ld-article-col">
+      <div class="ld-crumbs">
+        <span>Docs</span>
+        {%- for crumb in page.ancestors | reverse %}
+        <span>/</span><span>{{ crumb.title }}</span>
+        {%- endfor %}
+        <span>/</span><b>{{ page.title }}</b>
+      </div>
+
+      <article class="ld-article">
+        {{ page.content }}
+      </article>
+
+      <div class="ld-pagenav">
+        {%- if page.previous_page %}
+        <a href="{{ page.previous_page.url | url }}">
+          <div class="ld-pagenav__k">← Previous</div>
+          <div class="ld-pagenav__t">{{ page.previous_page.title }}</div>
+        </a>
+        {%- else %}<span></span>{% endif %}
+        {%- if page.next_page %}
+        <a class="next" href="{{ page.next_page.url | url }}">
+          <div class="ld-pagenav__k">Next →</div>
+          <div class="ld-pagenav__t">{{ page.next_page.title }}</div>
+        </a>
+        {%- endif %}
+      </div>
+    </div>
+
+    <aside class="ld-toc" aria-label="On this page">
+      {%- set toc_items = namespace(list=[]) -%}
+      {%- for h1 in page.toc -%}
+        {%- if loop.first and page.toc | length == 1 -%}
+          {%- set toc_items.list = h1.children -%}
+        {%- endif -%}
+      {%- endfor -%}
+      {%- if page.toc | length > 1 %}{% set toc_items.list = page.toc %}{% endif -%}
+      {%- if toc_items.list %}
+      <div class="ld-toc__label">On this page</div>
+      {%- for h2 in toc_items.list %}
+      <a class="ld-toc__h2" href="{{ h2.url }}">{{ h2.title }}</a>
+      {%- for h3 in h2.children %}
+      <a class="ld-toc__h3" href="{{ h3.url }}">{{ h3.title }}</a>
+      {%- endfor %}
+      {%- endfor %}
+      {%- endif %}
+    </aside>
+  </main>
+{% endblock %}
diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock
index 79d554807bd..cd7ba798a10 100644
--- a/java/lance-jni/Cargo.lock
+++ b/java/lance-jni/Cargo.lock
@@ -42,21 +42,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "alloc-no-stdlib"
-version = "2.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
-
-[[package]]
-name = "alloc-stdlib"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
-dependencies = [
- "alloc-no-stdlib",
-]
-
 [[package]]
 name = "allocator-api2"
 version = "0.2.21"
@@ -139,9 +124,9 @@ dependencies = [
 
 [[package]]
 name = "arc-swap"
-version = "1.9.1"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
 dependencies = [
  "rustversion",
 ]
@@ -426,7 +411,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -437,7 +422,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -491,7 +476,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "ring",
  "time",
  "tokio",
@@ -551,7 +536,7 @@ dependencies = [
  "bytes",
  "bytes-utils",
  "fastrand",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "percent-encoding",
  "pin-project-lite",
@@ -578,7 +563,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -602,7 +587,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -627,7 +612,7 @@ dependencies = [
  "aws-types",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -647,7 +632,7 @@ dependencies = [
  "hex",
  "hmac 0.12.1",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "percent-encoding",
  "sha2 0.10.9",
  "time",
@@ -677,7 +662,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "percent-encoding",
@@ -696,7 +681,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-rustls",
  "hyper-util",
@@ -753,7 +738,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -773,7 +758,7 @@ dependencies = [
  "aws-smithy-types",
  "bytes",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "pin-project-lite",
  "tokio",
  "tracing",
@@ -790,7 +775,7 @@ dependencies = [
  "bytes",
  "bytes-utils",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -836,7 +821,7 @@ dependencies = [
  "axum-core",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -869,7 +854,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "mime",
@@ -929,24 +914,15 @@ dependencies = [
 
 [[package]]
 name = "bitflags"
-version = "2.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
-
-[[package]]
-name = "bitpacking"
-version = "0.9.3"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
-dependencies = [
- "crunchy",
-]
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 
 [[package]]
 name = "bitvec"
-version = "1.0.1"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837"
 dependencies = [
  "funty",
  "radium",
@@ -988,9 +964,9 @@ dependencies = [
 
 [[package]]
 name = "block-buffer"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
 dependencies = [
  "hybrid-array",
 ]
@@ -1004,27 +980,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "brotli"
-version = "8.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor",
-]
-
-[[package]]
-name = "brotli-decompressor"
-version = "5.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
-]
-
 [[package]]
 name = "bs58"
 version = "0.5.1"
@@ -1056,6 +1011,20 @@ name = "bytemuck"
 version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
 
 [[package]]
 name = "byteorder"
@@ -1065,9 +1034,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.11.1"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1090,9 +1059,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.63"
+version = "1.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
+checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1137,9 +1106,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
  "iana-time-zone",
  "js-sys",
@@ -1200,7 +1169,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1430,18 +1399,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1563,7 +1532,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1574,7 +1543,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1593,14 +1562,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -1610,7 +1578,6 @@ dependencies = [
  "datafusion-datasource-arrow",
  "datafusion-datasource-csv",
  "datafusion-datasource-json",
- "datafusion-datasource-parquet",
  "datafusion-execution",
  "datafusion-expr",
  "datafusion-expr-common",
@@ -1628,13 +1595,11 @@ dependencies = [
  "datafusion-session",
  "datafusion-sql",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
  "object_store",
  "parking_lot",
- "parquet",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
@@ -1644,9 +1609,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1669,9 +1634,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1692,33 +1657,33 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
- "parquet",
- "paste",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
 dependencies = [
  "futures",
  "log",
@@ -1727,9 +1692,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1749,6 +1714,7 @@ dependencies = [
  "itertools 0.14.0",
  "log",
  "object_store",
+ "parking_lot",
  "rand 0.9.4",
  "tokio",
  "url",
@@ -1756,9 +1722,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -1780,9 +1746,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1803,9 +1769,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1820,57 +1786,25 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
-[[package]]
-name = "datafusion-datasource-parquet"
-version = "53.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
-dependencies = [
- "arrow",
- "async-trait",
- "bytes",
- "datafusion-common",
- "datafusion-common-runtime",
- "datafusion-datasource",
- "datafusion-execution",
- "datafusion-expr",
- "datafusion-functions-aggregate-common",
- "datafusion-physical-expr",
- "datafusion-physical-expr-adapter",
- "datafusion-physical-expr-common",
- "datafusion-physical-plan",
- "datafusion-pruning",
- "datafusion-session",
- "futures",
- "itertools 0.14.0",
- "log",
- "object_store",
- "parking_lot",
- "parquet",
- "tokio",
-]
-
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -1886,11 +1820,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -1901,29 +1836,27 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -1938,26 +1871,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
  "rand 0.9.4",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -1967,19 +1899,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -1988,9 +1919,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2004,34 +1935,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2042,14 +1973,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2057,20 +1987,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
 dependencies = [
  "arrow",
  "chrono",
@@ -2087,11 +2017,10 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2099,20 +2028,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2125,26 +2053,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2160,12 +2088,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2180,7 +2109,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2192,9 +2121,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2203,15 +2132,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2223,9 +2151,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2241,9 +2169,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2259,26 +2187,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "deepsize"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
-dependencies = [
- "deepsize_derive",
-]
-
-[[package]]
-name = "deepsize_derive"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -2318,7 +2226,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
 dependencies = [
- "block-buffer 0.12.0",
+ "block-buffer 0.12.1",
  "const-oid 0.10.2",
  "crypto-common 0.2.2",
  "ctutils",
@@ -2353,7 +2261,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2419,9 +2327,9 @@ dependencies = [
 
 [[package]]
 name = "env_filter"
-version = "1.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
 dependencies = [
  "log",
  "regex",
@@ -2429,9 +2337,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.10"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2525,7 +2433,6 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
- "zlib-rs",
 ]
 
 [[package]]
@@ -2569,7 +2476,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "rand 0.9.4",
@@ -2646,7 +2553,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2797,12 +2704,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -2863,17 +2771,15 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
 dependencies = [
  "cfg-if 1.0.4",
  "js-sys",
  "libc",
  "r-efi 6.0.0",
  "rand_core 0.10.1",
- "wasip2",
- "wasip3",
  "wasm-bindgen",
 ]
 
@@ -2894,7 +2800,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2915,18 +2821,42 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "goosefs-sdk"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "dashmap",
+ "hostname",
+ "prost",
+ "prost-types",
+ "rand 0.9.4",
+ "reqwest 0.12.28",
+ "serde",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tonic-prost",
+ "tracing",
+ "uuid",
+]
+
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.4.1",
+ "http 1.4.2",
  "indexmap 2.14.0",
  "slab",
  "tokio",
@@ -2940,6 +2870,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -2992,6 +2923,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3035,7 +2971,7 @@ checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "more-asserts",
  "serde",
  "thiserror 2.0.18",
@@ -3067,6 +3003,17 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "hostname"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
+dependencies = [
+ "cfg-if 1.0.4",
+ "libc",
+ "windows-link",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3080,9 +3027,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.4.1"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -3106,7 +3053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
 ]
 
 [[package]]
@@ -3117,7 +3064,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
 dependencies = [
  "bytes",
  "futures-core",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "pin-project-lite",
 ]
@@ -3136,9 +3083,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
@@ -3160,7 +3107,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "httparse",
  "httpdate",
@@ -3177,7 +3124,7 @@ version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-util",
  "rustls",
@@ -3185,6 +3132,20 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tower-service",
+ "webpki-roots",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
 ]
 
 [[package]]
@@ -3197,7 +3158,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "hyper",
  "ipnet",
@@ -3415,12 +3376,6 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f"
 
-[[package]]
-name = "id-arena"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -3481,17 +3436,11 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
 [[package]]
 name = "io-uring"
-version = "0.7.12"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
+checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.4",
@@ -3519,15 +3468,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.14.0"
@@ -3568,7 +3508,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3629,7 +3569,7 @@ dependencies = [
  "quote",
  "rustc_version",
  "simd_cesu8",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3657,7 +3597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
 dependencies = [
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3672,13 +3612,12 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
 dependencies = [
  "cfg-if 1.0.4",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
@@ -3695,7 +3634,7 @@ dependencies = [
  "jiff",
  "nom",
  "num-traits",
- "ordered-float 5.3.0",
+ "ordered-float",
  "rand 0.9.4",
  "serde",
  "serde_json",
@@ -3721,7 +3660,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -3738,7 +3677,6 @@ dependencies = [
  "async-trait",
  "async_cell",
  "aws-credential-types",
- "bitpacking",
  "byteorder",
  "bytes",
  "chrono",
@@ -3750,14 +3688,14 @@ dependencies = [
  "datafusion-functions",
  "datafusion-physical-expr",
  "datafusion-physical-plan",
- "deepsize",
  "either",
  "fst",
  "futures",
  "half",
  "humantime",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-encoding",
@@ -3795,7 +3733,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -3804,6 +3742,7 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
@@ -3837,30 +3776,33 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrayref",
+ "crunchy",
  "paste",
  "seq-macro",
 ]
 
 [[package]]
 name = "lance-core"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
+ "arrow-data",
  "arrow-schema",
  "async-trait",
  "byteorder",
  "bytes",
  "datafusion-common",
  "datafusion-sql",
- "deepsize",
  "futures",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-derive",
  "libc",
+ "libm",
  "log",
  "moka",
  "num_cpus",
@@ -3876,12 +3818,13 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "tracing",
+ "twox-hash",
  "url",
 ]
 
 [[package]]
 name = "lance-datafusion"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3913,7 +3856,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3926,12 +3869,20 @@ dependencies = [
  "rand 0.9.4",
  "rand_distr",
  "rand_xoshiro",
- "random_word",
+]
+
+[[package]]
+name = "lance-derive"
+version = "9.1.0-beta.2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3947,7 +3898,7 @@ dependencies = [
  "futures",
  "hex",
  "hyperloglogplus",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
  "lance-bitpacking",
  "lance-core",
@@ -3966,7 +3917,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -3979,7 +3930,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "datafusion-common",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -3997,7 +3947,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4011,7 +3961,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4023,7 +3973,6 @@ dependencies = [
  "async-channel",
  "async-recursion",
  "async-trait",
- "bitpacking",
  "bitvec",
  "bytes",
  "chrono",
@@ -4032,7 +3981,6 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "deepsize",
  "dirs",
  "fst",
  "futures",
@@ -4040,22 +3988,24 @@ dependencies = [
  "geoarrow-array",
  "geoarrow-schema",
  "half",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "jsonb",
  "lance-arrow",
  "lance-arrow-stats",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-datagen",
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
  "lance-table",
  "lance-tokenizer",
- "libm",
+ "libsais-rs",
  "log",
  "ndarray",
  "num-traits",
@@ -4067,6 +4017,7 @@ dependencies = [
  "rand_distr",
  "rangemap",
  "rayon",
+ "regex-syntax",
  "roaring",
  "serde",
  "serde_json",
@@ -4074,13 +4025,34 @@ dependencies = [
  "tempfile",
  "tokio",
  "tracing",
- "twox-hash",
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "9.1.0-beta.2"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4097,9 +4069,9 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "deepsize",
  "futures",
- "http 1.4.1",
+ "goosefs-sdk",
+ "http 1.4.2",
  "io-uring",
  "lance-arrow",
  "lance-core",
@@ -4122,7 +4094,7 @@ dependencies = [
 
 [[package]]
 name = "lance-jni"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4158,23 +4130,23 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-schema",
  "cc",
- "deepsize",
  "half",
  "lance-arrow",
  "lance-core",
  "num-traits",
  "rand 0.9.4",
+ "rayon",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4186,7 +4158,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4194,6 +4166,8 @@ dependencies = [
  "async-trait",
  "axum",
  "bytes",
+ "datafusion-common",
+ "datafusion-physical-plan",
  "futures",
  "lance",
  "lance-core",
@@ -4206,19 +4180,22 @@ dependencies = [
  "object_store",
  "rand 0.9.4",
  "reqwest 0.12.28",
+ "roaring",
  "serde",
  "serde_json",
+ "time",
  "tokio",
  "tower",
  "tower-http 0.5.2",
  "url",
+ "uuid",
 ]
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.8.0"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3eefb02ded2c3d4b6b60669bb74822d9fa628e144fc748c79ee31f13f566e87b"
+checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -4230,15 +4207,14 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-schema",
  "byteorder",
  "bytes",
- "deepsize",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-core",
  "roaring",
  "tracing",
@@ -4246,7 +4222,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4257,7 +4233,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -4284,11 +4259,12 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "icu_segmenter",
  "rust-stemmers",
  "serde",
+ "stop-words",
  "unicode-normalization",
 ]
 
@@ -4301,12 +4277,6 @@ dependencies = [
  "spin",
 ]
 
-[[package]]
-name = "leb128fmt"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -4385,11 +4355,20 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libsais-rs"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40fe164dbd47ea0c20e78a121c980ef673326905f1d4fba55e3645a20ef6717f"
+dependencies = [
+ "rayon",
+]
+
 [[package]]
 name = "link-section"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "014e440054ce8170890229eeef5bcda955305e056ec713de40ed366944483f09"
+checksum = "c2b1dd6fe32e55c0fc0ea9493aa57459ca3cf4ff3c857c7d0302290150da6e4f"
 
 [[package]]
 name = "linktime-proc-macro"
@@ -4420,9 +4399,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.31"
+version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
 [[package]]
 name = "loom"
@@ -4530,9 +4509,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.8.1"
+version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "mime"
@@ -4745,7 +4724,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -4790,7 +4769,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body-util",
  "httparse",
  "humantime",
@@ -4867,9 +4846,11 @@ dependencies = [
  "opendal-service-azdls",
  "opendal-service-cos",
  "opendal-service-gcs",
+ "opendal-service-goosefs",
  "opendal-service-hf",
  "opendal-service-oss",
  "opendal-service-s3",
+ "opendal-service-tos",
 ]
 
 [[package]]
@@ -4882,7 +4863,7 @@ dependencies = [
  "base64",
  "bytes",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "jiff",
  "log",
@@ -4907,7 +4888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
 dependencies = [
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "mea",
  "opendal-core",
 ]
@@ -4951,7 +4932,7 @@ checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
 dependencies = [
  "base64",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -4972,7 +4953,7 @@ checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
 dependencies = [
  "base64",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -4990,7 +4971,7 @@ version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "opendal-core",
 ]
 
@@ -5001,7 +4982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -5019,7 +5000,7 @@ checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -5032,6 +5013,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "opendal-service-goosefs"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
+dependencies = [
+ "bytes",
+ "goosefs-sdk",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+]
+
 [[package]]
 name = "opendal-service-hf"
 version = "0.57.0"
@@ -5040,7 +5035,7 @@ checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
 dependencies = [
  "bytes",
  "hf-xet",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -5056,7 +5051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -5075,7 +5070,7 @@ dependencies = [
  "base64",
  "bytes",
  "crc32c",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "md-5 0.11.0",
  "opendal-core",
@@ -5087,6 +5082,23 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "opendal-service-tos"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1"
+dependencies = [
+ "bytes",
+ "http 1.4.2",
+ "opendal-core",
+ "quick-xml 0.39.4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-volcengine-tos",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -5099,15 +5111,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
 
-[[package]]
-name = "ordered-float"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
-dependencies = [
- "num-traits",
-]
-
 [[package]]
 name = "ordered-float"
 version = "5.3.0"
@@ -5171,42 +5174,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "parquet"
-version = "58.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
-dependencies = [
- "ahash",
- "arrow-array",
- "arrow-buffer",
- "arrow-data",
- "arrow-ipc",
- "arrow-schema",
- "arrow-select",
- "base64",
- "brotli",
- "bytes",
- "chrono",
- "flate2",
- "futures",
- "half",
- "hashbrown 0.17.1",
- "lz4_flex",
- "num-bigint",
- "num-integer",
- "num-traits",
- "object_store",
- "paste",
- "seq-macro",
- "simdutf8",
- "snap",
- "thrift",
- "tokio",
- "twox-hash",
- "zstd",
-]
-
 [[package]]
 name = "paste"
 version = "1.0.15"
@@ -5350,7 +5317,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5457,7 +5424,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5480,9 +5447,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -5490,9 +5457,9 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
+checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
  "itertools 0.14.0",
@@ -5503,28 +5470,28 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn 2.0.117",
+ "syn",
  "tempfile",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
 dependencies = [
  "prost",
 ]
@@ -5659,7 +5626,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
 dependencies = [
  "chacha20",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "rand_core 0.10.1",
 ]
 
@@ -5726,19 +5693,6 @@ dependencies = [
  "rand_core 0.9.5",
 ]
 
-[[package]]
-name = "random_word"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81"
-dependencies = [
- "ahash",
- "brotli",
- "paste",
- "rand 0.9.4",
- "unicase",
-]
-
 [[package]]
 name = "rangemap"
 version = "1.7.1"
@@ -5817,14 +5771,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -5851,9 +5805,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "regress"
@@ -5873,7 +5827,7 @@ checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d"
 dependencies = [
  "anyhow",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -5892,7 +5846,7 @@ dependencies = [
  "bytes",
  "form_urlencoded",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "quick-xml 0.40.1",
@@ -5914,7 +5868,7 @@ dependencies = [
  "base64",
  "bytes",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "pem",
  "percent-encoding",
@@ -5938,7 +5892,7 @@ dependencies = [
  "futures",
  "hex",
  "hmac 0.13.0",
- "http 1.4.1",
+ "http 1.4.2",
  "jiff",
  "log",
  "percent-encoding",
@@ -5968,7 +5922,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2"
 dependencies = [
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-aws-v4",
@@ -5986,7 +5940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb"
 dependencies = [
  "anyhow",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -5994,6 +5948,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "reqsign-volcengine-tos"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3"
+dependencies = [
+ "anyhow",
+ "http 1.4.2",
+ "log",
+ "percent-encoding",
+ "reqsign-core",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.12.28"
@@ -6006,7 +5973,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -6037,6 +6004,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams 0.4.2",
  "web-sys",
+ "webpki-roots",
 ]
 
 [[package]]
@@ -6049,7 +6017,7 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -6087,7 +6055,7 @@ checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58"
 dependencies = [
  "anyhow",
  "async-trait",
- "http 1.4.1",
+ "http 1.4.2",
  "reqwest 0.13.4",
  "thiserror 2.0.18",
  "tower-service",
@@ -6177,9 +6145,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -6232,9 +6200,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
 dependencies = [
  "web-time",
  "zeroize",
@@ -6369,7 +6337,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6461,7 +6429,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6472,7 +6440,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6481,6 +6449,7 @@ version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -6507,7 +6476,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6519,7 +6488,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6536,9 +6505,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"
+checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
 dependencies = [
  "base64",
  "bs58",
@@ -6556,14 +6525,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac"
+checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6704,9 +6673,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "smallvec"
-version = "1.15.1"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
@@ -6726,15 +6695,9 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
-[[package]]
-name = "snap"
-version = "1.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
-
 [[package]]
 name = "socket2"
 version = "0.6.4"
@@ -6775,9 +6738,9 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
  "sqlparser_derive",
@@ -6791,7 +6754,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6828,6 +6791,15 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
 
+[[package]]
+name = "stop-words"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d68df56303396bcfb639455b3c166804aeb7994005010aab5e9e8a1277b8871d"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -6853,16 +6825,17 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -6876,7 +6849,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.117",
+ "syn",
  "typify",
  "walkdir",
 ]
@@ -6895,20 +6868,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
+version = "2.0.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.117"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6932,7 +6894,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6989,7 +6951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -7021,7 +6983,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7032,7 +6994,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7053,17 +7015,6 @@ dependencies = [
  "cfg-if 1.0.4",
 ]
 
-[[package]]
-name = "thrift"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
-dependencies = [
- "byteorder",
- "integer-encoding",
- "ordered-float 2.10.1",
-]
-
 [[package]]
 name = "time"
 version = "0.3.47"
@@ -7155,14 +7106,14 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "tokio-retry"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640"
+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"
 dependencies = [
  "pin-project-lite",
  "rand 0.10.1",
@@ -7234,6 +7185,45 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
+dependencies = [
+ "async-trait",
+ "base64",
+ "bytes",
+ "h2",
+ "http 1.4.2",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "socket2",
+ "sync_wrapper",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tonic-prost"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
+dependencies = [
+ "bytes",
+ "prost",
+ "tonic",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.3"
@@ -7242,9 +7232,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap 2.14.0",
  "pin-project-lite",
+ "slab",
  "sync_wrapper",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -7258,7 +7251,7 @@ checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
  "bitflags",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -7278,7 +7271,7 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -7335,7 +7328,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7442,7 +7435,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.117",
+ "syn",
  "thiserror 2.0.18",
  "unicode-ident",
 ]
@@ -7460,7 +7453,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.117",
+ "syn",
  "typify-impl",
 ]
 
@@ -7497,12 +7490,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.11"
@@ -7553,11 +7540,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.2"
+version = "1.23.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
+checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
 dependencies = [
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "js-sys",
  "serde_core",
  "wasm-bindgen",
@@ -7617,20 +7604,11 @@ dependencies = [
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
- "wit-bindgen 0.57.1",
-]
-
-[[package]]
-name = "wasip3"
-version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
-dependencies = [
- "wit-bindgen 0.51.0",
+ "wit-bindgen",
 ]
 
 [[package]]
@@ -7644,9 +7622,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
 dependencies = [
  "cfg-if 1.0.4",
  "once_cell",
@@ -7657,9 +7635,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.72"
+version = "0.4.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -7667,9 +7645,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -7677,48 +7655,26 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
 dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "wasm-encoder"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
-dependencies = [
- "leb128fmt",
- "wasmparser",
-]
-
-[[package]]
-name = "wasm-metadata"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
-dependencies = [
- "anyhow",
- "indexmap 2.14.0",
- "wasm-encoder",
- "wasmparser",
-]
-
 [[package]]
 name = "wasm-streams"
 version = "0.4.2"
@@ -7745,23 +7701,11 @@ dependencies = [
  "web-sys",
 ]
 
-[[package]]
-name = "wasmparser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
-dependencies = [
- "bitflags",
- "hashbrown 0.15.5",
- "indexmap 2.14.0",
- "semver",
-]
-
 [[package]]
 name = "web-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -7779,9 +7723,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.7"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -7883,7 +7836,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7894,7 +7847,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -8182,100 +8135,12 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
-dependencies = [
- "wit-bindgen-rust-macro",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.57.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
-[[package]]
-name = "wit-bindgen-core"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
-dependencies = [
- "anyhow",
- "heck",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-bindgen-rust"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
-dependencies = [
- "anyhow",
- "heck",
- "indexmap 2.14.0",
- "prettyplease",
- "syn 2.0.117",
- "wasm-metadata",
- "wit-bindgen-core",
- "wit-component",
-]
-
-[[package]]
-name = "wit-bindgen-rust-macro"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
-dependencies = [
- "anyhow",
- "prettyplease",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "wit-bindgen-core",
- "wit-bindgen-rust",
-]
-
-[[package]]
-name = "wit-component"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
-dependencies = [
- "anyhow",
- "bitflags",
- "indexmap 2.14.0",
- "log",
- "serde",
- "serde_derive",
- "serde_json",
- "wasm-encoder",
- "wasm-metadata",
- "wasmparser",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-parser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
-dependencies = [
- "anyhow",
- "id-arena",
- "indexmap 2.14.0",
- "log",
- "semver",
- "serde",
- "serde_derive",
- "serde_json",
- "unicode-xid",
- "wasmparser",
-]
-
 [[package]]
 name = "wkb"
 version = "0.9.2"
@@ -8329,7 +8194,7 @@ dependencies = [
  "clap",
  "crc32fast",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "lazy_static",
  "more-asserts",
@@ -8370,7 +8235,7 @@ dependencies = [
  "csv",
  "futures",
  "futures-util",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "heapify",
  "itertools 0.14.0",
  "lazy_static",
@@ -8403,7 +8268,7 @@ dependencies = [
  "chrono",
  "clap",
  "gearhash",
- "http 1.4.1",
+ "http 1.4.2",
  "itertools 0.14.0",
  "lazy_static",
  "more-asserts",
@@ -8471,15 +8336,15 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.15"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -8494,28 +8359,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "synstructure",
 ]
 
 [[package]]
 name = "zerocopy"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -8535,15 +8400,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "synstructure",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"
@@ -8577,15 +8442,9 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
-[[package]]
-name = "zlib-rs"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
-
 [[package]]
 name = "zmij"
 version = "1.0.21"
diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml
index e1478b27aeb..b10fe07beb8 100644
--- a/java/lance-jni/Cargo.toml
+++ b/java/lance-jni/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lance-jni"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 rust-version = "1.91"
@@ -9,11 +9,18 @@ repository = "https://github.com/lance-format/lance"
 readme = "../../README.md"
 description = "JNI bindings for Lance Columnar format"
 
+# `java/lance-jni/` is excluded from the top-level workspace (see ../../Cargo.toml) and
+# keeps its own Cargo.lock. Declaring an empty workspace here makes this crate an explicit
+# workspace root, so cargo resolves it standalone even when the checkout is nested inside
+# another cargo project (e.g. a git worktree placed under the main checkout).
+[workspace]
+
 [lib]
 crate-type = ["cdylib"]
 
 [features]
 default = []
+backtrace = ["lance/backtrace", "lance-core/backtrace"]
 
 [dependencies]
 lance = { path = "../../rust/lance", features = ["substrait"] }
@@ -23,15 +30,15 @@ lance-linalg = { path = "../../rust/lance-linalg" }
 lance-index = { path = "../../rust/lance-index" }
 lance-io = { path = "../../rust/lance-io" }
 lance-namespace = { path = "../../rust/lance-namespace" }
-lance-namespace-impls = { path = "../../rust/lance-namespace-impls", features = ["rest", "rest-adapter"] }
+lance-namespace-impls = { path = "../../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] }
 lance-core = { path = "../../rust/lance-core" }
 lance-file = { path = "../../rust/lance-file" }
 lance-table = { path = "../../rust/lance-table" }
 arrow = { version = "58.0.0", features = ["ffi"] }
 arrow-array = "58.0.0"
 arrow-schema = "58.0.0"
-datafusion = { version = "53.0.0", default-features = false }
-datafusion-common = "53.0.0"
+datafusion = { version = "54.0.0", default-features = false }
+datafusion-common = "54.0.0"
 object_store = { version = "0.13.2" }
 tokio = { version = "1.23", features = [
     "rt-multi-thread",
diff --git a/java/lance-jni/src/async_scanner.rs b/java/lance-jni/src/async_scanner.rs
index 7cb71c37086..6da10479266 100644
--- a/java/lance-jni/src/async_scanner.rs
+++ b/java/lance-jni/src/async_scanner.rs
@@ -193,6 +193,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local
     use_scalar_index: jboolean,
     fast_search: jboolean,
     substrait_aggregate_obj: JObject<'local>,
+    include_deleted_rows: jboolean,
+    strict_batch_size: jboolean,
+    disable_scoring_autoprojection: jboolean,
 ) -> JObject<'local> {
     crate::ok_or_throw!(
         env,
@@ -216,6 +219,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local
             use_scalar_index,
             fast_search,
             substrait_aggregate_obj,
+            include_deleted_rows,
+            strict_batch_size,
+            disable_scoring_autoprojection,
         )
     )
 }
@@ -241,6 +247,9 @@ fn inner_create_async_scanner<'local>(
     use_scalar_index: jboolean,
     fast_search: jboolean,
     substrait_aggregate_obj: JObject<'local>,
+    include_deleted_rows: jboolean,
+    strict_batch_size: jboolean,
+    disable_scoring_autoprojection: jboolean,
 ) -> Result<JObject<'local>> {
     let dataset_guard =
         unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
@@ -265,6 +274,9 @@ fn inner_create_async_scanner<'local>(
         use_scalar_index,
         fast_search,
         substrait_aggregate_obj,
+        include_deleted_rows,
+        strict_batch_size,
+        disable_scoring_autoprojection,
     };
 
     let scanner = build_scanner_with_options(env, &dataset, options)?;
diff --git a/java/lance-jni/src/blocking_dataset.rs b/java/lance-jni/src/blocking_dataset.rs
index cd43c69d61a..caf837b371a 100644
--- a/java/lance-jni/src/blocking_dataset.rs
+++ b/java/lance-jni/src/blocking_dataset.rs
@@ -29,7 +29,10 @@ use jni::sys::{jboolean, jint};
 use jni::sys::{jbyteArray, jlong};
 use jni::{JNIEnv, objects::JObject};
 use lance::dataset::builder::DatasetBuilder;
-use lance::dataset::cleanup::{CleanupPolicy, RemovalStats};
+use lance::dataset::cleanup::{
+    CleanupCandidateFile, CleanupExplanation, CleanupFileKind, CleanupPolicy,
+    CleanupReferencedBranch, RemovalStats,
+};
 use lance::dataset::optimize::{CompactionOptions as RustCompactionOptions, compact_files};
 use lance::dataset::refs::{Ref, TagContents};
 use lance::dataset::statistics::{DataStatistics, DatasetStatisticsExt};
@@ -61,6 +64,7 @@ use std::iter::empty;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::{Duration, UNIX_EPOCH};
+use uuid::Uuid;
 
 pub const NATIVE_DATASET: &str = "nativeDatasetHandle";
 
@@ -200,7 +204,8 @@ impl BlockingDataset {
         if namespace_client_managed_versioning
             && let (Some(namespace_client), Some(tid)) = (namespace, table_id)
         {
-            let external_store = LanceNamespaceExternalManifestStore::new(namespace_client, tid);
+            let external_store =
+                LanceNamespaceExternalManifestStore::for_table_uri(namespace_client, tid, uri)?;
             let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
                 external_manifest_store: Arc::new(external_store),
             });
@@ -412,6 +417,18 @@ impl BlockingDataset {
         Ok(RT.block_on(self.inner.cleanup_with_policy(policy))?)
     }
 
+    pub fn explain_cleanup_with_policy(
+        &self,
+        policy: CleanupPolicy,
+        max_candidate_files: Option<usize>,
+    ) -> Result<CleanupExplanation> {
+        let mut op = self.inner.cleanup(policy);
+        if let Some(limit) = max_candidate_files {
+            op = op.with_max_candidate_files(limit);
+        }
+        Ok(RT.block_on(op.explain())?)
+    }
+
     pub fn close(&self) {}
 }
 
@@ -688,8 +705,11 @@ fn create_dataset<'local>(
     if let Some((namespace, table_id)) = namespace_info {
         // Set up commit handler only if namespace manages versioning
         if namespace_client_managed_versioning {
-            let external_store =
-                LanceNamespaceExternalManifestStore::new(namespace.clone(), table_id.clone());
+            let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
+                namespace.clone(),
+                table_id.clone(),
+                &path_str,
+            )?;
             let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
                 external_manifest_store: Arc::new(external_store),
             });
@@ -951,7 +971,13 @@ fn inner_create_index<'local>(
     let fragment_ids = env
         .get_ints_opt(&fragments_jobj)?
         .map(|vec| vec.into_iter().map(|i| i as u32).collect());
-    let index_uuid = env.get_string_opt(&index_uuid_jobj)?;
+    let index_uuid = env
+        .get_string_opt(&index_uuid_jobj)?
+        .map(|s| {
+            Uuid::parse_str(&s)
+                .map_err(|e| Error::input_error(format!("Invalid UUID string for index_uuid: {e}")))
+        })
+        .transpose()?;
     let arrow_stream_addr_opt = env.get_long_opt(&arrow_stream_addr_jobj)?;
     let batch_reader = if let Some(arrow_stream_addr) = arrow_stream_addr_opt {
         let stream_ptr = arrow_stream_addr as *mut FFI_ArrowArrayStream;
@@ -974,6 +1000,7 @@ fn inner_create_index<'local>(
         | IndexType::NGram
         | IndexType::ZoneMap
         | IndexType::BloomFilter
+        | IndexType::Fm
         | IndexType::RTree => {
             // For scalar indices, create a scalar IndexParams
             let (index_type_str, params_opt) = get_scalar_index_params(env, params_jobj)?;
@@ -1088,7 +1115,9 @@ fn inner_merge_index_metadata(
     index_type_code_jobj: jint,
     batch_readhead_jobj: JObject, // Optional<Integer>
 ) -> Result<()> {
-    let index_uuid = index_uuid.extract(env)?;
+    let index_uuid_str = index_uuid.extract(env)?;
+    let index_uuid = Uuid::parse_str(&index_uuid_str)
+        .map_err(|e| Error::input_error(format!("Invalid UUID string for index_uuid: {e}")))?;
     let index_type = IndexType::try_from(index_type_code_jobj)?;
     let batch_readhead = env
         .get_int_opt(&batch_readhead_jobj)?
@@ -3049,51 +3078,104 @@ fn inner_cleanup_with_policy<'local>(
     jdataset: JObject,
     jpolicy: JObject,
 ) -> Result<JObject<'local>> {
-    let before_ts_millis =
-        env.get_optional_u64_from_method(&jpolicy, "getBeforeTimestampMillis")?;
+    let policy = extract_cleanup_policy(env, &jpolicy)?;
+
+    let stats = {
+        let mut dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
+        dataset.cleanup_with_policy(policy)
+    }?;
+
+    cleanup_stats_to_java(env, stats)
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_Dataset_nativeExplainCleanupWithPolicy<'local>(
+    mut env: JNIEnv<'local>,
+    jdataset: JObject,
+    jpolicy: JObject,
+    jmax_candidate_files: JObject,
+) -> JObject<'local> {
+    ok_or_throw!(
+        env,
+        inner_explain_cleanup_with_policy(&mut env, jdataset, jpolicy, jmax_candidate_files)
+    )
+}
+
+fn inner_explain_cleanup_with_policy<'local>(
+    env: &mut JNIEnv<'local>,
+    jdataset: JObject,
+    jpolicy: JObject,
+    jmax_candidate_files: JObject,
+) -> Result<JObject<'local>> {
+    let policy = extract_cleanup_policy(env, &jpolicy)?;
+    let max_candidate_files = env
+        .get_optional(&jmax_candidate_files, |env, inner| {
+            Ok(env.call_method(inner, "longValue", "()J", &[])?.j()?)
+        })?
+        .map(|v| {
+            usize::try_from(v).map_err(|e| {
+                Error::input_error(format!(
+                    "maxCandidateFiles must be a non-negative usize value, got {}: {:?}",
+                    v, e
+                ))
+            })
+        })
+        .transpose()?;
+
+    let explanation = {
+        let dataset =
+            unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
+        dataset.explain_cleanup_with_policy(policy, max_candidate_files)
+    }?;
+
+    cleanup_explanation_to_java(env, explanation)
+}
+
+fn extract_cleanup_policy(env: &mut JNIEnv<'_>, jpolicy: &JObject) -> Result<CleanupPolicy> {
+    let before_ts_millis = env.get_optional_u64_from_method(jpolicy, "getBeforeTimestampMillis")?;
     let before_timestamp = before_ts_millis.map(|millis| {
         let st = UNIX_EPOCH + Duration::from_millis(millis);
         DateTime::<Utc>::from(st)
     });
 
-    let before_version = env.get_optional_u64_from_method(&jpolicy, "getBeforeVersion")?;
+    let before_version = env.get_optional_u64_from_method(jpolicy, "getBeforeVersion")?;
 
     let delete_unverified = env
-        .get_optional_from_method(&jpolicy, "getDeleteUnverified", |env, obj| {
+        .get_optional_from_method(jpolicy, "getDeleteUnverified", |env, obj| {
             Ok(env.call_method(obj, "booleanValue", "()Z", &[])?.z()?)
         })?
         .unwrap_or(false);
 
     let error_if_tagged_old_versions = env
-        .get_optional_from_method(&jpolicy, "getErrorIfTaggedOldVersions", |env, obj| {
+        .get_optional_from_method(jpolicy, "getErrorIfTaggedOldVersions", |env, obj| {
             Ok(env.call_method(obj, "booleanValue", "()Z", &[])?.z()?)
         })?
         .unwrap_or(true);
 
     let clean_referenced_branches = env
-        .get_optional_from_method(&jpolicy, "getCleanReferencedBranches", |env, obj| {
+        .get_optional_from_method(jpolicy, "getCleanReferencedBranches", |env, obj| {
             Ok(env.call_method(obj, "booleanValue", "()Z", &[])?.z()?)
         })?
         .unwrap_or(false);
 
-    let delete_rate_limit = env.get_optional_u64_from_method(&jpolicy, "getDeleteRateLimit")?;
+    let delete_rate_limit = env.get_optional_u64_from_method(jpolicy, "getDeleteRateLimit")?;
 
-    let policy = CleanupPolicy {
+    Ok(CleanupPolicy {
         before_timestamp,
         before_version,
         delete_unverified,
         error_if_tagged_old_versions,
         clean_referenced_branches,
         delete_rate_limit,
-    };
-
-    let stats = {
-        let mut dataset =
-            unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
-        dataset.cleanup_with_policy(policy)
-    }?;
+    })
+}
 
-    let jstats = env.new_object(
+fn cleanup_stats_to_java<'local>(
+    env: &mut JNIEnv<'local>,
+    stats: RemovalStats,
+) -> Result<JObject<'local>> {
+    Ok(env.new_object(
         "org/lance/cleanup/RemovalStats",
         "(JJJJJJ)V",
         &[
@@ -3104,9 +3186,126 @@ fn inner_cleanup_with_policy<'local>(
             JValue::Long(stats.index_files_removed as i64),
             JValue::Long(stats.deletion_files_removed as i64),
         ],
-    )?;
+    )?)
+}
 
-    Ok(jstats)
+fn cleanup_file_kind_to_java(kind: CleanupFileKind) -> &'static str {
+    match kind {
+        CleanupFileKind::Manifest => "manifest",
+        CleanupFileKind::Data => "data",
+        CleanupFileKind::Transaction => "transaction",
+        CleanupFileKind::Index => "index",
+        CleanupFileKind::Deletion => "deletion",
+        CleanupFileKind::TemporaryManifest => "temporary_manifest",
+    }
+}
+
+fn cleanup_candidate_files_to_java<'local>(
+    env: &mut JNIEnv<'local>,
+    files: Vec<CleanupCandidateFile>,
+) -> Result<JObject<'local>> {
+    let list = env.new_object("java/util/ArrayList", "()V", &[])?;
+    // Wrap each iteration in a local frame so the temporary path/kind/candidate
+    // references do not accumulate on the JNI local reference table for large
+    // explanations (default limit is 1000 candidate files, but users can raise it).
+    for file in files {
+        env.with_local_frame(8, |env| {
+            let path = env.new_string(file.path)?;
+            let kind = env.new_string(cleanup_file_kind_to_java(file.kind))?;
+            let candidate = env.new_object(
+                "org/lance/cleanup/CleanupCandidateFile",
+                "(Ljava/lang/String;Ljava/lang/String;ZJ)V",
+                &[
+                    JValue::Object(&path),
+                    JValue::Object(&kind),
+                    JValue::Bool(file.unverified as jboolean),
+                    JValue::Long(file.size_bytes as i64),
+                ],
+            )?;
+            env.call_method(
+                &list,
+                "add",
+                "(Ljava/lang/Object;)Z",
+                &[JValue::Object(&candidate)],
+            )?;
+            Ok::<(), Error>(())
+        })?;
+    }
+    Ok(list)
+}
+
+fn cleanup_referenced_branches_to_java<'local>(
+    env: &mut JNIEnv<'local>,
+    branches: Vec<CleanupReferencedBranch>,
+) -> Result<JObject<'local>> {
+    let list = env.new_object("java/util/ArrayList", "()V", &[])?;
+    for branch in branches {
+        env.with_local_frame(8, |env| {
+            let name = env.new_string(branch.name)?;
+            let referenced_branch = env.new_object(
+                "org/lance/cleanup/CleanupReferencedBranch",
+                "(Ljava/lang/String;JZ)V",
+                &[
+                    JValue::Object(&name),
+                    JValue::Long(branch.referenced_version as i64),
+                    JValue::Bool(branch.cleanup_candidate as jboolean),
+                ],
+            )?;
+            env.call_method(
+                &list,
+                "add",
+                "(Ljava/lang/Object;)Z",
+                &[JValue::Object(&referenced_branch)],
+            )?;
+            Ok::<(), Error>(())
+        })?;
+    }
+    Ok(list)
+}
+
+fn cleanup_warnings_to_java<'local>(
+    env: &mut JNIEnv<'local>,
+    warnings: Vec<String>,
+) -> Result<JObject<'local>> {
+    let list = env.new_object("java/util/ArrayList", "()V", &[])?;
+    for warning in warnings {
+        env.with_local_frame(4, |env| {
+            let warning = env.new_string(warning)?;
+            env.call_method(
+                &list,
+                "add",
+                "(Ljava/lang/Object;)Z",
+                &[JValue::Object(&warning)],
+            )?;
+            Ok::<(), Error>(())
+        })?;
+    }
+    Ok(list)
+}
+
+fn cleanup_explanation_to_java<'local>(
+    env: &mut JNIEnv<'local>,
+    explanation: CleanupExplanation,
+) -> Result<JObject<'local>> {
+    let stats = cleanup_stats_to_java(env, explanation.stats)?;
+    let candidate_files = cleanup_candidate_files_to_java(env, explanation.candidate_files)?;
+    let referenced_branches =
+        cleanup_referenced_branches_to_java(env, explanation.referenced_branches)?;
+    let warnings = cleanup_warnings_to_java(env, explanation.warnings)?;
+
+    Ok(env.new_object(
+        "org/lance/cleanup/CleanupExplanation",
+        "(JLorg/lance/cleanup/RemovalStats;Ljava/util/List;ZJLjava/util/List;Ljava/util/List;)V",
+        &[
+            JValue::Long(explanation.read_version as i64),
+            JValue::Object(&stats),
+            JValue::Object(&candidate_files),
+            JValue::Bool(explanation.candidate_files_truncated as jboolean),
+            JValue::Long(explanation.candidate_file_limit as i64),
+            JValue::Object(&referenced_branches),
+            JValue::Object(&warnings),
+        ],
+    )?)
 }
 
 //////////////////////////////
diff --git a/java/lance-jni/src/blocking_scanner.rs b/java/lance-jni/src/blocking_scanner.rs
index f18b0d92a27..335cb2a4fa3 100644
--- a/java/lance-jni/src/blocking_scanner.rs
+++ b/java/lance-jni/src/blocking_scanner.rs
@@ -30,6 +30,7 @@ use crate::{
     RT,
     blocking_dataset::{BlockingDataset, NATIVE_DATASET},
     traits::IntoJava,
+    utils::parse_approx_mode,
 };
 
 pub const NATIVE_SCANNER: &str = "nativeScannerHandle";
@@ -247,6 +248,9 @@ pub(crate) struct ScannerOptions<'a> {
     pub use_scalar_index: jboolean,
     pub fast_search: jboolean,
     pub substrait_aggregate_obj: JObject<'a>,
+    pub include_deleted_rows: jboolean,
+    pub strict_batch_size: jboolean,
+    pub disable_scoring_autoprojection: jboolean,
 }
 
 /// Build a scanner with options applied - shared by blocking and async scanners
@@ -354,6 +358,9 @@ pub(crate) fn build_scanner_with_options<'a>(
             .call_method(&java_obj, "getQueryParallelism", "()I", &[])?
             .i()?;
         scanner.query_parallelism(query_parallelism);
+
+        let approx_mode_str = env.get_string_from_method(&java_obj, "getApproxModeString")?;
+        scanner.approx_mode(parse_approx_mode(&approx_mode_str)?);
         Ok(())
     })?;
 
@@ -394,6 +401,16 @@ pub(crate) fn build_scanner_with_options<'a>(
         scanner.aggregate(AggregateExpr::substrait(substrait_aggregate))?;
     }
 
+    if options.include_deleted_rows == JNI_TRUE {
+        scanner.include_deleted_rows();
+    }
+
+    scanner.strict_batch_size(options.strict_batch_size == JNI_TRUE);
+
+    if options.disable_scoring_autoprojection == JNI_TRUE {
+        scanner.disable_scoring_autoprojection();
+    }
+
     Ok(scanner)
 }
 
@@ -423,6 +440,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>(
     fast_search: jboolean,             // boolean
     substrait_aggregate_obj: JObject<'local>, // Optional<ByteBuffer>
     collect_stats: jboolean,           // boolean
+    include_deleted_rows: jboolean,    // boolean
+    strict_batch_size: jboolean,       // boolean
+    disable_scoring_autoprojection: jboolean, // boolean
 ) -> JObject<'local> {
     ok_or_throw!(
         env,
@@ -447,6 +467,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>(
             fast_search,
             substrait_aggregate_obj,
             collect_stats,
+            include_deleted_rows,
+            strict_batch_size,
+            disable_scoring_autoprojection,
         )
     )
 }
@@ -473,6 +496,9 @@ fn inner_create_scanner<'local>(
     fast_search: jboolean,
     substrait_aggregate_obj: JObject<'local>,
     collect_stats: jboolean,
+    include_deleted_rows: jboolean,
+    strict_batch_size: jboolean,
+    disable_scoring_autoprojection: jboolean,
 ) -> Result<JObject<'local>> {
     let dataset_guard =
         unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?;
@@ -497,6 +523,9 @@ fn inner_create_scanner<'local>(
         use_scalar_index,
         fast_search,
         substrait_aggregate_obj,
+        include_deleted_rows,
+        strict_batch_size,
+        disable_scoring_autoprojection,
     };
 
     let scanner = build_scanner_with_options(env, &dataset, options)?;
diff --git a/java/lance-jni/src/error.rs b/java/lance-jni/src/error.rs
index cdb922a3cef..0affdca8f98 100644
--- a/java/lance-jni/src/error.rs
+++ b/java/lance-jni/src/error.rs
@@ -181,29 +181,36 @@ impl std::fmt::Display for Error {
 
 impl From<LanceError> for Error {
     fn from(err: LanceError) -> Self {
+        let backtrace_suffix = err
+            .backtrace()
+            .map(|bt| format!("\n\nRust backtrace:\n{}", bt))
+            .unwrap_or_default();
+        let message = format!("{}{}", err, backtrace_suffix);
+
         match &err {
             LanceError::DatasetNotFound { .. }
             | LanceError::DatasetAlreadyExists { .. }
             | LanceError::CommitConflict { .. }
-            | LanceError::InvalidInput { .. } => Self::input_error(err.to_string()),
-            LanceError::IO { .. } => Self::io_error(err.to_string()),
-            LanceError::Timeout { .. } => Self::timeout_error(err.to_string()),
-            LanceError::NotSupported { .. } => Self::unsupported_error(err.to_string()),
-            LanceError::NotFound { .. } => Self::io_error(err.to_string()),
+            | LanceError::InvalidInput { .. } => Self::input_error(message),
+            LanceError::IO { .. } => Self::io_error(message),
+            LanceError::Timeout { .. } => Self::timeout_error(message),
+            LanceError::NotSupported { .. } => Self::unsupported_error(message),
+            LanceError::NotFound { .. } => Self::io_error(message),
             LanceError::Namespace { source, .. } => {
                 // Try to downcast to NamespaceError and get the error code
                 if let Some(ns_err) = source.downcast_ref::<NamespaceError>() {
-                    Self::namespace_error(ns_err.code().as_u32(), ns_err.to_string())
+                    let ns_message = format!("{}{}", ns_err, backtrace_suffix);
+                    Self::namespace_error(ns_err.code().as_u32(), ns_message)
                 } else {
                     log::warn!(
                         "Failed to downcast NamespaceError source, falling back to runtime error. \
                          This may indicate a version mismatch. Source type: {:?}",
                         source
                     );
-                    Self::runtime_error(err.to_string())
+                    Self::runtime_error(message)
                 }
             }
-            _ => Self::runtime_error(err.to_string()),
+            _ => Self::runtime_error(message),
         }
     }
 }
@@ -241,3 +248,104 @@ impl From<Utf8Error> for Error {
         Self::input_error(err.to_string())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // Helper: extract the java_class from an Error via Display output
+    fn java_class(err: &Error) -> &JavaExceptionClass {
+        &err.java_class
+    }
+
+    #[test]
+    fn test_invalid_input_maps_to_illegal_argument() {
+        let lance_err = LanceError::invalid_input("bad input");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("bad input"));
+    }
+
+    #[test]
+    fn test_dataset_not_found_maps_to_illegal_argument() {
+        let lance_err = LanceError::dataset_not_found("my_dataset", "not found".to_string().into());
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("my_dataset"));
+    }
+
+    #[test]
+    fn test_dataset_already_exists_maps_to_illegal_argument() {
+        let lance_err = LanceError::dataset_already_exists("my_dataset");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+        assert!(jni_err.message.contains("my_dataset"));
+    }
+
+    #[test]
+    fn test_commit_conflict_maps_to_illegal_argument() {
+        let lance_err = LanceError::commit_conflict_source(42, "conflict".to_string().into());
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::IllegalArgumentException
+        );
+    }
+
+    #[test]
+    fn test_io_maps_to_ioexception() {
+        let lance_err = LanceError::io("disk failure");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException);
+        assert!(jni_err.message.contains("disk failure"));
+    }
+
+    #[test]
+    fn test_not_supported_maps_to_unsupported() {
+        let lance_err = LanceError::not_supported("nope");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(
+            *java_class(&jni_err),
+            JavaExceptionClass::UnsupportedOperationException
+        );
+        assert!(jni_err.message.contains("nope"));
+    }
+
+    #[test]
+    fn test_not_found_maps_to_ioexception() {
+        let lance_err = LanceError::not_found("missing_uri");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException);
+        assert!(jni_err.message.contains("missing_uri"));
+    }
+
+    #[test]
+    fn test_fallthrough_maps_to_runtime() {
+        let lance_err = LanceError::internal("internal oops");
+        let jni_err: Error = lance_err.into();
+        assert_eq!(*java_class(&jni_err), JavaExceptionClass::RuntimeException);
+        assert!(jni_err.message.contains("internal oops"));
+    }
+
+    #[test]
+    fn test_no_backtrace_suffix_when_backtrace_is_none() {
+        // Without the backtrace feature enabled in lance-core default tests,
+        // backtrace() returns None, so no suffix should be appended.
+        let lance_err = LanceError::io("clean message");
+        let jni_err: Error = lance_err.into();
+        assert!(
+            !jni_err.message.contains("Rust backtrace:"),
+            "Expected no backtrace suffix, got: {}",
+            jni_err.message
+        );
+    }
+}
diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs
index a6798c2f237..59ce5e553ff 100644
--- a/java/lance-jni/src/fragment.rs
+++ b/java/lance-jni/src/fragment.rs
@@ -4,7 +4,7 @@
 use arrow::array::{RecordBatch, RecordBatchIterator, StructArray};
 use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi_and_data_type};
 use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
-use arrow_schema::DataType;
+use arrow_schema::{DataType, Schema as ArrowSchema};
 use jni::objects::{JIntArray, JValue, JValueGen};
 use jni::{
     JNIEnv,
@@ -19,7 +19,7 @@ use lance_io::utils::CachedFileSize;
 use lance_table::rowids::{RowIdSequence, write_row_ids};
 use std::iter::once;
 
-use lance::dataset::fragment::FileFragment;
+use lance::dataset::fragment::write::FragmentCreateBuilder;
 use lance::io::ObjectStoreParams;
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
@@ -108,6 +108,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>(
     table_id_obj: JObject,                      // List<String> (can be null)
     allow_external_blob_outside_bases: JObject, // Optional<Boolean>
     blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    schema_addr: jlong,
 ) -> JObject<'local> {
     ok_or_throw_with_return!(
         env,
@@ -130,6 +131,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>(
             table_id_obj,
             allow_external_blob_outside_bases,
             blob_pack_file_size_threshold,
+            schema_addr,
         ),
         JObject::default()
     )
@@ -155,6 +157,7 @@ fn inner_create_with_ffi_array<'local>(
     table_id_obj: JObject,                      // List<String> (can be null)
     allow_external_blob_outside_bases: JObject, // Optional<Boolean>
     blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    schema_addr: jlong,
 ) -> Result<JObject<'local>> {
     let c_array_ptr = arrow_array_addr as *mut FFI_ArrowArray;
     let c_schema_ptr = arrow_schema_addr as *mut FFI_ArrowSchema;
@@ -186,6 +189,7 @@ fn inner_create_with_ffi_array<'local>(
         table_id_obj,
         allow_external_blob_outside_bases,
         blob_pack_file_size_threshold,
+        schema_addr,
         reader,
     )
 }
@@ -210,6 +214,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>(
     table_id_obj: JObject,                      // List<String> (can be null)
     allow_external_blob_outside_bases: JObject, // Optional<Boolean>
     blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    schema_addr: jlong,
 ) -> JObject<'a> {
     ok_or_throw_with_return!(
         env,
@@ -231,6 +236,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>(
             table_id_obj,
             allow_external_blob_outside_bases,
             blob_pack_file_size_threshold,
+            schema_addr,
         ),
         JObject::null()
     )
@@ -255,6 +261,7 @@ fn inner_create_with_ffi_stream<'local>(
     table_id_obj: JObject,                      // List<String> (can be null)
     allow_external_blob_outside_bases: JObject, // Optional<Boolean>
     blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    schema_addr: jlong,
 ) -> Result<JObject<'local>> {
     let stream_ptr = arrow_array_stream_addr as *mut FFI_ArrowArrayStream;
     let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
@@ -276,6 +283,7 @@ fn inner_create_with_ffi_stream<'local>(
         table_id_obj,
         allow_external_blob_outside_bases,
         blob_pack_file_size_threshold,
+        schema_addr,
         reader,
     )
 }
@@ -298,6 +306,7 @@ fn create_fragment<'a>(
     table_id_obj: JObject,                      // List<String> (can be null)
     allow_external_blob_outside_bases: JObject, // Optional<Boolean>
     blob_pack_file_size_threshold: JObject,     // Optional<Long>
+    schema_addr: jlong,
     source: impl StreamingWriteSource,
 ) -> Result<JObject<'a>> {
     let path_str = dataset_uri.extract(env)?;
@@ -345,11 +354,19 @@ fn create_fragment<'a>(
         });
     }
 
-    let fragments = RT.block_on(FileFragment::create_fragments(
-        &path_str,
-        source,
-        Some(write_params),
-    ))?;
+    let mut builder = FragmentCreateBuilder::new(&path_str).write_params(&write_params);
+    let schema;
+    if schema_addr != 0 {
+        let c_schema_ptr = schema_addr as *mut FFI_ArrowSchema;
+        let c_schema = unsafe { FFI_ArrowSchema::from_raw(c_schema_ptr) };
+        let arrow_schema = ArrowSchema::try_from(&c_schema)?;
+        // Schema::try_from restores Lance field IDs from the LANCE_FIELD_ID_KEY
+        // metadata inserted by LanceSchema.asArrowSchemaWithFieldIds().
+        schema = Schema::try_from(&arrow_schema)?;
+        builder = builder.schema(&schema);
+    }
+
+    let fragments = RT.block_on(builder.write_fragments(source))?;
     export_vec(env, &fragments)
 }
 
@@ -811,6 +828,9 @@ impl FromJObjectWithEnv<Fragment> for JObject<'_> {
             row_id_meta,
             created_at_version_meta,
             last_updated_at_version_meta,
+            // Overlays are not exposed to Java yet, and the reverse conversion
+            // does not export them, so this round-trip is overlay-free.
+            overlays: vec![],
         })
     }
 }
diff --git a/java/lance-jni/src/index.rs b/java/lance-jni/src/index.rs
index 1e533eed9fc..6cb64a05a81 100644
--- a/java/lance-jni/src/index.rs
+++ b/java/lance-jni/src/index.rs
@@ -173,6 +173,8 @@ fn determine_index_type<'local>(
             Some("ZONEMAP")
         } else if lower.contains("bloomfilter") {
             Some("BLOOM_FILTER")
+        } else if lower.contains("rtree") {
+            Some("RTREE")
         } else if lower.contains("ivfhnsw") {
             if lower.contains("sq") {
                 Some("IVF_HNSW_SQ")
diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs
index 9ba3fdd7440..c4f56d7b97d 100644
--- a/java/lance-jni/src/mem_wal.rs
+++ b/java/lance-jni/src/mem_wal.rs
@@ -27,6 +27,7 @@ use jni::sys::{jdouble, jint, jlong};
 use lance::dataset::Dataset as LanceDataset;
 use lance::dataset::mem_wal::scanner::{
     FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner,
+    parse_filter_expr as parse_lsm_filter_expr, write_pk_sidecar,
 };
 use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot};
 use lance::dataset::mem_wal::{
@@ -180,6 +181,65 @@ fn inner_put(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()>
     Ok(())
 }
 
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_memwal_ShardWriter_nativeDelete(
+    mut env: JNIEnv,
+    this: JObject,
+    stream_addr: jlong,
+) {
+    ok_or_throw_without_return!(env, inner_delete(&mut env, this, stream_addr));
+}
+
+fn inner_delete(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> {
+    let stream_ptr = stream_addr as *mut FFI_ArrowArrayStream;
+    let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
+    let batches: Vec<RecordBatch> = reader.collect::<std::result::Result<_, _>>()?;
+    if batches.is_empty() {
+        return Ok(());
+    }
+
+    let guard =
+        unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?;
+    RT.block_on(guard.writer.delete(batches))?;
+    Ok(())
+}
+
+/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for a
+/// flushed-generation dataset already staged at `gen_path`, mirroring what
+/// production flush emits. Lets Java tests stage a *faithful* flushed
+/// generation (dataset + sidecar); production always writes the sidecar during
+/// flush, so a dataset-without-sidecar is not a state the system produces.
+/// Mirrors the Python `_write_pk_sidecar` binding.
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_org_lance_memwal_MemWalTest_nativeWritePkSidecar(
+    mut env: JNIEnv,
+    _class: JClass,
+    gen_path: JString,
+    stream_addr: jlong,
+    pk_columns: JObject,
+) {
+    ok_or_throw_without_return!(
+        env,
+        inner_write_pk_sidecar(&mut env, gen_path, stream_addr, pk_columns)
+    );
+}
+
+fn inner_write_pk_sidecar(
+    env: &mut JNIEnv,
+    gen_path: JString,
+    stream_addr: jlong,
+    pk_columns: JObject,
+) -> Result<()> {
+    let gen_path: String = env.get_string(&gen_path)?.into();
+    let pk_columns = env.get_strings(&pk_columns)?;
+    let stream_ptr = stream_addr as *mut FFI_ArrowArrayStream;
+    let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?;
+    let batches: Vec<RecordBatch> = reader.collect::<std::result::Result<_, _>>()?;
+    let pk_refs: Vec<&str> = pk_columns.iter().map(String::as_str).collect();
+    RT.block_on(write_pk_sidecar(&gen_path, &batches, &pk_refs))?;
+    Ok(())
+}
+
 #[unsafe(no_mangle)]
 pub extern "system" fn Java_org_lance_memwal_ShardWriter_nativeStats<'local>(
     mut env: JNIEnv<'local>,
@@ -361,7 +421,7 @@ fn inner_scanner_project(env: &mut JNIEnv, this: JObject, columns: JObject) -> R
     let columns = env.get_strings(&columns)?;
     with_lsm_scanner(env, &this, |scanner| {
         let cols: Vec<&str> = columns.iter().map(String::as_str).collect();
-        Ok(scanner.project(&cols))
+        Ok(scanner.project(&cols)?)
     })
 }
 
@@ -383,7 +443,7 @@ fn inner_scanner_filter(env: &mut JNIEnv, this: JObject, expr: JString) -> Resul
 pub extern "system" fn Java_org_lance_memwal_LsmScanner_nativeLimit(
     mut env: JNIEnv,
     this: JObject,
-    limit: jlong,
+    limit: JObject,
     offset: JObject,
 ) {
     ok_or_throw_without_return!(env, inner_scanner_limit(&mut env, this, limit, offset));
@@ -392,13 +452,12 @@ pub extern "system" fn Java_org_lance_memwal_LsmScanner_nativeLimit(
 fn inner_scanner_limit(
     env: &mut JNIEnv,
     this: JObject,
-    limit: jlong,
+    limit: JObject,
     offset: JObject,
 ) -> Result<()> {
-    let offset = env.get_u64_opt(&offset)?.map(|v| v as usize);
-    with_lsm_scanner(env, &this, |scanner| {
-        Ok(scanner.limit(limit as usize, offset))
-    })
+    let limit = env.get_u64_opt(&limit)?.map(|v| v as i64);
+    let offset = env.get_u64_opt(&offset)?.map(|v| v as i64);
+    with_lsm_scanner(env, &this, |scanner| Ok(scanner.limit(limit, offset)?))
 }
 
 #[unsafe(no_mangle)]
@@ -788,6 +847,7 @@ pub extern "system" fn Java_org_lance_memwal_LsmVectorSearchPlanner_nativeCreate
     vector_column: JString,
     pk_columns: JObject,
     distance_type: JObject,
+    filter: JObject,
 ) {
     ok_or_throw_without_return!(
         env,
@@ -799,6 +859,7 @@ pub extern "system" fn Java_org_lance_memwal_LsmVectorSearchPlanner_nativeCreate
             vector_column,
             pk_columns,
             distance_type,
+            filter,
         )
     );
 }
@@ -812,11 +873,13 @@ fn inner_create_vector_planner(
     vector_column: JString,
     pk_columns: JObject,
     distance_type: JObject,
+    filter: JObject,
 ) -> Result<()> {
     let snapshots = read_shard_snapshots(env, &shard_snapshots)?;
     let vector_column: String = env.get_string(&vector_column)?.into();
     let pk_columns = env.get_strings_opt(&pk_columns)?;
     let distance_type = env.get_string_opt(&distance_type)?;
+    let filter = env.get_string_opt(&filter)?;
     let dataset = {
         let guard =
             unsafe { env.get_rust_field::<_, _, BlockingDataset>(&dataset, NATIVE_DATASET) }?;
@@ -829,9 +892,13 @@ fn inner_create_vector_planner(
     let base_schema = Arc::new(ArrowSchema::from(dataset.schema()));
     let dist_type = parse_distance_type(distance_type.as_deref().unwrap_or("l2"))?;
     let vector_dim = get_vector_dim(&dataset, &vector_column)?;
+    let filter = filter
+        .as_deref()
+        .map(|filter| parse_lsm_filter_expr(base_schema.as_ref(), filter).map_err(Error::from))
+        .transpose()?;
 
     let collector = LsmDataSourceCollector::new(dataset.clone(), snapshots);
-    let planner = LsmVectorSearchPlanner::new(
+    let mut planner = LsmVectorSearchPlanner::new(
         collector,
         pk_columns,
         base_schema.clone(),
@@ -839,6 +906,9 @@ fn inner_create_vector_planner(
         dist_type,
     )
     .with_dataset(dataset);
+    if let Some(filter) = filter {
+        planner = planner.with_filter(Some(filter));
+    }
 
     let blocking = BlockingLsmVectorSearchPlanner {
         planner,
diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs
index 0898d1b049f..df4d63bd2f6 100644
--- a/java/lance-jni/src/merge_insert.rs
+++ b/java/lance-jni/src/merge_insert.rs
@@ -51,6 +51,7 @@ fn inner_merge_insert<'local>(
     let conflict_retries = extract_conflict_retries(env, &jparam)?;
     let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?;
     let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?;
+    let use_index = extract_use_index(env, &jparam)?;
     let marked_generations = extract_marked_generations(env, &jparam)?;
 
     let (new_ds, merge_stats) = unsafe {
@@ -69,6 +70,7 @@ fn inner_merge_insert<'local>(
             .conflict_retries(conflict_retries)
             .retry_timeout(Duration::from_millis(retry_timeout_ms as u64))
             .skip_auto_cleanup(skip_auto_cleanup)
+            .use_index(use_index)
             .mark_generations_as_merged(marked_generations)
             .try_build()?;
 
@@ -234,6 +236,11 @@ fn extract_skip_auto_cleanup<'local>(env: &mut JNIEnv<'local>, jparam: &JObject)
     Ok(skip_auto_cleanup)
 }
 
+fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Result<bool> {
+    let use_index = env.call_method(jparam, "useIndex", "()Z", &[])?.z()?;
+    Ok(use_index)
+}
+
 fn extract_marked_generations<'local>(
     env: &mut JNIEnv<'local>,
     jparam: &JObject,
diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs
index 6bc1948ae6a..4f899f56ff2 100644
--- a/java/lance-jni/src/transaction.rs
+++ b/java/lance-jni/src/transaction.rs
@@ -774,12 +774,18 @@ fn inner_commit_to_dataset<'local>(
     // Set namespace commit handler only if namespace_client_managed_versioning is true
     let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?;
     let commit_handler = if namespace_client_managed_versioning {
-        namespace_info.map(|(ns, tid)| {
-            let external_store = LanceNamespaceExternalManifestStore::new(ns, tid);
-            Arc::new(ExternalManifestCommitHandler {
-                external_manifest_store: Arc::new(external_store),
-            }) as Arc<dyn CommitHandler>
-        })
+        match namespace_info {
+            Some((ns, tid)) => {
+                // The store derives the branch a request targets from the base
+                // path it is handed, resolved against the table root.
+                let table_root = java_blocking_ds.inner.branch_location().find_main()?.path;
+                let external_store = LanceNamespaceExternalManifestStore::new(ns, tid, table_root);
+                Some(Arc::new(ExternalManifestCommitHandler {
+                    external_manifest_store: Arc::new(external_store),
+                }) as Arc<dyn CommitHandler>)
+            }
+            None => None,
+        }
     } else {
         None
     };
@@ -1560,7 +1566,8 @@ fn inner_commit_to_uri<'local>(
 
     // Set namespace commit handler only if namespace_client_managed_versioning is true
     if namespace_client_managed_versioning && let Some((namespace_client, tid)) = namespace_info {
-        let external_store = LanceNamespaceExternalManifestStore::new(namespace_client, tid);
+        let external_store =
+            LanceNamespaceExternalManifestStore::for_table_uri(namespace_client, tid, &uri_str)?;
         let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
             external_manifest_store: Arc::new(external_store),
         });
diff --git a/java/lance-jni/src/utils.rs b/java/lance-jni/src/utils.rs
index 1321e8e71e3..94372ef27cc 100644
--- a/java/lance-jni/src/utils.rs
+++ b/java/lance-jni/src/utils.rs
@@ -25,7 +25,7 @@ use crate::error::{Error, Result};
 use crate::ffi::JNIEnvExt;
 
 use crate::traits::FromJObjectWithEnv;
-use lance_index::vector::Query;
+use lance_index::vector::{ApproxMode, Query};
 use std::collections::HashMap;
 use std::str::FromStr;
 
@@ -76,6 +76,18 @@ pub fn extract_base_store_params(
     })
 }
 
+pub(crate) fn parse_approx_mode(value: &str) -> Result<ApproxMode> {
+    match value {
+        "fast" => Ok(ApproxMode::Fast),
+        "normal" => Ok(ApproxMode::Normal),
+        "accurate" => Ok(ApproxMode::Accurate),
+        _ => Err(Error::input_error(format!(
+            "Invalid approx mode '{}'. Expected one of: fast, normal, accurate",
+            value
+        ))),
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 pub fn extract_write_params(
     env: &mut JNIEnv,
@@ -253,6 +265,8 @@ pub fn get_query(env: &mut JNIEnv, query_obj: JObject) -> Result<Option<Query>>
         let query_parallelism = env
             .call_method(&java_obj, "getQueryParallelism", "()I", &[])?
             .i()?;
+        let approx_mode_str = env.get_string_from_method(&java_obj, "getApproxModeString")?;
+        let approx_mode = parse_approx_mode(&approx_mode_str)?;
 
         Ok(Query {
             column,
@@ -268,6 +282,7 @@ pub fn get_query(env: &mut JNIEnv, query_obj: JObject) -> Result<Option<Query>>
             use_index,
             dist_q_c: 0.0,
             query_parallelism,
+            approx_mode,
         })
     })?;
 
diff --git a/java/pom.xml b/java/pom.xml
index 4639cd74d2b..3cfe2a208c0 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -7,7 +7,7 @@
     <groupId>org.lance</groupId>
     <artifactId>lance-core</artifactId>
     <name>Lance Core</name>
-    <version>8.0.0-beta.1</version>
+    <version>9.1.0-beta.2</version>
     <packaging>jar</packaging>
 
     <description>Lance Format Java API</description>
@@ -39,6 +39,7 @@
         <spotless.scala.scalafmt.version>3.7.5</spotless.scala.scalafmt.version>
         <spotless.delimiter>package</spotless.delimiter>
         <rust.release.build>false</rust.release.build>
+        <rust.features></rust.features>
         <skip.build.jni>false</skip.build.jni>
         <shade.base>org.lance.shaded</shade.base>
         <spotless.license.header>
@@ -396,6 +397,9 @@
                                 <configuration>
                                     <path>lance-jni</path>
                                     <release>${rust.release.build}</release>
+                                    <features>
+                                        <feature>${rust.features}</feature>
+                                    </features>
                                     <!-- Copy native libraries to target/classes for runtime access -->
                                     <copyTo>${project.build.directory}/classes/nativelib</copyTo>
                                     <copyWithPlatformDir>true</copyWithPlatformDir>
@@ -409,6 +413,9 @@
                                 <configuration>
                                     <path>lance-jni</path>
                                     <release>${rust.release.build}</release>
+                                    <features>
+                                        <feature>${rust.features}</feature>
+                                    </features>
                                     <verbosity>-v</verbosity>
                                 </configuration>
                             </execution>
diff --git a/java/src/main/java/org/lance/CleanupOperation.java b/java/src/main/java/org/lance/CleanupOperation.java
new file mode 100644
index 00000000000..1f64473f43e
--- /dev/null
+++ b/java/src/main/java/org/lance/CleanupOperation.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance;
+
+import org.lance.cleanup.CleanupExplanation;
+import org.lance.cleanup.CleanupPolicy;
+import org.lance.cleanup.RemovalStats;
+
+import org.apache.arrow.util.Preconditions;
+
+import java.util.Optional;
+
+/**
+ * A cleanup operation with separate read-only explain and destructive execute actions.
+ *
+ * <p>This is not a deletion plan. Calling {@link #execute()} re-evaluates the current dataset and
+ * reference state before deleting files.
+ */
+public class CleanupOperation {
+  private final Dataset dataset;
+  private final CleanupPolicy policy;
+  private Optional<Long> maxCandidateFiles = Optional.empty();
+
+  CleanupOperation(Dataset dataset, CleanupPolicy policy) {
+    this.dataset = Preconditions.checkNotNull(dataset, "dataset cannot be null");
+    this.policy = Preconditions.checkNotNull(policy, "policy cannot be null");
+  }
+
+  /**
+   * Set the maximum number of candidate files included in the {@link #explain()} result.
+   *
+   * <p>Defaults to 1000 if not set. The aggregate {@link RemovalStats} returned by {@link
+   * #explain()} still account for all files that would be removed regardless of this limit; only
+   * the per-file {@code candidateFiles} list is truncated.
+   *
+   * @param maxCandidateFiles maximum number of candidate files to include; must be positive
+   * @return this operation for chaining
+   */
+  public CleanupOperation withMaxCandidateFiles(long maxCandidateFiles) {
+    Preconditions.checkArgument(
+        maxCandidateFiles > 0, "maxCandidateFiles must be positive, got %s", maxCandidateFiles);
+    this.maxCandidateFiles = Optional.of(maxCandidateFiles);
+    return this;
+  }
+
+  /**
+   * Explain what cleanup would remove without deleting files.
+   *
+   * @return cleanup explanation
+   */
+  public CleanupExplanation explain() {
+    return dataset.explainCleanup(policy, maxCandidateFiles);
+  }
+
+  /**
+   * Execute cleanup, re-evaluating the current dataset and reference state before deleting files.
+   *
+   * @return removal stats
+   */
+  public RemovalStats execute() {
+    return dataset.executeCleanup(policy);
+  }
+}
diff --git a/java/src/main/java/org/lance/Dataset.java b/java/src/main/java/org/lance/Dataset.java
index 23341283861..8f79d7cbaea 100644
--- a/java/src/main/java/org/lance/Dataset.java
+++ b/java/src/main/java/org/lance/Dataset.java
@@ -13,6 +13,7 @@
  */
 package org.lance;
 
+import org.lance.cleanup.CleanupExplanation;
 import org.lance.cleanup.CleanupPolicy;
 import org.lance.cleanup.RemovalStats;
 import org.lance.compaction.CompactionOptions;
@@ -2142,6 +2143,20 @@ public Dataset shallowClone(String targetPath, Ref ref, Map<String, String> stor
   private native Dataset nativeShallowClone(
       String targetPath, Ref ref, Optional<Map<String, String>> storageOptions);
 
+  /**
+   * Create a cleanup operation for the specified policy.
+   *
+   * <p>Use {@link CleanupOperation#explain()} to inspect what cleanup would remove without deleting
+   * files, or {@link CleanupOperation#execute()} to perform cleanup.
+   *
+   * @param policy cleanup policy
+   * @return cleanup operation
+   */
+  public CleanupOperation cleanup(CleanupPolicy policy) {
+    Preconditions.checkNotNull(policy, "policy cannot be null");
+    return new CleanupOperation(this, policy);
+  }
+
   /**
    * Cleanup dataset based on a specified policy.
    *
@@ -2149,6 +2164,10 @@ private native Dataset nativeShallowClone(
    * @return removal stats
    */
   public RemovalStats cleanupWithPolicy(CleanupPolicy policy) {
+    return cleanup(policy).execute();
+  }
+
+  RemovalStats executeCleanup(CleanupPolicy policy) {
     try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) {
       Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
       return nativeCleanupWithPolicy(policy);
@@ -2156,4 +2175,14 @@ public RemovalStats cleanupWithPolicy(CleanupPolicy policy) {
   }
 
   private native RemovalStats nativeCleanupWithPolicy(CleanupPolicy policy);
+
+  CleanupExplanation explainCleanup(CleanupPolicy policy, Optional<Long> maxCandidateFiles) {
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed");
+      return nativeExplainCleanupWithPolicy(policy, maxCandidateFiles);
+    }
+  }
+
+  private native CleanupExplanation nativeExplainCleanupWithPolicy(
+      CleanupPolicy policy, Optional<Long> maxCandidateFiles);
 }
diff --git a/java/src/main/java/org/lance/Fragment.java b/java/src/main/java/org/lance/Fragment.java
index b27b189bf48..3b12e158617 100644
--- a/java/src/main/java/org/lance/Fragment.java
+++ b/java/src/main/java/org/lance/Fragment.java
@@ -18,6 +18,7 @@
 import org.lance.ipc.LanceScanner;
 import org.lance.ipc.ScanOptions;
 import org.lance.namespace.LanceNamespace;
+import org.lance.schema.LanceSchema;
 
 import org.apache.arrow.c.ArrowArray;
 import org.apache.arrow.c.ArrowArrayStream;
@@ -260,6 +261,18 @@ static List<FragmentMetadata> create(
       WriteParams params,
       LanceNamespace namespaceClient,
       List<String> tableId) {
+    return create(datasetUri, allocator, root, params, namespaceClient, tableId, null);
+  }
+
+  /** Create a fragment from the given arrow array and schema. */
+  static List<FragmentMetadata> create(
+      String datasetUri,
+      BufferAllocator allocator,
+      VectorSchemaRoot root,
+      WriteParams params,
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      LanceSchema schema) {
     Preconditions.checkNotNull(datasetUri);
     Preconditions.checkNotNull(allocator);
     Preconditions.checkNotNull(root);
@@ -267,6 +280,30 @@ static List<FragmentMetadata> create(
     try (ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator);
         ArrowArray arrowArray = ArrowArray.allocateNew(allocator)) {
       Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchema);
+      if (schema != null) {
+        try (ArrowSchema lanceSchema = ArrowSchema.allocateNew(allocator)) {
+          Data.exportSchema(allocator, schema.asArrowSchemaWithFieldIds(), null, lanceSchema);
+          return createWithFfiArray(
+              datasetUri,
+              arrowArray.memoryAddress(),
+              arrowSchema.memoryAddress(),
+              params.getMaxRowsPerFile(),
+              params.getMaxRowsPerGroup(),
+              params.getMaxBytesPerFile(),
+              params.getMode(),
+              params.getEnableStableRowIds(),
+              params.getDataStorageVersion(),
+              params.getStorageOptions(),
+              params.getBaseStoreParams(),
+              params.getInitialBases(),
+              params.getTargetBases(),
+              namespaceClient,
+              tableId,
+              params.getAllowExternalBlobOutsideBases(),
+              params.getBlobPackFileSizeThreshold(),
+              lanceSchema.memoryAddress());
+        }
+      }
       return createWithFfiArray(
           datasetUri,
           arrowArray.memoryAddress(),
@@ -284,7 +321,8 @@ static List<FragmentMetadata> create(
           namespaceClient,
           tableId,
           params.getAllowExternalBlobOutsideBases(),
-          params.getBlobPackFileSizeThreshold());
+          params.getBlobPackFileSizeThreshold(),
+          0L);
     }
   }
 
@@ -295,9 +333,45 @@ static List<FragmentMetadata> create(
       WriteParams params,
       LanceNamespace namespaceClient,
       List<String> tableId) {
+    return create(datasetUri, null, stream, params, namespaceClient, tableId, null);
+  }
+
+  /** Create a fragment from the given arrow stream. */
+  static List<FragmentMetadata> create(
+      String datasetUri,
+      BufferAllocator allocator,
+      ArrowArrayStream stream,
+      WriteParams params,
+      LanceNamespace namespaceClient,
+      List<String> tableId,
+      LanceSchema schema) {
     Preconditions.checkNotNull(datasetUri);
     Preconditions.checkNotNull(stream);
     Preconditions.checkNotNull(params);
+    if (schema != null) {
+      Preconditions.checkNotNull(allocator, "allocator is required with schema");
+      try (ArrowSchema lanceSchema = ArrowSchema.allocateNew(allocator)) {
+        Data.exportSchema(allocator, schema.asArrowSchemaWithFieldIds(), null, lanceSchema);
+        return createWithFfiStream(
+            datasetUri,
+            stream.memoryAddress(),
+            params.getMaxRowsPerFile(),
+            params.getMaxRowsPerGroup(),
+            params.getMaxBytesPerFile(),
+            params.getMode(),
+            params.getEnableStableRowIds(),
+            params.getDataStorageVersion(),
+            params.getStorageOptions(),
+            params.getBaseStoreParams(),
+            params.getInitialBases(),
+            params.getTargetBases(),
+            namespaceClient,
+            tableId,
+            params.getAllowExternalBlobOutsideBases(),
+            params.getBlobPackFileSizeThreshold(),
+            lanceSchema.memoryAddress());
+      }
+    }
     return createWithFfiStream(
         datasetUri,
         stream.memoryAddress(),
@@ -314,7 +388,8 @@ static List<FragmentMetadata> create(
         namespaceClient,
         tableId,
         params.getAllowExternalBlobOutsideBases(),
-        params.getBlobPackFileSizeThreshold());
+        params.getBlobPackFileSizeThreshold(),
+        0L);
   }
 
   /** Create a fragment from the given arrow array and schema. */
@@ -335,7 +410,8 @@ private static native List<FragmentMetadata> createWithFfiArray(
       LanceNamespace namespaceClient,
       List<String> tableId,
       Optional<Boolean> allowExternalBlobOutsideBases,
-      Optional<Long> blobPackFileSizeThreshold);
+      Optional<Long> blobPackFileSizeThreshold,
+      long schemaMemoryAddress);
 
   /** Create a fragment from the given arrow stream. */
   private static native List<FragmentMetadata> createWithFfiStream(
@@ -354,5 +430,6 @@ private static native List<FragmentMetadata> createWithFfiStream(
       LanceNamespace namespaceClient,
       List<String> tableId,
       Optional<Boolean> allowExternalBlobOutsideBases,
-      Optional<Long> blobPackFileSizeThreshold);
+      Optional<Long> blobPackFileSizeThreshold,
+      long schemaMemoryAddress);
 }
diff --git a/java/src/main/java/org/lance/OpenDatasetBuilder.java b/java/src/main/java/org/lance/OpenDatasetBuilder.java
index baece0767a1..32fd5ca7635 100644
--- a/java/src/main/java/org/lance/OpenDatasetBuilder.java
+++ b/java/src/main/java/org/lance/OpenDatasetBuilder.java
@@ -216,8 +216,8 @@ private Dataset buildFromNamespaceClient() {
     // Call describe_table to get location and storage options
     DescribeTableRequest request = new DescribeTableRequest();
     request.setId(tableId);
-    // Only set version if present
-    options.getVersion().ifPresent(v -> request.setVersion(Long.valueOf(v)));
+    // Do not set the dataset version here. Some namespace implementations only support describing
+    // the latest table metadata; the requested version is applied when opening the dataset below.
 
     DescribeTableResponse response = namespaceClient.describeTable(request);
 
diff --git a/java/src/main/java/org/lance/ReadOptions.java b/java/src/main/java/org/lance/ReadOptions.java
index a2e60e0e75b..5d6447539f0 100644
--- a/java/src/main/java/org/lance/ReadOptions.java
+++ b/java/src/main/java/org/lance/ReadOptions.java
@@ -189,6 +189,13 @@ public Builder setMetadataCacheSize(int metadataCacheSize) {
      * Set storage options. Extra options that make sense for a particular storage connection. This
      * is used to store connection parameters like credentials, endpoint, etc.
      *
+     * <p>For datasets with additional registered base paths, a key of the form {@code
+     * base_<id>.<key>} applies {@code <key>} only to the base path with that manifest id,
+     * overriding the unscoped options that every base inherits. For example {@code
+     * base_1.account_key = abc} makes base 1 use {@code account_key = abc} while all other options
+     * are shared. Exact per-base bindings set via {@link #setBaseStoreParams(Map)} take precedence
+     * over base-scoped keys.
+     *
      * @param storageOptions the storage options
      * @return this builder
      */
diff --git a/java/src/main/java/org/lance/WriteDatasetBuilder.java b/java/src/main/java/org/lance/WriteDatasetBuilder.java
index 96a894fc944..9d406406291 100644
--- a/java/src/main/java/org/lance/WriteDatasetBuilder.java
+++ b/java/src/main/java/org/lance/WriteDatasetBuilder.java
@@ -70,6 +70,7 @@ public class WriteDatasetBuilder {
   private WriteParams.WriteMode mode = WriteParams.WriteMode.CREATE;
   private Schema schema;
   private Map<String, String> storageOptions = new HashMap<>();
+  private Map<String, String> properties = new HashMap<>();
   private Map<String, Map<String, String>> baseStoreParams = new HashMap<>();
   private boolean ignoreNamespaceStorageOptions = false;
   private Optional<Integer> maxRowsPerFile = Optional.empty();
@@ -207,6 +208,27 @@ public WriteDatasetBuilder storageOptions(Map<String, String> storageOptions) {
     return this;
   }
 
+  /**
+   * Sets the table properties to forward to the namespace on table creation.
+   *
+   * <p>These are Lance-namespace <b>properties</b>: catalog-level key-value metadata stored by the
+   * namespace outside the Lance table (available even if the table manifest does not exist), as
+   * distinct from the manifest-stored {@code config} (read/write behavior) and {@code metadata}
+   * (business metadata), and from non-persisted {@code storageOptions}. They are attached to the
+   * underlying declareTable request via its {@code properties} field.
+   *
+   * <p>Only used when a namespace client is configured via namespaceClient()+tableId() and the
+   * write creates the table (CREATE mode). Ignored for direct-URI writes and for APPEND/OVERWRITE
+   * modes.
+   *
+   * @param properties Table properties to forward on declareTable
+   * @return this builder instance
+   */
+  public WriteDatasetBuilder properties(Map<String, String> properties) {
+    this.properties = new HashMap<>(properties);
+    return this;
+  }
+
   /**
    * Sets runtime-only object store parameters for registered base paths.
    *
@@ -410,6 +432,9 @@ private Dataset executeWithNamespaceClient() {
     if (mode == WriteParams.WriteMode.CREATE) {
       DeclareTableRequest declareRequest = new DeclareTableRequest();
       declareRequest.setId(tableId);
+      if (properties != null && !properties.isEmpty()) {
+        declareRequest.setProperties(properties);
+      }
       DeclareTableResponse declareResponse = namespaceClient.declareTable(declareRequest);
 
       tableUri = declareResponse.getLocation();
diff --git a/java/src/main/java/org/lance/WriteFragmentBuilder.java b/java/src/main/java/org/lance/WriteFragmentBuilder.java
index 5d7dc1a42b2..2dbef873849 100644
--- a/java/src/main/java/org/lance/WriteFragmentBuilder.java
+++ b/java/src/main/java/org/lance/WriteFragmentBuilder.java
@@ -14,6 +14,7 @@
 package org.lance;
 
 import org.lance.namespace.LanceNamespace;
+import org.lance.schema.LanceSchema;
 
 import org.apache.arrow.c.ArrowArrayStream;
 import org.apache.arrow.memory.BufferAllocator;
@@ -45,6 +46,7 @@ public class WriteFragmentBuilder {
   private BufferAllocator allocator;
   private VectorSchemaRoot vectorSchemaRoot;
   private ArrowArrayStream arrowArrayStream;
+  private LanceSchema schema;
   private WriteParams writeParams;
   private WriteParams.Builder writeParamsBuilder;
   private LanceNamespace namespaceClient;
@@ -100,6 +102,22 @@ public WriteFragmentBuilder data(ArrowArrayStream stream) {
     return this;
   }
 
+  /**
+   * Set the Lance dataset schema to use when writing fragments.
+   *
+   * <p>This is useful for distributed writes where workers create uncommitted fragments and a
+   * coordinator commits them later. When this schema is supplied, lance-core does not need to open
+   * the existing dataset to infer the schema in APPEND mode. The schema should come from the target
+   * dataset so Lance field IDs are preserved.
+   *
+   * @param schema the target Lance dataset schema
+   * @return this builder
+   */
+  public WriteFragmentBuilder schema(LanceSchema schema) {
+    this.schema = schema;
+    return this;
+  }
+
   /**
    * Set the write parameters.
    *
@@ -278,10 +296,22 @@ public List<FragmentMetadata> execute() {
     // storage options provider when these are non-null for credential refresh
     if (vectorSchemaRoot != null) {
       return Fragment.create(
-          datasetUri, allocator, vectorSchemaRoot, finalWriteParams, namespaceClient, tableId);
+          datasetUri,
+          allocator,
+          vectorSchemaRoot,
+          finalWriteParams,
+          namespaceClient,
+          tableId,
+          schema);
     } else {
       return Fragment.create(
-          datasetUri, arrowArrayStream, finalWriteParams, namespaceClient, tableId);
+          datasetUri,
+          allocator,
+          arrowArrayStream,
+          finalWriteParams,
+          namespaceClient,
+          tableId,
+          schema);
     }
   }
 
@@ -312,6 +342,8 @@ private void validate() {
     Preconditions.checkState(
         vectorSchemaRoot == null || allocator != null,
         "allocator is required when using VectorSchemaRoot");
+    Preconditions.checkState(
+        schema == null || allocator != null, "allocator is required with schema");
     Preconditions.checkState(
         writeParams == null || writeParamsBuilder == null,
         "Cannot use both writeParams() and individual parameter methods");
diff --git a/java/src/main/java/org/lance/WriteParams.java b/java/src/main/java/org/lance/WriteParams.java
index d8757006975..90d48feafb7 100644
--- a/java/src/main/java/org/lance/WriteParams.java
+++ b/java/src/main/java/org/lance/WriteParams.java
@@ -197,6 +197,17 @@ public Builder withDataStorageVersion(String dataStorageVersion) {
       return this;
     }
 
+    /**
+     * Set storage options for the write.
+     *
+     * <p>For writes involving additional registered base paths, a key of the form {@code
+     * base_<id>.<key>} applies {@code <key>} only to the base path with that id, overriding the
+     * unscoped options that every base inherits. Exact per-base bindings set via {@link
+     * #withBaseStoreParams(Map)} take precedence over base-scoped keys.
+     *
+     * @param storageOptions the storage options
+     * @return this builder
+     */
     public Builder withStorageOptions(Map<String, String> storageOptions) {
       this.storageOptions = storageOptions;
       return this;
diff --git a/java/src/main/java/org/lance/cleanup/CleanupCandidateFile.java b/java/src/main/java/org/lance/cleanup/CleanupCandidateFile.java
new file mode 100644
index 00000000000..a7925a746af
--- /dev/null
+++ b/java/src/main/java/org/lance/cleanup/CleanupCandidateFile.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.cleanup;
+
+/** A file that cleanup identified as removable. */
+public class CleanupCandidateFile {
+  private final String path;
+  private final CleanupFileKind kind;
+  private final boolean unverified;
+  private final long sizeBytes;
+
+  public CleanupCandidateFile(String path, String kind, boolean unverified, long sizeBytes) {
+    this(path, CleanupFileKind.fromRustString(kind), unverified, sizeBytes);
+  }
+
+  public CleanupCandidateFile(
+      String path, CleanupFileKind kind, boolean unverified, long sizeBytes) {
+    this.path = path;
+    this.kind = kind;
+    this.unverified = unverified;
+    this.sizeBytes = sizeBytes;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  public CleanupFileKind getKind() {
+    return kind;
+  }
+
+  public boolean isUnverified() {
+    return unverified;
+  }
+
+  public long getSizeBytes() {
+    return sizeBytes;
+  }
+}
diff --git a/java/src/main/java/org/lance/cleanup/CleanupExplanation.java b/java/src/main/java/org/lance/cleanup/CleanupExplanation.java
new file mode 100644
index 00000000000..320d555c3b5
--- /dev/null
+++ b/java/src/main/java/org/lance/cleanup/CleanupExplanation.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.cleanup;
+
+import java.util.List;
+
+/** Read-only explanation of what cleanup would remove. */
+public class CleanupExplanation {
+  private final long readVersion;
+  private final RemovalStats stats;
+  private final List<CleanupCandidateFile> candidateFiles;
+  private final boolean candidateFilesTruncated;
+  private final long candidateFileLimit;
+  private final List<CleanupReferencedBranch> referencedBranches;
+  private final List<String> warnings;
+
+  public CleanupExplanation(
+      long readVersion,
+      RemovalStats stats,
+      List<CleanupCandidateFile> candidateFiles,
+      boolean candidateFilesTruncated,
+      long candidateFileLimit,
+      List<CleanupReferencedBranch> referencedBranches,
+      List<String> warnings) {
+    this.readVersion = readVersion;
+    this.stats = stats;
+    this.candidateFiles = candidateFiles;
+    this.candidateFilesTruncated = candidateFilesTruncated;
+    this.candidateFileLimit = candidateFileLimit;
+    this.referencedBranches = referencedBranches;
+    this.warnings = warnings;
+  }
+
+  public long getReadVersion() {
+    return readVersion;
+  }
+
+  public RemovalStats getStats() {
+    return stats;
+  }
+
+  public List<CleanupCandidateFile> getCandidateFiles() {
+    return candidateFiles;
+  }
+
+  public boolean isCandidateFilesTruncated() {
+    return candidateFilesTruncated;
+  }
+
+  public long getCandidateFileLimit() {
+    return candidateFileLimit;
+  }
+
+  public List<CleanupReferencedBranch> getReferencedBranches() {
+    return referencedBranches;
+  }
+
+  public List<String> getWarnings() {
+    return warnings;
+  }
+}
diff --git a/java/src/main/java/org/lance/cleanup/CleanupFileKind.java b/java/src/main/java/org/lance/cleanup/CleanupFileKind.java
new file mode 100644
index 00000000000..695e778fc76
--- /dev/null
+++ b/java/src/main/java/org/lance/cleanup/CleanupFileKind.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.cleanup;
+
+/** Kind of file identified by cleanup. */
+public enum CleanupFileKind {
+  MANIFEST,
+  DATA,
+  TRANSACTION,
+  INDEX,
+  DELETION,
+  TEMPORARY_MANIFEST;
+
+  public static CleanupFileKind fromRustString(String value) {
+    switch (value) {
+      case "manifest":
+        return MANIFEST;
+      case "data":
+        return DATA;
+      case "transaction":
+        return TRANSACTION;
+      case "index":
+        return INDEX;
+      case "deletion":
+        return DELETION;
+      case "temporary_manifest":
+        return TEMPORARY_MANIFEST;
+      default:
+        throw new IllegalArgumentException("Unknown cleanup file kind: " + value);
+    }
+  }
+}
diff --git a/java/src/main/java/org/lance/cleanup/CleanupReferencedBranch.java b/java/src/main/java/org/lance/cleanup/CleanupReferencedBranch.java
new file mode 100644
index 00000000000..47fae25cd5a
--- /dev/null
+++ b/java/src/main/java/org/lance/cleanup/CleanupReferencedBranch.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.cleanup;
+
+/** A branch that references the current branch lineage. */
+public class CleanupReferencedBranch {
+  private final String name;
+  private final long referencedVersion;
+  private final boolean cleanupCandidate;
+
+  public CleanupReferencedBranch(String name, long referencedVersion, boolean cleanupCandidate) {
+    this.name = name;
+    this.referencedVersion = referencedVersion;
+    this.cleanupCandidate = cleanupCandidate;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public long getReferencedVersion() {
+    return referencedVersion;
+  }
+
+  public boolean isCleanupCandidate() {
+    return cleanupCandidate;
+  }
+}
diff --git a/java/src/main/java/org/lance/index/IndexType.java b/java/src/main/java/org/lance/index/IndexType.java
index 3a03934effd..1fff86fc7e0 100644
--- a/java/src/main/java/org/lance/index/IndexType.java
+++ b/java/src/main/java/org/lance/index/IndexType.java
@@ -24,6 +24,7 @@ public enum IndexType {
   MEM_WAL(7),
   ZONEMAP(8),
   BLOOM_FILTER(9),
+  RTREE(10),
   VECTOR(100),
   IVF_FLAT(101),
   IVF_SQ(102),
diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
index ca0a7a46c70..82513a89ee7 100755
--- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
+++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java
@@ -53,7 +53,9 @@ public static final class Builder {
     private Integer minNgramLength;
     private Integer maxNgramLength;
     private Boolean prefixOnly;
+    private Integer blockSize = 128;
     private Boolean skipMerge;
+    private Integer formatVersion;
 
     /**
      * Configure the base tokenizer.
@@ -65,6 +67,8 @@ public static final class Builder {
      *   <li>{@code "whitespace"}: splits tokens on whitespace
      *   <li>{@code "raw"}: no tokenization
      *   <li>{@code "ngram"}: N-Gram tokenizer
+     *   <li>{@code "icu"}: ICU dictionary-based Unicode word segmentation
+     *   <li>{@code "icu/split"}: ICU segmentation with simple-style delimiter splitting
      *   <li>{@code "lindera/*"}: Lindera tokenizer
      *   <li>{@code "jieba/*"}: Jieba tokenizer
      * </ul>
@@ -223,6 +227,27 @@ public Builder prefixOnly(boolean prefixOnly) {
       return this;
     }
 
+    /**
+     * Configure the number of documents in each compressed posting block.
+     *
+     * <p>Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when
+     * this is not set.
+     *
+     * <p>{@code blockSize = 256} is experimental and may introduce breaking changes. Use {@code
+     * 128} when stable compatibility with the legacy posting layout is required.
+     *
+     * @param blockSize posting block size
+     * @return this builder
+     * @throws IllegalArgumentException if {@code blockSize} is unsupported
+     */
+    public Builder blockSize(int blockSize) {
+      if (blockSize != 128 && blockSize != 256) {
+        throw new IllegalArgumentException("blockSize must be one of 128 or 256");
+      }
+      this.blockSize = blockSize;
+      return this;
+    }
+
     /**
      * Configure whether to skip the partition merge stage after indexing. If true, skip the
      * partition merge stage after indexing. This can be useful for distributed indexing where merge
@@ -236,8 +261,31 @@ public Builder skipMerge(boolean skipMerge) {
       return this;
     }
 
+    /**
+     * Configure the on-disk FTS format version to write when creating a new index.
+     *
+     * <p>If unset, Lance writes v2 for {@code blockSize = 128} and v3 for {@code blockSize = 256}.
+     * {@code formatVersion = 3} is experimental and is only valid with {@code blockSize = 256}.
+     *
+     * @param formatVersion FTS format version, must be 1, 2, or 3
+     * @return this builder
+     * @throws IllegalArgumentException
+     */
+    public Builder formatVersion(int formatVersion) {
+      if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) {
+        throw new IllegalArgumentException("formatVersion must be 1, 2, or 3");
+      }
+      this.formatVersion = formatVersion;
+      return this;
+    }
+
     /** Build a {@link ScalarIndexParams} instance for an inverted index. */
     public ScalarIndexParams build() {
+      if (formatVersion != null) {
+        Preconditions.checkArgument(
+            (blockSize == 256 && formatVersion == 3) || (blockSize == 128 && formatVersion != 3),
+            "formatVersion 3 requires blockSize 256, and blockSize 256 requires formatVersion 3");
+      }
       Map<String, Object> params = new HashMap<>();
       if (baseTokenizer != null) {
         params.put("base_tokenizer", baseTokenizer);
@@ -280,9 +328,15 @@ public ScalarIndexParams build() {
       if (prefixOnly != null) {
         params.put("prefix_only", prefixOnly);
       }
+      if (blockSize != null) {
+        params.put("block_size", blockSize);
+      }
       if (skipMerge != null) {
         params.put("skip_merge", skipMerge);
       }
+      if (formatVersion != null) {
+        params.put("format_version", formatVersion);
+      }
 
       String json = JsonUtils.toJson(params);
       return ScalarIndexParams.create(INDEX_TYPE, json);
diff --git a/java/src/main/java/org/lance/index/scalar/ScalarIndexParams.java b/java/src/main/java/org/lance/index/scalar/ScalarIndexParams.java
index 345a55f20b2..b3408e2d68d 100644
--- a/java/src/main/java/org/lance/index/scalar/ScalarIndexParams.java
+++ b/java/src/main/java/org/lance/index/scalar/ScalarIndexParams.java
@@ -31,7 +31,7 @@ private ScalarIndexParams(Builder builder) {
    * Create a new ScalarIndexParams with the given index type and no parameters.
    *
    * @param indexType the index type (e.g., "btree", "zonemap", "bitmap", "inverted", "labellist",
-   *     "ngram")
+   *     "ngram", "rtree")
    * @return ScalarIndexParams
    */
   public static ScalarIndexParams create(String indexType) {
@@ -42,7 +42,7 @@ public static ScalarIndexParams create(String indexType) {
    * Create a new ScalarIndexParams with the given index type and JSON parameters.
    *
    * @param indexType the index type (e.g., "btree", "zonemap", "bitmap", "inverted", "labellist",
-   *     "ngram")
+   *     "ngram", "rtree")
    * @param jsonParams JSON string containing index-specific parameters
    * @return ScalarIndexParams
    */
@@ -58,7 +58,7 @@ public static class Builder {
      * Create a new builder for scalar index parameters.
      *
      * @param indexType the index type (e.g., "btree", "zonemap", "bitmap", "inverted", "labellist",
-     *     "ngram")
+     *     "ngram", "rtree")
      */
     public Builder(String indexType) {
       this.indexType = indexType;
diff --git a/java/src/main/java/org/lance/ipc/ApproxMode.java b/java/src/main/java/org/lance/ipc/ApproxMode.java
new file mode 100644
index 00000000000..e6aa7fc3552
--- /dev/null
+++ b/java/src/main/java/org/lance/ipc/ApproxMode.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.ipc;
+
+/**
+ * Controls the speed / accuracy tradeoff for approximate vector search.
+ *
+ * <p>This setting currently only affects RQ-quantized vector indexes, such as IVF_RQ. Other index
+ * types ignore this setting.
+ */
+public enum ApproxMode {
+  /** Prefer faster approximate scoring when supported by the RQ index. */
+  FAST("fast"),
+
+  /** Use the index's default approximation behavior. */
+  NORMAL("normal"),
+
+  /** Prefer more accurate approximate scoring when supported by the RQ index. */
+  ACCURATE("accurate");
+
+  private final String value;
+
+  ApproxMode(String value) {
+    this.value = value;
+  }
+
+  /** Returns the lowercase value passed across the JNI boundary. */
+  public String toRustString() {
+    return value;
+  }
+}
diff --git a/java/src/main/java/org/lance/ipc/AsyncScanner.java b/java/src/main/java/org/lance/ipc/AsyncScanner.java
index 2ec317cb245..6e515e3546c 100644
--- a/java/src/main/java/org/lance/ipc/AsyncScanner.java
+++ b/java/src/main/java/org/lance/ipc/AsyncScanner.java
@@ -80,7 +80,10 @@ public static AsyncScanner create(
             options.getColumnOrderings(),
             options.isUseScalarIndex(),
             options.isFastSearch(),
-            options.getSubstraitAggregate());
+            options.getSubstraitAggregate(),
+            options.isIncludeDeletedRows(),
+            options.isStrictBatchSize(),
+            options.isDisableScoringAutoprojection());
     scanner.allocator = allocator;
     return scanner;
   }
@@ -103,7 +106,10 @@ static native AsyncScanner createAsyncScanner(
       Optional<List<ColumnOrdering>> columnOrderings,
       boolean useScalarIndex,
       boolean fastSearch,
-      Optional<ByteBuffer> substraitAggregate);
+      Optional<ByteBuffer> substraitAggregate,
+      boolean includeDeletedRows,
+      boolean strictBatchSize,
+      boolean disableScoringAutoprojection);
 
   /**
    * Asynchronously scan batches and return a CompletableFuture.
diff --git a/java/src/main/java/org/lance/ipc/LanceScanner.java b/java/src/main/java/org/lance/ipc/LanceScanner.java
index edd3ebc22cc..3a413e0ccfd 100644
--- a/java/src/main/java/org/lance/ipc/LanceScanner.java
+++ b/java/src/main/java/org/lance/ipc/LanceScanner.java
@@ -77,7 +77,10 @@ public static LanceScanner create(
             options.isUseScalarIndex(),
             options.isFastSearch(),
             options.getSubstraitAggregate(),
-            options.isCollectStats());
+            options.isCollectStats(),
+            options.isIncludeDeletedRows(),
+            options.isStrictBatchSize(),
+            options.isDisableScoringAutoprojection());
     scanner.allocator = allocator;
     scanner.dataset = dataset;
     scanner.options = options;
@@ -103,7 +106,10 @@ static native LanceScanner createScanner(
       boolean useScalarIndex,
       boolean fastSearch,
       Optional<ByteBuffer> substraitAggregate,
-      boolean collectStats);
+      boolean collectStats,
+      boolean includeDeletedRows,
+      boolean strictBatchSize,
+      boolean disableScoringAutoprojection);
 
   /**
    * Closes this scanner and releases any system resources associated with it. If the scanner is
diff --git a/java/src/main/java/org/lance/ipc/Query.java b/java/src/main/java/org/lance/ipc/Query.java
index 48013b375ee..215865310df 100644
--- a/java/src/main/java/org/lance/ipc/Query.java
+++ b/java/src/main/java/org/lance/ipc/Query.java
@@ -32,6 +32,7 @@ public class Query {
   private final Optional<DistanceType> distanceType;
   private final boolean useIndex;
   private final int queryParallelism;
+  private final ApproxMode approxMode;
 
   private Query(Builder builder) {
     this.column = Preconditions.checkNotNull(builder.column, "Columns must be set");
@@ -52,6 +53,7 @@ private Query(Builder builder) {
     this.distanceType = builder.distanceType;
     this.useIndex = builder.useIndex;
     this.queryParallelism = builder.queryParallelism;
+    this.approxMode = builder.approxMode;
   }
 
   public String getColumn() {
@@ -98,6 +100,14 @@ public int getQueryParallelism() {
     return queryParallelism;
   }
 
+  public ApproxMode getApproxMode() {
+    return approxMode;
+  }
+
+  public String getApproxModeString() {
+    return approxMode.toRustString();
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
@@ -111,6 +121,7 @@ public String toString() {
         .add("distanceType", distanceType.orElse(null))
         .add("useIndex", useIndex)
         .add("queryParallelism", queryParallelism)
+        .add("approxMode", approxMode)
         .toString();
   }
 
@@ -125,6 +136,7 @@ public static class Builder {
     private Optional<DistanceType> distanceType = Optional.empty();
     private boolean useIndex = true;
     private int queryParallelism = 0;
+    private ApproxMode approxMode = ApproxMode.NORMAL;
 
     /**
      * Sets the column to be searched.
@@ -275,6 +287,20 @@ public Builder setQueryParallelism(int queryParallelism) {
       return this;
     }
 
+    /**
+     * Sets the speed / accuracy tradeoff for approximate vector search.
+     *
+     * <p>This setting currently only affects RQ-quantized vector indexes, such as IVF_RQ. Other
+     * index types ignore this setting.
+     *
+     * @param approxMode The approximate search mode to use for the query.
+     * @return The Builder instance for method chaining.
+     */
+    public Builder setApproxMode(ApproxMode approxMode) {
+      this.approxMode = Preconditions.checkNotNull(approxMode, "ApproxMode must not be null");
+      return this;
+    }
+
     /**
      * Builds the Query object.
      *
diff --git a/java/src/main/java/org/lance/ipc/ScanOptions.java b/java/src/main/java/org/lance/ipc/ScanOptions.java
index 68c485e39a3..7e1b4222262 100644
--- a/java/src/main/java/org/lance/ipc/ScanOptions.java
+++ b/java/src/main/java/org/lance/ipc/ScanOptions.java
@@ -40,6 +40,9 @@ public class ScanOptions {
   private final Optional<ByteBuffer> substraitAggregate;
   private final boolean collectStats;
   private final boolean fastSearch;
+  private final boolean includeDeletedRows;
+  private final boolean strictBatchSize;
+  private final boolean disableScoringAutoprojection;
 
   public ScanOptions(
       Optional<List<Integer>> fragmentIds,
@@ -77,6 +80,9 @@ public ScanOptions(
         useScalarIndex,
         substraitAggregate,
         collectStats,
+        false,
+        false,
+        false,
         false);
   }
 
@@ -121,10 +127,15 @@ public ScanOptions(
       boolean useScalarIndex,
       Optional<ByteBuffer> substraitAggregate,
       boolean collectStats,
-      boolean fastSearch) {
+      boolean fastSearch,
+      boolean includeDeletedRows,
+      boolean strictBatchSize,
+      boolean disableScoringAutoprojection) {
     Preconditions.checkArgument(
         !(filter.isPresent() && substraitFilter.isPresent()),
         "cannot set both substrait filter and string filter");
+    Preconditions.checkArgument(
+        batchReadahead > 0, "batchReadahead must be greater than 0, got %s", batchReadahead);
     this.fragmentIds = fragmentIds;
     this.batchSize = batchSize;
     this.columns = columns;
@@ -143,6 +154,9 @@ public ScanOptions(
     this.substraitAggregate = substraitAggregate;
     this.collectStats = collectStats;
     this.fastSearch = fastSearch;
+    this.includeDeletedRows = includeDeletedRows;
+    this.strictBatchSize = strictBatchSize;
+    this.disableScoringAutoprojection = disableScoringAutoprojection;
   }
 
   /**
@@ -297,6 +311,33 @@ public boolean isCollectStats() {
     return collectStats;
   }
 
+  /**
+   * Get whether to include deleted rows in scan results.
+   *
+   * @return true if deleted rows should be included, false otherwise.
+   */
+  public boolean isIncludeDeletedRows() {
+    return includeDeletedRows;
+  }
+
+  /**
+   * Get whether to enforce strict batch sizing.
+   *
+   * @return true if batch sizes must be strictly enforced, false otherwise.
+   */
+  public boolean isStrictBatchSize() {
+    return strictBatchSize;
+  }
+
+  /**
+   * Get whether to disable scoring autoprojection.
+   *
+   * @return true if scoring column autoprojection is disabled, false otherwise.
+   */
+  public boolean isDisableScoringAutoprojection() {
+    return disableScoringAutoprojection;
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
@@ -322,6 +363,9 @@ public String toString() {
             "substraitAggregate",
             substraitAggregate.map(buf -> "ByteBuffer[" + buf.remaining() + " bytes]").orElse(null))
         .add("collectStats", collectStats)
+        .add("includeDeletedRows", includeDeletedRows)
+        .add("strictBatchSize", strictBatchSize)
+        .add("disableScoringAutoprojection", disableScoringAutoprojection)
         .toString();
   }
 
@@ -345,6 +389,9 @@ public static class Builder {
     private boolean fastSearch = false;
     private Optional<ByteBuffer> substraitAggregate = Optional.empty();
     private boolean collectStats = false;
+    private boolean includeDeletedRows = false;
+    private boolean strictBatchSize = false;
+    private boolean disableScoringAutoprojection = false;
 
     public Builder() {}
 
@@ -372,6 +419,9 @@ public Builder(ScanOptions options) {
       this.fastSearch = options.isFastSearch();
       this.substraitAggregate = options.getSubstraitAggregate();
       this.collectStats = options.isCollectStats();
+      this.includeDeletedRows = options.isIncludeDeletedRows();
+      this.strictBatchSize = options.isStrictBatchSize();
+      this.disableScoringAutoprojection = options.isDisableScoringAutoprojection();
     }
 
     /**
@@ -577,6 +627,39 @@ public Builder collectStats(boolean collectStats) {
       return this;
     }
 
+    /**
+     * Set whether to include deleted rows in scan results. Default is false.
+     *
+     * @param includeDeletedRows whether to include deleted rows
+     * @return Builder instance for method chaining.
+     */
+    public Builder includeDeletedRows(boolean includeDeletedRows) {
+      this.includeDeletedRows = includeDeletedRows;
+      return this;
+    }
+
+    /**
+     * Set whether to enforce strict batch sizing. Default is false.
+     *
+     * @param strictBatchSize whether to enforce strict batch sizing
+     * @return Builder instance for method chaining.
+     */
+    public Builder strictBatchSize(boolean strictBatchSize) {
+      this.strictBatchSize = strictBatchSize;
+      return this;
+    }
+
+    /**
+     * Set whether to disable scoring column autoprojection. Default is false.
+     *
+     * @param disableScoringAutoprojection whether to disable autoprojection
+     * @return Builder instance for method chaining.
+     */
+    public Builder disableScoringAutoprojection(boolean disableScoringAutoprojection) {
+      this.disableScoringAutoprojection = disableScoringAutoprojection;
+      return this;
+    }
+
     /**
      * Build the LanceScanOptions instance.
      *
@@ -601,7 +684,10 @@ public ScanOptions build() {
           useScalarIndex,
           substraitAggregate,
           collectStats,
-          fastSearch);
+          fastSearch,
+          includeDeletedRows,
+          strictBatchSize,
+          disableScoringAutoprojection);
     }
   }
 }
diff --git a/java/src/main/java/org/lance/memwal/LsmScanner.java b/java/src/main/java/org/lance/memwal/LsmScanner.java
index ddbed76da8e..29bfb097375 100644
--- a/java/src/main/java/org/lance/memwal/LsmScanner.java
+++ b/java/src/main/java/org/lance/memwal/LsmScanner.java
@@ -101,9 +101,11 @@ public LsmScanner filter(String expr) {
   private native void nativeFilter(String expr);
 
   /** Limit the number of rows returned, optionally skipping {@code offset} rows first. */
-  public LsmScanner limit(long limit, Optional<Long> offset) {
+  public LsmScanner limit(Optional<Long> limit, Optional<Long> offset) {
+    Preconditions.checkNotNull(limit, "limit must not be null");
     Preconditions.checkNotNull(offset, "offset must not be null");
-    Preconditions.checkArgument(limit >= 0, "limit must not be negative, got %s", limit);
+    limit.ifPresent(
+        l -> Preconditions.checkArgument(l >= 0, "limit must not be negative, got %s", l));
     offset.ifPresent(
         o -> Preconditions.checkArgument(o >= 0, "offset must not be negative, got %s", o));
     try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) {
@@ -113,12 +115,17 @@ public LsmScanner limit(long limit, Optional<Long> offset) {
     }
   }
 
+  /** Limit the number of rows returned, optionally skipping {@code offset} rows first. */
+  public LsmScanner limit(long limit, Optional<Long> offset) {
+    return limit(Optional.of(limit), offset);
+  }
+
   /** Limit the number of rows returned. */
   public LsmScanner limit(long limit) {
     return limit(limit, Optional.empty());
   }
 
-  private native void nativeLimit(long limit, Optional<Long> offset);
+  private native void nativeLimit(Optional<Long> limit, Optional<Long> offset);
 
   /** Include the {@code _rowaddr} internal column in results. */
   public LsmScanner withRowAddress() {
diff --git a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
index 8517d2b2498..75c0d86c475 100644
--- a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
+++ b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java
@@ -49,7 +49,7 @@ public class LsmVectorSearchPlanner implements AutoCloseable {
    */
   public LsmVectorSearchPlanner(
       Dataset dataset, List<ShardSnapshot> shardSnapshots, String vectorColumn) {
-    this(dataset, shardSnapshots, vectorColumn, null, null);
+    this(dataset, shardSnapshots, vectorColumn, null, null, null);
   }
 
   /**
@@ -66,6 +66,26 @@ public LsmVectorSearchPlanner(
       String vectorColumn,
       List<String> pkColumns,
       String distanceType) {
+    this(dataset, shardSnapshots, vectorColumn, pkColumns, distanceType, null);
+  }
+
+  /**
+   * @param dataset the base dataset
+   * @param shardSnapshots shard snapshots specifying the flushed generations to include
+   * @param vectorColumn name of the {@code FixedSizeList<float32>} vector column
+   * @param pkColumns primary key column names; inferred from schema metadata when {@code null}
+   * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"},
+   *     {@code "hamming"}; defaults to {@code "l2"} when {@code null}
+   * @param filter SQL predicate applied as a prefilter to every LSM source before vector candidate
+   *     selection; pass {@code null} for no predicate
+   */
+  public LsmVectorSearchPlanner(
+      Dataset dataset,
+      List<ShardSnapshot> shardSnapshots,
+      String vectorColumn,
+      List<String> pkColumns,
+      String distanceType,
+      String filter) {
     Preconditions.checkNotNull(dataset, "dataset must not be null");
     Preconditions.checkNotNull(shardSnapshots, "shardSnapshots must not be null");
     Preconditions.checkNotNull(vectorColumn, "vectorColumn must not be null");
@@ -75,7 +95,8 @@ public LsmVectorSearchPlanner(
         shardSnapshots,
         vectorColumn,
         Optional.ofNullable(pkColumns),
-        Optional.ofNullable(distanceType));
+        Optional.ofNullable(distanceType),
+        Optional.ofNullable(filter));
   }
 
   private native void nativeCreate(
@@ -83,7 +104,8 @@ private native void nativeCreate(
       List<ShardSnapshot> shardSnapshots,
       String vectorColumn,
       Optional<List<String>> pkColumns,
-      Optional<String> distanceType);
+      Optional<String> distanceType,
+      Optional<String> filter);
 
   /**
    * Plan a KNN vector search.
@@ -96,17 +118,17 @@ private native void nativeCreate(
    * @param refineBaseTable when true, the base-table arm re-ranks candidates with exact distances
    *     (refine factor 1). Useful when the base table uses an approximate index (IVF-PQ).
    *     Auto-enabled whenever stale filtering is on (see {@code overfetchFactor}).
-   * @param overfetchFactor single knob controlling both stale-read filtering and over-fetch:
+   * @param overfetchFactor over-fetch multiple for sources with rows superseded by newer
+   *     generations. Must be at least {@code 1.0}:
    *     <ul>
-   *       <li>{@code < 1.0} (e.g. {@code 0.0}): stale filtering off — rows superseded by a newer
-   *           generation may surface (the global primary-key dedup still runs).
-   *       <li>{@code == 1.0}: filtering on, no over-fetch — a source with superseded rows fetches
-   *           exactly {@code k} candidates and may return fewer than {@code k} live rows.
-   *       <li>{@code > 1.0}: filtering on, with over-fetch — such a source fetches {@code ceil(k *
-   *           overfetchFactor)} candidates so dropping the stale ones still leaves {@code k} live
-   *           rows.
+   *       <li>{@code == 1.0}: no over-fetch — an affected source fetches exactly {@code k}
+   *           candidates and may return fewer than {@code k} live rows after stale rows are
+   *           dropped.
+   *       <li>{@code > 1.0}: with over-fetch — such a source fetches {@code ceil(k *
+   *           overfetchFactor)} candidates so dropping stale rows is less likely to under-fill the
+   *           result.
    *     </ul>
-   *     There is no separate on/off flag: over-fetch is only meaningful while filtering.
+   *
    * @return an executable plan
    */
   public ExecutionPlan planSearch(
@@ -119,6 +141,10 @@ public ExecutionPlan planSearch(
     Preconditions.checkNotNull(query, "query must not be null");
     Preconditions.checkArgument(k > 0, "k must be positive, got %s", k);
     Preconditions.checkArgument(nprobes > 0, "nprobes must be positive, got %s", nprobes);
+    Preconditions.checkArgument(
+        Double.isFinite(overfetchFactor) && overfetchFactor >= 1.0,
+        "overfetchFactor must be finite and >= 1.0, got %s",
+        overfetchFactor);
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
       Preconditions.checkArgument(
           nativeVectorPlannerHandle != 0, "LsmVectorSearchPlanner is closed");
diff --git a/java/src/main/java/org/lance/memwal/ShardWriter.java b/java/src/main/java/org/lance/memwal/ShardWriter.java
index a5a3000bdba..3414ebda616 100644
--- a/java/src/main/java/org/lance/memwal/ShardWriter.java
+++ b/java/src/main/java/org/lance/memwal/ShardWriter.java
@@ -36,6 +36,7 @@
  * <pre>{@code
  * try (ShardWriter writer = dataset.memWalWriter(shardId)) {
  *   writer.put(reader);
+ *   writer.delete(keys);
  * }
  * }</pre>
  *
@@ -99,6 +100,34 @@ public void put(ArrowReader reader) {
 
   private native void nativePut(long streamAddress);
 
+  /**
+   * Delete rows from the MemWAL by primary key.
+   *
+   * <p>Each batch in {@code reader} must carry this shard's primary key column(s); other columns
+   * are ignored. Lance builds a tombstone row per key — the primary key plus {@code _tombstone =
+   * true} and null in every other column — and appends it like an ordinary write. The tombstone is
+   * the newest value for its key: it wins newest-per-PK resolution (suppressing the older real row)
+   * and is then dropped from query results.
+   *
+   * <p>Only supported in memtable mode. Because a tombstone nulls every non-PK column, those
+   * columns must be nullable in the base schema; deleting against a schema with a non-nullable
+   * non-PK column errors. Deleting on a shard with no primary key columns also errors.
+   *
+   * @param reader the keys to delete; consumed fully by this call
+   */
+  public void delete(ArrowReader reader) {
+    Preconditions.checkNotNull(reader, "reader must not be null");
+    try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
+      Preconditions.checkArgument(nativeShardWriterHandle != 0, "ShardWriter is closed");
+      try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) {
+        Data.exportArrayStream(allocator, reader, stream);
+        nativeDelete(stream.memoryAddress());
+      }
+    }
+  }
+
+  private native void nativeDelete(long streamAddress);
+
   /** Return a snapshot of cumulative write statistics. */
   public WriteStats stats() {
     try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) {
diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java
index de40c9e4f1c..2ae27b67cba 100644
--- a/java/src/main/java/org/lance/merge/MergeInsertParams.java
+++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java
@@ -38,6 +38,7 @@ public class MergeInsertParams {
   private int conflictRetries = 10;
   private long retryTimeoutMs = 30 * 1000;
   private boolean skipAutoCleanup = false;
+  private boolean useIndex = true;
   private List<MergedGeneration> markedGenerations = Collections.emptyList();
 
   public MergeInsertParams(List<String> on) {
@@ -227,6 +228,22 @@ public MergeInsertParams withSkipAutoCleanup(boolean skipAutoCleanup) {
     return this;
   }
 
+  /**
+   * Controls whether to use indices for the merge operation.
+   *
+   * <p>When set to false, forces a full table scan even if an index exists on the join key. This
+   * can be useful for benchmarking or when the optimizer chooses a suboptimal path.
+   *
+   * <p>Default is true (use index if available).
+   *
+   * @param useIndex Whether to use indices for the merge join
+   * @return This MergeInsertParams instance
+   */
+  public MergeInsertParams withUseIndex(boolean useIndex) {
+    this.useIndex = useIndex;
+    return this;
+  }
+
   /**
    * Mark MemWAL generations as merged into the base table.
    *
@@ -298,6 +315,10 @@ public boolean skipAutoCleanup() {
     return skipAutoCleanup;
   }
 
+  public boolean useIndex() {
+    return useIndex;
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
@@ -315,6 +336,7 @@ public String toString() {
         .add("conflictRetries", conflictRetries)
         .add("retryTimeoutMs", retryTimeoutMs)
         .add("skipAutoCleanup", skipAutoCleanup)
+        .add("useIndex", useIndex)
         .toString();
   }
 
diff --git a/java/src/main/java/org/lance/schema/LanceField.java b/java/src/main/java/org/lance/schema/LanceField.java
index 9c7014092fa..4dbb3a0ea38 100644
--- a/java/src/main/java/org/lance/schema/LanceField.java
+++ b/java/src/main/java/org/lance/schema/LanceField.java
@@ -25,6 +25,7 @@
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -156,6 +157,25 @@ public Field asArrowField() {
         name, new FieldType(nullable, type, dictionaryEncoding, metadata), arrowChildren);
   }
 
+  Field asArrowFieldWithFieldIds() {
+    List<Field> arrowChildren =
+        children.stream().map(LanceField::asArrowFieldWithFieldIds).collect(Collectors.toList());
+
+    if (type instanceof ArrowType.FixedSizeList) {
+      arrowChildren.addAll(childrenForFixedSizeList());
+    }
+
+    if (id < 0) {
+      throw new IllegalStateException("Lance field id is required for schema override: " + name);
+    }
+    Map<String, String> metadataWithFieldId = new HashMap<>(metadata);
+    metadataWithFieldId.put(LanceSchema.LANCE_FIELD_ID_KEY, Integer.toString(id));
+    return new Field(
+        name,
+        new FieldType(nullable, type, dictionaryEncoding, metadataWithFieldId),
+        arrowChildren);
+  }
+
   private List<Field> childrenForFixedSizeList() {
     if (logicalType == null || logicalType.isEmpty()) {
       return Collections.emptyList();
diff --git a/java/src/main/java/org/lance/schema/LanceSchema.java b/java/src/main/java/org/lance/schema/LanceSchema.java
index 9492ef45d5e..50a48e578af 100644
--- a/java/src/main/java/org/lance/schema/LanceSchema.java
+++ b/java/src/main/java/org/lance/schema/LanceSchema.java
@@ -23,6 +23,7 @@
 import java.util.stream.Collectors;
 
 public class LanceSchema {
+  static final String LANCE_FIELD_ID_KEY = "lance:field_id";
   private final List<LanceField> fields;
   private final Map<String, String> metadata;
 
@@ -68,6 +69,12 @@ public Schema asArrowSchema() {
         fields.stream().map(LanceField::asArrowField).collect(Collectors.toList()), metadata);
   }
 
+  public Schema asArrowSchemaWithFieldIds() {
+    return new Schema(
+        fields.stream().map(LanceField::asArrowFieldWithFieldIds).collect(Collectors.toList()),
+        metadata);
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
diff --git a/java/src/test/java/org/lance/AsyncScannerTest.java b/java/src/test/java/org/lance/AsyncScannerTest.java
index 578bf000755..fc786ff57c2 100644
--- a/java/src/test/java/org/lance/AsyncScannerTest.java
+++ b/java/src/test/java/org/lance/AsyncScannerTest.java
@@ -192,6 +192,75 @@ private static int countRows(ArrowReader reader) throws Exception {
     return rowCount;
   }
 
+  @Test
+  void testIncludeDeletedRowsAsync(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_include_deleted_rows").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      try (Dataset dataset = testDataset.write(1, 10)) {
+        assertEquals(10, dataset.countRows());
+
+        // Delete half the rows
+        dataset.delete("id >= 5");
+        assertEquals(5, dataset.countRows());
+
+        // Async scan without includeDeletedRows — should only see live rows
+        ScanOptions defaultOptions = new ScanOptions.Builder().batchSize(20L).build();
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, defaultOptions, allocator)) {
+          ArrowReader reader = scanner.scanBatchesAsync().get(10, TimeUnit.SECONDS);
+          assertEquals(5, countRows(reader), "default async scan: should exclude deleted rows");
+          reader.close();
+        }
+
+        // Async scan with includeDeletedRows=true — should see all rows
+        ScanOptions includeDeletedOptions =
+            new ScanOptions.Builder()
+                .batchSize(20L)
+                .withRowId(true) // required by includeDeletedRows
+                .includeDeletedRows(true)
+                .build();
+        try (AsyncScanner scanner =
+            AsyncScanner.create(dataset, includeDeletedOptions, allocator)) {
+          ArrowReader reader = scanner.scanBatchesAsync().get(10, TimeUnit.SECONDS);
+          assertEquals(
+              10, countRows(reader), "includeDeletedRows async: should include deleted rows");
+          reader.close();
+        }
+      }
+    }
+  }
+
+  @Test
+  void testStrictBatchSizeAsync(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("async_scanner_strict_batch_size").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      try (Dataset dataset = testDataset.write(1, 25)) {
+        int batchSize = 10;
+
+        ScanOptions strictOptions =
+            new ScanOptions.Builder().batchSize(batchSize).strictBatchSize(true).build();
+
+        try (AsyncScanner scanner = AsyncScanner.create(dataset, strictOptions, allocator)) {
+          ArrowReader reader = scanner.scanBatchesAsync().get(10, TimeUnit.SECONDS);
+          int totalRows = 0;
+          while (reader.loadNextBatch()) {
+            int rows = reader.getVectorSchemaRoot().getRowCount();
+            assertTrue(
+                rows <= batchSize, "strict async: batch " + rows + " should be <= " + batchSize);
+            totalRows += rows;
+          }
+          assertEquals(25, totalRows, "strictBatchSize async: should read all rows");
+          reader.close();
+        }
+      }
+    }
+  }
+
   /**
    * Example 3: Multiple concurrent async scans.
    *
diff --git a/java/src/test/java/org/lance/CleanupTest.java b/java/src/test/java/org/lance/CleanupTest.java
index f287f1d0f0a..5fc8ceeaa3f 100644
--- a/java/src/test/java/org/lance/CleanupTest.java
+++ b/java/src/test/java/org/lance/CleanupTest.java
@@ -13,6 +13,7 @@
  */
 package org.lance;
 
+import org.lance.cleanup.CleanupExplanation;
 import org.lance.cleanup.CleanupPolicy;
 import org.lance.cleanup.RemovalStats;
 
@@ -53,6 +54,39 @@ public void testCleanupBeforeVersion(@TempDir Path tempDir) {
     }
   }
 
+  @Test
+  public void testExplainCleanupBeforeVersion(@TempDir Path tempDir) {
+    String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+
+      testDataset.createEmptyDataset().close();
+
+      testDataset.write(1, 10).close();
+      testDataset.write(2, 10).close();
+
+      try (Dataset dataset = testDataset.write(3, 10)) {
+        CleanupPolicy policy = CleanupPolicy.builder().withBeforeVersion(3L).build();
+        CleanupOperation cleanup = dataset.cleanup(policy);
+        CleanupExplanation explanation = cleanup.explain();
+
+        assertEquals(2L, explanation.getStats().getOldVersions());
+        assertEquals(2L, explanation.getStats().getTransactionFilesRemoved());
+        assertTrue(explanation.getStats().getBytesRemoved() > 0);
+        assertTrue(explanation.getReadVersion() > 0);
+        assertTrue(explanation.getCandidateFiles().size() > 0);
+        assertTrue(explanation.getReferencedBranches().isEmpty());
+
+        List<Version> versions = dataset.listVersions();
+        assertEquals(4, versions.size());
+
+        RemovalStats stats = cleanup.execute();
+        assertEquals(explanation.getStats().getOldVersions(), stats.getOldVersions());
+      }
+    }
+  }
+
   @Test
   public void testCleanupBeforeTimestamp(@TempDir Path tempDir) throws Exception {
     String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString();
@@ -118,6 +152,39 @@ public void testCleanupTaggedVersion(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  public void testExplainCleanupWithMaxCandidateFiles(@TempDir Path tempDir) {
+    String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+
+      testDataset.createEmptyDataset().close();
+
+      testDataset.write(1, 10).close();
+      testDataset.write(2, 10).close();
+
+      try (Dataset dataset = testDataset.write(3, 10)) {
+        CleanupPolicy policy = CleanupPolicy.builder().withBeforeVersion(3L).build();
+        CleanupExplanation full = dataset.cleanup(policy).explain();
+        assertTrue(full.getCandidateFiles().size() > 1);
+        assertEquals(1000L, full.getCandidateFileLimit());
+
+        CleanupExplanation truncated = dataset.cleanup(policy).withMaxCandidateFiles(1L).explain();
+        assertEquals(1L, truncated.getCandidateFileLimit());
+        assertEquals(1, truncated.getCandidateFiles().size());
+        assertTrue(truncated.isCandidateFilesTruncated());
+        assertTrue(!truncated.getWarnings().isEmpty());
+        // Aggregate stats stay accurate even when the per-file list is truncated.
+        assertEquals(full.getStats().getOldVersions(), truncated.getStats().getOldVersions());
+
+        Assertions.assertThrows(
+            IllegalArgumentException.class,
+            () -> dataset.cleanup(policy).withMaxCandidateFiles(0L));
+      }
+    }
+  }
+
   @Test
   public void testCleanupWithRateLimit(@TempDir Path tempDir) throws Exception {
     String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString();
diff --git a/java/src/test/java/org/lance/DatasetTest.java b/java/src/test/java/org/lance/DatasetTest.java
index 3ea6a0812e1..45466a0367c 100644
--- a/java/src/test/java/org/lance/DatasetTest.java
+++ b/java/src/test/java/org/lance/DatasetTest.java
@@ -1993,18 +1993,20 @@ void testOptimizingIndices(@TempDir Path tempDir) throws Exception {
           OptimizeOptions options = OptimizeOptions.builder().numIndicesToMerge(0).build();
           dsAppended.optimizeIndices(options);
 
-          List<Index> afterIndexes = dsAppended.getIndexes();
-          Index idIndexAfter =
-              afterIndexes.stream()
+          List<Index> idIndexes =
+              dsAppended.getIndexes().stream()
                   .filter(idx -> "id_idx".equals(idx.name()))
-                  .findFirst()
-                  .orElse(null);
-          assertNotNull(idIndexAfter);
-          List<Integer> afterFragments = idIndexAfter.fragments().orElse(Collections.emptyList());
-
-          assertTrue(afterFragments.contains(0));
-          assertTrue(afterFragments.contains(1));
-          assertEquals(2, afterFragments.size());
+                  .collect(Collectors.toList());
+          assertEquals(
+              2,
+              idIndexes.size(),
+              "append-only optimize must add a delta segment instead of merging");
+
+          Set<Integer> coveredFragments =
+              idIndexes.stream()
+                  .flatMap(idx -> idx.fragments().orElse(Collections.emptyList()).stream())
+                  .collect(Collectors.toSet());
+          assertEquals(new HashSet<>(Arrays.asList(0, 1)), coveredFragments);
         }
       }
     }
diff --git a/java/src/test/java/org/lance/FileReaderWriterTest.java b/java/src/test/java/org/lance/FileReaderWriterTest.java
index a849a87c576..85c7430f087 100644
--- a/java/src/test/java/org/lance/FileReaderWriterTest.java
+++ b/java/src/test/java/org/lance/FileReaderWriterTest.java
@@ -256,7 +256,9 @@ void testInvalidPath() {
       LanceFileReader.open("/tmp/does_not_exist.lance", allocator);
       fail("Expected LanceException to be thrown");
     } catch (IOException e) {
-      assertTrue(e.getMessage().contains("Object at location /tmp/does_not_exist.lance not found"));
+      String message = e.getMessage();
+      assertTrue(message.contains("/tmp/does_not_exist.lance"));
+      assertTrue(message.toLowerCase().contains("not found"));
     }
     try {
       LanceFileReader.open("", allocator);
diff --git a/java/src/test/java/org/lance/FragmentTest.java b/java/src/test/java/org/lance/FragmentTest.java
index 61bfc439290..29a21b5258a 100644
--- a/java/src/test/java/org/lance/FragmentTest.java
+++ b/java/src/test/java/org/lance/FragmentTest.java
@@ -17,9 +17,12 @@
 import org.lance.ipc.LanceScanner;
 import org.lance.ipc.ScanOptions;
 import org.lance.operation.Merge;
+import org.lance.operation.Project;
 import org.lance.operation.Update;
+import org.lance.schema.LanceField;
 
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -29,6 +32,7 @@
 import org.junit.jupiter.api.io.TempDir;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -37,6 +41,7 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -81,6 +86,70 @@ void testFragmentCreate(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  void testWriteFragmentWithSchemaOverride(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("fragment_schema_override").toString();
+    try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      try (Dataset dataset = testDataset.createEmptyDataset()) {
+        List<org.apache.arrow.vector.types.pojo.Field> fieldList =
+            new ArrayList<>(testDataset.getSchema().getFields());
+        Collections.reverse(fieldList);
+
+        try (Transaction projectTxn =
+                new Transaction.Builder()
+                    .readVersion(dataset.version())
+                    .operation(Project.builder().schema(new Schema(fieldList)).build())
+                    .build();
+            Dataset evolvedDataset = new CommitBuilder(dataset).execute(projectTxn);
+            VectorSchemaRoot root =
+                VectorSchemaRoot.create(evolvedDataset.getSchema(), allocator)) {
+          root.allocateNew();
+          VarCharVector nameVector = (VarCharVector) root.getVector("name");
+          IntVector idVector = (IntVector) root.getVector("id");
+          nameVector.setSafe(0, "Person 1".getBytes(StandardCharsets.UTF_8));
+          idVector.setSafe(0, 1);
+          root.setRowCount(1);
+
+          List<FragmentMetadata> fragments =
+              Fragment.write()
+                  .datasetUri(datasetPath)
+                  .allocator(allocator)
+                  .data(root)
+                  .schema(evolvedDataset.getLanceSchema())
+                  .mode(WriteParams.WriteMode.APPEND)
+                  .execute();
+
+          assertEquals(1, fragments.size());
+          assertEquals(1, fragments.get(0).getPhysicalRows());
+          assertArrayEquals(
+              evolvedDataset.getLanceSchema().fields().stream()
+                  .mapToInt(LanceField::getId)
+                  .toArray(),
+              fragments.get(0).getFiles().get(0).getFields());
+
+          FragmentOperation.Append appendOp = new FragmentOperation.Append(fragments);
+          try (Dataset appendedDataset =
+                  Dataset.commit(
+                      allocator, datasetPath, appendOp, Optional.of(evolvedDataset.version()));
+              ArrowReader reader = appendedDataset.newScan().scanBatches()) {
+            assertEquals(3, appendedDataset.version());
+            assertEquals(1, appendedDataset.countRows());
+            assertTrue(reader.loadNextBatch());
+            VectorSchemaRoot batch = reader.getVectorSchemaRoot();
+            assertEquals(1, batch.getRowCount());
+            assertEquals(
+                "Person 1",
+                new String(
+                    ((VarCharVector) batch.getVector("name")).get(0), StandardCharsets.UTF_8));
+            assertEquals(1, ((IntVector) batch.getVector("id")).get(0));
+          }
+        }
+      }
+    }
+  }
+
   @Test
   void commitWithoutVersion(@TempDir Path tempDir) {
     String datasetPath = tempDir.resolve("commit_without_version").toString();
diff --git a/java/src/test/java/org/lance/JNITest.java b/java/src/test/java/org/lance/JNITest.java
index c0e5f900edc..daa123b3200 100644
--- a/java/src/test/java/org/lance/JNITest.java
+++ b/java/src/test/java/org/lance/JNITest.java
@@ -20,6 +20,7 @@
 import org.lance.index.vector.PQBuildParams;
 import org.lance.index.vector.SQBuildParams;
 import org.lance.index.vector.VectorIndexParams;
+import org.lance.ipc.ApproxMode;
 import org.lance.ipc.Query;
 import org.lance.test.JniTestHelper;
 
@@ -28,6 +29,7 @@
 import java.util.Arrays;
 import java.util.Optional;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class JNITest {
@@ -48,6 +50,10 @@ public void testIntsOpt() {
 
   @Test
   public void testQuery() {
+    Query defaultQuery =
+        new Query.Builder().setColumn("column").setKey(new float[] {1.0f, 2.0f, 3.0f}).build();
+    assertEquals(ApproxMode.NORMAL, defaultQuery.getApproxMode());
+
     JniTestHelper.parseQuery(
         Optional.of(
             new Query.Builder()
@@ -60,6 +66,7 @@ public void testQuery() {
                 .setDistanceType(DistanceType.L2)
                 .setUseIndex(true)
                 .setQueryParallelism(-1)
+                .setApproxMode(ApproxMode.ACCURATE)
                 .build()));
   }
 
diff --git a/java/src/test/java/org/lance/MergeInsertTest.java b/java/src/test/java/org/lance/MergeInsertTest.java
index c36ec26b4fa..b738ef8852d 100644
--- a/java/src/test/java/org/lance/MergeInsertTest.java
+++ b/java/src/test/java/org/lance/MergeInsertTest.java
@@ -275,6 +275,29 @@ private ArrowArrayStream convertToStream(VectorSchemaRoot root, RootAllocator al
     return stream;
   }
 
+  @Test
+  public void testMergeInsertWithoutIndex() throws Exception {
+    // Verify that merge insert with useIndex=false still completes and
+    // produces results consistent with the default (useIndex=true).
+
+    try (VectorSchemaRoot source = buildSource(testDataset.getSchema(), allocator)) {
+      try (ArrowArrayStream sourceStream = convertToStream(source, allocator)) {
+        MergeInsertResult result =
+            dataset.mergeInsert(
+                new MergeInsertParams(Collections.singletonList("id"))
+                    .withMatchedUpdateAll()
+                    .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll)
+                    .withUseIndex(false),
+                sourceStream);
+
+        Assertions.assertEquals(
+            "{0=Source 0, 1=Source 1, 2=Source 2, 3=Person 3, 4=Person 4, 7=Source 7, 8=Source 8, 9=Source 9}",
+            readAll(result.dataset()).toString(),
+            "merge insert with useIndex=false should produce correct upsert results");
+      }
+    }
+  }
+
   private TreeMap<Integer, String> readAll(Dataset dataset) throws Exception {
     try (ArrowReader reader = dataset.newScan().scanBatches()) {
       TreeMap<Integer, String> map = new TreeMap<>();
diff --git a/java/src/test/java/org/lance/ScannerTest.java b/java/src/test/java/org/lance/ScannerTest.java
index 894b208e8af..0b07026bc68 100644
--- a/java/src/test/java/org/lance/ScannerTest.java
+++ b/java/src/test/java/org/lance/ScannerTest.java
@@ -422,25 +422,32 @@ void testDatasetScannerBatchReadahead(@TempDir Path tempDir) throws Exception {
       TestUtils.SimpleTestDataset testDataset =
           new TestUtils.SimpleTestDataset(allocator, datasetPath);
       testDataset.createEmptyDataset().close();
-      int totalRows = 1000;
-      int batchSize = 100;
-      int batchReadahead = 5;
-      try (Dataset dataset = testDataset.write(1, totalRows)) {
+
+      int totalRows = 2000;
+      int maxRowsPerFile = 100; // ~20 fragments
+      List<FragmentMetadata> fragments = testDataset.createNewFragment(totalRows, maxRowsPerFile);
+      assertTrue(fragments.size() > 1, "expected multiple fragments, got " + fragments.size());
+
+      FragmentOperation.Append append = new FragmentOperation.Append(fragments);
+      try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) {
+        int batchReadahead = 2; // far below the default (num compute CPUs)
         try (LanceScanner scanner =
             dataset.newScan(
-                new ScanOptions.Builder()
-                    .batchSize(batchSize)
-                    .batchReadahead(batchReadahead)
-                    .build())) {
-          // This test is more about ensuring that the batchReadahead parameter is accepted
-          // and doesn't cause errors. The actual effect of batchReadahead might not be
-          // directly observable in this test.
+                new ScanOptions.Builder().batchSize(50).batchReadahead(batchReadahead).build())) {
           try (ArrowReader reader = scanner.scanBatches()) {
             int rowCount = 0;
+            long idSum = 0;
             while (reader.loadNextBatch()) {
-              rowCount += reader.getVectorSchemaRoot().getRowCount();
+              VectorSchemaRoot root = reader.getVectorSchemaRoot();
+              IntVector ids = (IntVector) root.getVector("id");
+              for (int i = 0; i < root.getRowCount(); i++) {
+                idSum += ids.get(i);
+              }
+              rowCount += root.getRowCount();
             }
             assertEquals(totalRows, rowCount);
+            // ids are the contiguous range [0, totalRows)
+            assertEquals((long) totalRows * (totalRows - 1) / 2, idSum);
           }
         }
       }
@@ -697,6 +704,120 @@ void testFastSearchSkipsUnindexedFragments(@TempDir Path tempDir) throws Excepti
     }
   }
 
+  @Test
+  void testIncludeDeletedRows(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("include_deleted_rows").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      try (Dataset dataset = testDataset.write(1, 10)) {
+        assertEquals(10, dataset.countRows());
+
+        // Delete rows where id >= 5
+        dataset.delete("id >= 5");
+        assertEquals(5, dataset.countRows());
+
+        // Default scan should exclude deleted rows
+        try (LanceScanner scanner =
+            dataset.newScan(new ScanOptions.Builder().batchSize(20).build())) {
+          assertEquals(5, scanner.countRows(), "default scan: should exclude deleted rows");
+        }
+
+        // includeDeletedRows=true should surface deleted rows
+        // NOTE: includeDeletedRows requires withRowId=true
+        try (LanceScanner scanner =
+            dataset.newScan(
+                new ScanOptions.Builder()
+                    .batchSize(20)
+                    .withRowId(true)
+                    .includeDeletedRows(true)
+                    .build())) {
+          assertEquals(10, scanner.countRows(), "includeDeletedRows: should include deleted rows");
+        }
+      }
+    }
+  }
+
+  @Test
+  void testStrictBatchSize(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("strict_batch_size").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      try (Dataset dataset = testDataset.write(1, 25)) {
+        int batchSize = 10;
+
+        // With strictBatchSize=true, no batch should exceed batchSize
+        try (Scanner scanner =
+            dataset.newScan(
+                new ScanOptions.Builder().batchSize(batchSize).strictBatchSize(true).build())) {
+          try (ArrowReader reader = scanner.scanBatches()) {
+            int totalRows = 0;
+            while (reader.loadNextBatch()) {
+              int rows = reader.getVectorSchemaRoot().getRowCount();
+              assertTrue(rows <= batchSize, "strict: batch " + rows + " should be <= " + batchSize);
+              totalRows += rows;
+            }
+            assertEquals(25, totalRows);
+          }
+        }
+
+        // strictBatchSize=false (default) — batch size may vary
+        try (Scanner scanner =
+            dataset.newScan(new ScanOptions.Builder().batchSize(batchSize).build())) {
+          try (ArrowReader reader = scanner.scanBatches()) {
+            int totalRows = 0;
+            while (reader.loadNextBatch()) {
+              totalRows += reader.getVectorSchemaRoot().getRowCount();
+            }
+            assertEquals(25, totalRows);
+          }
+        }
+      }
+    }
+  }
+
+  @Test
+  void testDisableScoringAutoprojection(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("disable_scoring_autoprojection").toString();
+    try (BufferAllocator allocator = new RootAllocator()) {
+      TestUtils.SimpleTestDataset testDataset =
+          new TestUtils.SimpleTestDataset(allocator, datasetPath);
+      testDataset.createEmptyDataset().close();
+      try (Dataset dataset = testDataset.write(1, 10)) {
+        // Smoke test: verify the option is accepted and scan still works
+        ScanOptions options =
+            new ScanOptions.Builder().batchSize(20).disableScoringAutoprojection(true).build();
+
+        try (LanceScanner scanner = dataset.newScan(options)) {
+          assertEquals(
+              10,
+              scanner.countRows(),
+              "scan with disableScoringAutoprojection should return all rows");
+        }
+
+        // Also verify it doesn't break when combined with other options
+        ScanOptions combinedOptions =
+            new ScanOptions.Builder()
+                .batchSize(20)
+                .filter("id < 5")
+                .disableScoringAutoprojection(true)
+                .includeDeletedRows(false)
+                .strictBatchSize(false)
+                .build();
+
+        try (LanceScanner scanner = dataset.newScan(combinedOptions)) {
+          assertEquals(
+              5,
+              scanner.countRows(),
+              "scan with disableScoringAutoprojection + filter should work");
+        }
+      }
+    }
+  }
+
   private void validScanResult(Dataset dataset, int fragmentId, int rowCount) throws Exception {
     try (Scanner scanner =
         dataset.newScan(
diff --git a/java/src/test/java/org/lance/index/ScalarIndexTest.java b/java/src/test/java/org/lance/index/ScalarIndexTest.java
index b993a7e8a5f..cb090e7c955 100644
--- a/java/src/test/java/org/lance/index/ScalarIndexTest.java
+++ b/java/src/test/java/org/lance/index/ScalarIndexTest.java
@@ -25,14 +25,18 @@
 import org.apache.arrow.c.Data;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.UInt8Vector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.StructVector;
 import org.apache.arrow.vector.ipc.ArrowReader;
 import org.apache.arrow.vector.ipc.ArrowStreamReader;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.types.pojo.Schema;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -318,4 +322,78 @@ public void testCreateZonemapIndex(@TempDir Path tempDir) throws Exception {
       }
     }
   }
+
+  @Test
+  public void testCreateRTreeIndex(@TempDir Path tempDir) throws Exception {
+    String datasetPath = tempDir.resolve("rtree_test").toString();
+    ArrowType f64 = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
+    Field geometryField =
+        new Field(
+            "geometry",
+            new FieldType(
+                true,
+                new ArrowType.Struct(),
+                null,
+                Collections.singletonMap("ARROW:extension:name", "geoarrow.point")),
+            Arrays.asList(Field.notNullable("x", f64), Field.notNullable("y", f64)));
+    Schema schema = new Schema(Collections.singletonList(geometryField), null);
+
+    int rowCount = 3;
+    try (RootAllocator allocator = new RootAllocator();
+        VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+      root.allocateNew();
+      StructVector geometry = (StructVector) root.getVector("geometry");
+      Float8Vector x = (Float8Vector) geometry.getChild("x");
+      Float8Vector y = (Float8Vector) geometry.getChild("y");
+      for (int i = 0; i < rowCount; i++) {
+        geometry.setIndexDefined(i);
+        x.setSafe(i, (double) i);
+        y.setSafe(i, i * 2.0);
+      }
+      geometry.setValueCount(rowCount);
+      root.setRowCount(rowCount);
+
+      ByteArrayOutputStream out = new ByteArrayOutputStream();
+      try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) {
+        writer.start();
+        writer.writeBatch();
+        writer.end();
+      }
+
+      try (ArrowStreamReader reader =
+              new ArrowStreamReader(new ByteArrayInputStream(out.toByteArray()), allocator);
+          Dataset dataset =
+              Dataset.write()
+                  .reader(reader)
+                  .uri(datasetPath)
+                  .allocator(allocator)
+                  .mode(WriteParams.WriteMode.CREATE)
+                  .execute()) {
+        // The point data round-trips through Lance.
+        assertEquals(rowCount, dataset.countRows());
+        try (ArrowReader scan = dataset.newScan(new ScanOptions.Builder().build()).scanBatches()) {
+          assertTrue(scan.loadNextBatch());
+          StructVector readGeometry =
+              (StructVector) scan.getVectorSchemaRoot().getVector("geometry");
+          assertEquals(2.0, ((Float8Vector) readGeometry.getChild("x")).get(2));
+          assertEquals(4.0, ((Float8Vector) readGeometry.getChild("y")).get(2));
+        }
+
+        // Creating and listing an RTree index via the typed IndexType works end-to-end.
+        Index index =
+            dataset.createIndex(
+                Collections.singletonList("geometry"),
+                IndexType.RTREE,
+                Optional.of("rtree_geometry_index"),
+                IndexParams.builder()
+                    .setScalarIndexParams(ScalarIndexParams.create("rtree"))
+                    .build(),
+                true);
+        assertEquals(IndexType.RTREE, index.indexType());
+        assertTrue(
+            dataset.listIndexes().contains("rtree_geometry_index"),
+            "Expected 'rtree_geometry_index' in: " + dataset.listIndexes());
+      }
+    }
+  }
 }
diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
new file mode 100644
index 00000000000..0ccc429ec57
--- /dev/null
+++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lance.index.scalar;
+
+import org.lance.util.JsonUtils;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class InvertedIndexParamsTest {
+
+  @Test
+  void testIcuSplitTokenizerVariant() {
+    ScalarIndexParams params = InvertedIndexParams.builder().baseTokenizer("icu/split").build();
+
+    assertEquals("inverted", params.getIndexType());
+    String jsonParams = params.getJsonParams().orElseThrow(AssertionError::new);
+    assertTrue(jsonParams.contains("\"base_tokenizer\":\"icu/split\""));
+  }
+
+  @Test
+  void defaultBlockSizeIsSerialized() {
+    ScalarIndexParams params = InvertedIndexParams.builder().build();
+
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(128, ((Number) json.get("block_size")).intValue());
+  }
+
+  @Test
+  void blockSizeIsSerialized() {
+    ScalarIndexParams params = InvertedIndexParams.builder().blockSize(128).build();
+
+    assertEquals("inverted", params.getIndexType());
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(128, ((Number) json.get("block_size")).intValue());
+  }
+
+  @Test
+  void invalidBlockSizeIsRejected() {
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(129));
+    assertThrows(
+        IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(512));
+  }
+
+  @Test
+  void formatVersionThreeRequiresBlockSize256() {
+    ScalarIndexParams params =
+        InvertedIndexParams.builder().blockSize(256).formatVersion(3).build();
+
+    Map<String, Object> json = JsonUtils.fromJson(params.getJsonParams().orElseThrow());
+    assertEquals(256, ((Number) json.get("block_size")).intValue());
+    assertEquals(3, ((Number) json.get("format_version")).intValue());
+
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> InvertedIndexParams.builder().formatVersion(3).build());
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build());
+  }
+}
diff --git a/java/src/test/java/org/lance/memwal/MemWalTest.java b/java/src/test/java/org/lance/memwal/MemWalTest.java
index ee26932dd59..57d0b5b81ad 100644
--- a/java/src/test/java/org/lance/memwal/MemWalTest.java
+++ b/java/src/test/java/org/lance/memwal/MemWalTest.java
@@ -50,6 +50,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
@@ -96,6 +97,22 @@ private static VectorSchemaRoot lookupRoot(BufferAllocator allocator, long[] ids
     return root;
   }
 
+  /** Build a single-batch root carrying only the {@code id} primary key, for deletes. */
+  private static VectorSchemaRoot keysRoot(BufferAllocator allocator, long[] ids) {
+    VectorSchemaRoot root =
+        VectorSchemaRoot.create(
+            new Schema(
+                Collections.singletonList(Field.nullable("id", new ArrowType.Int(64, true)))),
+            allocator);
+    BigIntVector idVector = (BigIntVector) root.getVector("id");
+    idVector.allocateNew(ids.length);
+    for (int i = 0; i < ids.length; i++) {
+      idVector.set(i, ids[i]);
+    }
+    root.setRowCount(ids.length);
+    return root;
+  }
+
   /** Build a single-batch append-only root without primary-key metadata. */
   private static VectorSchemaRoot appendOnlyRoot(
       BufferAllocator allocator, long[] ids, String prefix) {
@@ -142,6 +159,30 @@ private static Dataset writeAppendOnlyDataset(
     }
   }
 
+  /**
+   * Stage a <em>faithful</em> flushed generation at {@code genPath}: the Lance dataset plus its
+   * primary-key dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM
+   * scanner's cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not
+   * a state production produces. Mirrors the Python {@code _write_flushed_gen} test helper.
+   */
+  private static void writeFlushedGen(
+      BufferAllocator allocator, String genPath, long[] ids, String prefix) throws Exception {
+    writeLookupDataset(allocator, genPath, ids, prefix).close();
+    try (VectorSchemaRoot root = lookupRoot(allocator, ids, prefix);
+        ArrowReader reader = toReader(allocator, root);
+        ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) {
+      Data.exportArrayStream(allocator, reader, stream);
+      nativeWritePkSidecar(genPath, stream.memoryAddress(), Collections.singletonList("id"));
+    }
+  }
+
+  /**
+   * Test-support native: write the primary-key dedup sidecar for a flushed-generation dataset
+   * already staged at {@code genPath}. See {@link #writeFlushedGen}.
+   */
+  private static native void nativeWritePkSidecar(
+      String genPath, long streamAddress, List<String> pkColumns);
+
   /** Read an LSM scanner fully into an {@code id -> name} map. */
   private static Map<Long, String> readByName(ArrowReader reader) throws Exception {
     Map<Long, String> byId = new HashMap<>();
@@ -357,6 +398,52 @@ void testShardWriterPutAndLsmScanner(@TempDir Path tempDir) throws Exception {
     }
   }
 
+  @Test
+  void testShardWriterDeleteMasksBaseRow(@TempDir Path tempDir) throws Exception {
+    String path = tempDir.resolve("base").toString();
+    String shardId = UUID.randomUUID().toString();
+    try (BufferAllocator allocator = new RootAllocator();
+        Dataset dataset = writeLookupDataset(allocator, path, new long[] {1, 2, 3}, "base")) {
+      dataset.initializeMemWal(new InitializeMemWalParams());
+
+      ShardWriterConfig config =
+          new ShardWriterConfig()
+              .withDurableWrite(true)
+              .withSyncIndexedWrite(true)
+              .withMaxWalBufferSize(1)
+              .withMaxWalFlushIntervalMs(10);
+
+      try (ShardWriter writer = dataset.memWalWriter(shardId, config)) {
+        try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {4}, "writer");
+            ArrowReader reader = toReader(allocator, root)) {
+          writer.put(reader);
+        }
+        try (VectorSchemaRoot keys = keysRoot(allocator, new long[] {2});
+            ArrowReader reader = toReader(allocator, keys)) {
+          writer.delete(reader);
+        }
+
+        Map<Long, String> byId = Collections.emptyMap();
+        long deadline = System.currentTimeMillis() + 10_000;
+        while (System.currentTimeMillis() < deadline) {
+          try (LsmScanner scanner = writer.lsmScanner();
+              ArrowReader reader = scanner.scanBatches()) {
+            byId = readByName(reader);
+          }
+          if (!byId.containsKey(2L) && "writer_4".equals(byId.get(4L))) {
+            break;
+          }
+          Thread.sleep(50);
+        }
+
+        assertEquals("base_1", byId.get(1L));
+        assertFalse(byId.containsKey(2L), "deleted base row should be masked by the tombstone");
+        assertEquals("base_3", byId.get(3L));
+        assertEquals("writer_4", byId.get(4L));
+      }
+    }
+  }
+
   @Test
   void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
     String basePath = tempDir.resolve("base").toString();
@@ -367,7 +454,7 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
 
       // Flushed generation overwrites id=2.
       String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1";
-      writeLookupDataset(allocator, genPath, new long[] {2}, "gen1").close();
+      writeFlushedGen(allocator, genPath, new long[] {2}, "gen1");
 
       ShardSnapshot snapshot =
           new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2);
@@ -381,6 +468,14 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception {
         assertEquals("gen1_2", byId.get(2L), "Flushed generation must win over base");
         assertEquals("base_3", byId.get(3L));
       }
+
+      try (LsmScanner scanner =
+              LsmScanner.fromSnapshots(dataset, Collections.singletonList(snapshot))
+                  .limit(Optional.empty(), Optional.of(1L));
+          ArrowReader reader = scanner.scanBatches()) {
+        Map<Long, String> byId = readByName(reader);
+        assertEquals(2, byId.size(), "Offset-only LSM scan should not require a limit");
+      }
     }
   }
 
@@ -393,7 +488,7 @@ void testPointLookup(@TempDir Path tempDir) throws Exception {
       dataset.initializeMemWal(new InitializeMemWalParams());
 
       String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1";
-      writeLookupDataset(allocator, genPath, new long[] {2}, "gen1").close();
+      writeFlushedGen(allocator, genPath, new long[] {2}, "gen1");
 
       ShardSnapshot snapshot =
           new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2);
@@ -545,6 +640,40 @@ void testVectorSearch(@TempDir Path tempDir) throws Exception {
           double recall = (double) found / queryIds.length;
           assertTrue(recall >= 0.5, "vector search recall too low: " + recall);
         }
+
+        try (LsmVectorSearchPlanner planner =
+                new LsmVectorSearchPlanner(
+                    dataset, Collections.emptyList(), "vec", null, null, "id >= 200");
+            Float4Vector query = new Float4Vector("q", allocator)) {
+          int filteredQueryId = 250;
+          query.allocateNew(VDIM);
+          for (int d = 0; d < VDIM; d++) {
+            query.set(d, (float) (filteredQueryId * VDIM + d));
+          }
+          query.setValueCount(VDIM);
+
+          int filteredRows = 0;
+          boolean foundFilteredNearest = false;
+          try (ExecutionPlan plan =
+                  planner.planSearch(query, 20, 2, Collections.singletonList("id"), false);
+              ArrowReader reader = plan.toReader()) {
+            while (reader.loadNextBatch()) {
+              VectorSchemaRoot result = reader.getVectorSchemaRoot();
+              IntVector ids = (IntVector) result.getVector("id");
+              for (int i = 0; i < result.getRowCount(); i++) {
+                int id = ids.get(i);
+                assertTrue(id >= 200, "filtered vector search returned id " + id);
+                if (id == filteredQueryId) {
+                  foundFilteredNearest = true;
+                }
+                filteredRows++;
+              }
+            }
+          }
+          assertTrue(filteredRows > 0, "filtered vector search should return rows");
+          assertTrue(
+              foundFilteredNearest, "filtered vector search recall missed id " + filteredQueryId);
+        }
       } finally {
         dataset.close();
       }
diff --git a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
index f425ddcc4f9..c622bac9fcd 100644
--- a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
+++ b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java
@@ -189,6 +189,33 @@ void testNamespaceId() {
         "namespaceId should contain 'DirectoryNamespace', got: " + namespaceId);
   }
 
+  @Test
+  void testOpenSpecificVersionDoesNotPassVersionToDescribeTable() throws Exception {
+    VersionRejectingNamespace versionRejectingNamespace =
+        new VersionRejectingNamespace(innerNamespaceClient);
+    namespaceClient = versionRejectingNamespace;
+    List<String> tableId = Arrays.asList("test_table");
+
+    namespaceClient.createTable(new CreateTableRequest().id(tableId), createTestTableData());
+    namespaceClient.insertIntoTable(
+        new InsertIntoTableRequest().id(tableId).mode("append"), createTestTableData());
+
+    try (Dataset versionOne =
+        Dataset.open()
+            .allocator(allocator)
+            .namespaceClient(namespaceClient)
+            .tableId(tableId)
+            .readOptions(new ReadOptions.Builder().setVersion(1L).build())
+            .build()) {
+      assertEquals(1, versionOne.version());
+      assertEquals(3, versionOne.countRows());
+    }
+
+    assertTrue(
+        versionRejectingNamespace.getDescribeTableCallCount() > 0,
+        "Expected describeTable to be called when opening through namespace");
+  }
+
   @Test
   void testCreateAndListNamespaces() {
     // Create a namespace
@@ -1439,4 +1466,25 @@ private byte[] createVectorTableData(int numRows, int dim) throws Exception {
       return out.toByteArray();
     }
   }
+
+  private static class VersionRejectingNamespace extends CustomNamespace {
+    private final AtomicInteger describeTableCallCount = new AtomicInteger();
+
+    VersionRejectingNamespace(DirectoryNamespace inner) {
+      super(inner);
+    }
+
+    @Override
+    public DescribeTableResponse describeTable(DescribeTableRequest request) {
+      describeTableCallCount.incrementAndGet();
+      assertNull(
+          request.getVersion(),
+          "Dataset version should be passed to dataset open, not describeTable");
+      return super.describeTable(request);
+    }
+
+    int getDescribeTableCallCount() {
+      return describeTableCallCount.get();
+    }
+  }
 }
diff --git a/memtest/pyproject.toml b/memtest/pyproject.toml
index 396d7c442e0..4418d0e19c8 100644
--- a/memtest/pyproject.toml
+++ b/memtest/pyproject.toml
@@ -7,7 +7,7 @@ name = "lance-memtest"
 version = "0.1.0"
 description = "Memory allocation testing utilities for Python test suites"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = { text = "Apache-2.0" }
 authors = [
     { name = "Lance Developers" }
@@ -17,7 +17,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
diff --git a/protos/AGENTS.md b/protos/AGENTS.md
index 23aef9fc196..2dba0e23dcb 100644
--- a/protos/AGENTS.md
+++ b/protos/AGENTS.md
@@ -4,7 +4,8 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 
 ## Compatibility
 
-- All changes must be backwards compatible. Never re-use or change field numbers of existing fields.
+- Protobuf schemas that are part of a stable file format or any other stable persisted contract must remain backwards compatible. Never reuse or change their existing field numbers.
+- Protobuf schemas used exclusively by an unstable file format follow the root file-format stability contract: do not preserve compatibility with prior unstable revisions. Before making a breaking protobuf change, verify that the schema is not shared with a stable format or another persisted contract.
 
 ## Schema Design
 
diff --git a/protos/ann.proto b/protos/ann.proto
index c9d3b4dcc2f..f5de5e25e7b 100644
--- a/protos/ann.proto
+++ b/protos/ann.proto
@@ -9,6 +9,20 @@ import "table_identifier.proto";
 import "table.proto";
 import "index.proto";
 
+// Query-time approximation mode for vector search.
+//
+// This currently only affects RQ-quantized vector indexes, such as IVF_RQ.
+// Other index types ignore this setting.
+enum VectorApproxMode {
+  // Use all RQ bits for query-time scoring with u8-quantized lookup tables.
+  Normal = 0;
+  // Use only one RQ bit for query-time scoring, even for multi-bit indexes.
+  Fast = 1;
+  // Use all RQ bits for query-time scoring with u16-quantized lookup tables
+  // to reduce estimator quantization error.
+  Accurate = 2;
+}
+
 // Serialized vector query parameters.
 message VectorQueryProto {
   // Query vector as Arrow IPC bytes (supports Float16, Float32, Float64, UInt8, etc.)
@@ -26,6 +40,9 @@ message VectorQueryProto {
   bool use_index = 11;
   optional float dist_q_c = 12;
   optional int32 query_parallelism = 13;
+  // Query-time approximation mode. Currently only affects RQ-quantized vector
+  // indexes, such as IVF_RQ. Other index types ignore this setting.
+  VectorApproxMode approx_mode = 14;
 }
 
 // Serializable form of ANNIvfSubIndexExec — the IVF sub-index search node.
diff --git a/protos/index.proto b/protos/index.proto
index ea21c70387d..a72207b59fe 100644
--- a/protos/index.proto
+++ b/protos/index.proto
@@ -246,4 +246,6 @@ message JsonIndexDetails {
 }
 message BloomFilterIndexDetails {}
 
-message RTreeIndexDetails {}
\ No newline at end of file
+message RTreeIndexDetails {}
+
+message FMIndexDetails {}
\ No newline at end of file
diff --git a/protos/index_old.proto b/protos/index_old.proto
index 601aa2681da..eb984d6fe29 100644
--- a/protos/index_old.proto
+++ b/protos/index_old.proto
@@ -39,4 +39,9 @@ message InvertedIndexDetails {
   uint32 min_ngram_length = 9;
   uint32 max_ngram_length = 10;
   bool prefix_only = 11;
+  // Number of documents per compressed posting block. An absent value means
+  // the index predates this field and must use the legacy block size of 128.
+  // A present value records the block size used by the index; 256 is only
+  // valid with format version 3.
+  optional uint32 block_size = 12;
 }
diff --git a/protos/table.proto b/protos/table.proto
index d298809d5d8..7d4fb19d50b 100644
--- a/protos/table.proto
+++ b/protos/table.proto
@@ -109,10 +109,15 @@ message Manifest {
   // should not attempt to read the dataset.
   //
   // Known flags:
-  // * 1: deletion files are present
-  // * 2: row ids are stable and stored as part of the fragment metadata.
-  // * 4: use v2 format (deprecated)
-  // * 8: table config is present
+  // * 1 << 0: deletion files are present
+  // * 1 << 1: row ids are stable and stored as part of the fragment metadata.
+  // * 1 << 2: use v2 format (deprecated)
+  // * 1 << 3: table config is present
+  // * 1 << 4: dataset uses multiple base paths
+  // * 1 << 5: transaction file writes are disabled
+  // * 1 << 6: data overlay files are present (see DataOverlayFile). Readers that do
+  //   not understand overlays must refuse the dataset, since ignoring an overlay
+  //   would silently return stale base values.
   uint64 reader_feature_flags = 9;
 
   // Feature flags for writers.
@@ -311,6 +316,15 @@ message DataFragment {
 
   repeated DataFile files = 2;
 
+  // Optional overlay files for this fragment, which supply new values for a
+  // subset of cells without rewriting the base data files. This MUST be empty
+  // if the data overlay files feature flag (64) is not set in the manifest.
+  //
+  // Order is significant: a later entry is newer than an earlier one. When two
+  // overlays cover the same (offset, field) and share a `committed_version`, the
+  // later entry wins. See DataOverlayFile for the full resolution rules.
+  repeated DataOverlayFile overlays = 11;
+
   // File that indicates which rows, if any, should be considered deleted.
   DeletionFile deletion_file = 3;
 
@@ -433,6 +447,66 @@ message DataFile {
   optional uint32 base_id = 7;
 } // DataFile
 
+// An overlay file supplies new values for a subset of (row offset, field) cells
+// within a fragment, without rewriting the fragment's base data files. It is
+// used for efficient updates when only a small fraction of rows and/or columns
+// change.
+//
+// On read, a cell is resolved by consulting the fragment's overlays from newest
+// to oldest: the first overlay that covers that (offset, field) wins; if none
+// cover it, the value falls through to the base data file. Because deletions
+// take precedence over overlays, an overlay value for an offset that is also
+// marked deleted is dead and is ignored.
+//
+// The overlay's data file does NOT store a row-offset key column. Within a value
+// column, the position of a covered offset's value is the rank (0-based count of
+// set bits below it) of that offset within the field's coverage bitmap. Because
+// fields may cover different offset sets, the value columns of a single overlay
+// data file may have different lengths (which the Lance file format permits).
+message DataOverlayFile {
+  // The data file storing the overlay's new cell values, one value column per
+  // field in `data_file.fields`. No row-offset key column is stored.
+  DataFile data_file = 1;
+
+  // Which (offset, field) cells this overlay provides values for.
+  oneof coverage {
+    // A single 32-bit Roaring bitmap of physical row offsets that applies to
+    // every field in `data_file.fields` (a "dense" / rectangular overlay).
+    // Every covered offset has a value for every field. This is the common case
+    // for a plain UPDATE, where one SET list is applied to one set of rows.
+    bytes shared_offset_bitmap = 2;
+    // Per-field coverage for a "sparse" overlay, used when different fields cover
+    // different offset sets (e.g. a MERGE with multiple WHEN MATCHED branches).
+    FieldCoverage field_coverage = 4;
+  }
+
+  // The dataset version at which this overlay became effective: the version of
+  // the commit that introduced it, NOT the version it was read from. It is
+  // stamped at commit time and re-stamped if the commit is retried, in the same
+  // way as the created-at / last-updated-at version sequences.
+  //
+  // This drives two orderings:
+  //  * Versus index builds: an index whose `dataset_version` >= this value
+  //    already incorporates this overlay. Otherwise the overlay's covered cells
+  //    are excluded from index results for the affected fields and re-evaluated
+  //    against their current values (see the Data Overlay Files specification).
+  //  * Versus other overlays: when two overlays cover the same (offset, field),
+  //    the one with the higher `committed_version` wins. Overlays that share a
+  //    `committed_version` are ordered by their position in
+  //    `DataFragment.overlays`, where a later entry is newer and wins.
+  uint64 committed_version = 3;
+}
+
+// Per-field coverage for a sparse overlay.
+message FieldCoverage {
+  // One entry per field in the overlay's `data_file.fields`, in the same order.
+  // Each is a 32-bit Roaring bitmap of the physical row offsets covered for that
+  // field. An offset present in a field's bitmap but mapped to a NULL value
+  // means the cell is overridden to NULL (distinct from an offset that is absent,
+  // which falls through to the base data file).
+  repeated bytes offset_bitmaps = 1;
+}
+
 // Deletion File
 //
 // The path of the deletion file is constructed as:
@@ -521,6 +595,16 @@ message FragmentReuseIndexDetails {
 // MemWAL Index Types
 // ============================================================================
 
+// Lifecycle status of a WAL shard. Drives drop-table two-phase commit:
+// a SEALED shard refuses new writer claims (reversible) until the drop
+// commits (the shard dir is deleted) or rolls back (status -> ACTIVE).
+enum ShardStatus {
+  // Normal: the shard accepts writer claims.
+  ACTIVE = 0;
+  // A drop is in flight: claims are refused. Reversible to ACTIVE.
+  SEALED = 1;
+}
+
 // Shard manifest containing epoch-based fencing and WAL state.
 // Each shard has exactly one active writer at any time.
 message ShardManifest {
@@ -566,6 +650,10 @@ message ShardManifest {
 
   // List of flushed MemTable generations and their directory paths.
   repeated FlushedGeneration flushed_generations = 8;
+
+  // Lifecycle status. Default ACTIVE; SEALED marks an in-flight drop
+  // (drop-table 2PC). A SEALED manifest refuses claims at claim_epoch.
+  ShardStatus status = 15;
 }
 
 // A shard field value stored as raw Arrow scalar bytes.
diff --git a/protos/transaction.proto b/protos/transaction.proto
index e72e95025a4..13d11915af4 100644
--- a/protos/transaction.proto
+++ b/protos/transaction.proto
@@ -315,6 +315,28 @@ message Transaction {
     repeated DataReplacementGroup replacements = 1;
   }
 
+  // Overlay files to append to a single fragment, in order (the last entry is
+  // newest). The overlays are appended to the fragment's existing `overlays`
+  // list; they do not replace it, so overlays written by concurrent commits are
+  // preserved.
+  message DataOverlayGroup {
+    uint64 fragment_id = 1;
+    // Each DataOverlayFile.committed_version is left 0 by the writer and stamped
+    // to the new dataset version at commit time (re-stamped on retry), in the
+    // same way as the created-at / last-updated-at version sequences. The fields
+    // touched are read from each overlay's `data_file.fields`.
+    repeated DataOverlayFile overlays = 2;
+  }
+
+  // Attach overlay files to fragments, supplying new values for a subset of
+  // (row offset, field) cells without rewriting the fragments' base data files.
+  // See the DataOverlayFile message in table.proto for resolution, coverage, and
+  // versioning rules, and the Data Overlay Files and Transactions specifications
+  // for the (intentionally permissive) conflict semantics.
+  message DataOverlay {
+    repeated DataOverlayGroup groups = 1;
+  }
+
   // Update the merged generations in MemWAL index.
   // This operation is used during merge-insert to atomically record which
   // generations have been merged to the base table.
@@ -346,6 +368,7 @@ message Transaction {
     UpdateMemWalState update_mem_wal_state = 112;
     Clone clone = 113;
     UpdateBases update_bases = 114;
+    DataOverlay data_overlay = 115;
   }
 
   // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops.
diff --git a/python/AGENTS.md b/python/AGENTS.md
index 5f5ee0134c4..2025f4ef9df 100644
--- a/python/AGENTS.md
+++ b/python/AGENTS.md
@@ -20,6 +20,16 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 * Lint: `uv run make lint`
 * Format: `uv run make format`
 
+## Beta Releases
+
+- Python RC and beta preview wheels are published on fury.io, not only PyPI. When a task needs a beta version such as `7.2.0b4`, use a disposable venv and install with `--pre` and the Lance fury indices:
+  ```shell
+  uv venv /path/to/venv
+  uv pip install --python /path/to/venv/bin/python --pre \
+      --extra-index-url https://pypi.fury.io/lance-format/ \
+      "pylance==7.2.0b4"
+  ```
+
 ## API Design
 
 - Keep bindings as thin wrappers — centralize validation and logic in Rust core.
diff --git a/python/Cargo.lock b/python/Cargo.lock
index 162a221907f..e9a1f66ebde 100644
--- a/python/Cargo.lock
+++ b/python/Cargo.lock
@@ -2,54 +2,6 @@
 # It is not intended for manual editing.
 version = 4
 
-[[package]]
-name = "abi_stable"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445"
-dependencies = [
- "abi_stable_derive",
- "abi_stable_shared",
- "const_panic",
- "core_extensions",
- "crossbeam-channel",
- "generational-arena",
- "libloading",
- "lock_api",
- "parking_lot",
- "paste",
- "repr_offset",
- "rustc_version",
- "serde",
- "serde_derive",
- "serde_json",
-]
-
-[[package]]
-name = "abi_stable_derive"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898"
-dependencies = [
- "abi_stable_shared",
- "as_derive_utils",
- "core_extensions",
- "proc-macro2",
- "quote",
- "rustc_version",
- "syn 1.0.109",
- "typed-arena",
-]
-
-[[package]]
-name = "abi_stable_shared"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63"
-dependencies = [
- "core_extensions",
-]
-
 [[package]]
 name = "adler2"
 version = "2.0.1"
@@ -185,20 +137,11 @@ dependencies = [
  "num-traits",
 ]
 
-[[package]]
-name = "ar_archive_writer"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
-dependencies = [
- "object",
-]
-
 [[package]]
 name = "arc-swap"
-version = "1.9.1"
+version = "1.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
+checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
 dependencies = [
  "rustversion",
 ]
@@ -417,7 +360,7 @@ version = "58.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "serde_core",
  "serde_json",
 ]
@@ -453,18 +396,6 @@ dependencies = [
  "regex-syntax",
 ]
 
-[[package]]
-name = "as_derive_utils"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4"
-dependencies = [
- "core_extensions",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "async-channel"
 version = "2.5.0"
@@ -494,9 +425,6 @@ name = "async-ffi"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50"
-dependencies = [
- "abi_stable",
-]
 
 [[package]]
 name = "async-lock"
@@ -517,7 +445,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -528,7 +456,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -582,7 +510,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "ring",
  "time",
  "tokio",
@@ -642,7 +570,7 @@ dependencies = [
  "bytes",
  "bytes-utils",
  "fastrand",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "percent-encoding",
  "pin-project-lite",
@@ -669,7 +597,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -693,7 +621,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -717,7 +645,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -742,7 +670,7 @@ dependencies = [
  "aws-types",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "regex-lite",
  "tracing",
 ]
@@ -762,7 +690,7 @@ dependencies = [
  "hex",
  "hmac 0.12.1",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "percent-encoding",
  "sha2 0.10.9",
  "time",
@@ -792,7 +720,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "percent-encoding",
@@ -811,7 +739,7 @@ dependencies = [
  "aws-smithy-runtime-api",
  "aws-smithy-types",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-rustls",
  "hyper-util",
@@ -868,7 +796,7 @@ dependencies = [
  "bytes",
  "fastrand",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -888,7 +816,7 @@ dependencies = [
  "aws-smithy-types",
  "bytes",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "pin-project-lite",
  "tokio",
  "tracing",
@@ -906,7 +834,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "http 0.2.12",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -954,7 +882,7 @@ dependencies = [
  "axum-core",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -987,7 +915,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "mime",
@@ -1053,24 +981,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a"
-
-[[package]]
-name = "bitpacking"
-version = "0.9.3"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
-dependencies = [
- "crunchy",
-]
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 
 [[package]]
 name = "bitvec"
-version = "1.0.1"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c"
+checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837"
 dependencies = [
  "funty",
  "radium",
@@ -1112,9 +1031,9 @@ dependencies = [
 
 [[package]]
 name = "block-buffer"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
 dependencies = [
  "hybrid-array",
 ]
@@ -1130,9 +1049,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "8.0.3"
+version = "8.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
+checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -1141,9 +1060,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "5.0.1"
+version = "5.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
+checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -1195,7 +1114,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1209,6 +1128,20 @@ name = "bytemuck"
 version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
 
 [[package]]
 name = "byteorder"
@@ -1218,9 +1151,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.11.1"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "bytes-utils"
@@ -1232,15 +1165,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "bzip2"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
-dependencies = [
- "libbz2-rs-sys",
-]
-
 [[package]]
 name = "cbc"
 version = "0.1.2"
@@ -1252,9 +1176,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.63"
+version = "1.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
+checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1302,9 +1226,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
 dependencies = [
  "iana-time-zone",
  "js-sys",
@@ -1365,7 +1289,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1430,13 +1354,9 @@ version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
 dependencies = [
- "bzip2",
  "compression-core",
  "flate2",
- "liblzma",
  "memchr",
- "zstd",
- "zstd-safe",
 ]
 
 [[package]]
@@ -1533,21 +1453,6 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
-[[package]]
-name = "core_extensions"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003"
-dependencies = [
- "core_extensions_proc_macros",
-]
-
-[[package]]
-name = "core_extensions_proc_macros"
-version = "1.5.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea"
-
 [[package]]
 name = "countio"
 version = "0.3.0"
@@ -1614,18 +1519,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.18"
+version = "0.9.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
 dependencies = [
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-queue"
-version = "0.3.12"
+version = "0.3.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
+checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1764,7 +1669,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1777,7 +1682,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1788,7 +1693,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
 dependencies = [
  "darling_core 0.20.11",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1799,7 +1704,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
 dependencies = [
  "darling_core 0.23.0",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -1818,15 +1723,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
 dependencies = [
  "arrow",
  "arrow-schema",
  "async-trait",
- "bytes",
- "bzip2",
  "chrono",
  "datafusion-catalog",
  "datafusion-catalog-listing",
@@ -1836,7 +1739,6 @@ dependencies = [
  "datafusion-datasource-arrow",
  "datafusion-datasource-csv",
  "datafusion-datasource-json",
- "datafusion-datasource-parquet",
  "datafusion-execution",
  "datafusion-expr",
  "datafusion-expr-common",
@@ -1853,29 +1755,24 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-session",
  "datafusion-sql",
- "flate2",
  "futures",
+ "indexmap 2.14.0",
  "itertools 0.14.0",
- "liblzma",
  "log",
  "object_store",
  "parking_lot",
- "parquet",
- "rand 0.9.4",
- "regex",
  "sqlparser",
  "tempfile",
  "tokio",
  "url",
  "uuid",
- "zstd",
 ]
 
 [[package]]
 name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1898,9 +1795,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1921,34 +1818,34 @@ dependencies = [
 
 [[package]]
 name = "datafusion-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
 dependencies = [
- "ahash",
  "arrow",
  "arrow-ipc",
+ "arrow-schema",
  "chrono",
+ "foldhash 0.2.0",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "libc",
  "log",
  "object_store",
  "parquet",
- "paste",
- "recursive",
  "sqlparser",
  "tokio",
+ "uuid",
  "web-time",
 ]
 
 [[package]]
 name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
 dependencies = [
  "futures",
  "log",
@@ -1957,15 +1854,13 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
 dependencies = [
  "arrow",
- "async-compression",
  "async-trait",
  "bytes",
- "bzip2",
  "chrono",
  "datafusion-common",
  "datafusion-common-runtime",
@@ -1976,25 +1871,22 @@ dependencies = [
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
  "datafusion-session",
- "flate2",
  "futures",
  "glob",
  "itertools 0.14.0",
- "liblzma",
  "log",
  "object_store",
+ "parking_lot",
  "rand 0.9.4",
  "tokio",
- "tokio-util",
  "url",
- "zstd",
 ]
 
 [[package]]
 name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -2016,9 +1908,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2039,9 +1931,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2056,16 +1948,15 @@ dependencies = [
  "datafusion-session",
  "futures",
  "object_store",
- "serde_json",
  "tokio",
  "tokio-stream",
 ]
 
 [[package]]
 name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450"
 dependencies = [
  "arrow",
  "async-trait",
@@ -2075,6 +1966,7 @@ dependencies = [
  "datafusion-datasource",
  "datafusion-execution",
  "datafusion-expr",
+ "datafusion-functions",
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-adapter",
@@ -2093,20 +1985,19 @@ dependencies = [
 
 [[package]]
 name = "datafusion-doc"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
 
 [[package]]
 name = "datafusion-execution"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
 dependencies = [
  "arrow",
  "arrow-buffer",
  "async-trait",
- "chrono",
  "dashmap",
  "datafusion-common",
  "datafusion-expr",
@@ -2122,11 +2013,12 @@ dependencies = [
 
 [[package]]
 name = "datafusion-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
 dependencies = [
  "arrow",
+ "arrow-schema",
  "async-trait",
  "chrono",
  "datafusion-common",
@@ -2137,36 +2029,33 @@ dependencies = [
  "datafusion-physical-expr-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
- "recursive",
  "serde_json",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
 dependencies = [
  "arrow",
  "datafusion-common",
  "indexmap 2.14.0",
  "itertools 0.14.0",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-ffi"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c"
+checksum = "e5660e8fa79fd51e29ce46f3026b67317ef738ebd633e106beb1a1907a406152"
 dependencies = [
- "abi_stable",
  "arrow",
  "arrow-schema",
  "async-ffi",
  "async-trait",
+ "chrono",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-datasource",
@@ -2175,22 +2064,25 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
  "datafusion-physical-plan",
  "datafusion-proto",
  "datafusion-proto-common",
  "datafusion-session",
  "futures",
+ "libloading",
  "log",
  "prost",
  "semver",
+ "stabby",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-functions"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
 dependencies = [
  "arrow",
  "arrow-buffer",
@@ -2205,26 +2097,25 @@ dependencies = [
  "datafusion-expr",
  "datafusion-expr-common",
  "datafusion-macros",
+ "datafusion-physical-expr-common",
  "hex",
  "itertools 0.14.0",
  "log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
  "memchr",
  "num-traits",
  "rand 0.9.4",
  "regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
  "uuid",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-doc",
@@ -2234,19 +2125,18 @@ dependencies = [
  "datafusion-macros",
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
+ "foldhash 0.2.0",
  "half",
  "log",
  "num-traits",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr-common",
@@ -2255,9 +2145,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
 dependencies = [
  "arrow",
  "arrow-ord",
@@ -2271,34 +2161,34 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-macros",
  "datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "itertools 0.14.0",
  "itoa",
  "log",
- "paste",
+ "memchr",
 ]
 
 [[package]]
 name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
 dependencies = [
  "arrow",
  "async-trait",
  "datafusion-catalog",
  "datafusion-common",
  "datafusion-expr",
+ "datafusion-physical-expr",
  "datafusion-physical-plan",
  "parking_lot",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2309,14 +2199,13 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "log",
- "paste",
 ]
 
 [[package]]
 name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
 dependencies = [
  "datafusion-common",
  "datafusion-physical-expr-common",
@@ -2324,20 +2213,20 @@ dependencies = [
 
 [[package]]
 name = "datafusion-macros"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
 dependencies = [
  "datafusion-doc",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
 dependencies = [
  "arrow",
  "chrono",
@@ -2348,18 +2237,16 @@ dependencies = [
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
- "recursive",
  "regex",
  "regex-syntax",
 ]
 
 [[package]]
 name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
 dependencies = [
- "ahash",
  "arrow",
  "datafusion-common",
  "datafusion-expr",
@@ -2367,21 +2254,19 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-physical-expr-common",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
- "paste",
  "petgraph",
- "recursive",
  "tokio",
 ]
 
 [[package]]
 name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2394,26 +2279,26 @@ dependencies = [
 
 [[package]]
 name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
 dependencies = [
- "ahash",
  "arrow",
  "chrono",
  "datafusion-common",
  "datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "parking_lot",
+ "pin-project",
 ]
 
 [[package]]
 name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2425,17 +2310,17 @@ dependencies = [
  "datafusion-physical-plan",
  "datafusion-pruning",
  "itertools 0.14.0",
- "recursive",
 ]
 
 [[package]]
 name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
 dependencies = [
- "ahash",
  "arrow",
+ "arrow-data",
+ "arrow-ipc",
  "arrow-ord",
  "arrow-schema",
  "async-trait",
@@ -2450,7 +2335,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "futures",
  "half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
  "indexmap 2.14.0",
  "itertools 0.14.0",
  "log",
@@ -2462,9 +2347,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-proto"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae"
 dependencies = [
  "arrow",
  "chrono",
@@ -2485,14 +2370,13 @@ dependencies = [
  "datafusion-proto-common",
  "object_store",
  "prost",
- "rand 0.9.4",
 ]
 
 [[package]]
 name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2501,9 +2385,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -2512,15 +2396,14 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "datafusion-physical-plan",
- "itertools 0.14.0",
  "log",
 ]
 
 [[package]]
 name = "datafusion-session"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
 dependencies = [
  "async-trait",
  "datafusion-common",
@@ -2532,9 +2415,9 @@ dependencies = [
 
 [[package]]
 name = "datafusion-sql"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
 dependencies = [
  "arrow",
  "bigdecimal",
@@ -2544,16 +2427,15 @@ dependencies = [
  "datafusion-functions-nested",
  "indexmap 2.14.0",
  "log",
- "recursive",
  "regex",
  "sqlparser",
 ]
 
 [[package]]
 name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "3b22c8f8c72d317e54fad6f85c0ef6d1e1da53cc7faadc7eea8daf0f8d86d4f2"
 dependencies = [
  "async-recursion",
  "async-trait",
@@ -2569,26 +2451,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "deepsize"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c"
-dependencies = [
- "deepsize_derive",
-]
-
-[[package]]
-name = "deepsize_derive"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -2628,7 +2490,7 @@ dependencies = [
  "darling 0.20.11",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2638,7 +2500,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
 dependencies = [
  "derive_builder_core",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2659,7 +2521,7 @@ version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
 dependencies = [
- "block-buffer 0.12.0",
+ "block-buffer 0.12.1",
  "const-oid 0.10.2",
  "crypto-common 0.2.2",
  "ctutils",
@@ -2694,7 +2556,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -2767,11 +2629,17 @@ dependencies = [
  "encoding_rs",
 ]
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "env_filter"
-version = "1.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
+checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217"
 dependencies = [
  "log",
  "regex",
@@ -2779,9 +2647,9 @@ dependencies = [
 
 [[package]]
 name = "env_logger"
-version = "0.11.10"
+version = "0.11.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a"
+checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -2863,7 +2731,7 @@ version = "25.12.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "rustc_version",
 ]
 
@@ -2919,7 +2787,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "rand 0.9.4",
@@ -2996,7 +2864,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3037,15 +2905,6 @@ dependencies = [
  "cfg-if 0.1.10",
 ]
 
-[[package]]
-name = "generational-arena"
-version = "0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7"
-dependencies = [
- "cfg-if 1.0.4",
-]
-
 [[package]]
 name = "generator"
 version = "0.8.9"
@@ -3156,12 +3015,13 @@ dependencies = [
 
 [[package]]
 name = "geodatafusion"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b"
+checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552"
 dependencies = [
  "arrow-arith",
  "arrow-array",
+ "arrow-buffer",
  "arrow-schema",
  "datafusion",
  "geo",
@@ -3222,17 +3082,15 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
 dependencies = [
  "cfg-if 1.0.4",
  "js-sys",
  "libc",
  "r-efi 6.0.0",
  "rand_core 0.10.1",
- "wasip2",
- "wasip3",
  "wasm-bindgen",
 ]
 
@@ -3253,7 +3111,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3274,18 +3132,42 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "goosefs-sdk"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "dashmap",
+ "hostname",
+ "prost",
+ "prost-types",
+ "rand 0.9.4",
+ "reqwest 0.12.28",
+ "serde",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tonic-prost",
+ "tracing",
+ "uuid",
+]
+
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.4.1",
+ "http 1.4.2",
  "indexmap 2.14.0",
  "slab",
  "tokio",
@@ -3299,6 +3181,7 @@ version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
 dependencies = [
+ "bytemuck",
  "cfg-if 1.0.4",
  "crunchy",
  "num-traits",
@@ -3351,6 +3234,11 @@ name = "hashbrown"
 version = "0.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
 
 [[package]]
 name = "heapify"
@@ -3394,7 +3282,7 @@ checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "more-asserts",
  "serde",
  "thiserror 2.0.18",
@@ -3426,6 +3314,17 @@ dependencies = [
  "digest 0.11.3",
 ]
 
+[[package]]
+name = "hostname"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
+dependencies = [
+ "cfg-if 1.0.4",
+ "libc",
+ "windows-link",
+]
+
 [[package]]
 name = "http"
 version = "0.2.12"
@@ -3439,9 +3338,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.4.1"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -3465,7 +3364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
 ]
 
 [[package]]
@@ -3476,7 +3375,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
 dependencies = [
  "bytes",
  "futures-core",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "pin-project-lite",
 ]
@@ -3495,9 +3394,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
 
 [[package]]
 name = "hybrid-array"
@@ -3519,7 +3418,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "httparse",
  "httpdate",
@@ -3536,7 +3435,7 @@ version = "0.27.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "hyper-util",
  "rustls",
@@ -3544,6 +3443,20 @@ dependencies = [
  "tokio",
  "tokio-rustls",
  "tower-service",
+ "webpki-roots",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
 ]
 
 [[package]]
@@ -3556,7 +3469,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "hyper",
  "ipnet",
@@ -3774,12 +3687,6 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f"
 
-[[package]]
-name = "id-arena"
-version = "2.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -3848,11 +3755,11 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "io-uring"
-version = "0.7.12"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
+checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "cfg-if 1.0.4",
  "libc",
 ]
@@ -3878,15 +3785,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.14.0"
@@ -3904,18 +3802,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
 
 [[package]]
 name = "jieba-macros"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82"
+checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5"
 dependencies = [
  "phf_codegen",
 ]
 
 [[package]]
 name = "jieba-rs"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9"
+checksum = "a813bbf185c8c62eb6fcf54a223177b644824d91612045dfd80bb779acd080eb"
 dependencies = [
  "bytecount",
  "cedarwood",
@@ -3950,7 +3848,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -3995,7 +3893,7 @@ dependencies = [
  "quote",
  "rustc_version",
  "simd_cesu8",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -4014,7 +3912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
 dependencies = [
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -4029,13 +3927,12 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
 dependencies = [
  "cfg-if 1.0.4",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
@@ -4087,7 +3984,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4105,7 +4002,6 @@ dependencies = [
  "async_cell",
  "aws-credential-types",
  "aws-sdk-dynamodb",
- "bitpacking",
  "byteorder",
  "bytes",
  "chrono",
@@ -4117,14 +4013,14 @@ dependencies = [
  "datafusion-functions",
  "datafusion-physical-expr",
  "datafusion-physical-plan",
- "deepsize",
  "either",
  "fst",
  "futures",
  "half",
  "humantime",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-encoding",
@@ -4162,7 +4058,7 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4171,6 +4067,7 @@ dependencies = [
  "arrow-ord",
  "arrow-schema",
  "arrow-select",
+ "bytemuck",
  "bytes",
  "futures",
  "getrandom 0.2.17",
@@ -4204,30 +4101,33 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrayref",
+ "crunchy",
  "paste",
  "seq-macro",
 ]
 
 [[package]]
 name = "lance-core"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
+ "arrow-data",
  "arrow-schema",
  "async-trait",
  "byteorder",
  "bytes",
  "datafusion-common",
  "datafusion-sql",
- "deepsize",
  "futures",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
+ "lance-derive",
  "libc",
+ "libm",
  "log",
  "moka",
  "num_cpus",
@@ -4243,12 +4143,13 @@ dependencies = [
  "tokio-stream",
  "tokio-util",
  "tracing",
+ "twox-hash",
  "url",
 ]
 
 [[package]]
 name = "lance-datafusion"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4280,7 +4181,7 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4293,12 +4194,20 @@ dependencies = [
  "rand 0.9.4",
  "rand_distr",
  "rand_xoshiro",
- "random_word",
+]
+
+[[package]]
+name = "lance-derive"
+version = "9.1.0-beta.2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
 name = "lance-encoding"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4314,7 +4223,7 @@ dependencies = [
  "futures",
  "hex",
  "hyperloglogplus",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-arrow",
  "lance-bitpacking",
  "lance-core",
@@ -4333,7 +4242,7 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4346,7 +4255,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "datafusion-common",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -4364,7 +4272,7 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -4378,7 +4286,7 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4390,7 +4298,6 @@ dependencies = [
  "async-channel",
  "async-recursion",
  "async-trait",
- "bitpacking",
  "bitvec",
  "bytes",
  "chrono",
@@ -4399,7 +4306,6 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "deepsize",
  "dirs",
  "fst",
  "futures",
@@ -4407,23 +4313,25 @@ dependencies = [
  "geoarrow-array",
  "geoarrow-schema",
  "half",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "jieba-rs",
  "jsonb",
  "lance-arrow",
  "lance-arrow-stats",
+ "lance-bitpacking",
  "lance-core",
  "lance-datafusion",
  "lance-datagen",
  "lance-encoding",
  "lance-file",
  "lance-geo",
+ "lance-index-core",
  "lance-io",
  "lance-linalg",
  "lance-select",
  "lance-table",
  "lance-tokenizer",
- "libm",
+ "libsais-rs",
  "log",
  "ndarray",
  "num-traits",
@@ -4435,6 +4343,7 @@ dependencies = [
  "rand_distr",
  "rangemap",
  "rayon",
+ "regex-syntax",
  "roaring",
  "serde",
  "serde_json",
@@ -4442,13 +4351,34 @@ dependencies = [
  "tempfile",
  "tokio",
  "tracing",
- "twox-hash",
  "uuid",
 ]
 
+[[package]]
+name = "lance-index-core"
+version = "9.1.0-beta.2"
+dependencies = [
+ "arrow-array",
+ "arrow-schema",
+ "arrow-select",
+ "async-trait",
+ "bytes",
+ "datafusion",
+ "datafusion-common",
+ "datafusion-expr",
+ "futures",
+ "lance-core",
+ "lance-io",
+ "lance-select",
+ "prost-types",
+ "roaring",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "lance-io"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4465,14 +4395,15 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "deepsize",
  "futures",
- "http 1.4.1",
+ "goosefs-sdk",
+ "http 1.4.2",
  "io-uring",
  "lance-arrow",
  "lance-core",
  "lance-namespace",
  "log",
+ "metrics",
  "moka",
  "object_store",
  "object_store_opendal",
@@ -4490,23 +4421,23 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-schema",
  "cc",
- "deepsize",
  "half",
  "lance-arrow",
  "lance-core",
  "num-traits",
  "rand 0.9.4",
+ "rayon",
 ]
 
 [[package]]
 name = "lance-namespace"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4518,7 +4449,7 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4526,6 +4457,8 @@ dependencies = [
  "async-trait",
  "axum",
  "bytes",
+ "datafusion-common",
+ "datafusion-physical-plan",
  "futures",
  "lance",
  "lance-core",
@@ -4538,19 +4471,22 @@ dependencies = [
  "object_store",
  "rand 0.9.4",
  "reqwest 0.12.28",
+ "roaring",
  "serde",
  "serde_json",
+ "time",
  "tokio",
  "tower",
  "tower-http 0.5.2",
  "url",
+ "uuid",
 ]
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.8.0"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3eefb02ded2c3d4b6b60669bb74822d9fa628e144fc748c79ee31f13f566e87b"
+checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -4562,15 +4498,14 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-schema",
  "byteorder",
  "bytes",
- "deepsize",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lance-core",
  "roaring",
  "tracing",
@@ -4578,7 +4513,7 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4591,7 +4526,6 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
- "deepsize",
  "futures",
  "lance-arrow",
  "lance-core",
@@ -4618,13 +4552,14 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",
  "lindera",
  "rust-stemmers",
  "serde",
+ "stop-words",
  "unicode-normalization",
 ]
 
@@ -4637,12 +4572,6 @@ dependencies = [
  "spin",
 ]
 
-[[package]]
-name = "leb128fmt"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
-
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -4700,12 +4629,6 @@ dependencies = [
  "lexical-util",
 ]
 
-[[package]]
-name = "libbz2-rs-sys"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
-
 [[package]]
 name = "libc"
 version = "0.2.186"
@@ -4714,32 +4637,12 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "libloading"
-version = "0.7.4"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
 dependencies = [
  "cfg-if 1.0.4",
- "winapi",
-]
-
-[[package]]
-name = "liblzma"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899"
-dependencies = [
- "liblzma-sys",
-]
-
-[[package]]
-name = "liblzma-sys"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
+ "windows-link",
 ]
 
 [[package]]
@@ -4757,6 +4660,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libsais-rs"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40fe164dbd47ea0c20e78a121c980ef673326905f1d4fba55e3645a20ef6717f"
+dependencies = [
+ "rayon",
+]
+
 [[package]]
 name = "lindera"
 version = "3.0.7"
@@ -4813,9 +4725,9 @@ dependencies = [
 
 [[package]]
 name = "link-section"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "014e440054ce8170890229eeef5bcda955305e056ec713de40ed366944483f09"
+checksum = "c2b1dd6fe32e55c0fc0ea9493aa57459ca3cf4ff3c857c7d0302290150da6e4f"
 
 [[package]]
 name = "linktime-proc-macro"
@@ -4846,9 +4758,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.31"
+version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f"
+checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
 
 [[package]]
 name = "loom"
@@ -4956,9 +4868,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.8.1"
+version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "memmap2"
@@ -4969,6 +4881,36 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "metrics"
+version = "0.24.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
+dependencies = [
+ "portable-atomic",
+ "rapidhash",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.19.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
+dependencies = [
+ "aho-corasick",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown 0.15.5",
+ "indexmap 2.14.0",
+ "metrics",
+ "ordered-float 4.6.0",
+ "quanta",
+ "radix_trie",
+ "rand 0.9.4",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -5055,7 +4997,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5073,6 +5015,15 @@ dependencies = [
  "rawpointer",
 ]
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "nom"
 version = "8.0.0"
@@ -5200,7 +5151,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5209,7 +5160,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
 ]
 
 [[package]]
@@ -5231,15 +5182,6 @@ dependencies = [
  "objc2-core-foundation",
 ]
 
-[[package]]
-name = "object"
-version = "0.37.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "object_store"
 version = "0.13.2"
@@ -5254,7 +5196,7 @@ dependencies = [
  "futures-channel",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body-util",
  "httparse",
  "humantime",
@@ -5331,9 +5273,11 @@ dependencies = [
  "opendal-service-azdls",
  "opendal-service-cos",
  "opendal-service-gcs",
+ "opendal-service-goosefs",
  "opendal-service-hf",
  "opendal-service-oss",
  "opendal-service-s3",
+ "opendal-service-tos",
 ]
 
 [[package]]
@@ -5346,7 +5290,7 @@ dependencies = [
  "base64",
  "bytes",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "jiff",
  "log",
@@ -5371,7 +5315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
 dependencies = [
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "mea",
  "opendal-core",
 ]
@@ -5415,7 +5359,7 @@ checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
 dependencies = [
  "base64",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -5436,7 +5380,7 @@ checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
 dependencies = [
  "base64",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "opendal-service-azure-common",
@@ -5454,7 +5398,7 @@ version = "0.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
 dependencies = [
- "http 1.4.1",
+ "http 1.4.2",
  "opendal-core",
 ]
 
@@ -5465,7 +5409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -5483,7 +5427,7 @@ checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
 dependencies = [
  "async-trait",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -5496,6 +5440,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "opendal-service-goosefs"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
+dependencies = [
+ "bytes",
+ "goosefs-sdk",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+]
+
 [[package]]
 name = "opendal-service-hf"
 version = "0.57.0"
@@ -5504,7 +5462,7 @@ checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
 dependencies = [
  "bytes",
  "hf-xet",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "percent-encoding",
@@ -5520,7 +5478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
 dependencies = [
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "opendal-core",
  "quick-xml 0.39.4",
@@ -5539,7 +5497,7 @@ dependencies = [
  "base64",
  "bytes",
  "crc32c",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "md-5 0.11.0",
  "opendal-core",
@@ -5551,6 +5509,23 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "opendal-service-tos"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1"
+dependencies = [
+ "bytes",
+ "http 1.4.2",
+ "opendal-core",
+ "quick-xml 0.39.4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-volcengine-tos",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.2.1"
@@ -5572,6 +5547,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "ordered-float"
 version = "5.3.0"
@@ -5853,7 +5837,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5960,7 +5944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
 dependencies = [
  "proc-macro2",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -5983,9 +5967,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -5993,9 +5977,9 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
+checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
 dependencies = [
  "heck",
  "itertools 0.14.0",
@@ -6006,42 +5990,32 @@ dependencies = [
  "prost",
  "prost-types",
  "regex",
- "syn 2.0.117",
+ "syn",
  "tempfile",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
 dependencies = [
  "prost",
 ]
 
-[[package]]
-name = "psm"
-version = "0.1.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea"
-dependencies = [
- "ar_archive_writer",
- "cc",
-]
-
 [[package]]
 name = "ptr_meta"
 version = "0.3.1"
@@ -6059,13 +6033,14 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "pylance"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 dependencies = [
+ "alloc-stdlib",
  "arrow",
  "arrow-array",
  "arrow-cast",
@@ -6095,6 +6070,8 @@ dependencies = [
  "lance-table",
  "libc",
  "log",
+ "metrics",
+ "metrics-util",
  "object_store",
  "prost",
  "prost-types",
@@ -6154,7 +6131,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6167,7 +6144,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6180,6 +6157,21 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
+]
+
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -6283,6 +6275,16 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rancor"
 version = "0.1.1"
@@ -6319,7 +6321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
 dependencies = [
  "chacha20",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "rand_core 0.10.1",
 ]
 
@@ -6387,23 +6389,28 @@ dependencies = [
 ]
 
 [[package]]
-name = "random_word"
-version = "0.5.2"
+name = "rangemap"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81"
+checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+
+[[package]]
+name = "rapidhash"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37"
 dependencies = [
- "ahash",
- "brotli",
- "paste",
- "rand 0.9.4",
- "unicase",
+ "rustversion",
 ]
 
 [[package]]
-name = "rangemap"
-version = "1.7.1"
+name = "raw-cpuid"
+version = "11.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
+checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186"
+dependencies = [
+ "bitflags 2.13.0",
+]
 
 [[package]]
 name = "rawpointer"
@@ -6431,26 +6438,6 @@ dependencies = [
  "crossbeam-utils",
 ]
 
-[[package]]
-name = "recursive"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e"
-dependencies = [
- "recursive-proc-macro-impl",
- "stacker",
-]
-
-[[package]]
-name = "recursive-proc-macro-impl"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
-dependencies = [
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "redb"
 version = "3.1.3"
@@ -6466,7 +6453,7 @@ version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
 ]
 
 [[package]]
@@ -6497,14 +6484,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -6531,9 +6518,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "regress"
@@ -6554,15 +6541,6 @@ dependencies = [
  "bytecheck",
 ]
 
-[[package]]
-name = "repr_offset"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea"
-dependencies = [
- "tstr",
-]
-
 [[package]]
 name = "reqsign-aliyun-oss"
 version = "3.1.0"
@@ -6571,7 +6549,7 @@ checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d"
 dependencies = [
  "anyhow",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -6590,7 +6568,7 @@ dependencies = [
  "bytes",
  "form_urlencoded",
  "hex",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "quick-xml 0.40.1",
@@ -6612,7 +6590,7 @@ dependencies = [
  "base64",
  "bytes",
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "pem",
  "percent-encoding",
@@ -6636,7 +6614,7 @@ dependencies = [
  "futures",
  "hex",
  "hmac 0.13.0",
- "http 1.4.1",
+ "http 1.4.2",
  "jiff",
  "log",
  "percent-encoding",
@@ -6666,7 +6644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2"
 dependencies = [
  "form_urlencoded",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-aws-v4",
@@ -6684,7 +6662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb"
 dependencies = [
  "anyhow",
- "http 1.4.1",
+ "http 1.4.2",
  "log",
  "percent-encoding",
  "reqsign-core",
@@ -6692,6 +6670,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "reqsign-volcengine-tos"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3"
+dependencies = [
+ "anyhow",
+ "http 1.4.2",
+ "log",
+ "percent-encoding",
+ "reqsign-core",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.12.28"
@@ -6704,7 +6695,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -6735,6 +6726,7 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams 0.4.2",
  "web-sys",
+ "webpki-roots",
 ]
 
 [[package]]
@@ -6747,7 +6739,7 @@ dependencies = [
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "hyper",
@@ -6785,7 +6777,7 @@ checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58"
 dependencies = [
  "anyhow",
  "async-trait",
- "http 1.4.1",
+ "http 1.4.2",
  "reqwest 0.13.4",
  "thiserror 2.0.18",
  "tower-service",
@@ -6832,7 +6824,7 @@ checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -6905,9 +6897,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -6924,7 +6916,7 @@ version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -6960,9 +6952,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
 dependencies = [
  "web-time",
  "zeroize",
@@ -7097,7 +7089,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7129,7 +7121,7 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "core-foundation 0.10.1",
  "core-foundation-sys",
  "libc",
@@ -7189,7 +7181,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7200,7 +7192,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7209,6 +7201,7 @@ version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -7235,7 +7228,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7247,7 +7240,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7264,9 +7257,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2"
+checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
 dependencies = [
  "base64",
  "bs58",
@@ -7284,14 +7277,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.20.0"
+version = "3.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac"
+checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
 dependencies = [
  "darling 0.23.0",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7363,6 +7356,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "sha2-const-stable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9"
+
 [[package]]
 name = "sharded-slab"
 version = "0.1.7"
@@ -7437,6 +7436,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
+[[package]]
+name = "sketches-ddsketch"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
+
 [[package]]
 name = "slab"
 version = "0.4.12"
@@ -7445,9 +7450,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "smallvec"
-version = "1.15.1"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "snafu"
@@ -7467,7 +7472,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7516,12 +7521,11 @@ dependencies = [
 
 [[package]]
 name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
 dependencies = [
  "log",
- "recursive",
  "sqlparser_derive",
 ]
 
@@ -7533,28 +7537,49 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
-name = "stable_deref_trait"
-version = "1.2.1"
+name = "stabby"
+version = "72.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec"
+dependencies = [
+ "rustversion",
+ "stabby-abi",
+]
 
 [[package]]
-name = "stacker"
-version = "0.1.24"
+name = "stabby-abi"
+version = "72.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190"
+checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4"
 dependencies = [
- "cc",
- "cfg-if 1.0.4",
- "libc",
- "psm",
- "windows-sys 0.61.2",
+ "rustc_version",
+ "rustversion",
+ "sha2-const-stable",
+ "stabby-macros",
 ]
 
+[[package]]
+name = "stabby-macros"
+version = "72.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+
 [[package]]
 name = "static_assertions"
 version = "1.1.0"
@@ -7583,6 +7608,15 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
 
+[[package]]
+name = "stop-words"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d68df56303396bcfb639455b3c166804aeb7994005010aab5e9e8a1277b8871d"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -7617,7 +7651,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7629,16 +7663,17 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
 dependencies = [
  "heck",
+ "indexmap 2.14.0",
  "pbjson",
  "pbjson-build",
  "pbjson-types",
@@ -7652,7 +7687,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.117",
+ "syn",
  "typify",
  "walkdir",
 ]
@@ -7671,20 +7706,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.117"
+version = "2.0.118"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -7708,7 +7732,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7731,7 +7755,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
@@ -7771,7 +7795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -7803,7 +7827,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7814,7 +7838,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -7937,14 +7961,14 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
 name = "tokio-retry"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40f644c762e9d396831ae2f8935c954b0d758c4532e924bead0f666d0c1c8640"
+checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470"
 dependencies = [
  "pin-project-lite",
  "rand 0.10.1",
@@ -8016,6 +8040,45 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tonic"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
+dependencies = [
+ "async-trait",
+ "base64",
+ "bytes",
+ "h2",
+ "http 1.4.2",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "socket2",
+ "sync_wrapper",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tonic-prost"
+version = "0.14.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
+dependencies = [
+ "bytes",
+ "prost",
+ "tonic",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.3"
@@ -8024,9 +8087,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
 dependencies = [
  "futures-core",
  "futures-util",
+ "indexmap 2.14.0",
  "pin-project-lite",
+ "slab",
  "sync_wrapper",
  "tokio",
+ "tokio-util",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -8038,9 +8104,9 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "bytes",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -8056,11 +8122,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
 dependencies = [
  "async-compression",
- "bitflags 2.12.1",
+ "bitflags 2.13.0",
  "bytes",
  "futures-core",
  "futures-util",
- "http 1.4.1",
+ "http 1.4.2",
  "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
@@ -8117,7 +8183,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -8189,21 +8255,6 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "tstr"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7"
-dependencies = [
- "tstr_proc_macros",
-]
-
-[[package]]
-name = "tstr_proc_macros"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a"
-
 [[package]]
 name = "twox-hash"
 version = "2.1.2"
@@ -8213,12 +8264,6 @@ dependencies = [
  "rand 0.9.4",
 ]
 
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
-
 [[package]]
 name = "typenum"
 version = "1.20.1"
@@ -8256,7 +8301,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.117",
+ "syn",
  "thiserror 2.0.18",
  "unicode-ident",
 ]
@@ -8274,7 +8319,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.117",
+ "syn",
  "typify-impl",
 ]
 
@@ -8317,12 +8362,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.11"
@@ -8373,11 +8412,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.23.2"
+version = "1.23.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
+checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
 dependencies = [
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "js-sys",
  "serde_core",
  "wasm-bindgen",
@@ -8437,20 +8476,11 @@ dependencies = [
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
- "wit-bindgen 0.57.1",
-]
-
-[[package]]
-name = "wasip3"
-version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
-dependencies = [
- "wit-bindgen 0.51.0",
+ "wit-bindgen",
 ]
 
 [[package]]
@@ -8464,9 +8494,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
 dependencies = [
  "cfg-if 1.0.4",
  "once_cell",
@@ -8477,9 +8507,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.72"
+version = "0.4.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -8487,9 +8517,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -8497,48 +8527,26 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
 dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.122"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
 dependencies = [
  "unicode-ident",
 ]
 
-[[package]]
-name = "wasm-encoder"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
-dependencies = [
- "leb128fmt",
- "wasmparser",
-]
-
-[[package]]
-name = "wasm-metadata"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
-dependencies = [
- "anyhow",
- "indexmap 2.14.0",
- "wasm-encoder",
- "wasmparser",
-]
-
 [[package]]
 name = "wasm-streams"
 version = "0.4.2"
@@ -8565,23 +8573,11 @@ dependencies = [
  "web-sys",
 ]
 
-[[package]]
-name = "wasmparser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
-dependencies = [
- "bitflags 2.12.1",
- "hashbrown 0.15.5",
- "indexmap 2.14.0",
- "semver",
-]
-
 [[package]]
 name = "web-sys"
-version = "0.3.99"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -8599,9 +8595,18 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -8703,7 +8708,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -8714,7 +8719,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -8936,100 +8941,12 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
-dependencies = [
- "wit-bindgen-rust-macro",
-]
-
 [[package]]
 name = "wit-bindgen"
 version = "0.57.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
 
-[[package]]
-name = "wit-bindgen-core"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
-dependencies = [
- "anyhow",
- "heck",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-bindgen-rust"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
-dependencies = [
- "anyhow",
- "heck",
- "indexmap 2.14.0",
- "prettyplease",
- "syn 2.0.117",
- "wasm-metadata",
- "wit-bindgen-core",
- "wit-component",
-]
-
-[[package]]
-name = "wit-bindgen-rust-macro"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
-dependencies = [
- "anyhow",
- "prettyplease",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
- "wit-bindgen-core",
- "wit-bindgen-rust",
-]
-
-[[package]]
-name = "wit-component"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
-dependencies = [
- "anyhow",
- "bitflags 2.12.1",
- "indexmap 2.14.0",
- "log",
- "serde",
- "serde_derive",
- "serde_json",
- "wasm-encoder",
- "wasm-metadata",
- "wasmparser",
- "wit-parser",
-]
-
-[[package]]
-name = "wit-parser"
-version = "0.244.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
-dependencies = [
- "anyhow",
- "id-arena",
- "indexmap 2.14.0",
- "log",
- "semver",
- "serde",
- "serde_derive",
- "serde_json",
- "unicode-xid",
- "wasmparser",
-]
-
 [[package]]
 name = "wkb"
 version = "0.9.2"
@@ -9083,7 +9000,7 @@ dependencies = [
  "clap",
  "crc32fast",
  "futures",
- "http 1.4.1",
+ "http 1.4.2",
  "hyper",
  "lazy_static",
  "more-asserts",
@@ -9124,7 +9041,7 @@ dependencies = [
  "csv",
  "futures",
  "futures-util",
- "getrandom 0.4.2",
+ "getrandom 0.4.3",
  "heapify",
  "itertools 0.14.0",
  "lazy_static",
@@ -9157,7 +9074,7 @@ dependencies = [
  "chrono",
  "clap",
  "gearhash",
- "http 1.4.1",
+ "http 1.4.2",
  "itertools 0.14.0",
  "lazy_static",
  "more-asserts",
@@ -9225,15 +9142,15 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
 [[package]]
 name = "xxhash-rust"
-version = "0.8.15"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576"
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -9248,28 +9165,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "synstructure",
 ]
 
 [[package]]
 name = "zerocopy"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.50"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
@@ -9289,15 +9206,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
  "synstructure",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"
@@ -9331,7 +9248,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.117",
+ "syn",
 ]
 
 [[package]]
diff --git a/python/Cargo.toml b/python/Cargo.toml
index 2ea0d46764b..2540114b925 100644
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pylance"
-version = "8.0.0-beta.1"
+version = "9.1.0-beta.2"
 edition = "2024"
 authors = ["Lance Devs <dev@lance.org>"]
 license = "Apache-2.0"
@@ -8,6 +8,12 @@ rust-version = "1.91"
 exclude = ["python/lance/conftest.py"]
 publish = false
 
+# `python/` is excluded from the top-level workspace (see ../Cargo.toml) and keeps
+# its own Cargo.lock. Declaring an empty workspace here makes this crate an explicit
+# workspace root, so cargo resolves it standalone even when the checkout is nested
+# inside another cargo project (e.g. a git worktree placed under the main checkout).
+[workspace]
+
 [lib]
 name = "lance"
 crate-type = ["cdylib"]
@@ -19,9 +25,13 @@ arrow-cast = "58.0.0"
 arrow-data = "58.0.0"
 arrow-schema = "58.0.0"
 object_store = "0.13.2"
-datafusion = "53.0.0"
-datafusion-ffi = "53.0.0"
-datafusion-common = "53.0.0"
+datafusion = { version = "54.0.0", default-features = false }
+datafusion-ffi = "54.0.0"
+datafusion-common = "54.0.0"
+# Keep the Python FFI build on the working Brotli allocator resolution until
+# datafusion-ffi no longer enables datafusion-proto/default.
+# See https://github.com/lance-format/lance/issues/7271.
+alloc-stdlib = "=0.2.2"
 async-trait = "0.1"
 chrono = "0.4.42"
 env_logger = "0.11.7"
@@ -31,8 +41,10 @@ half = { version = "2.5", default-features = false, features = [
     "std",
 ] }
 lance = { path = "../rust/lance", features = [
+    "goosefs",
     "dynamodb",
     "substrait",
+    "metrics",
 ] }
 lance-arrow = { path = "../rust/lance-arrow" }
 lance-core = { path = "../rust/lance-core" }
@@ -43,19 +55,21 @@ lance-index = { path = "../rust/lance-index", features = [
     "tokenizer-lindera",
     "tokenizer-jieba",
 ] }
-lance-io = { path = "../rust/lance-io" }
+lance-io = { path = "../rust/lance-io", features = ["metrics"] }
 lance-linalg = { path = "../rust/lance-linalg" }
 lance-namespace = { path = "../rust/lance-namespace" }
-lance-namespace-impls = { path = "../rust/lance-namespace-impls", features = ["rest", "rest-adapter"] }
+lance-namespace-impls = { path = "../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] }
 lance-table = { path = "../rust/lance-table" }
 lance-datafusion = { path = "../rust/lance-datafusion" }
 libc = "0.2.176"
 log = "0.4"
+metrics = "0.24"
+metrics-util = "0.19"
 prost = "0.14.1"
 prost-types = "0.14.1"
 pyo3 = { version = "0.28", features = [
     "extension-module",
-    "abi3-py39",
+    "abi3-py310",
     "py-clone",
     "chrono",
 ] }
@@ -69,7 +83,7 @@ serde_yaml = "0.9.34"
 tracing-chrome = "0.7.1"
 tracing-subscriber = "0.3.17"
 tracing = { version = "0.1" }
-bytes = "1.4"
+bytes = "1.11.1"
 
 [features]
 default = []
diff --git a/python/pyproject.toml b/python/pyproject.toml
index a1e69855a0f..60b4bec5b11 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,13 +1,13 @@
 [project]
 name = "pylance"
 dynamic = ["version"]
-dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.8.0,<0.9"]
+dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.8.5,<0.9"]
 description = "python wrapper for Lance columnar format"
 authors = [{ name = "Lance Devs", email = "dev@lance.org" }]
 license = { file = "LICENSE" }
 repository = "https://github.com/lancedb/lance"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 keywords = [
     "data-format",
     "data-science",
@@ -30,7 +30,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -58,13 +57,12 @@ tests = [
     "polars[pyarrow,pandas]",
     "psutil",
     "pytest",
-    # Only test tensorflow on linux for now. We will deprecate tensorflow soon.
-    "tensorflow; sys_platform == 'linux'",
     "tqdm",
-    "datafusion>=53,<54; python_version >= '3.10'",
+    "datafusion>=54,<55",
 ]
 dev = ["ruff==0.11.2", "pyright"]
 benchmarks = ["pytest-benchmark"]
+otel = ["opentelemetry-api", "opentelemetry-sdk"]
 torch = ["torch>=2.0"]
 geo = [
     "geoarrow-rust-core",
@@ -74,7 +72,7 @@ geo = [
 [dependency-groups]
 tests = [
     "boto3==1.40.43",
-    "datasets==4.1.1; python_version >= '3.10'",
+    "datasets==4.1.1",
     "duckdb==1.4.0",
     "ml_dtypes==0.5.3",
     "pillow==11.3.0",
@@ -82,9 +80,9 @@ tests = [
     "polars[pyarrow,pandas]==1.34.0",
     "psutil==7.1.0",
     "pytest==8.4.2",
-    "tensorflow==2.20.0; sys_platform == 'linux' and python_version >= '3.10'",
     "tqdm==4.67.1",
-    "datafusion==53.0.0; python_version >= '3.10'",
+    "datafusion==54.0.0",
+    "opentelemetry-sdk==1.30.0",
 ]
 dev = [
     "maturin==1.13.3",
@@ -115,6 +113,8 @@ include = [
     "python/lance/schema.py",
     "python/lance/file.py",
     "python/lance/util.py",
+    "python/lance/arrow.py",
+    "python/tests/test_arrow.py",
 ]
 # Dependencies like pyarrow make this difficult to enforce strictly.
 reportMissingTypeStubs = "warning"
@@ -136,9 +136,6 @@ markers = [
 filterwarnings = [
     'error::FutureWarning',
     'error::DeprecationWarning',
-    # TensorFlow import can emit NumPy deprecation FutureWarnings in some environments.
-    # We keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning.
-    'ignore:.*`np\\.object` will be defined as the corresponding NumPy scalar\\..*:FutureWarning',
     # Boto3
     'ignore:.*datetime\.datetime\.utcnow\(\) is deprecated.*:DeprecationWarning',
     # Hugging Face Hub calls this deprecated hf-xet API internally.
@@ -152,9 +149,8 @@ filterwarnings = [
     'ignore:.*the load_module\(\) method is deprecated.*:DeprecationWarning',
     # Pytorch uses deprecated jit.script_method internally (torch/utils/mkldnn.py)
     'ignore:.*torch\.jit\.script_method.*is deprecated.*:DeprecationWarning',
+    # Pytorch uses the same API internally on Python 3.14+ with a different warning message.
+    'ignore:.*torch\.jit\.script_method.*is not supported in Python 3\.14\+.*:DeprecationWarning',
     # huggingface_hub still calls the deprecated hf_xet.download_files() during Xet downloads
     'ignore:.*hf_xet\.download_files\(\) is deprecated.*:DeprecationWarning',
-    # TensorFlow/Keras import can emit NumPy deprecation FutureWarnings in some environments.
-    # Keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning.
-    'ignore:.*np\.object.*:FutureWarning',
 ]
diff --git a/python/python/benchmarks/test_blob.py b/python/python/benchmarks/test_blob.py
new file mode 100644
index 00000000000..c2465f36fad
--- /dev/null
+++ b/python/python/benchmarks/test_blob.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+from itertools import count
+from time import process_time
+
+import pyarrow as pa
+import pytest
+from lance.file import LanceFileSession
+
+# Many small blobs isolate per-row Python overhead; fewer large blobs show how
+# the bulk path behaves once payload copying accounts for more of the CPU time.
+WORKLOADS = [
+    pytest.param(50_000, 256, id="50000x256b"),
+    pytest.param(2_000, 64 * 1024, id="2000x64kib"),
+]
+
+
+def _packed_blob_benchmark(benchmark, tmpdir_factory, row_count, payload_size, mode):
+    payload = b"x" * payload_size
+    payloads = pa.repeat(payload, row_count)
+    files = LanceFileSession(str(tmpdir_factory.mktemp("packed_blob_writer")))
+    file_number = count()
+
+    if mode == "scalar_preconverted":
+        python_payloads = payloads.to_pylist()
+
+        def write():
+            writer = files.open_packed_blob_writer(
+                f"scalar-{next(file_number)}.lance", 1
+            )
+            for value in python_payloads:
+                writer.write_blob(value)
+            return writer.finish()
+
+    elif mode == "scalar_from_arrow":
+
+        def write():
+            writer = files.open_packed_blob_writer(
+                f"scalar-arrow-{next(file_number)}.lance", 1
+            )
+            for value in payloads.to_pylist():
+                writer.write_blob(value)
+            return writer.finish()
+
+    elif mode == "bulk":
+
+        def write():
+            writer = files.open_packed_blob_writer(f"bulk-{next(file_number)}.lance", 1)
+            writer.write_blobs(payloads)
+            return writer.finish_array("blob")
+
+    else:
+        raise ValueError(f"Unknown benchmark mode: {mode}")
+
+    result = benchmark.pedantic(write, iterations=1, rounds=5)
+    assert len(result) == row_count
+
+
+@pytest.mark.benchmark(group="packed_blob_writer_cpu", timer=process_time)
+@pytest.mark.parametrize("row_count,payload_size", WORKLOADS)
+@pytest.mark.parametrize(
+    "mode",
+    ["scalar_preconverted", "scalar_from_arrow", "bulk"],
+)
+def test_packed_blob_writer(benchmark, tmpdir_factory, row_count, payload_size, mode):
+    _packed_blob_benchmark(
+        benchmark,
+        tmpdir_factory,
+        row_count,
+        payload_size,
+        mode,
+    )
diff --git a/python/python/benchmarks/test_search.py b/python/python/benchmarks/test_search.py
index 61076e61687..b4e33338cb1 100644
--- a/python/python/benchmarks/test_search.py
+++ b/python/python/benchmarks/test_search.py
@@ -78,10 +78,12 @@ def create_base_dataset(data_dir: Path) -> lance.LanceDataset:
         rows_remaining -= next_batch_length
         table = create_table(next_batch_length, offset)
         if offset == 0:
-            dataset = lance.write_dataset(table, tmp_path, use_legacy_format=False)
+            dataset = lance.write_dataset(
+                table, tmp_path, data_storage_version="stable"
+            )
         else:
             dataset = lance.write_dataset(
-                table, tmp_path, mode="append", use_legacy_format=False
+                table, tmp_path, mode="append", data_storage_version="stable"
             )
         offset += next_batch_length
 
@@ -98,7 +100,7 @@ def create_base_dataset(data_dir: Path) -> lance.LanceDataset:
     dataset.create_scalar_index("category", "BITMAP")
     dataset.create_scalar_index("genres", "LABEL_LIST")
 
-    return lance.dataset(tmp_path, index_cache_size=64 * 1024)
+    return lance.dataset(tmp_path, index_cache_size_bytes=512 * 1024 * 1024)
 
 
 def create_delete_dataset(data_dir):
@@ -113,7 +115,7 @@ def create_delete_dataset(data_dir):
     dataset = lance.dataset(tmp_path)
     dataset.delete("filterable % 2 != 0")
 
-    return lance.dataset(tmp_path, index_cache_size=64 * 1024)
+    return lance.dataset(tmp_path, index_cache_size_bytes=512 * 1024 * 1024)
 
 
 def create_new_rows_dataset(data_dir):
@@ -129,7 +131,7 @@ def create_new_rows_dataset(data_dir):
     table = create_table(NEW_ROWS, offset=NUM_ROWS)
     dataset = lance.write_dataset(table, tmp_path, mode="append")
 
-    return lance.dataset(tmp_path, index_cache_size=64 * 1024)
+    return lance.dataset(tmp_path, index_cache_size_bytes=512 * 1024 * 1024)
 
 
 class Datasets(NamedTuple):
diff --git a/python/python/ci_benchmarks/benchmarks/test_count_rows.py b/python/python/ci_benchmarks/benchmarks/test_count_rows.py
new file mode 100644
index 00000000000..f6228b04f13
--- /dev/null
+++ b/python/python/ci_benchmarks/benchmarks/test_count_rows.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Benchmark count_rows acceleration for IS NULL / IS NOT NULL filters.
+
+Tests five index configurations against an int32 dataset with ~1% NULL
+values.  Each configuration stores the same data in a separate column so
+that only one index type is active per measurement:
+
+  none        — no index, full column scan (baseline)
+  BITMAP      — bitmap index
+  BTREE       — btree index
+  ZONEMAP     — zone-map index
+  BLOOMFILTER — bloom-filter index
+
+Two filters are exercised for each configuration:
+  IS NULL     — count the ~1% null rows
+  IS NOT NULL — count the ~99% non-null rows
+
+Indexed configurations are tested in two cache states to separate first-load
+latency from steady-state throughput:
+
+  warm — one prewarm call is made before measuring; the same dataset instance
+         is reused so its in-memory index cache is already populated.
+  cold — a fresh ``lance.dataset()`` instance is created inside each measured
+         round so the in-memory index cache starts empty every time.  No
+         prewarm pass is performed.
+"""
+
+from __future__ import annotations
+
+import lance
+import pytest
+from ci_benchmarks.datasets import get_dataset_uri
+
+# ---------------------------------------------------------------------------
+# Parameters
+# ---------------------------------------------------------------------------
+
+# Indexed configs only (warm/cold dimension applies to these)
+_INDEXED_CONFIGS: list[tuple[str, str]] = [
+    ("bitmap", "value_bitmap"),
+    ("btree", "value_btree"),
+    ("zonemap", "value_zonemap"),
+    ("bloomfilter", "value_bloomfilter"),
+]
+_INDEXED_IDS = [cfg[0] for cfg in _INDEXED_CONFIGS]
+
+_FILTERS = ["is_null", "is_not_null"]
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture(scope="module")
+def count_rows_ds() -> lance.LanceDataset:
+    """Shared dataset instance (index cache persists across rounds — use for warm)."""
+    return lance.dataset(get_dataset_uri("count_rows"))
+
+
+@pytest.fixture(scope="module")
+def count_rows_uri() -> str:
+    return get_dataset_uri("count_rows")
+
+
+# ---------------------------------------------------------------------------
+# No-index baseline (no warm/cold — there is no index cache to speak of)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("filter_type", _FILTERS)
+def test_count_rows_no_index(
+    benchmark,
+    count_rows_ds: lance.LanceDataset,
+    filter_type: str,
+) -> None:
+    """Full-scan baseline with no scalar index."""
+    filt = (
+        "value_none IS NULL" if filter_type == "is_null" else "value_none IS NOT NULL"
+    )
+
+    def bench() -> int:
+        return count_rows_ds.count_rows(filter=filt)
+
+    benchmark.pedantic(bench, warmup_rounds=1, rounds=5)
+
+
+# ---------------------------------------------------------------------------
+# Indexed benchmarks — warm vs cold
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("warm", [True, False], ids=["warm", "cold"])
+@pytest.mark.parametrize("filter_type", _FILTERS)
+@pytest.mark.parametrize("index_id,column", _INDEXED_CONFIGS, ids=_INDEXED_IDS)
+def test_count_rows_indexed(
+    benchmark,
+    count_rows_ds: lance.LanceDataset,
+    count_rows_uri: str,
+    index_id: str,
+    column: str,
+    filter_type: str,
+    warm: bool,
+) -> None:
+    """Benchmark count_rows with a scalar index, in warm and cold cache states.
+
+    Args:
+        index_id: Human-readable index name (parametrize label only).
+        column: Dataset column that carries this index type.
+        filter_type: ``"is_null"`` or ``"is_not_null"``.
+        warm: If True, prewarm the index cache before measuring and reuse the
+              shared dataset instance.  If False, create a fresh dataset
+              instance on every round so the index cache starts empty.
+    """
+    filt = f"{column} IS NULL" if filter_type == "is_null" else f"{column} IS NOT NULL"
+
+    if warm:
+
+        def bench() -> int:
+            return count_rows_ds.count_rows(filter=filt)
+
+        # warmup_rounds=1 makes one unmeasured call that populates the cache.
+        benchmark.pedantic(bench, warmup_rounds=1, rounds=5)
+    else:
+
+        def bench() -> int:
+            # Fresh instance → empty in-memory index cache every round.
+            ds = lance.dataset(count_rows_uri)
+            return ds.count_rows(filter=filt)
+
+        benchmark.pedantic(bench, warmup_rounds=0, rounds=5)
diff --git a/python/python/ci_benchmarks/datagen/count_rows.py b/python/python/ci_benchmarks/datagen/count_rows.py
new file mode 100644
index 00000000000..ff5fa02cf8a
--- /dev/null
+++ b/python/python/ci_benchmarks/datagen/count_rows.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Generate the count_rows benchmark dataset.
+
+Creates a 10-million-row Lance dataset with five int32 columns that all hold the
+same values and the same ~1% null mask.  Each column carries a different scalar
+index so the benchmark can compare no-index, BITMAP, BTREE, ZONEMAP, and
+BLOOMFILTER side-by-side on the same underlying data.
+
+Column layout
+-------------
+value_none        — no index (full-scan baseline)
+value_bitmap      — BITMAP index
+value_btree       — BTREE index
+value_zonemap     — ZONEMAP index
+value_bloomfilter — BLOOMFILTER index
+
+Null pattern: row i is null when i % 100 == 0 (~1% of rows exactly).
+"""
+
+import lance
+import numpy as np
+import pyarrow as pa
+from lance.log import LOGGER
+
+from ci_benchmarks.datasets import get_dataset_uri
+
+NUM_ROWS = 10_000_000
+BATCH_SIZE = 1_000_000  # 1 M rows per batch → 10 batches total
+
+COLUMNS = [
+    "value_none",
+    "value_bitmap",
+    "value_btree",
+    "value_zonemap",
+    "value_bloomfilter",
+]
+
+SCHEMA = pa.schema([(col, pa.int32()) for col in COLUMNS])
+
+
+def _gen_data():
+    num_batches = NUM_ROWS // BATCH_SIZE
+    for batch_idx in range(num_batches):
+        offset = batch_idx * BATCH_SIZE
+        values = np.arange(offset, offset + BATCH_SIZE, dtype=np.int32)
+        # Null mask: True where the value should be null (~1% of rows)
+        null_mask = (np.arange(BATCH_SIZE) + offset) % 100 == 0
+        col = pa.array(values, type=pa.int32(), mask=null_mask)
+        yield pa.record_batch([col] * len(COLUMNS), schema=SCHEMA)
+
+
+def gen_count_rows() -> lance.LanceDataset:
+    dataset_uri = get_dataset_uri("count_rows")
+
+    try:
+        ds = lance.dataset(dataset_uri)
+        if ds.count_rows() == NUM_ROWS:
+            LOGGER.info(
+                "count_rows dataset already exists at %s (%d rows)",
+                dataset_uri,
+                NUM_ROWS,
+            )
+            return ds
+        LOGGER.warning(
+            "count_rows dataset at %s has unexpected row count %d; regenerating",
+            dataset_uri,
+            ds.count_rows(),
+        )
+    except Exception:
+        pass
+
+    LOGGER.info(
+        "Writing count_rows dataset (%d rows, %d columns) to %s",
+        NUM_ROWS,
+        len(COLUMNS),
+        dataset_uri,
+    )
+    ds = lance.write_dataset(
+        _gen_data(),
+        dataset_uri,
+        schema=SCHEMA,
+        mode="overwrite",
+    )
+    LOGGER.info("Dataset written; building scalar indexes …")
+
+    for index_type, column in [
+        ("BITMAP", "value_bitmap"),
+        ("BTREE", "value_btree"),
+        ("ZONEMAP", "value_zonemap"),
+        ("BLOOMFILTER", "value_bloomfilter"),
+    ]:
+        LOGGER.info("  Creating %s index on %s …", index_type, column)
+        ds.create_scalar_index(column, index_type)
+
+    LOGGER.info("count_rows dataset ready.")
+    return ds
diff --git a/python/python/ci_benchmarks/datagen/gen_all.py b/python/python/ci_benchmarks/datagen/gen_all.py
index 1da7c05fd9b..d5120d20ff7 100644
--- a/python/python/ci_benchmarks/datagen/gen_all.py
+++ b/python/python/ci_benchmarks/datagen/gen_all.py
@@ -6,6 +6,7 @@
 from lance.log import LOGGER
 
 from ci_benchmarks.datagen.basic import gen_basic
+from ci_benchmarks.datagen.count_rows import gen_count_rows
 from ci_benchmarks.datagen.lineitems import gen_tcph
 from ci_benchmarks.datagen.wikipedia import gen_wikipedia
 
@@ -40,6 +41,9 @@ def setup_logging():
     LOGGER.info("Generating Wikipedia dataset...")
     gen_wikipedia()
 
+    LOGGER.info("Generating count_rows benchmark dataset...")
+    gen_count_rows()
+
     LOGGER.info("=" * 80)
     LOGGER.info("All datasets generated successfully!")
     LOGGER.info("=" * 80)
diff --git a/python/python/lance/__init__.py b/python/python/lance/__init__.py
index f58b169a47a..3479a177824 100644
--- a/python/python/lance/__init__.py
+++ b/python/python/lance/__init__.py
@@ -9,7 +9,18 @@
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from . import io, log
-from .blob import Blob, BlobArray, BlobColumn, BlobFile, blob_array, blob_field
+from .blob import (
+    Blob,
+    BlobArray,
+    BlobColumn,
+    BlobDescriptor,
+    BlobDescriptorArrayBuilder,
+    BlobFile,
+    DedicatedBlobWriter,
+    PackedBlobWriter,
+    blob_array,
+    blob_field,
+)
 from .dataset import (
     DataStatistics,
     FieldStatistics,
@@ -27,11 +38,16 @@
 )
 from .fragment import FragmentMetadata, LanceFragment
 from .lance import (
+    CleanupCandidateFile,
+    CleanupExplanation,
+    CleanupReferencedBranch,
+    CleanupStats,
     DatasetBasePath,
     FFILanceTableProvider,
     ScanStatistics,
     bytes_read_counter,
     iops_counter,
+    simd_info,
 )
 from .mem_wal import (
     ExecutionPlan,
@@ -68,8 +84,16 @@
     "BlobArray",
     "BlobColumn",
     "BlobFile",
+    "DedicatedBlobWriter",
+    "BlobDescriptorArrayBuilder",
+    "PackedBlobWriter",
+    "BlobDescriptor",
     "blob_array",
     "blob_field",
+    "CleanupCandidateFile",
+    "CleanupExplanation",
+    "CleanupReferencedBranch",
+    "CleanupStats",
     "DatasetBasePath",
     "DataStatistics",
     "FieldStatistics",
@@ -92,6 +116,7 @@
     "json_to_schema",
     "schema_to_json",
     "set_logger",
+    "simd_info",
     "write_dataset",
     "FFILanceTableProvider",
     "IndexProgress",
@@ -158,6 +183,13 @@ def dataset(
     storage_options : optional, dict
         Extra options that make sense for a particular storage connection. This is
         used to store connection parameters like credentials, endpoint, etc.
+
+        For datasets with additional registered base paths, a key of the form
+        ``base_<id>.<key>`` applies ``<key>`` only to the base path with that
+        manifest id, overriding the unscoped options that every base inherits.
+        For example ``{"account_key": "shared", "base_1.account_key": "abc"}``
+        makes base 1 use ``account_key = abc`` while all other options are
+        shared.
     default_scan_options : optional, dict
         Default scan options that are used when scanning the dataset.  This accepts
         the same arguments described in :py:meth:`lance.LanceDataset.scanner`.  The
@@ -195,9 +227,10 @@ def dataset(
     base_store_params : dict of str to dict, optional
         Runtime-only object store parameters keyed by base path URI. Each key
         is a base path URI (e.g., "s3://bucket/path") and each value is a dict
-        of storage options (credentials, endpoint, etc.) for that base.  When a base
-        has no explicit entry here, the top-level ``storage_options`` is
-        used as a fallback.
+        of storage options (credentials, endpoint, etc.) for that base.  These
+        take precedence over ``base_<id>.<key>`` entries in ``storage_options``.
+        When a base has no explicit entry here, the top-level
+        ``storage_options`` is used as a fallback.
 
     Notes
     -----
@@ -230,7 +263,9 @@ def dataset(
                 "Both 'namespace_client' and 'table_id' must be provided together."
             )
 
-        request = DescribeTableRequest(id=table_id, version=version)
+        # Resolve the latest table metadata here. The requested dataset version is
+        # applied by the lower-level dataset open path after namespace resolution.
+        request = DescribeTableRequest(id=table_id, version=None)
         response = namespace_client.describe_table(request)
 
         uri = response.location
diff --git a/python/python/lance/arrow.py b/python/python/lance/arrow.py
index 54da15705f8..5e9f671ab80 100644
--- a/python/python/lance/arrow.py
+++ b/python/python/lance/arrow.py
@@ -4,10 +4,12 @@
 """Extensions to PyArrows."""
 
 import json
+import typing
 from pathlib import Path
 from typing import Callable, Iterable, Optional, Union
 
 import pyarrow as pa
+import pyarrow.compute as pc
 
 from ._arrow.bf16 import (  # noqa: F401
     BFloat16,
@@ -19,8 +21,10 @@
 from .lance import bfloat16_array
 
 __all__ = [
+    "BFloat16",
     "BFloat16Array",
     "BFloat16Type",
+    "PandasBFloat16Array",
     "bfloat16_array",
     "cast",
     "EncodedImageType",
@@ -220,14 +224,15 @@ def from_uris(
         <lance.arrow.ImageURIArray object at 0x...>
         ['file::///tmp/1.png']
         """
+        storage: pa.Array
         if isinstance(uris, (pa.StringArray, pa.LargeStringArray)):
-            pass
+            storage = uris
         elif isinstance(uris, Iterable):
-            uris = pa.array((str(uri) for uri in uris), type=pa.string())
+            storage = pa.array((str(uri) for uri in uris), type=pa.string())
         else:
             raise TypeError("Cannot build a ImageURIArray from {}".format(type(uris)))
 
-        return cls.from_storage(ImageURIType(uris.type), uris)
+        return cls.from_storage(ImageURIType(storage.type), storage)
 
     def read_uris(self, storage_type=pa.binary()) -> "EncodedImageArray":
         """
@@ -268,7 +273,8 @@ def download(url):
                     print("Failed to reach the server: ", e.reason)
                 elif hasattr(e, "code"):
                     print(
-                        "The server could not fulfill the request. Error code: ", e.code
+                        "The server could not fulfill the request. Error code: ",
+                        getattr(e, "code"),
                     )
 
         images = []
@@ -277,7 +283,9 @@ def download(url):
             if parsed_uri.scheme in ("http", "https"):
                 images.append(download(uri))
             else:
-                filesystem, path = fs.FileSystem.from_uri(uri.as_py())
+                filesystem, path = fs.FileSystem.from_uri(  # pyright: ignore[reportPrivateImportUsage]
+                    uri.as_py()
+                )
                 with filesystem.open_input_stream(path) as f:
                     images.append(f.read())
 
@@ -297,21 +305,12 @@ def __repr__(self):
         def pillow_metadata_decoder(images):
             import io
 
-            from PIL import Image
+            from PIL import Image  # pyright: ignore[reportMissingImports]
 
             img = Image.open(io.BytesIO(images[0].as_py()))
             return img
 
-        def tensorflow_metadata_decoder(images):
-            import tensorflow as tf
-
-            img = tf.io.decode_image(images[0].as_py())
-            return img
-
-        decoders = (
-            ("tensorflow", tensorflow_metadata_decoder),
-            ("PIL", pillow_metadata_decoder),
-        )
+        decoders = (("PIL", pillow_metadata_decoder),)
         decoder = None
 
         for libname, metadata_decoder in decoders:
@@ -343,7 +342,7 @@ def to_tensor(
         decoder : Callable[pa.binary()], optional
             A function that takes a binary array and returns a numpy.ndarray
             or pa.fixed_shape_tensor. If not provided, will attempt to use
-            tensorflow and then pillow decoder in that order.
+            pillow.
 
         Returns
         -------
@@ -365,27 +364,19 @@ def to_tensor(
 
         if not decoder:
 
-            def pillow_decoder(images):
+            def pillow_decoder(images) -> "np.ndarray":
                 import io
 
-                from PIL import Image
+                from PIL import Image  # pyright: ignore[reportMissingImports]
 
                 return np.stack(
-                    [Image.open(io.BytesIO(img)) for img in images.to_pylist()]
+                    [
+                        np.asarray(Image.open(io.BytesIO(img)))
+                        for img in images.to_pylist()
+                    ]
                 )
 
-            def tensorflow_decoder(images):
-                import tensorflow as tf
-
-                decoded_to_tensor = tuple(
-                    tf.io.decode_image(img) for img in images.to_pylist()
-                )
-                return tf.stack(decoded_to_tensor, axis=0).numpy()
-
-            decoders = [
-                ("tensorflow", tensorflow_decoder),
-                ("PIL", pillow_decoder),
-            ]
+            decoders = [("PIL", pillow_decoder)]
             for libname, decoder_function in decoders:
                 try:
                     __import__(libname)
@@ -395,15 +386,16 @@ def tensorflow_decoder(images):
                     pass
             else:
                 raise ValueError(
-                    "No image decoder available. Please either install one of "
-                    "tensorflow, pillow, or pass a decoder argument."
+                    "No image decoder available. Please install pillow or pass a "
+                    "decoder argument."
                 )
 
         image_array = decoder(self.storage)
         if isinstance(image_array, pa.FixedShapeTensorType):
-            shape = image_array.shape
-            arrow_type = image_array.storage_type
-            tensor_array = image_array
+            tensor = typing.cast("pa.Array", image_array)
+            shape = tensor.shape
+            arrow_type = tensor.storage_type
+            tensor_array = tensor
         else:
             shape = image_array.shape[1:]
             arrow_type = pa.from_numpy_dtype(image_array.dtype)
@@ -476,7 +468,7 @@ def to_encoded(self, encoder=None, storage_type=pa.binary()) -> "EncodedImageArr
         def pillow_encoder(x):
             import io
 
-            from PIL import Image
+            from PIL import Image  # pyright: ignore[reportMissingImports]
 
             encoded_images = []
             for y in x:
@@ -485,19 +477,8 @@ def pillow_encoder(x):
                     encoded_images.append(buf.getvalue())
             return pa.array(encoded_images, type=storage_type)
 
-        def tensorflow_encoder(x):
-            import tensorflow as tf
-
-            encoded_images = (
-                tf.io.encode_png(y).numpy() for y in tf.convert_to_tensor(x)
-            )
-            return pa.array(encoded_images, type=storage_type)
-
         if not encoder:
-            encoders = (
-                ("PIL", pillow_encoder),
-                ("tensorflow", tensorflow_encoder),
-            )
+            encoders = (("PIL", pillow_encoder),)
             for libname, encoder_function in encoders:
                 try:
                     __import__(libname)
@@ -507,8 +488,8 @@ def tensorflow_encoder(x):
                     pass
             else:
                 raise ValueError(
-                    "No image encoder available. Please either install one of "
-                    "tensorflow, pillow, or pass an encoder argument."
+                    "No image encoder available. Please install pillow or pass an "
+                    "encoder argument."
                 )
 
         return EncodedImageArray.from_storage(
@@ -571,7 +552,8 @@ def cast(
                 + f"got: {target_type}"
             )
         np_arr = arr.to_numpy()
-        float_arr = np_arr.astype(target_type.to_pandas_dtype())
+        float_type = typing.cast("pa.DataType", target_type)
+        float_arr = np_arr.astype(float_type.to_pandas_dtype())
         return pa.array(float_arr)
     elif isinstance(target_type, BFloat16Type) or target_type in ["bfloat16", "bf16"]:
         if not pa.types.is_floating(arr.type):
@@ -586,15 +568,16 @@ def cast(
         target_type
     ):
         # Casting fixed size list to fixed size list
-        if arr.type.list_size != target_type.list_size:
+        list_type = typing.cast("pa.DataType", target_type)
+        if arr.type.list_size != list_type.list_size:
             raise ValueError(
                 "Only support casting fixed size list to fixed size list "
                 f"with the same size, got: {arr.type} to {target_type}"
             )
-        values = cast(arr.values, target_type.value_type)
+        values = cast(arr.values, list_type.value_type)
         return pa.FixedSizeListArray.from_arrays(
-            values=values, list_size=target_type.list_size
+            values=values, list_size=list_type.list_size
         )
 
     # Fallback to normal cast.
-    return pa.compute.cast(arr, target_type, *args, **kwargs)
+    return pc.cast(arr, target_type, *args, **kwargs)
diff --git a/python/python/lance/blob.py b/python/python/lance/blob.py
index 46faf760cdd..6d1af6797dc 100644
--- a/python/python/lance/blob.py
+++ b/python/python/lance/blob.py
@@ -1,13 +1,37 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+import ctypes
 import io
 from dataclasses import dataclass
 from typing import IO, Any, Iterator, Optional, Union
 
 import pyarrow as pa
 
-from .lance import LanceBlobFile
+from .lance import (
+    BlobDescriptor as BlobDescriptor,
+)
+from .lance import (
+    BlobDescriptorArrayBuilder as BlobDescriptorArrayBuilder,
+)
+from .lance import (
+    DedicatedBlobWriter as DedicatedBlobWriter,
+)
+from .lance import (
+    LanceBlobFile,
+)
+from .lance import (
+    PackedBlobWriter as PackedBlobWriter,
+)
+
+_BLOB_INLINE_SIZE_THRESHOLD_META_KEY = b"lance-encoding:blob-inline-size-threshold"
+_BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY = (
+    b"lance-encoding:blob-dedicated-size-threshold"
+)
+_BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY = (
+    b"lance-encoding:blob-pack-file-size-threshold"
+)
+_MAX_RUST_USIZE = ctypes.c_size_t(-1).value
 
 
 @dataclass(frozen=True)
@@ -190,9 +214,78 @@ def blob_array(values: list[Any]) -> BlobArray:
     return BlobArray.from_pylist(values)
 
 
-def blob_field(name: str, *, nullable: bool = True) -> pa.Field:
-    """Construct an Arrow field for a Lance blob column."""
-    return pa.field(name, BlobType(), nullable=nullable)
+def _validate_threshold(name: str, value: Optional[int], *, allow_zero: bool) -> None:
+    if value is None:
+        return
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise TypeError(f"{name} must be an int, got {type(value).__name__}")
+    if allow_zero:
+        if value < 0:
+            raise ValueError(f"{name} must be non-negative")
+    elif value <= 0:
+        raise ValueError(f"{name} must be positive")
+    if value > _MAX_RUST_USIZE:
+        raise OverflowError(f"{name} must fit in a Rust usize")
+
+
+def blob_field(
+    name: str,
+    *,
+    nullable: bool = True,
+    inline_size_threshold: Optional[int] = None,
+    dedicated_size_threshold: Optional[int] = None,
+    pack_file_size_threshold: Optional[int] = None,
+) -> pa.Field:
+    """
+    Construct an Arrow field for a Lance blob column.
+
+    Parameters
+    ----------
+    name : str
+        Field name.
+    nullable : bool, default True
+        Whether the blob column accepts null values.
+    inline_size_threshold : optional, int
+        Maximum payload size in bytes to keep inline in the data file before
+        using packed blob storage.
+    dedicated_size_threshold : optional, int
+        Maximum payload size in bytes to store in packed blob storage before
+        using dedicated blob storage. This threshold is checked before
+        ``inline_size_threshold``.
+    pack_file_size_threshold : optional, int
+        Maximum size in bytes of a single packed blob sidecar (``.pack``) file.
+        Once a sidecar reaches this size a new one is started.
+    """
+    _validate_threshold("inline_size_threshold", inline_size_threshold, allow_zero=True)
+    _validate_threshold(
+        "dedicated_size_threshold", dedicated_size_threshold, allow_zero=False
+    )
+    _validate_threshold(
+        "pack_file_size_threshold", pack_file_size_threshold, allow_zero=False
+    )
+
+    field = pa.field(name, BlobType(), nullable=nullable)
+    if (
+        inline_size_threshold is None
+        and dedicated_size_threshold is None
+        and pack_file_size_threshold is None
+    ):
+        return field
+
+    metadata = dict(field.metadata or {})
+    if inline_size_threshold is not None:
+        metadata[_BLOB_INLINE_SIZE_THRESHOLD_META_KEY] = str(
+            inline_size_threshold
+        ).encode()
+    if dedicated_size_threshold is not None:
+        metadata[_BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY] = str(
+            dedicated_size_threshold
+        ).encode()
+    if pack_file_size_threshold is not None:
+        metadata[_BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY] = str(
+            pack_file_size_threshold
+        ).encode()
+    return field.with_metadata(metadata)
 
 
 class BlobIterator:
@@ -212,9 +305,11 @@ class BlobColumn:
     file-like objects.
 
     This can be useful for working with medium-to-small binary objects that need
-    to interface with APIs that expect file-like objects.  For very large binary
-    objects (4-8MB or more per value) you might be better off creating a blob column
-    and using :py:meth:`lance.Dataset.take_blobs` to access the blob data.
+    to interface with APIs that expect file-like objects. For very large binary
+    objects (4-8MB or more per value) you might be better off creating a blob
+    column. Use :py:meth:`lance.Dataset.read_blobs` when you need complete blob
+    bytes, or :py:meth:`lance.Dataset.take_blobs` when you need lazy file-like
+    access.
     """
 
     def __init__(self, blob_column: Union[pa.Array, pa.ChunkedArray]):
diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py
index 4f7ed434436..def635d58a6 100644
--- a/python/python/lance/dataset.py
+++ b/python/python/lance/dataset.py
@@ -51,6 +51,7 @@
 from .fragment import DataFile, FragmentMetadata, LanceFragment
 from .indices import IndexConfig, IndexSegment, SupportedDistributedIndices
 from .lance import (
+    CleanupExplanation,
     CleanupStats,
     Compaction,
     CompactionMetrics,
@@ -74,7 +75,11 @@
 from .udf import BatchUDF, normalize_transform
 from .udf import BatchUDFCheckpoint as BatchUDFCheckpoint
 from .udf import batch_udf as batch_udf
-from .util import _target_partition_size_to_num_partitions, td_to_micros
+from .util import (
+    _normalize_index_segment_ids,
+    _target_partition_size_to_num_partitions,
+    td_to_micros,
+)
 
 if TYPE_CHECKING:
     from pyarrow._compute import Expression
@@ -535,6 +540,54 @@ def use_index(self, use_index: bool) -> "MergeInsertBuilder":
         """
         return super(MergeInsertBuilder, self).use_index(use_index)
 
+    def target_bases(self, bases: List[str]) -> "MergeInsertBuilder":
+        """
+        Write new fragments produced by this merge insert to these bases.
+
+        Each entry references a base path registered in the dataset manifest,
+        by name or by path URI, like the ``target_bases`` parameter of
+        :func:`~lance.write_dataset`. An entry equal to the dataset's URI
+        includes the dataset's primary storage in the rotation, e.g.
+        ``[ds.uri, "base1", "base2"]`` spreads new data files across primary
+        storage and both bases. New data files are distributed across the
+        target bases round-robin. Data files that patch existing fragments
+        and deletion files are always written to the dataset's primary
+        storage.
+
+        Parameters
+        ----------
+        bases : List[str]
+            Base names or path URIs to write new data files to.
+
+        Returns
+        -------
+        MergeInsertBuilder
+            The builder instance for method chaining.
+        """
+        return super(MergeInsertBuilder, self).target_bases(bases)
+
+    def target_all_bases(self, include_primary: bool = True) -> "MergeInsertBuilder":
+        """
+        Write new fragments to every base registered in the dataset manifest.
+
+        The bases are resolved when the merge insert executes, so bases added
+        later are picked up automatically. When ``include_primary`` is True
+        (the default), the dataset's primary storage participates in the
+        round-robin rotation as the first slot. Cannot be combined with
+        :meth:`target_bases`.
+
+        Parameters
+        ----------
+        include_primary : bool, default True
+            Whether the dataset's primary storage is part of the rotation.
+
+        Returns
+        -------
+        MergeInsertBuilder
+            The builder instance for method chaining.
+        """
+        return super(MergeInsertBuilder, self).target_all_bases(include_primary)
+
     def explain_plan(
         self, schema: Optional[pa.Schema] = None, verbose: bool = False
     ) -> str:
@@ -950,6 +1003,9 @@ def create_branch(
         ds._base_store_params = self._base_store_params
         ds._namespace_client = self._namespace_client
         ds._table_id = self._table_id
+        ds._namespace_client_managed_versioning = (
+            self._namespace_client_managed_versioning
+        )
         ds._default_scan_options = self._default_scan_options
         ds._read_params = self._read_params
         return ds
@@ -958,14 +1014,14 @@ def checkout_latest(self):
         """Check out the latest version of the current branch."""
         self._ds.checkout_latest()
 
-    def list_indices(self) -> List[Index]:
+    def list_indices(self) -> List[IndexInformation]:
         """
         Returns index information for all indices in the dataset.
 
-        This method is deprecated as it requires loading the statistics for each index
-        which can be a very expensive operation.  Instead use describe_indices() to
-        list index information and index_statistics() to get the statistics for
-        individual indexes of interest.
+        This method is deprecated.  Use describe_indices() instead, which returns
+        richer per-index information.
+
+        Each returned :class:`IndexInformation` describes one index segment.
         """
         warnings.warn(
             "The 'list_indices' method is deprecated. It may be removed in a future "
@@ -973,12 +1029,34 @@ def list_indices(self) -> List[Index]:
             DeprecationWarning,
         )
 
-        return self._ds.load_indices()
+        return [
+            {
+                "name": desc.name,
+                "type": desc.index_type,
+                "uuid": segment.uuid,
+                "fields": desc.field_names,
+                "version": segment.dataset_version_at_last_update,
+                "fragment_ids": segment.fragment_ids,
+                "base_id": segment.base_id,
+            }
+            for desc in self.describe_indices()
+            for segment in desc.segments
+        ]
 
     def describe_indices(self) -> List[IndexDescription]:
         """Returns index information for all indices in the dataset."""
         return self._ds.describe_indices()
 
+    def remap_row_addrs(self, addrs: "pa.Array") -> "Optional[pa.Array]":
+        """Remap row addresses across compactions still recorded in the
+        fragment-reuse index. Rows a compaction dropped become null. The index
+        retains only recent rounds (older ones are pruned as index remap catches
+        up), so remap promptly: an address whose round was pruned is returned
+        unchanged, not remapped. Returns ``None`` when there is no fragment-reuse
+        index.
+        """
+        return self._ds.remap_row_addrs(addrs)
+
     def index_statistics(self, index_name: str) -> Dict[str, Any]:
         warnings.warn(
             "LanceDataset.index_statistics() is deprecated, "
@@ -1338,7 +1416,10 @@ def data_storage_version(self) -> str:
     @property
     def has_stable_row_ids(self) -> bool:
         """
-        Whether this dataset has stable row IDs enabled
+        Whether this dataset has stable row IDs enabled.
+
+        This is based on the dataset manifest feature flag and does not depend on
+        whether the current version has any fragments.
         """
         return self._ds.has_stable_row_ids
 
@@ -2093,6 +2174,10 @@ def take_blobs(
         this API allows you to open binary blob data as a regular Python file-like
         object. For more details, see :py:class:`lance.BlobFile`.
 
+        If you plan to read each selected blob completely with ``read()`` or
+        ``readall()``, use :py:meth:`read_blobs` instead. It materializes blob
+        payloads with Lance's planned batched reader.
+
         Exactly one of ids, addresses, or indices must be specified.
 
         Parameters
@@ -2141,7 +2226,8 @@ def read_blobs(
 
         Unlike :py:meth:`take_blobs`, which returns file-like :py:class:`lance.BlobFile`
         handles for random access, this API plans and executes batched reads and
-        returns materialized blob payloads.
+        returns materialized blob payloads. Use this API for training loaders,
+        batch preprocessing, and other workflows that need complete blob bytes.
 
         Exactly one of ids, addresses, or indices must be specified.
 
@@ -2271,9 +2357,9 @@ def alter_columns(self, *alterations: Iterable[AlterColumn]):
                 not changed.
             - "nullable": bool, optional
                 Whether the column should be nullable. If not specified, the column
-                nullability is not changed. Only non-nullable columns can be changed
-                to nullable. Currently, you cannot change a nullable column to
-                non-nullable.
+                nullability is not changed. A non-nullable column can always be made
+                nullable. A nullable column can be made non-nullable only if it
+                contains no NULL values; otherwise an error is raised.
             - "data_type": pyarrow.DataType, optional
                 The new data type to cast the column to. If not specified, the column
                 data type is not changed.
@@ -2961,6 +3047,64 @@ def cleanup_old_versions(
             delete_rate_limit,
         )
 
+    def explain_cleanup_old_versions(
+        self,
+        older_than: Optional[timedelta] = None,
+        retain_versions: Optional[int] = None,
+        *,
+        delete_unverified: bool = False,
+        error_if_tagged_old_versions: bool = True,
+        delete_rate_limit: Optional[int] = None,
+        include_files: bool = False,
+        max_files: int = 1000,
+    ) -> CleanupExplanation:
+        """
+        Explain what :meth:`cleanup_old_versions` would remove without deleting files.
+
+        Parameters
+        ----------
+
+        older_than: timedelta, optional
+            Only versions older than this would be removed. If ``older_than`` and
+            ``retain_versions`` are not specified, this will default to two weeks.
+
+        retain_versions: int, optional
+            Retain the last N versions of the dataset.
+
+        delete_unverified: bool, default False
+            Include unverified files that cleanup would remove when this is set.
+
+        error_if_tagged_old_versions: bool, default True
+            If set to `True`, an exception will be raised if any tagged versions
+            match the parameters. Otherwise, tagged versions will be ignored.
+
+        delete_rate_limit: int, optional
+            Accepted for parity with :meth:`cleanup_old_versions`; no deletes are
+            issued by explain.
+
+        include_files: bool, default False
+            If `True`, include candidate files in the explanation up to
+            ``max_files`` entries. Aggregate stats always include all candidates.
+
+        max_files: int, default 1000
+            Maximum number of candidate files to include when ``include_files``
+            is `True`.
+        """
+        if older_than is None and retain_versions is None:
+            older_than = timedelta(days=14)
+        if max_files <= 0:
+            raise ValueError("max_files must be positive")
+
+        return self._ds.explain_cleanup_old_versions(
+            td_to_micros(older_than) if older_than else None,
+            retain_versions,
+            delete_unverified,
+            error_if_tagged_old_versions,
+            delete_rate_limit,
+            include_files,
+            max_files,
+        )
+
     def _prepare_scalar_index_request(
         self,
         column: Union[str, List[str]],
@@ -3017,12 +3161,13 @@ def _prepare_scalar_index_request(
                     and not pa.types.is_floating(field_type)
                     and not pa.types.is_boolean(field_type)
                     and not pa.types.is_string(field_type)
+                    and not pa.types.is_large_string(field_type)
                     and not pa.types.is_temporal(field_type)
                     and not pa.types.is_fixed_size_binary(field_type)
                 ):
                     raise TypeError(
-                        f"BTREE/BITMAP index column {column} must be int",
-                        ", float, bool, str, fixed-size-binary, or temporal ",
+                        f"BTREE/BITMAP/ZONEMAP index column {column} must be int",
+                        ", float, bool, str, large_str, fixed-size-binary, or temporal",
                     )
             elif index_type == "LABEL_LIST":
                 if not pa.types.is_list(field_type):
@@ -3062,6 +3207,38 @@ def _prepare_scalar_index_request(
         else:
             raise Exception("index_type must be str or IndexConfig")
 
+    @staticmethod
+    def _normalized_index_type(
+        index_type: Union[str, IndexConfig],
+    ) -> str:
+        if isinstance(index_type, IndexConfig):
+            index_type = index_type.index_type
+        return index_type.upper()
+
+    @classmethod
+    def _is_segment_native_scalar_index_type(
+        cls,
+        index_type: Union[str, IndexConfig],
+    ) -> bool:
+        return cls._normalized_index_type(index_type) in {
+            "BTREE",
+            "BITMAP",
+            "INVERTED",
+            "FTS",
+            "ZONEMAP",
+        }
+
+    @classmethod
+    def _requires_uncommitted_scalar_index(
+        cls,
+        index_type: Union[str, IndexConfig],
+    ) -> bool:
+        return cls._normalized_index_type(index_type) in {
+            "BTREE",
+            "BITMAP",
+            "ZONEMAP",
+        }
+
     def create_scalar_index(
         self,
         column: str,
@@ -3084,6 +3261,7 @@ def create_scalar_index(
         fragment_ids: Optional[List[int]] = None,
         index_uuid: Optional[str] = None,
         progress_callback: Optional[Callable[[IndexProgress], None]] = None,
+        format_version: Optional[Union[int, str]] = None,
         **kwargs,
     ):
         """Create a scalar index on a column.
@@ -3118,7 +3296,7 @@ def create_scalar_index(
             )
 
 
-        There are 5 types of scalar indices available today.
+        Lance supports the following scalar index types:
 
         * ``BTREE``. The most common type is ``BTREE``. This index is inspired
           by the btree data structure although only the first few layers of the btree
@@ -3189,6 +3367,13 @@ def create_scalar_index(
         progress_callback : callable, optional
             A callback that receives :class:`lance.progress.IndexProgress` events while
             the index is being built.
+        format_version: int or str, optional
+            This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS
+            format version to write when creating a new index. Accepts ``1``,
+            ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. If unset, Lance
+            writes v2 for ``block_size=128`` and v3 for ``block_size=256``.
+            ``format_version=3`` is experimental and is only valid with
+            ``block_size=256``.
 
         with_position: bool, default False
             This is for the ``INVERTED`` index. If True, the index will store the
@@ -3196,6 +3381,12 @@ def create_scalar_index(
             query. This will significantly increase the index size.
             It won't impact the performance of non-phrase queries even if it is set to
             True.
+        block_size: int, default 128
+            This is for the ``INVERTED`` index. Number of documents per compressed
+            posting block. Must be one of ``128`` or ``256``.
+            ``block_size=256`` is experimental and may introduce breaking changes.
+            Use ``128`` when stable compatibility with the legacy posting layout is
+            required.
         memory_limit: int, optional
             This is for the ``INVERTED`` index. Total build-time memory limit in MiB.
             If set, Lance divides this budget evenly across the workers. If unset,
@@ -3217,6 +3408,8 @@ def create_scalar_index(
             * "simple": splits tokens on whitespace and punctuation.
             * "whitespace": splits tokens on whitespace.
             * "raw": no tokenization.
+            * "icu": ICU dictionary-based Unicode word segmentation.
+            * "icu/split": ICU segmentation with simple-style delimiter splitting.
         language: str, default "English"
             This is for the ``INVERTED`` index. The language for stemming
             and stop words. This is only used when `stem` or `remove_stop_words` is true
@@ -3250,7 +3443,7 @@ def create_scalar_index(
             import lance
 
             dataset = lance.dataset("/tmp/images.lance")
-            dataset.create_index(
+            dataset.create_scalar_index(
                 "category",
                 "BTREE",
             )
@@ -3279,12 +3472,14 @@ def create_scalar_index(
             column, index_type, kwargs
         )
 
-        if fragment_ids is not None and logical_index_type == "BTREE":
+        if fragment_ids is not None and self._requires_uncommitted_scalar_index(
+            logical_index_type
+        ):
             raise ValueError(
-                "BTree distributed indexing uses create_index_uncommitted(..., "
-                'index_type="BTREE", fragment_ids=...)'
+                f"{logical_index_type} distributed indexing uses "
+                "create_index_uncommitted(..., "
+                f'index_type="{logical_index_type}", fragment_ids=...)'
             )
-
         # Add fragment_ids and index_uuid to kwargs if provided
         if fragment_ids is not None:
             kwargs["fragment_ids"] = fragment_ids
@@ -3292,6 +3487,8 @@ def create_scalar_index(
             kwargs["index_uuid"] = index_uuid
         if progress_callback is not None:
             kwargs["progress_callback"] = progress_callback
+        if format_version is not None:
+            kwargs["format_version"] = format_version
 
         self._ds.create_index([column], index_type, name, replace, train, None, kwargs)
 
@@ -3327,6 +3524,7 @@ def _create_index_impl(
         streaming_coreset_rate: Optional[int] = None,
         streaming_refine_passes: Optional[int] = None,
         skip_transpose: bool = False,
+        rabitq_model: Optional[str] = None,
         require_commit: bool = True,
         **kwargs,
     ) -> Index:
@@ -3617,13 +3815,17 @@ def _create_index_impl(
                 if _check_for_numpy(pq_codebook) and isinstance(
                     pq_codebook, np.ndarray
                 ):
+                    num_bits = kwargs.get("num_bits", 8)
+                    expected_centroids = 2**num_bits
                     if (
                         len(pq_codebook.shape) != 3
                         or pq_codebook.shape[0] != num_sub_vectors
-                        or pq_codebook.shape[1] != 256
+                        or pq_codebook.shape[1] != expected_centroids
                     ):
                         raise ValueError(
-                            f"PQ codebook must be 3D array: (sub_vectors, 256, dim), "
+                            "PQ codebook must be 3D array: "
+                            f"(sub_vectors, {expected_centroids}, dim) "
+                            f"for num_bits={num_bits}, "
                             f"got {pq_codebook.shape}"
                         )
                     if pq_codebook.dtype not in [np.float16, np.float32, np.float64]:
@@ -3648,6 +3850,9 @@ def _create_index_impl(
         if skip_transpose:
             kwargs["skip_transpose"] = True
 
+        if rabitq_model is not None:
+            kwargs["rabitq_model"] = rabitq_model
+
         # Add fragment_ids and index_uuid to kwargs if provided for
         # distributed indexing
         if fragment_ids is not None:
@@ -3742,10 +3947,9 @@ def create_index(
         pq_codebook : optional,
             It can be :py:class:`np.ndarray`, :py:class:`pyarrow.FixedSizeListArray`,
             or :py:class:`pyarrow.FixedShapeTensorArray`.
-            A ``num_sub_vectors x (2 ^ nbits * dimensions // num_sub_vectors)``
-            array of K-mean centroids for PQ codebook.
-
-            Note: ``nbits`` is always 8 for now.
+            A ``num_sub_vectors x (2 ^ num_bits) x
+            (dimensions // num_sub_vectors)`` array of K-mean centroids for PQ
+            codebook. ``num_bits`` defaults to 8.
             If not provided, a new PQ model will be trained.
         num_sub_vectors : int, optional
             The number of sub-vectors for PQ (Product Quantization).
@@ -3785,6 +3989,17 @@ def create_index(
             to the dataset. The returned metadata can be passed to
             ``merge_existing_index_segments(...)`` if grouping is needed and then
             committed with ``commit_existing_index_segments(...)``.
+
+            Vector segments support both shared and independent model scopes. If
+            the caller provides the same IVF centroids, and for IVF_PQ the same
+            PQ codebook, to each worker, the resulting segments share model
+            semantics and are suitable for workflows that physically merge
+            compatible segments. If those artifacts are omitted, each segment can
+            train its own IVF/PQ model for its assigned fragments. Such segments
+            can be committed together and are queried independently by segment
+            UUID; partition ids are interpreted within each segment's own model.
+            Keep independently trained segments as separate physical segments
+            unless the merge workflow can preserve or reconcile the model state.
         index_uuid : str, optional
             A UUID to use for the segment written by this call.
             If not provided, a new UUID will be generated.
@@ -3970,15 +4185,18 @@ def create_index_uncommitted(
         streaming_coreset_rate: Optional[int] = None,
         streaming_refine_passes: Optional[int] = None,
         skip_transpose: bool = False,
+        rabitq_model: Optional[str] = None,
         **kwargs,
     ) -> Index:
         """
         Create one segment without publishing it and return its metadata.
 
-        This is the public distributed-build API for vector and BTREE scalar
-        index construction. Unlike :meth:`create_index`, this method does not
-        publish the index into the dataset manifest. Instead, it writes one
-        segment under ``_indices/<segment_uuid>/`` and returns the resulting
+        This is the public distributed-build API for vector, BTREE scalar,
+        canonical bitmap scalar, INVERTED scalar, and ZONEMAP scalar index
+        construction. Unlike
+        :meth:`create_index`, this method does not publish the index into the
+        dataset manifest. Instead, it writes one segment under
+        ``_indices/<segment_uuid>/`` and returns the resulting
         :class:`Index` metadata.
 
         Callers should:
@@ -3991,27 +4209,37 @@ def create_index_uncommitted(
         4. commit the final segment list with
            :meth:`commit_existing_index_segments`
 
-        BTREE segments do not yet support the segment builder (steps 3-4); collect
-        the returned segments and pass them straight to
-        :meth:`commit_existing_index_segments`.
-
+        BTREE, BITMAP, INVERTED, and ZONEMAP segments may
+        be merged with :meth:`merge_existing_index_segments` before commit.
         Parameters are the same as :meth:`create_index`, with one additional
         requirement:
 
         - ``fragment_ids`` must be provided
+        - Vector segments support both shared and independent model scopes. Pass
+          the same IVF centroids, and for IVF_PQ the same PQ codebook, to each
+          worker when segments need shared model semantics or physical merge
+          compatibility. If these artifacts are omitted, each segment may train
+          its own IVF/PQ model and can be committed with other segments as one
+          logical index; query execution searches each segment by UUID and
+          interprets partition ids within that segment. Keep independently
+          trained segments as separate physical segments unless the merge
+          workflow can preserve or reconcile the model state.
+        - ``rabitq_model`` (``IVF_RQ`` only): a JSON string produced by
+          ``lance.lance.indices.build_rq_model``. It must be identical across all
+          workers for their segments to be mergeable, since it pins the RaBitQ
+          rotation so every segment rotates vectors the same way. If omitted, each
+          call generates its own random rotation, which is only safe for a single,
+          non-merged segment.
 
         Returns
         -------
         Index
             Metadata for the segment that was written by this call.
         """
-        is_btree_request = (
-            isinstance(index_type, str) and index_type.upper() == "BTREE"
-        ) or (
-            isinstance(index_type, IndexConfig)
-            and index_type.index_type.upper() == "BTREE"
+        is_scalar_segment_request = self._is_segment_native_scalar_index_type(
+            index_type
         )
-        if is_btree_request:
+        if is_scalar_segment_request:
             if fragment_ids is None:
                 raise ValueError(
                     "create_index_uncommitted requires fragment_ids "
@@ -4062,6 +4290,7 @@ def create_index_uncommitted(
             streaming_coreset_rate=streaming_coreset_rate,
             streaming_refine_passes=streaming_refine_passes,
             skip_transpose=skip_transpose,
+            rabitq_model=rabitq_model,
             require_commit=False,
             **kwargs,
         )
@@ -4077,13 +4306,22 @@ def drop_index(self, name: str):
         """
         return self._ds.drop_index(name)
 
-    def prewarm_index(self, name: str, *, with_position: bool = False):
+    def prewarm_index(
+        self,
+        name: str,
+        *,
+        with_position: bool = False,
+        index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
+    ):
         """
         Prewarm an index
 
-        This will load the entire index into memory.  This can help avoid cold start
-        issues with index queries.  If the index does not fit in the index cache, then
-        this will result in wasted I/O.
+        By default, this will load the entire index into memory. This can help
+        avoid cold start issues with index queries. If the index does not fit in
+        the index cache, then this will result in wasted I/O.
+
+        Use ``session().index_cache_size_bytes()`` before and after prewarm to
+        inspect how much the index cache grew.
 
         Parameters
         ----------
@@ -4093,8 +4331,16 @@ def prewarm_index(self, name: str, *, with_position: bool = False):
             This is only supported for ``INVERTED`` indices. If True, positions are
             also loaded into the cache during prewarm so phrase queries do not need a
             separate lazy positions read.
-        """
-        return self._ds.prewarm_index(name, with_position=with_position)
+        index_segments: iterable of str or uuid.UUID, default None
+            If specified, prewarm only these physical index segment UUIDs from the
+            named logical index. Use :meth:`describe_indices` to inspect logical
+            indices and obtain segment UUIDs from ``IndexDescription.segments``.
+        """
+        return self._ds.prewarm_index(
+            name,
+            with_position=with_position,
+            index_segments=_normalize_index_segment_ids(index_segments),
+        )
 
     def merge_index_metadata(
         self,
@@ -4106,9 +4352,10 @@ def merge_index_metadata(
         """
         Merge distributed scalar index metadata.
 
-        Vector distributed indexing no longer uses this API. For vector indices,
-        build segments with :meth:`create_index_uncommitted`, optionally merge
-        them with :meth:`merge_existing_index_segments`, and publish them with
+        Vector and Bitmap distributed indexing no longer use this API. For
+        those index families, build segments with
+        :meth:`create_index_uncommitted`, optionally merge caller-defined
+        groups with :meth:`merge_existing_index_segments`, and publish them with
         :meth:`commit_existing_index_segments`.
 
         This method does NOT commit changes.
@@ -4503,6 +4750,7 @@ def commit_batch(
         ds._base_store_params = base_store_params
         ds._namespace_client = None
         ds._table_id = None
+        ds._namespace_client_managed_versioning = False
         ds._default_scan_options = None
         ds._read_params = None
         return BulkCommitResult(
@@ -5053,6 +5301,65 @@ def mem_wal_writer(
         raw = self._ds.mem_wal_writer(shard_id, **kwargs)
         return _mw.ShardWriter(raw)
 
+    def tracked_files(
+        self,
+        *,
+        min_version: Optional[int] = None,
+        progress: Optional[Callable] = None,
+    ) -> pa.RecordBatchReader:
+        """Stream all files referenced by any manifest version of this dataset.
+
+        Parameters
+        ----------
+        min_version : int, optional
+            If set, only include manifests with version >= min_version.
+        progress : callable, optional
+            Called after each manifest is processed with two arguments:
+            ``(manifests_processed: int, manifests_total: Optional[int])``.
+            ``manifests_total`` is ``None`` until all manifest locations
+            have been listed. Works well with ``tqdm``::
+
+                from tqdm import tqdm
+                pbar = tqdm(unit="manifest")
+                def on_progress(processed, total):
+                    if total is not None:
+                        pbar.total = total
+                    pbar.update(1)
+                reader = ds.tracked_files(progress=on_progress)
+                table = reader.read_all()
+                pbar.close()
+
+        Returns
+        -------
+        pyarrow.RecordBatchReader
+            Schema:
+
+            - **version** (int64): manifest version number
+            - **base_uri** (dictionary<int32, utf8>): storage root URI
+            - **path** (utf8): file path relative to ``base_uri``
+            - **type** (dictionary<int8, utf8>): one of ``manifest``,
+              ``data file``, ``deletion file``, ``transaction file``,
+              ``index file``
+
+            Output order is non-deterministic.
+        """
+        return self._ds.tracked_files(min_version=min_version, progress=progress)
+
+    def all_files(self) -> pa.RecordBatchReader:
+        """Stream all files physically present at this dataset's base URI.
+
+        Returns a :class:`pyarrow.RecordBatchReader` with schema:
+
+        - **base_uri** (dictionary<int32, utf8>): storage root URI
+        - **path** (utf8): file path relative to ``base_uri``
+        - **size_bytes** (int64): file size in bytes
+        - **last_modified** (timestamp[us, UTC]): last modification time
+
+        Only the primary object store is scanned; alternate ``base_paths``
+        entries are not included.
+        """
+        return self._ds.all_files()
+
 
 class SqlQuery:
     """
@@ -5287,6 +5594,19 @@ class Index:
     index_details: Optional[Tuple[str, bytes]] = None
 
 
+class IndexInformation(TypedDict):
+    """Information about a single index segment, as returned by
+    :meth:`LanceDataset.list_indices`."""
+
+    name: str
+    type: str
+    uuid: str
+    fields: List[str]
+    version: int
+    fragment_ids: Set[int]
+    base_id: Optional[int]
+
+
 class AutoCleanupConfig(TypedDict):
     interval: int
     older_than_seconds: int
@@ -5671,6 +5991,76 @@ class DataReplacement(BaseOperation):
 
         replacements: List[LanceOperation.DataReplacementGroup]
 
+    @dataclass
+    class DataOverlayFile:
+        """
+        An overlay file supplying new values for a subset of
+        ``(physical offset, field)`` cells of a fragment, resolved on read and
+        layered over the base data without rewriting the base files.
+
+        The overlay is dense or sparse depending on the shape of ``offsets``:
+        pass a flat ``List[int]`` for a dense overlay (one offset list shared by
+        every field in ``data_file``) or a ``List[List[int]]`` for a sparse
+        overlay (one offset list per field, in the order of the file's fields).
+        Offsets are **physical** row offsets (positions in the base files,
+        counting deleted rows), like deletion vectors.
+
+        Attributes
+        ----------
+        data_file : DataFile
+            The Lance data file storing the overlay's new cell values — one
+            value column per covered field. The value at each covered offset is
+            stored at the rank (0-based count of covered offsets below it) of
+            that offset in the field's coverage.
+        offsets : Union[List[int], List[List[int]]]
+            The covered physical row offsets. A flat list is dense coverage
+            (shared by every field); a list of per-field lists is sparse
+            coverage (in field order). Each list must be strictly ascending
+            with no duplicates, since the Nth offset maps to the Nth value row
+            in ``data_file``; a non-ascending list raises ``ValueError``.
+        committed_version : Optional[int]
+            The dataset version at which this overlay became effective. Leave as
+            ``None`` when creating an overlay to commit — the commit stamps it.
+            It is populated when reading an existing fragment's overlays so they
+            round-trip through :class:`FragmentMetadata`.
+        """
+
+        data_file: DataFile
+        offsets: Union[List[int], List[List[int]]]
+        committed_version: Optional[int] = None
+
+    @dataclass
+    class DataOverlayGroup:
+        """
+        Overlay files to append to a single fragment.
+
+        Attributes
+        ----------
+        fragment_id : int
+            The id of the fragment the overlays apply to.
+        overlays : List[LanceOperation.DataOverlayFile]
+            The overlay files to append, ordered oldest-first (a later entry is
+            newer and wins where coverage overlaps).
+        """
+
+        fragment_id: int
+        overlays: List[LanceOperation.DataOverlayFile]
+
+    @dataclass
+    class DataOverlay(BaseOperation):
+        """
+        Operation that appends data overlay files to fragments.
+
+        Overlays are appended to each fragment's existing overlays (overlays
+        written by concurrent commits are preserved) and resolved on read
+        over the base data without rewriting it.
+
+        If multiple groups target the same data then the values in the
+        latest group take precedence.
+        """
+
+        groups: List[LanceOperation.DataOverlayGroup]
+
     @dataclass
     class Project(BaseOperation):
         """
@@ -5876,18 +6266,14 @@ def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder:
         used by the scanner.  If the buffer is full then the scanner will block until
         the buffer is processed.
 
-        Generally this should scale with the number of concurrent I/O threads.  The
-        default is 2GiB which comfortably provides enough space for somewhere between
-        32 and 256 concurrent I/O threads.
+        Generally this should scale with the number of concurrent I/O threads.  If
+        unset, v2 scans choose a default based on the object store and
+        ``LANCE_DEFAULT_IO_BUFFER_SIZE`` can override that default.
 
         This value is not a hard cap on the amount of RAM the scanner will use.  Some
         space is used for the compute (which can be controlled by the batch size) and
         Lance does not keep track of memory after it is returned to the user.
 
-        Currently, if there is a single batch of data which is larger than the io buffer
-        size then the scanner will deadlock.  This is a known issue and will be fixed in
-        a future release.
-
         This parameter is only used when reading v2 files
         """
         self._io_buffer_size = io_buffer_size
@@ -5895,10 +6281,12 @@ def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder:
 
     def batch_readahead(self, nbatches: Optional[int] = None) -> ScannerBuilder:
         """
-        This parameter is ignored when reading v2 files
+        Set the maximum number of batches to decode concurrently.
+
+        This parameter must be greater than zero.
         """
-        if nbatches is not None and int(nbatches) < 0:
-            raise ValueError("batch_readahead must be non-negative")
+        if nbatches is not None and int(nbatches) <= 0:
+            raise ValueError("batch_readahead must be greater than 0")
         self._batch_readahead = nbatches
         return self
 
@@ -6107,19 +6495,7 @@ def with_fragments(
     def with_index_segments(
         self, index_segments: Optional[Iterable[Union[str, uuid.UUID]]]
     ) -> ScannerBuilder:
-        if index_segments is not None:
-            segment_ids = []
-            for segment_id in index_segments:
-                if isinstance(segment_id, (str, uuid.UUID)):
-                    segment_ids.append(str(segment_id))
-                else:
-                    raise TypeError(
-                        "index_segments must be an iterable of str or uuid.UUID. "
-                        f"Got {type(segment_id)} instead."
-                    )
-            index_segments = segment_ids
-
-        self._index_segments = index_segments
+        self._index_segments = _normalize_index_segment_ids(index_segments)
         return self
 
     def nearest(
@@ -6135,6 +6511,7 @@ def nearest(
         use_index: bool = True,
         ef: Optional[int] = None,
         query_parallelism: Optional[int] = None,
+        approx_mode: Literal["fast", "normal", "accurate"] = "normal",
         distance_range: Optional[tuple[Optional[float], Optional[float]]] = None,
     ) -> ScannerBuilder:
         """Configure nearest neighbor search.
@@ -6158,6 +6535,13 @@ def nearest(
             the CPU pool size. Value 1 uses the single-worker sequential path.
             Values >= 2 use the partition-parallel path and are clamped to the
             CPU pool size.
+        approx_mode: {"fast", "normal", "accurate"}, default "normal"
+            Controls the speed / accuracy tradeoff for approximate vector search
+            when supported by the selected index. This currently only affects
+            RQ-quantized indexes, such as IVF_RQ. Other index types ignore this
+            setting. ``fast`` favors lower latency and may reduce recall,
+            ``normal`` uses the default balance, and ``accurate`` favors higher
+            recall and may increase latency.
         """
         self._nearest = _build_vector_search_query(
             column,
@@ -6172,6 +6556,7 @@ def nearest(
             use_index=use_index,
             ef=ef,
             query_parallelism=query_parallelism,
+            approx_mode=approx_mode,
             distance_range=distance_range,
         )
         return self
@@ -6496,20 +6881,22 @@ def explain_plan(self, verbose=False) -> str:
 
         return self._scanner.explain_plan(verbose=verbose)
 
-    def analyze_plan(self) -> str:
+    def analyze_plan(self, count_rows: bool = False) -> str:
         """Execute the plan for this scanner and display with runtime metrics.
 
         Parameters
         ----------
-        verbose : bool, default False
-            Use a verbose output format.
+        count_rows : bool, default False
+            If True, auto-apply a ``COUNT(*)`` aggregate before analyzing so
+            the returned plan reflects what :py:meth:`count_rows` would
+            execute (including the optimizer's count-pushdown decisions).
 
         Returns
         -------
         plan : str
         """
 
-        return self._scanner.analyze_plan()
+        return self._scanner.analyze_plan(count_rows=count_rows)
 
 
 class DatasetOptimizer:
@@ -6938,6 +7325,7 @@ def write_dataset(
     transaction_properties: Optional[Dict[str, str]] = None,
     initial_bases: Optional[List[DatasetBasePath]] = None,
     target_bases: Optional[List[str]] = None,
+    target_all_bases: Optional[bool] = None,
     base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
     external_blob_mode: Literal["reference", "ingest"] = "reference",
     allow_external_blob_outside_bases: bool = False,
@@ -6985,6 +7373,13 @@ def write_dataset(
     storage_options : optional, dict
         Extra options that make sense for a particular storage connection. This is
         used to store connection parameters like credentials, endpoint, etc.
+
+        For writes involving additional base paths, a key of the form
+        ``base_<id>.<key>`` applies ``<key>`` only to the base path with that
+        id, overriding the unscoped options that every base inherits. For
+        example ``{"account_key": "shared", "base_1.account_key": "abc"}``
+        makes base 1 use ``account_key = abc`` while all other options are
+        shared.
     data_storage_version: optional, str, default None
         The version of the data storage format to use. Newer versions are more
         efficient but require newer versions of lance to read.  The default (None)
@@ -7036,12 +7431,19 @@ def write_dataset(
 
         **CREATE mode**: References must match bases in `initial_bases`
         **APPEND/OVERWRITE modes**: References must match bases in the existing manifest
+    target_all_bases: bool, optional
+        Write new data files round-robin across every base registered in the
+        manifest. When True (include primary), the dataset's primary storage
+        participates in the rotation as the first slot; when False, only the
+        registered bases are used. Cannot be combined with ``target_bases``.
     base_store_params : dict of str to dict, optional
         Runtime-only object store parameters keyed by base path URI. Each key
         is a base path URI (e.g., "s3://bucket/path") and each value is a dict
         of storage options (credentials, endpoint, etc.) for that base. These
-        are not persisted to the manifest. When a base has no explicit entry
-        here, the top-level ``storage_options`` is used as a fallback.
+        are not persisted to the manifest. These take precedence over
+        ``base_<id>.<key>`` entries in ``storage_options``. When a base has no
+        explicit entry here, the top-level ``storage_options`` is used as a
+        fallback.
     external_blob_mode: {"reference", "ingest"}, default "reference"
         How external blob URIs are handled on write.
 
@@ -7184,6 +7586,7 @@ def write_dataset(
         "transaction_properties": merged_properties,
         "initial_bases": initial_bases,
         "target_bases": target_bases,
+        "target_all_bases": target_all_bases,
         "base_store_params": base_store_params,
         "external_blob_mode": external_blob_mode,
         "allow_external_blob_outside_bases": allow_external_blob_outside_bases,
@@ -7250,8 +7653,7 @@ def _coerce_query_vector(query: QueryVectorLike) -> tuple[pa.Array, int]:
         if isinstance(query.type, pa.FixedSizeListType):
             query = query.values
     elif isinstance(query, (list, tuple)) or (
-        _check_for_numpy(query),
-        isinstance(query, np.ndarray),
+        _check_for_numpy(query) and isinstance(query, np.ndarray)
     ):
         query = np.array(query).astype("float64")  # workaround for GH-608
         query = pa.FloatingPointArray.from_pandas(query, type=pa.float32())
@@ -7292,6 +7694,7 @@ def _build_vector_search_query(
     use_index: bool = True,
     ef: Optional[int] = None,
     query_parallelism: Optional[int] = None,
+    approx_mode: Literal["fast", "normal", "accurate"] = "normal",
     distance_range: Optional[tuple[Optional[float], Optional[float]]] = None,
 ) -> dict:
     """Configure nearest neighbor search.
@@ -7333,6 +7736,13 @@ def _build_vector_search_query(
         maps to the single-worker sequential path. Value -1 uses the CPU pool
         size. Value 1 uses the single-worker sequential path. Values >= 2 use
         the partition-parallel path and are clamped to the CPU pool size.
+    approx_mode: {"fast", "normal", "accurate"}, default "normal"
+        Controls the speed / accuracy tradeoff for approximate vector search
+        when supported by the selected index. This currently only affects
+        RQ-quantized indexes, such as IVF_RQ. Other index types ignore this
+        setting. ``fast`` favors lower latency and may reduce recall,
+        ``normal`` uses the default balance, and ``accurate`` favors higher
+        recall and may increase latency.
     distance_range: tuple[Optional[float], Optional[float]], optional
         A tuple of (lower_bound, upper_bound) to filter results by distance.
         Both bounds are optional. The lower bound is inclusive and the upper
@@ -7406,6 +7816,12 @@ def _build_vector_search_query(
     if query_parallelism is not None and query_parallelism < -1:
         raise ValueError("query_parallelism must be >= -1")
 
+    if approx_mode not in {"fast", "normal", "accurate"}:
+        raise ValueError(
+            "approx_mode must be one of 'fast', 'normal', or 'accurate', "
+            f"got {approx_mode!r}"
+        )
+
     if distance_range is not None:
         if len(distance_range) != 2:
             raise ValueError(
@@ -7423,6 +7839,7 @@ def _build_vector_search_query(
         "use_index": use_index,
         "ef": ef,
         "query_parallelism": query_parallelism,
+        "approx_mode": approx_mode,
         "distance_range": distance_range,
     }
 
@@ -7579,6 +7996,7 @@ def __init__(
         use_index: bool = True,
         ef: Optional[int] = None,
         query_parallelism: Optional[int] = None,
+        approx_mode: Literal["fast", "normal", "accurate"] = "normal",
     ):
         self._inner = _build_vector_search_query(
             column,
@@ -7592,6 +8010,7 @@ def __init__(
             use_index=use_index,
             ef=ef,
             query_parallelism=query_parallelism,
+            approx_mode=approx_mode,
         )
 
     def inner(self):
diff --git a/python/python/lance/dependencies.py b/python/python/lance/dependencies.py
index df6b71cb22f..a739b7bc75d 100644
--- a/python/python/lance/dependencies.py
+++ b/python/python/lance/dependencies.py
@@ -27,7 +27,6 @@
 _CAGRA_AVAILABLE = True
 _RAFT_COMMON_AVAILABLE = True
 _HUGGING_FACE_AVAILABLE = True
-_TENSORFLOW_AVAILABLE = True
 
 
 class _LazyModule(ModuleType):
@@ -49,7 +48,6 @@ class _LazyModule(ModuleType):
         "pandas": "pd.",
         "polars": "pl.",
         "torch": "torch.",
-        "tensorflow": "tf.",
     }
 
     def __init__(
@@ -163,8 +161,7 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]:
     import numpy
     import pandas
     import polars
-    import tensorflow
-    import torch
+    import torch  # type: ignore[reportMissingImports]
 else:
     # heavy/optional third party libs
     numpy, _NUMPY_AVAILABLE = _lazy_import("numpy")
@@ -172,7 +169,6 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]:
     polars, _POLARS_AVAILABLE = _lazy_import("polars")
     torch, _TORCH_AVAILABLE = _lazy_import("torch")
     datasets, _HUGGING_FACE_AVAILABLE = _lazy_import("datasets")
-    tensorflow, _TENSORFLOW_AVAILABLE = _lazy_import("tensorflow")
 
 
 @lru_cache(maxsize=None)
@@ -215,26 +211,18 @@ def _check_for_hugging_face(obj: Any, *, check_type: bool = True) -> bool:
     )
 
 
-def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool:
-    return _TENSORFLOW_AVAILABLE and _might_be(
-        cast("Hashable", type(obj) if check_type else obj), "tensorflow"
-    )
-
-
 __all__ = [
     # lazy-load third party libs
     "datasets",
     "numpy",
     "pandas",
     "polars",
-    "tensorflow",
     "torch",
     # lazy utilities
     "_check_for_hugging_face",
     "_check_for_numpy",
     "_check_for_pandas",
     "_check_for_polars",
-    "_check_for_tensorflow",
     "_check_for_torch",
     "_LazyModule",
     # exported flags/guards
@@ -243,5 +231,4 @@ def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool:
     "_POLARS_AVAILABLE",
     "_TORCH_AVAILABLE",
     "_HUGGING_FACE_AVAILABLE",
-    "_TENSORFLOW_AVAILABLE",
 ]
diff --git a/python/python/lance/file.py b/python/python/lance/file.py
index 011fbe4a01d..5926241977e 100644
--- a/python/python/lance/file.py
+++ b/python/python/lance/file.py
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
@@ -25,6 +26,7 @@
 )
 
 if TYPE_CHECKING:
+    from .blob import DedicatedBlobWriter, PackedBlobWriter
     from .namespace import LanceNamespace
 
 
@@ -207,6 +209,26 @@ def num_rows(self) -> int:
         return self._reader.num_rows()
 
 
+@dataclass
+class ListResult:
+    """
+    Result of a non-recursive, delimited list (see
+    :meth:`LanceFileSession.list_with_delimiter`).
+
+    Attributes
+    ----------
+    common_prefixes : List[str]
+        The immediate child "directories" of the listed path, relative to the
+        session's base path.
+    objects : List[str]
+        The immediate child files of the listed path, relative to the session's
+        base path.
+    """
+
+    common_prefixes: List[str]
+    objects: List[str]
+
+
 class LanceFileSession:
     """
     A file session for reading and writing Lance files.
@@ -311,6 +333,24 @@ def open_writer(
             _inner_writer=inner,
         )
 
+    def open_packed_blob_writer(self, path: str, blob_id: int) -> "PackedBlobWriter":
+        """
+        Opens a packed blob writer for the given data file path.
+
+        The path will be appended to the base path of the session.
+        """
+        return self._session.open_packed_blob_writer(path, blob_id)
+
+    def open_dedicated_blob_writer(
+        self, path: str, blob_id: int
+    ) -> "DedicatedBlobWriter":
+        """
+        Opens a dedicated blob writer for the given data file path.
+
+        The path will be appended to the base path of the session.
+        """
+        return self._session.open_dedicated_blob_writer(path, blob_id)
+
     def contains(self, path: str) -> bool:
         """
         Check if a file exists at the given path (relative to this session's base path).
@@ -344,6 +384,66 @@ def list(self, path: Optional[str] = None) -> List[str]:
         """
         return self._session.list(path)
 
+    def list_with_delimiter(self, path: Optional[str] = None) -> ListResult:
+        """
+        Non-recursively list a single directory level (relative to this
+        session's base path).
+
+        Unlike :meth:`list`, which recurses into the entire subtree, this
+        returns only the immediate children of ``path``: the child
+        "directories" as ``common_prefixes`` and the direct child files as
+        ``objects``.
+
+        Parameters
+        ----------
+        path : str, optional
+            Path relative to `base_path` to list. If None, lists the base path.
+
+        Returns
+        -------
+        ListResult
+            The immediate child prefixes and objects of `path`.
+        """
+        common_prefixes, objects = self._session.list_with_delimiter(path)
+        return ListResult(common_prefixes=common_prefixes, objects=objects)
+
+    def read_range(self, path: str, offset: int, length: int) -> bytes:
+        """
+        Read a byte range from a file (relative to this session's base path).
+
+        Issues a single ranged read. Reading a missing object raises
+        ``OSError``, consistent with ``download_file``.
+
+        Parameters
+        ----------
+        path : str
+            Path relative to `base_path` to read from.
+        offset : int
+            Byte offset at which to start reading.
+        length : int
+            Number of bytes to read.
+
+        Returns
+        -------
+        bytes
+            The requested byte range.
+        """
+        return self._session.read_range(path, offset, length)
+
+    def delete_file(self, path: str) -> None:
+        """
+        Delete a file (relative to this session's base path).
+
+        Deleting a path that does not exist raises ``OSError``, consistent with
+        ``download_file``.
+
+        Parameters
+        ----------
+        path : str
+            Path relative to `base_path` to delete.
+        """
+        self._session.delete_file(path)
+
     def upload_file(self, local_path: Union[str, Path], remote_path: str) -> None:
         """
         Upload a file from local filesystem to the object store.
diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py
index c2661ce8f02..bca62e2b280 100644
--- a/python/python/lance/fragment.py
+++ b/python/python/lance/fragment.py
@@ -14,6 +14,7 @@
     Any,
     Callable,
     Dict,
+    Iterable,
     Iterator,
     List,
     Literal,
@@ -44,6 +45,7 @@
         ColumnOrdering,
         DatasetBasePath,
         LanceDataset,
+        LanceOperation,
         LanceScanner,
         ReaderLike,
         Transaction,
@@ -77,6 +79,11 @@ class FragmentMetadata:
         The row created at version metadata, if any.
     last_updated_at_version_meta : Optional[RowDatasetVersionMeta]
         The row last updated at version metadata, if any.
+    overlays : List[LanceOperation.DataOverlayFile]
+        The data overlay files layered over this fragment's base data, if any.
+        Overlays are created via :class:`LanceOperation.DataOverlay`; they are
+        carried here so they survive operations that round-trip fragment
+        metadata (e.g. a manual ``Delete``, ``Update``, or ``Merge`` commit).
     """
 
     id: int
@@ -86,6 +93,7 @@ class FragmentMetadata:
     row_id_meta: Optional[RowIdMeta] = None
     created_at_version_meta: Optional[RowDatasetVersionMeta] = None
     last_updated_at_version_meta: Optional[RowDatasetVersionMeta] = None
+    overlays: List["LanceOperation.DataOverlayFile"] = field(default_factory=list)
 
     @property
     def num_deletions(self) -> int:
@@ -109,12 +117,25 @@ def data_files(self) -> List[DataFile]:
 
     def to_json(self) -> dict:
         """Get this as a simple JSON-serializable dictionary."""
-        files = [asdict(f) for f in self.files]
-        for f in files:
-            f["path"] = f.pop("_path")
+
+        def _data_file_to_json(f: DataFile) -> dict:
+            d = asdict(f)
+            d["path"] = d.pop("_path")
+            return d
+
+        files = [_data_file_to_json(f) for f in self.files]
+        overlays = [
+            dict(
+                data_file=_data_file_to_json(o.data_file),
+                offsets=o.offsets,
+                committed_version=o.committed_version,
+            )
+            for o in self.overlays
+        ]
         return dict(
             id=self.id,
             files=files,
+            overlays=overlays,
             physical_rows=self.physical_rows,
             deletion_file=(
                 self.deletion_file.asdict() if self.deletion_file is not None else None
@@ -158,6 +179,20 @@ def from_json(json_data: str) -> FragmentMetadata:
                 json.dumps(last_updated_at_version_meta)
             )
 
+        overlays = []
+        overlays_json = json_data.get("overlays")
+        if overlays_json:
+            from .dataset import LanceOperation
+
+            overlays = [
+                LanceOperation.DataOverlayFile(
+                    data_file=DataFile(**o["data_file"]),
+                    offsets=o["offsets"],
+                    committed_version=o.get("committed_version"),
+                )
+                for o in overlays_json
+            ]
+
         return FragmentMetadata(
             id=json_data["id"],
             files=[DataFile(**f) for f in json_data["files"]],
@@ -166,6 +201,7 @@ def from_json(json_data: str) -> FragmentMetadata:
             row_id_meta=row_id_meta,
             created_at_version_meta=created_at_version_meta,
             last_updated_at_version_meta=last_updated_at_version_meta,
+            overlays=overlays,
         )
 
 
@@ -965,6 +1001,24 @@ def delete(self, predicate: str) -> FragmentMetadata | None:
             return None
         return raw_fragment.metadata()
 
+    def delete_rows(self, offsets: "Iterable[int]") -> FragmentMetadata | None:
+        """Delete rows by their local (within-fragment) physical row offsets.
+
+        Adds the given 0-based offsets to this fragment's deletion file and
+        returns a new fragment, or None if no rows are left. Unlike
+        :meth:`delete`, this deletes exactly the supplied rows without
+        re-evaluating a SQL predicate -- useful when the caller already knows
+        which rows to delete (e.g. offsets collected from a prior scan).
+
+        .. warning::
+
+            Internal API. This method is not intended to be used by end users.
+        """
+        raw_fragment = self._fragment.delete_rows([int(o) for o in offsets])
+        if raw_fragment is None:
+            return None
+        return raw_fragment.metadata()
+
     @property
     def schema(self) -> pa.Schema:
         """Return the schema of this fragment."""
@@ -1010,6 +1064,7 @@ def write_fragments(
         storage_options: Optional[Dict[str, str]] = None,
         enable_stable_row_ids: bool = False,
         target_bases: Optional[List[str]] = None,
+        target_all_bases: Optional[bool] = None,
         initial_bases: Optional[List["DatasetBasePath"]] = None,
         base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
         external_blob_mode: Literal["reference", "ingest"] = "reference",
@@ -1035,6 +1090,7 @@ def write_fragments(
         storage_options: Optional[Dict[str, str]] = None,
         enable_stable_row_ids: bool = False,
         target_bases: Optional[List[str]] = None,
+        target_all_bases: Optional[bool] = None,
         initial_bases: Optional[List["DatasetBasePath"]] = None,
         base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
         external_blob_mode: Literal["reference", "ingest"] = "reference",
@@ -1060,6 +1116,7 @@ def write_fragments(
     storage_options: Optional[Dict[str, str]] = None,
     enable_stable_row_ids: bool = False,
     target_bases: Optional[List[str]] = None,
+    target_all_bases: Optional[bool] = None,
     initial_bases: Optional[List["DatasetBasePath"]] = None,
     base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
     external_blob_mode: Literal["reference", "ingest"] = "reference",
@@ -1130,6 +1187,11 @@ def write_fragments(
         **CREATE mode**: References must match bases in `initial_bases`
         **APPEND/OVERWRITE modes**: References must match bases in the
         existing manifest
+    target_all_bases : bool, optional
+        Write new data files round-robin across every base registered in the
+        manifest, resolved at execution time. When True, the dataset's
+        primary storage participates as the first slot. Cannot be combined
+        with `target_bases`.
     initial_bases : list of DatasetBasePath, optional
         Base paths to register when creating a new dataset (CREATE mode only).
 
@@ -1235,6 +1297,7 @@ def write_fragments(
         table_id=table_id,
         enable_stable_row_ids=enable_stable_row_ids,
         target_bases=target_bases,
+        target_all_bases=target_all_bases,
         initial_bases=initial_bases,
         base_store_params=base_store_params,
         external_blob_mode=external_blob_mode,
diff --git a/python/python/lance/indices/__init__.py b/python/python/lance/indices/__init__.py
index 40dc9ed93ac..675754cc2d0 100644
--- a/python/python/lance/indices/__init__.py
+++ b/python/python/lance/indices/__init__.py
@@ -30,7 +30,6 @@ class IndexFileVersion(str, Enum):
 class SupportedDistributedIndices(str, Enum):
     # Scalar index types
     BTREE = "BTREE"
-    BITMAP = "BITMAP"
     INVERTED = "INVERTED"
 
     # Precise vector index types supported by distributed merge
diff --git a/python/python/lance/indices/builder.py b/python/python/lance/indices/builder.py
index d3d61c5f8ff..e235f348979 100644
--- a/python/python/lance/indices/builder.py
+++ b/python/python/lance/indices/builder.py
@@ -150,7 +150,7 @@ def train_ivf(
                 max_iters=max_iters,
             )
             num_dims = ivf_centroids.shape[1]
-            ivf_centroids.shape = -1
+            ivf_centroids = ivf_centroids.reshape(-1)
             flat_centroids_array = pa.array(ivf_centroids)
             centroids_array = pa.FixedSizeListArray.from_arrays(
                 flat_centroids_array, num_dims
@@ -164,6 +164,7 @@ def train_pq(
         *,
         sample_rate: int = 256,
         max_iters: int = 50,
+        num_bits: int = 8,
         fragment_ids: Optional[list[int]] = None,
     ) -> PqModel:
         """
@@ -195,6 +196,8 @@ def train_pq(
             This parameter is used in the same way as in the IVF model.
         max_iters: int
             This parameter is used in the same way as in the IVF model.
+        num_bits: int
+            The number of bits used to encode each PQ centroid.
         fragment_ids: list[int], optional
             If provided, train using only the specified fragments from the dataset.
         """
@@ -202,7 +205,7 @@ def train_pq(
 
         num_rows = self._count_rows(fragment_ids)
         num_subvectors = self._normalize_pq_params(num_subvectors, self.dimension)
-        self._verify_pq_sample_rate(num_rows, sample_rate)
+        self._verify_pq_sample_rate(num_rows, sample_rate, num_bits)
         distance_type = ivf_model.distance_type
         pq_codebook = indices.train_pq_model(
             self.dataset._ds,
@@ -214,8 +217,9 @@ def train_pq(
             max_iters,
             ivf_model.centroids,
             fragment_ids,
+            num_bits,
         )
-        return PqModel(num_subvectors, pq_codebook)
+        return PqModel(num_subvectors, pq_codebook, num_bits=num_bits)
 
     def prepare_global_ivf_pq(
         self,
@@ -226,6 +230,7 @@ def prepare_global_ivf_pq(
         accelerator: Optional[Union[str, "torch.Device"]] = None,
         sample_rate: int = 256,
         max_iters: int = 50,
+        num_bits: int = 8,
         fragment_ids: Optional[list[int]] = None,
     ) -> dict:
         """
@@ -267,6 +272,7 @@ def prepare_global_ivf_pq(
             num_subvectors,
             sample_rate=sample_rate,
             max_iters=max_iters,
+            num_bits=num_bits,
             fragment_ids=fragment_ids,
         )
 
@@ -381,6 +387,7 @@ def transform_vectors(
             dest_uri,
             fragments,
             partition_ds_uri,
+            pq.num_bits,
         )
 
     def shuffle_transformed_vectors(
@@ -471,6 +478,7 @@ def load_shuffled_vectors(
                 num_subvectors,
                 distance_type,
                 index_name,
+                pq.num_bits,
             )
         else:
             raise ValueError("filenames must be a list of strings")
@@ -526,13 +534,17 @@ def _verify_base_sample_rate(self, sample_rate: int):
                 f"The sample_rate must be an int greater than 1, got {sample_rate}"
             )
 
-    def _verify_pq_sample_rate(self, num_rows: int, sample_rate: int):
+    def _verify_pq_sample_rate(
+        self, num_rows: int, sample_rate: int, num_bits: int = 8
+    ):
         self._verify_base_sample_rate(sample_rate)
-        if 256 * sample_rate > num_rows:
+        required_rows = (2**num_bits) * sample_rate
+        if required_rows > num_rows:
             raise ValueError(
                 "There are not enough rows in the dataset to create PQ"
-                f" codebook with a sample rate of {sample_rate}.  {sample_rate * 256}"
-                f" rows needed and there are {num_rows}"
+                f" codebook with a sample rate of {sample_rate} and num_bits"
+                f" of {num_bits}.  {required_rows} rows needed and there are"
+                f" {num_rows}"
             )
 
     def _verify_ivf_sample_rate(
diff --git a/python/python/lance/indices/pq.py b/python/python/lance/indices/pq.py
index b3aeb50bcbe..e3d334ccf48 100644
--- a/python/python/lance/indices/pq.py
+++ b/python/python/lance/indices/pq.py
@@ -14,9 +14,13 @@ class PqModel:
     Can be saved / loaded to checkpoint progress.
     """
 
-    def __init__(self, num_subvectors: int, codebook: pa.FixedSizeListArray):
+    def __init__(
+        self, num_subvectors: int, codebook: pa.FixedSizeListArray, *, num_bits: int = 8
+    ):
         self.num_subvectors = num_subvectors
         """The number of subvectors to divide source vectors into"""
+        self.num_bits = num_bits
+        """The number of bits used to encode each PQ centroid"""
         self.codebook = codebook
         """The centroids of the PQ clusters"""
 
@@ -42,7 +46,10 @@ def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
             uri,
             pa.schema(
                 [pa.field("codebook", self.codebook.type)],
-                metadata={b"num_subvectors": str(self.num_subvectors).encode()},
+                metadata={
+                    b"num_subvectors": str(self.num_subvectors).encode(),
+                    b"num_bits": str(self.num_bits).encode(),
+                },
             ),
             storage_options=storage_options,
         ) as writer:
@@ -65,9 +72,10 @@ def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None):
         """
         reader = LanceFileReader(uri, storage_options=storage_options)
         num_rows = reader.metadata().num_rows
-        metadata = reader.metadata().schema.metadata
+        metadata = reader.metadata().schema.metadata or {}
         num_subvectors = int(metadata[b"num_subvectors"].decode())
+        num_bits = int(metadata.get(b"num_bits", b"8").decode())
         codebook = (
             reader.read_all(batch_size=num_rows).to_table().column("codebook").chunk(0)
         )
-        return cls(num_subvectors, codebook)
+        return cls(num_subvectors, codebook, num_bits=num_bits)
diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi
index f3bc9a681b2..03ec9f46db7 100644
--- a/python/python/lance/lance/__init__.pyi
+++ b/python/python/lance/lance/__init__.pyi
@@ -87,6 +87,25 @@ from .trace import capture_trace_events as capture_trace_events
 from .trace import shutdown_tracing as shutdown_tracing
 from .trace import trace_to_chrome as trace_to_chrome
 
+class MetricPoint:
+    name: str
+    kind: str
+    attributes: Dict[str, str]
+    value: Optional[float]
+    buckets: Optional[List[Tuple[str, int]]]
+    count: Optional[int]
+    sum: Optional[float]
+
+class MetricDescription:
+    name: str
+    kind: str
+    unit: Optional[str]
+    description: str
+
+def register_lance_metrics_recorder() -> bool: ...
+def lance_metrics_catalog() -> List[MetricDescription]: ...
+def snapshot_lance_metrics() -> List[MetricPoint]: ...
+
 class CleanupStats:
     bytes_removed: int
     old_versions: int
@@ -95,6 +114,26 @@ class CleanupStats:
     index_files_removed: int
     deletion_files_removed: int
 
+class CleanupCandidateFile:
+    path: str
+    kind: str
+    unverified: bool
+    size_bytes: int
+
+class CleanupReferencedBranch:
+    name: str
+    referenced_version: int
+    cleanup_candidate: bool
+
+class CleanupExplanation:
+    read_version: int
+    stats: CleanupStats
+    candidate_files: List[CleanupCandidateFile]
+    candidate_files_truncated: bool
+    candidate_file_limit: int
+    referenced_branches: List[CleanupReferencedBranch]
+    warnings: List[str]
+
 class LanceFileWriter:
     def __init__(
         self,
@@ -113,6 +152,46 @@ class LanceFileWriter:
     def add_schema_metadata(self, key: str, value: str) -> None: ...
     def add_global_buffer(self, data: bytes) -> int: ...
 
+class BlobDescriptor:
+    def __repr__(self) -> str: ...
+
+class PackedBlobWriter:
+    @property
+    def blob_id(self) -> int: ...
+    @property
+    def path(self) -> str: ...
+    @property
+    def field(self) -> pa.Field: ...
+    def write_blob(self, data: bytes) -> None: ...
+    def write_blobs(
+        self,
+        payloads: Union[pa.BinaryArray, pa.LargeBinaryArray, pa.ChunkedArray],
+    ) -> None: ...
+    def finish(self) -> List[BlobDescriptor]: ...
+    def finish_array(self, field_name: str) -> pa.StructArray: ...
+
+class DedicatedBlobWriter:
+    @property
+    def blob_id(self) -> int: ...
+    @property
+    def path(self) -> str: ...
+    def write(self, data: bytes) -> None: ...
+    def finish(self) -> BlobDescriptor: ...
+
+class BlobDescriptorArrayBuilder:
+    def __init__(self, column: str): ...
+    @property
+    def field(self) -> pa.Field: ...
+    def extend_packed(
+        self, blob_id: int, offsets: List[int], sizes: List[int]
+    ) -> None: ...
+    def append_dedicated(self, blob_id: int, size: int) -> None: ...
+    def append(self, value: BlobDescriptor) -> None: ...
+    def extend(self, values: List[BlobDescriptor]) -> None: ...
+    def append_inline(self, data: bytes) -> None: ...
+    def append_null(self) -> None: ...
+    def finish(self) -> pa.Array: ...
+
 class LanceFileSession:
     def __init__(
         self,
@@ -133,8 +212,17 @@ class LanceFileSession:
         keep_original_array: Optional[bool] = None,
         max_page_bytes: Optional[int] = None,
     ) -> LanceFileWriter: ...
+    def open_packed_blob_writer(self, path: str, blob_id: int) -> PackedBlobWriter: ...
+    def open_dedicated_blob_writer(
+        self, path: str, blob_id: int
+    ) -> DedicatedBlobWriter: ...
     def contains(self, path: str) -> bool: ...
     def list(self, path: Optional[str] = None) -> List[str]: ...
+    def list_with_delimiter(
+        self, path: Optional[str] = None
+    ) -> tuple[List[str], List[str]]: ...
+    def read_range(self, path: str, offset: int, length: int) -> bytes: ...
+    def delete_file(self, path: str) -> None: ...
     def upload_file(self, local_path: str, remote_path: str) -> None: ...
     def download_file(self, remote_path: str, local_path: str) -> None: ...
 
@@ -191,6 +279,7 @@ class LanceColumnStatistics:
 
 class _Session:
     def size_bytes(self) -> int: ...
+    def index_cache_size_bytes(self) -> int: ...
 
 class LanceBlobFile:
     def close(self): ...
@@ -226,10 +315,12 @@ class _Dataset:
     def replace_field_metadata(self, field_name: str, metadata: Dict[str, str]): ...
     @property
     def data_storage_version(self) -> str: ...
+    @property
+    def has_stable_row_ids(self) -> bool: ...
     def index_statistics(self, index_name: str) -> str: ...
     def serialized_manifest(self) -> bytes: ...
-    def load_indices(self) -> List[Index]: ...
     def describe_indices(self) -> List[IndexDescription]: ...
+    def remap_row_addrs(self, addrs: pa.Array) -> Optional[pa.Array]: ...
     def scanner(
         self,
         columns: Optional[List[str]] = None,
@@ -343,11 +434,22 @@ class _Dataset:
     def restore(self): ...
     def cleanup_old_versions(
         self,
-        older_than_micros: int,
+        older_than_micros: Optional[int] = None,
+        retain_versions: Optional[int] = None,
         delete_unverified: Optional[bool] = None,
         error_if_tagged_old_versions: Optional[bool] = None,
         delete_rate_limit: Optional[int] = None,
     ) -> CleanupStats: ...
+    def explain_cleanup_old_versions(
+        self,
+        older_than_micros: Optional[int] = None,
+        retain_versions: Optional[int] = None,
+        delete_unverified: Optional[bool] = None,
+        error_if_tagged_old_versions: Optional[bool] = None,
+        delete_rate_limit: Optional[int] = None,
+        include_files: bool = False,
+        max_files: int = 1000,
+    ) -> CleanupExplanation: ...
     def get_version(self, tag: str) -> int: ...
     # Tag operations
     def tags(self) -> Dict[str, Tag]: ...
@@ -395,7 +497,13 @@ class _Dataset:
         kwargs: Optional[Dict[str, Any]] = None,
     ): ...
     def drop_index(self, name: str): ...
-    def prewarm_index(self, name: str, *, with_position: bool = False): ...
+    def prewarm_index(
+        self,
+        name: str,
+        *,
+        with_position: bool = False,
+        index_segments: Optional[List[str]] = None,
+    ): ...
     def merge_index_metadata(
         self,
         index_uuid: str,
@@ -462,6 +570,32 @@ class _Dataset:
     def get_transactions(
         self, recent_transactions=10
     ) -> List[Optional[Transaction]]: ...
+    def hamming_clustering_for_ivf_partition(
+        self,
+        index_name: str,
+        partition_id: int,
+        hamming_threshold: int,
+        index_segments: Optional[List[str]] = None,
+    ) -> pa.RecordBatchReader: ...
+    def get_ivf_partition_info(
+        self,
+        index_name: str,
+        index_segments: Optional[List[str]] = None,
+    ) -> List[dict]: ...
+    def hamming_clustering_for_sample(
+        self,
+        column: str,
+        sample_size: Optional[int],
+        hamming_threshold: int,
+    ) -> pa.RecordBatchReader: ...
+    def hamming_clustering_for_range(
+        self,
+        column: str,
+        fragment_id: int,
+        start_row: int,
+        num_rows: int,
+        hamming_threshold: int,
+    ) -> pa.RecordBatchReader: ...
 
 class _MergeInsertBuilder:
     def __init__(self, dataset: _Dataset, on: str | Iterable[str]): ...
@@ -469,13 +603,15 @@ class _MergeInsertBuilder:
     def when_matched_fail(self) -> Self: ...
     def when_not_matched_insert_all(self) -> Self: ...
     def when_not_matched_by_source_delete(self, expr: Optional[str] = None) -> Self: ...
+    def target_bases(self, bases: list[str]) -> Self: ...
+    def target_all_bases(self, include_primary: bool = True) -> Self: ...
     def execute(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ...
 
 class _Scanner:
     @property
     def schema(self) -> pa.Schema: ...
     def explain_plan(self, verbose: bool) -> str: ...
-    def analyze_plan(self) -> str: ...
+    def analyze_plan(self, count_rows: bool = False) -> str: ...
     def count_rows(self) -> int: ...
     def to_pyarrow(self) -> pa.RecordBatchReader: ...
 
@@ -527,6 +663,7 @@ class _Fragment:
         batch_size: Optional[int],
     ) -> Tuple[FragmentMetadata, LanceSchema]: ...
     def delete(self, predicate: str) -> Optional[_Fragment]: ...
+    def delete_rows(self, offsets: List[int]) -> Optional[_Fragment]: ...
     def schema(self) -> pa.Schema: ...
     def data_files(self) -> List[DataFile]: ...
     def deletion_file(self) -> Optional[str]: ...
@@ -555,6 +692,7 @@ def _write_fragments(
     table_id: Optional[List[str]],
     enable_stable_row_ids: bool,
     target_bases: Optional[List[str]] = None,
+    target_all_bases: Optional[bool] = None,
     initial_bases: Optional[List[Any]] = None,
     base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
     external_blob_mode: Literal["reference", "ingest"] = "reference",
@@ -574,6 +712,7 @@ def _write_fragments_transaction(
     table_id: Optional[List[str]],
     enable_stable_row_ids: bool,
     target_bases: Optional[List[str]] = None,
+    target_all_bases: Optional[bool] = None,
     initial_bases: Optional[List[Any]] = None,
     base_store_params: Optional[Dict[str, Dict[str, str]]] = None,
     external_blob_mode: Literal["reference", "ingest"] = "reference",
@@ -604,6 +743,7 @@ class _ShardSnapshot:
 class _ShardWriter:
     shard_id: str
     def put(self, data: Any) -> None: ...
+    def delete(self, keys: Any) -> None: ...
     def close(self) -> None: ...
     def stats(self) -> Dict[str, Any]: ...
     def memtable_stats(self) -> Dict[str, Any]: ...
@@ -697,7 +837,7 @@ class BFloat16:
     def __gt__(self, other: BFloat16) -> bool: ...
     def __ge__(self, other: BFloat16) -> bool: ...
 
-def bfloat16_array(values: List[str | None]) -> BFloat16Array: ...
+def bfloat16_array(values: Sequence[float | None]) -> BFloat16Array: ...
 
 class PyFullTextQuery:
     @staticmethod
diff --git a/python/python/lance/lance/indices/__init__.pyi b/python/python/lance/lance/indices/__init__.pyi
index 384e2528e99..181ec93b2d1 100644
--- a/python/python/lance/lance/indices/__init__.pyi
+++ b/python/python/lance/lance/indices/__init__.pyi
@@ -17,6 +17,8 @@ from typing import Optional
 
 import pyarrow as pa
 
+from .. import _Fragment
+
 class IndexConfig:
     index_type: str
     config: str
@@ -48,6 +50,7 @@ def train_pq_model(
     max_iters: int,
     ivf_model: pa.Array,
     fragment_ids: Optional[list[int]] = None,
+    num_bits: int = 8,
 ) -> pa.Array: ...
 def transform_vectors(
     dataset,
@@ -58,7 +61,15 @@ def transform_vectors(
     ivf_centroids: pa.Array,
     pq_codebook: pa.Array,
     dst_uri: str,
+    fragments: list[_Fragment],
+    partitions_ds_uri: Optional[str] = None,
+    num_bits: int = 8,
 ): ...
+def build_rq_model(
+    dimension: int,
+    num_bits: int = 1,
+    dtype: str = "float32",
+) -> str: ...
 
 class IndexSegmentDescription:
     uuid: str
@@ -67,6 +78,7 @@ class IndexSegmentDescription:
     index_version: int
     created_at: Optional[datetime]
     size_bytes: Optional[int]
+    base_id: Optional[int]
 
     def __repr__(self) -> str: ...
 
diff --git a/python/python/lance/lance/optimize.pyi b/python/python/lance/lance/optimize.pyi
index 9a26d23c003..c4b6b6546e6 100644
--- a/python/python/lance/lance/optimize.pyi
+++ b/python/python/lance/lance/optimize.pyi
@@ -12,7 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from typing import List
+from typing import List, Optional
 
 from lance import LanceDataset
 from lance.fragment import FragmentMetadata
@@ -51,5 +51,7 @@ class Compaction:
     def plan(dataset: "LanceDataset", options: CompactionOptions) -> CompactionPlan: ...
     @staticmethod
     def commit(
-        dataset: "LanceDataset", rewrites: List[RewriteResult]
+        dataset: "LanceDataset",
+        rewrites: List[RewriteResult],
+        options: Optional[CompactionOptions] = None,
     ) -> CompactionMetrics: ...
diff --git a/python/python/lance/mem_wal.py b/python/python/lance/mem_wal.py
index ed0ce316da8..426a5f7ee50 100644
--- a/python/python/lance/mem_wal.py
+++ b/python/python/lance/mem_wal.py
@@ -15,6 +15,7 @@
 
 from __future__ import annotations
 
+import math
 from dataclasses import asdict, dataclass, field
 from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Union
 
@@ -186,6 +187,7 @@ class ShardWriter:
 
         with dataset.mem_wal_writer(shard_id) as writer:
             writer.put(batch)
+            writer.delete(pa.table({"id": [1]}))
 
     Parameters
     ----------
@@ -223,6 +225,36 @@ def put(self, data, *, schema: Optional[pa.Schema] = None) -> None:
         reader = _coerce_reader(data, schema)
         self._raw.put(reader)
 
+    def delete(self, keys, *, schema: Optional[pa.Schema] = None) -> None:
+        """Delete rows by primary key from the MemWAL.
+
+        Parameters
+        ----------
+        keys : ReaderLike
+            Any Arrow-compatible data containing this shard's primary key
+            column(s). Non-primary-key columns, if present, are ignored by
+            the Rust core delete path.
+        schema : pa.Schema, optional
+            Schema hint, needed when *keys* is a generator.
+
+        Raises
+        ------
+        IOError
+            If delete validation fails, WAL flush fails, or the writer has
+            already been closed. Delete validation is centralized in Rust and
+            includes checks for primary-key metadata and tombstone-compatible
+            nullable non-key columns.
+
+        Examples
+        --------
+        ::
+
+            with dataset.mem_wal_writer(shard_id) as writer:
+                writer.delete(pa.table({"id": [42]}))
+        """
+        reader = _coerce_reader(keys, schema)
+        self._raw.delete(reader)
+
     def close(self) -> None:
         """Flush and close the writer.
 
@@ -337,7 +369,9 @@ def filter(self, expr: str) -> "LsmScanner":
         self._raw = self._raw.filter(expr)
         return self
 
-    def limit(self, n: int, offset: Optional[int] = None) -> "LsmScanner":
+    def limit(
+        self, n: Optional[int] = None, offset: Optional[int] = None
+    ) -> "LsmScanner":
         """Limit rows returned, optionally with an offset."""
         self._raw = self._raw.limit(n, offset)
         return self
@@ -489,6 +523,9 @@ class LsmVectorSearchPlanner:
     distance_type : str, optional
         Distance metric — one of ``"l2"`` (default), ``"cosine"``,
         ``"dot"``, ``"hamming"``.
+    filter : str, optional
+        SQL predicate applied as a prefilter to every LSM source before vector
+        candidate selection.
 
     Examples
     --------
@@ -506,12 +543,15 @@ def __init__(
         vector_column: str,
         pk_columns: Optional[List[str]] = None,
         distance_type: Optional[str] = None,
+        filter: Optional[str] = None,
     ) -> None:
         kwargs = {}
         if pk_columns is not None:
             kwargs["pk_columns"] = pk_columns
         if distance_type is not None:
             kwargs["distance_type"] = distance_type
+        if filter is not None:
+            kwargs["filter"] = filter
         self._raw = _LsmVectorSearchPlanner(
             dataset._ds,
             [s._raw for s in shard_snapshots],
@@ -547,22 +587,11 @@ def plan_search(
             IVF-PQ).  Memtable arms use exact HNSW and are unaffected.
             Auto-enabled whenever stale filtering is on (see ``overfetch_factor``).
         overfetch_factor : float, optional
-            Single knob controlling **both** stale-read filtering and over-fetch
-            (default: ``1.0``):
-
-            - ``< 1.0`` (e.g. ``0.0``): stale filtering **off**.  Rows superseded
-              by a newer generation may surface.  (The global primary-key dedup
-              still runs, so it suppresses stale copies whenever both the stale
-              and the fresh row reach it.)
-            - ``== 1.0``: filtering **on**, no over-fetch.  Each source with
-              superseded rows fetches exactly ``k`` candidates and drops the stale
-              ones, so it may return fewer than ``k`` live rows.
-            - ``> 1.0``: filtering **on**, with over-fetch.  Such a source fetches
-              ``ceil(k * overfetch_factor)`` candidates so dropping the stale ones
-              still leaves ``k`` live rows.
-
-            There is no separate on/off flag: over-fetch is only meaningful while
-            filtering, so the factor encodes both.
+            Over-fetch multiple for sources with rows superseded by newer
+            generations. Must be at least ``1.0``. At ``1.0`` each affected
+            source fetches exactly ``k`` candidates; above ``1.0`` it fetches
+            ``ceil(k * overfetch_factor)`` candidates so dropping stale rows is
+            less likely to under-fill the result.
 
         Returns
         -------
@@ -570,6 +599,14 @@ def plan_search(
             Physical plan for the vector search. Execute it via
             `to_table`, `to_reader`, or `to_batches`.
         """
+        if k <= 0:
+            raise ValueError(f"k must be positive, got {k}")
+        if nprobes <= 0:
+            raise ValueError(f"nprobes must be positive, got {nprobes}")
+        if not math.isfinite(overfetch_factor) or overfetch_factor < 1.0:
+            raise ValueError(
+                f"overfetch_factor must be finite and >= 1.0, got {overfetch_factor}"
+            )
         return ExecutionPlan(
             self._raw.plan_search(
                 query,
diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py
index f448e5c3368..6df5b9aa4bc 100644
--- a/python/python/lance/namespace.py
+++ b/python/python/lance/namespace.py
@@ -32,6 +32,8 @@
     CreateMaterializedViewResponse,
     CreateNamespaceRequest,
     CreateNamespaceResponse,
+    CreateTableBranchRequest,
+    CreateTableBranchResponse,
     CreateTableIndexRequest,
     CreateTableIndexResponse,
     CreateTableRequest,
@@ -42,6 +44,8 @@
     DeclareTableResponse,
     DeleteFromTableRequest,
     DeleteFromTableResponse,
+    DeleteTableBranchRequest,
+    DeleteTableBranchResponse,
     DeleteTableTagRequest,
     DeleteTableTagResponse,
     DeregisterTableRequest,
@@ -70,6 +74,8 @@
     LanceNamespace,
     ListNamespacesRequest,
     ListNamespacesResponse,
+    ListTableBranchesRequest,
+    ListTableBranchesResponse,
     ListTableIndicesRequest,
     ListTableIndicesResponse,
     ListTablesRequest,
@@ -336,12 +342,13 @@ class DirectoryNamespace(LanceNamespace):
     >>>
     >>> # With AWS credential vending (requires credential-vendor-aws feature)
     >>> # Use **dict to pass property names with dots
-    >>> ns = lance.namespace.DirectoryNamespace(**{
+    >>> aws_properties = {
     ...     "root": "s3://my-bucket/data",
     ...     "credential_vendor.enabled": "true",
     ...     "credential_vendor.aws_role_arn": "arn:aws:iam::123456789012:role/MyRole",
     ...     "credential_vendor.aws_duration_millis": "3600000",
-    ... })
+    ... }
+    >>> # ns = lance.namespace.DirectoryNamespace(**aws_properties)
 
     With dynamic context provider:
 
@@ -850,6 +857,27 @@ def update_table_tag(
         response_dict = self._inner.update_table_tag(request.model_dump())
         return UpdateTableTagResponse.from_dict(response_dict)
 
+    def create_table_branch(
+        self, request: CreateTableBranchRequest
+    ) -> CreateTableBranchResponse:
+        """Create a new branch forked from a table version."""
+        response_dict = self._inner.create_table_branch(request.model_dump())
+        return CreateTableBranchResponse.from_dict(response_dict)
+
+    def list_table_branches(
+        self, request: ListTableBranchesRequest
+    ) -> ListTableBranchesResponse:
+        """List all branches of a table."""
+        response_dict = self._inner.list_table_branches(request.model_dump())
+        return ListTableBranchesResponse.from_dict(response_dict)
+
+    def delete_table_branch(
+        self, request: DeleteTableBranchRequest
+    ) -> DeleteTableBranchResponse:
+        """Delete a branch from a table."""
+        response_dict = self._inner.delete_table_branch(request.model_dump())
+        return DeleteTableBranchResponse.from_dict(response_dict)
+
     # Operation metrics methods
 
     def retrieve_ops_metrics(self) -> Dict[str, int]:
@@ -1420,6 +1448,27 @@ def update_table_tag(
         response_dict = self._inner.update_table_tag(request.model_dump())
         return UpdateTableTagResponse.from_dict(response_dict)
 
+    def create_table_branch(
+        self, request: CreateTableBranchRequest
+    ) -> CreateTableBranchResponse:
+        """Create a new branch forked from a table version."""
+        response_dict = self._inner.create_table_branch(request.model_dump())
+        return CreateTableBranchResponse.from_dict(response_dict)
+
+    def list_table_branches(
+        self, request: ListTableBranchesRequest
+    ) -> ListTableBranchesResponse:
+        """List all branches of a table."""
+        response_dict = self._inner.list_table_branches(request.model_dump())
+        return ListTableBranchesResponse.from_dict(response_dict)
+
+    def delete_table_branch(
+        self, request: DeleteTableBranchRequest
+    ) -> DeleteTableBranchResponse:
+        """Delete a branch from a table."""
+        response_dict = self._inner.delete_table_branch(request.model_dump())
+        return DeleteTableBranchResponse.from_dict(response_dict)
+
     # Operation metrics methods
 
     def retrieve_ops_metrics(self) -> Dict[str, int]:
diff --git a/python/python/lance/optimize.py b/python/python/lance/optimize.py
index 8b98308d442..3ac7547960b 100644
--- a/python/python/lance/optimize.py
+++ b/python/python/lance/optimize.py
@@ -57,6 +57,14 @@ class CompactionOptions(TypedDict):
     The batch size to use when scanning input fragments.  You may want
     to reduce this if you are running out of memory during compaction.
 
+    The default will use the same default from ``scanner``.
+    """
+    io_buffer_size: Optional[int]
+    """
+    The number of bytes to allow to queue up in the I/O buffer when scanning
+    input fragments.  Increasing this can avoid a deadlock that occurs when a
+    single batch of data is larger than the I/O buffer size.
+
     The default will use the same default from ``scanner``.
     """
     compaction_mode: Optional[
diff --git a/python/python/lance/otel.py b/python/python/lance/otel.py
new file mode 100644
index 00000000000..0c9ee1790fd
--- /dev/null
+++ b/python/python/lance/otel.py
@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Bridge Lance's internal metrics into OpenTelemetry.
+
+Lance core publishes metrics (currently object store request counts, bytes,
+latency, errors, and throttles) through the Rust ``metrics`` facade. This module
+installs a process-global recorder that aggregates them and registers
+OpenTelemetry observable instruments that report the aggregated values into the
+user's ``MeterProvider``.
+
+The bridge is generic: every metric Lance describes is surfaced automatically,
+with no per-metric Python code. Histograms have no asynchronous OpenTelemetry
+instrument, so each is exported Prometheus-style as cumulative ``le`` buckets
+plus ``_count`` and ``_sum`` observable counters.
+"""
+
+from __future__ import annotations
+
+import warnings
+from typing import TYPE_CHECKING, Optional
+
+from .lance import (
+    lance_metrics_catalog,
+    register_lance_metrics_recorder,
+    snapshot_lance_metrics,
+)
+
+if TYPE_CHECKING:
+    from opentelemetry.metrics import MeterProvider
+
+_INSTRUMENTED = False
+
+
+def instrument_lance_metrics(meter_provider: Optional["MeterProvider"] = None) -> bool:
+    """Register Lance metrics as OpenTelemetry observable instruments.
+
+    Installs a process-global metrics recorder and creates one observable
+    instrument per Lance metric on the given (or global) ``MeterProvider``. The
+    user's configured ``MetricReader`` then collects them on its own schedule.
+
+    Counters and gauges map directly to observable counters/gauges. Each
+    histogram is exported as cumulative ``le`` bucket counts (``<name>_bucket``,
+    with an ``le`` attribute) plus ``<name>_count`` and ``<name>_sum``.
+
+    Parameters
+    ----------
+    meter_provider : opentelemetry.metrics.MeterProvider, optional
+        The provider to register instruments on. Defaults to the global provider
+        from ``opentelemetry.metrics.get_meter_provider()``.
+
+    Returns
+    -------
+    bool
+        ``True`` if the recorder is installed and instruments are registered.
+        ``False`` if a different ``metrics`` recorder is already installed in
+        this process (``metrics`` permits only one global recorder), in which
+        case a warning is emitted and no instruments are created.
+
+    Notes
+    -----
+    Requires the OpenTelemetry SDK (``pip install pylance[otel]``). Calling this
+    more than once is safe; instruments are created only on the first successful
+    call.
+    """
+    global _INSTRUMENTED
+
+    try:
+        from opentelemetry.metrics import Observation, get_meter_provider
+    except ImportError as exc:
+        raise ImportError(
+            "instrument_lance_metrics requires the OpenTelemetry API/SDK. "
+            "Install it with `pip install pylance[otel]` or "
+            "`pip install opentelemetry-sdk`."
+        ) from exc
+
+    if not register_lance_metrics_recorder():
+        warnings.warn(
+            "Could not install the Lance metrics recorder: another `metrics` "
+            "recorder is already installed in this process. Lance metrics will "
+            "not be exported via OpenTelemetry.",
+            stacklevel=2,
+        )
+        return False
+
+    if _INSTRUMENTED:
+        return True
+
+    provider = meter_provider or get_meter_provider()
+    meter = provider.get_meter("lance")
+
+    def scalar_callback(metric_name: str):
+        def callback(_options):
+            return [
+                Observation(point.value, point.attributes)
+                for point in snapshot_lance_metrics()
+                if point.name == metric_name and point.value is not None
+            ]
+
+        return callback
+
+    def bucket_callback(metric_name: str):
+        def callback(_options):
+            observations = []
+            for point in snapshot_lance_metrics():
+                if point.name != metric_name or point.buckets is None:
+                    continue
+                for le, cumulative in point.buckets:
+                    attributes = dict(point.attributes)
+                    attributes["le"] = le
+                    observations.append(Observation(cumulative, attributes))
+            return observations
+
+        return callback
+
+    def field_callback(metric_name: str, field: str):
+        def callback(_options):
+            observations = []
+            for point in snapshot_lance_metrics():
+                if point.name != metric_name:
+                    continue
+                value = getattr(point, field)
+                if value is not None:
+                    observations.append(Observation(value, point.attributes))
+            return observations
+
+        return callback
+
+    for desc in lance_metrics_catalog():
+        unit = desc.unit or ""
+        if desc.kind == "counter":
+            meter.create_observable_counter(
+                desc.name,
+                callbacks=[scalar_callback(desc.name)],
+                unit=unit,
+                description=desc.description,
+            )
+        elif desc.kind == "gauge":
+            meter.create_observable_gauge(
+                desc.name,
+                callbacks=[scalar_callback(desc.name)],
+                unit=unit,
+                description=desc.description,
+            )
+        elif desc.kind == "histogram":
+            # `_bucket` and `_count` observe cumulative counts, so they are
+            # unitless; only `_sum` carries the histogram's unit (e.g. seconds).
+            meter.create_observable_counter(
+                f"{desc.name}_bucket",
+                callbacks=[bucket_callback(desc.name)],
+                description=f"{desc.description} (cumulative buckets)",
+            )
+            meter.create_observable_counter(
+                f"{desc.name}_count",
+                callbacks=[field_callback(desc.name, "count")],
+                description=f"{desc.description} (count)",
+            )
+            meter.create_observable_counter(
+                f"{desc.name}_sum",
+                callbacks=[field_callback(desc.name, "sum")],
+                unit=unit,
+                description=f"{desc.description} (sum)",
+            )
+
+    _INSTRUMENTED = True
+    return True
diff --git a/python/python/lance/tf/__init__.py b/python/python/lance/tf/__init__.py
deleted file mode 100644
index 1aa41beb99c..00000000000
--- a/python/python/lance/tf/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-import importlib.util
-
-if importlib.util.find_spec("tensorflow") is None:
-    raise ImportError(
-        "Tensorflow is not installed. Please install tensorflow"
-        + " to use lance.tf module.",
-    )
diff --git a/python/python/lance/tf/data.py b/python/python/lance/tf/data.py
deleted file mode 100644
index 68280f9211f..00000000000
--- a/python/python/lance/tf/data.py
+++ /dev/null
@@ -1,410 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-
-"""Tensorflow Dataset (`tf.data <https://www.tensorflow.org/guide/data>`_)
-implementation for Lance.
-
-.. warning::
-
-    Experimental feature. API stability is not guaranteed.
-"""
-
-from __future__ import annotations
-
-from functools import partial
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union
-
-import pyarrow as pa
-
-import lance
-from lance import LanceDataset
-from lance.arrow import EncodedImageType, FixedShapeImageTensorType, ImageURIType
-from lance.dependencies import _check_for_numpy
-from lance.dependencies import numpy as np
-from lance.dependencies import tensorflow as tf
-from lance.fragment import FragmentMetadata, LanceFragment
-from lance.log import LOGGER
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from lance import LanceNamespace
-
-
-def arrow_data_type_to_tf(dt: pa.DataType) -> tf.DType:
-    """Convert Pyarrow DataType to Tensorflow."""
-    if pa.types.is_boolean(dt):
-        return tf.bool
-    elif pa.types.is_int8(dt):
-        return tf.int8
-    elif pa.types.is_int16(dt):
-        return tf.int16
-    elif pa.types.is_int32(dt):
-        return tf.int32
-    elif pa.types.is_int64(dt):
-        return tf.int64
-    elif pa.types.is_uint8(dt):
-        return tf.uint8
-    elif pa.types.is_uint16(dt):
-        return tf.uint16
-    elif pa.types.is_uint32(dt):
-        return tf.uint32
-    elif pa.types.is_uint64(dt):
-        return tf.uint64
-    elif pa.types.is_float16(dt):
-        return tf.float16
-    elif pa.types.is_float32(dt):
-        return tf.float32
-    elif pa.types.is_float64(dt):
-        return tf.float64
-    elif (
-        pa.types.is_string(dt)
-        or pa.types.is_large_string(dt)
-        or pa.types.is_binary(dt)
-        or pa.types.is_large_binary(dt)
-    ):
-        return tf.string
-
-    raise TypeError(f"Arrow/Tf conversion: Unsupported arrow data type: {dt}")
-
-
-def data_type_to_tensor_spec(dt: pa.DataType) -> tf.TensorSpec:
-    """Convert PyArrow DataType to Tensorflow TensorSpec."""
-    if (
-        pa.types.is_boolean(dt)
-        or pa.types.is_integer(dt)
-        or pa.types.is_floating(dt)
-        or pa.types.is_string(dt)
-        or pa.types.is_binary(dt)
-    ):
-        return tf.TensorSpec(shape=(None,), dtype=arrow_data_type_to_tf(dt))
-    elif isinstance(dt, pa.FixedShapeTensorType):
-        return tf.TensorSpec(
-            shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.value_type)
-        )
-    elif pa.types.is_fixed_size_list(dt):
-        return tf.TensorSpec(
-            shape=(None, dt.list_size), dtype=arrow_data_type_to_tf(dt.value_type)
-        )
-    elif pa.types.is_list(dt) or pa.types.is_large_list(dt):
-        return tf.TensorSpec(
-            shape=(
-                None,
-                None,
-            ),
-            dtype=arrow_data_type_to_tf(dt.value_type),
-        )
-    elif pa.types.is_struct(dt):
-        return {field.name: data_type_to_tensor_spec(field.type) for field in dt}
-    elif isinstance(dt, (EncodedImageType, ImageURIType)):
-        return tf.TensorSpec(shape=(None,), dtype=tf.string)
-    elif isinstance(dt, FixedShapeImageTensorType):
-        return tf.TensorSpec(
-            shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.arrow_type)
-        )
-
-    raise TypeError("Unsupported data type: ", dt)
-
-
-def schema_to_spec(schema: pa.Schema) -> tf.TypeSpec:
-    """Convert PyArrow Schema to Tensorflow output signature."""
-    signature = {}
-    for name in schema.names:
-        field = schema.field(name)
-        signature[name] = data_type_to_tensor_spec(field.type)
-    return signature
-
-
-def column_to_tensor(array: pa.Array, tensor_spec: tf.TensorSpec) -> tf.Tensor:
-    """Convert a PyArrow array into a TensorFlow tensor."""
-    if isinstance(tensor_spec, tf.RaggedTensorSpec):
-        return tf.ragged.constant(array.to_pylist(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, pa.FixedShapeTensorType):
-        return tf.constant(array.to_numpy_ndarray(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, FixedShapeImageTensorType):
-        return tf.constant(array.to_numpy(), dtype=tensor_spec.dtype)
-    elif isinstance(array.type, pa.StructType):
-        return {
-            field.name: column_to_tensor(array.field(i), tensor_spec[field.name])
-            for (i, field) in enumerate(array.type)
-        }
-    else:
-        return tf.constant(array.to_pylist(), dtype=tensor_spec.dtype)
-
-
-def from_lance(
-    dataset: Optional[Union[str, Path, LanceDataset]] = None,
-    *,
-    columns: Optional[Union[List[str], Dict[str, str]]] = None,
-    batch_size: int = 256,
-    filter: Optional[str] = None,
-    fragments: Union[Iterable[int], Iterable[LanceFragment], tf.data.Dataset] = None,
-    output_signature: Optional[Dict[str, tf.TypeSpec]] = None,
-    namespace_client: Optional["LanceNamespace"] = None,
-    table_id: Optional[List[str]] = None,
-    ignore_namespace_table_storage_options: bool = False,
-) -> tf.data.Dataset:
-    """Create a ``tf.data.Dataset`` from a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset], optional
-        Lance dataset or dataset URI/path. Either ``dataset`` or both
-        ``namespace_client`` and ``table_id`` must be provided.
-    columns : Optional[List[str]], optional
-        List of columns to include in the output dataset.
-        If not set, all columns will be read.
-    batch_size : int, optional
-        Batch size, by default 256
-    filter : Optional[str], optional
-        SQL filter expression, by default None.
-    fragments : Union[List[LanceFragment], tf.data.Dataset], optional
-        If provided, only the fragments are read. It can be used to feed
-        for distributed training.
-    output_signature : Optional[tf.TypeSpec], optional
-        Override output signature of the returned tensors. If not provided,
-        the output signature is inferred from the projection Schema.
-    namespace_client : Optional[LanceNamespace], optional
-        Namespace client to resolve the table location when ``table_id`` is
-        provided.
-    table_id : Optional[List[str]], optional
-        Table identifier used together with ``namespace_client`` to locate
-        the table.
-    ignore_namespace_table_storage_options : bool, default False
-        When using ``namespace_client``/``table_id``, ignore storage options
-        returned by the namespace.
-
-    Examples
-    --------
-
-    .. code-block:: python
-
-        import tensorflow as tf
-        import lance.tf.data
-
-        ds = lance.tf.data.from_lance(
-            "s3://bucket/path",
-            columns=["image", "id"],
-            filter="catalog = 'train' AND split = 'train'",
-            batch_size=100)
-
-        for batch in ds.repeat(10).shuffle(128).map(io_decode):
-            print(batch["image"].shape)
-
-    ``from_lance`` can take an iterator or ``tf.data.Dataset`` of
-    Fragments. So that it can be used to feed for distributed training.
-
-    .. code-block:: python
-
-        import tensorflow as tf
-        import lance.tf.data
-
-        seed = 200  # seed to shuffle the fragments in distributed machines.
-        fragments = lance.tf.data.lance_fragments("s3://bucket/path")
-            repeat(10).shuffle(4, seed=seed)
-        ds = lance.tf.data.from_lance(
-            "s3://bucket/path",
-            columns=["image", "id"],
-            filter="catalog = 'train' AND split = 'train'",
-            fragments=fragments,
-            batch_size=100)
-        for batch in ds.shuffle(128).map(io_decode):
-            print(batch["image"].shape)
-
-    """
-    if isinstance(dataset, LanceDataset):
-        if namespace_client is not None or table_id is not None:
-            raise ValueError(
-                "Cannot specify 'namespace_client' or 'table_id' when passing "
-                "a LanceDataset instance"
-            )
-    else:
-        dataset = lance.dataset(
-            dataset,
-            namespace_client=namespace_client,
-            table_id=table_id,
-            ignore_namespace_table_storage_options=ignore_namespace_table_storage_options,
-        )
-
-    if isinstance(fragments, tf.data.Dataset):
-        fragments = list(fragments.as_numpy_iterator())
-    elif _check_for_numpy(fragments) and isinstance(fragments, np.ndarray):
-        fragments = list(fragments)
-
-    if fragments is not None:
-
-        def gen_fragments(fragments):
-            for f in fragments:
-                if isinstance(f, int) or (
-                    _check_for_numpy(f) and isinstance(f, np.integer)
-                ):
-                    yield LanceFragment(dataset, int(f))
-                elif isinstance(f, FragmentMetadata):
-                    yield LanceFragment(dataset, f.id)
-                elif isinstance(f, LanceFragment):
-                    yield f
-                else:
-                    raise TypeError(f"Invalid type passed to `fragments`: {type(f)}")
-
-        # A Generator of Fragments
-        fragments = gen_fragments(fragments)
-
-    scanner = dataset.scanner(
-        filter=filter, columns=columns, batch_size=batch_size, fragments=fragments
-    )
-
-    if output_signature is None:
-        schema = scanner.projected_schema
-        output_signature = schema_to_spec(schema)
-    LOGGER.debug("Output signature: %s", output_signature)
-
-    def generator():
-        for batch in scanner.to_batches():
-            yield {
-                name: column_to_tensor(batch[name], output_signature[name])
-                for name in batch.schema.names
-            }
-
-    return tf.data.Dataset.from_generator(generator, output_signature=output_signature)
-
-
-def lance_fragments(dataset: Union[str, Path, LanceDataset]) -> tf.data.Dataset:
-    """Create a ``tf.data.Dataset`` of Lance Fragments in the dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-    return tf.data.Dataset.from_tensor_slices(
-        [f.fragment_id for f in dataset.get_fragments()]
-    )
-
-
-def _ith_batch(i: int, batch_size: int, total_size: int) -> Tuple[int, int]:
-    """
-    Get the start and end index of the ith batch.
-
-    This takes into account the total_size, the total number of rows in the dataset.
-    """
-    start = i * batch_size
-    end = tf.math.minimum(start + batch_size, total_size)
-    return (start, end)
-
-
-def from_lance_batches(
-    dataset: Union[str, Path, LanceDataset],
-    *,
-    shuffle: bool = False,
-    seed: Optional[int] = None,
-    batch_size: int = 1024,
-    skip: int = 0,
-) -> tf.data.Dataset:
-    """
-    Create a ``tf.data.Dataset`` of batch indices for a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    shuffle : bool, optional
-        Shuffle the batches, by default False
-    seed : Optional[int], optional
-        Random seed for shuffling, by default None
-    batch_size : int, optional
-        Batch size, by default 1024
-    skip : int, optional
-        Number of batches to skip.
-
-    Returns
-    -------
-    tf.data.Dataset
-        A tensorflow dataset of batch slice ranges. These can be passed to
-        :func:`lance_take_batches` to create a Tensorflow dataset of batches.
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-    num_rows = dataset.count_rows()
-    num_batches = (num_rows + batch_size - 1) // batch_size
-    indices = tf.data.Dataset.range(num_batches, dtype=tf.int64)
-    if shuffle:
-        indices = indices.shuffle(num_batches, seed=seed)
-    if skip > 0:
-        indices = indices.skip(skip)
-    return indices.map(partial(_ith_batch, batch_size=batch_size, total_size=num_rows))
-
-
-def lance_take_batches(
-    dataset: Union[str, Path, LanceDataset],
-    batch_ranges: Iterable[Tuple[int, int]],
-    *,
-    columns: Optional[List[str]] = None,
-    output_signature: Optional[Dict[str, tf.TypeSpec]] = None,
-    batch_readahead: int = 10,
-) -> tf.data.Dataset:
-    """
-    Create a ``tf.data.Dataset`` of batches from a Lance dataset.
-
-    Parameters
-    ----------
-    dataset : Union[str, Path, LanceDataset]
-        A Lance Dataset or dataset URI/path.
-    batch_ranges : Iterable[Tuple[int, int]]
-        Iterable of batch indices.
-    columns : Optional[List[str]], optional
-        List of columns to include in the output dataset.
-        If not set, all columns will be read.
-    output_signature : Optional[tf.TypeSpec], optional
-        Override output signature of the returned tensors. If not provided,
-        the output signature is inferred from the projection Schema.
-    batch_readahead : int, default 10
-        The number of batches to read ahead in parallel.
-
-    Examples
-    --------
-    You can compose this with ``from_lance_batches`` to create a randomized Tensorflow
-    dataset. With ``from_lance_batches``, you can deterministically randomized the
-    batches by setting ``seed``.
-
-    .. code-block:: python
-
-        batch_iter = from_lance_batches(dataset, batch_size=100, shuffle=True, seed=200)
-        batch_iter = batch_iter.as_numpy_iterator()
-        lance_ds = lance_take_batches(dataset, batch_iter)
-        lance_ds = lance_ds.unbatch().shuffle(500, seed=42).batch(100)
-    """
-    if not isinstance(dataset, LanceDataset):
-        dataset = lance.dataset(dataset)
-
-    if output_signature is None:
-        schema = dataset.scanner(columns=columns).projected_schema
-        output_signature = schema_to_spec(schema)
-    LOGGER.debug("Output signature: %s", output_signature)
-
-    def gen_ranges():
-        for start, end in batch_ranges:
-            yield (start, end)
-
-    def gen_batches():
-        batches = dataset._ds.take_scan(
-            gen_ranges(),
-            columns=columns,
-            batch_readahead=batch_readahead,
-        )
-        for batch in batches:
-            yield {
-                name: column_to_tensor(batch[name], output_signature[name])
-                for name in batch.schema.names
-            }
-
-    return tf.data.Dataset.from_generator(
-        gen_batches, output_signature=output_signature
-    )
-
-
-# Register `from_lance` to ``tf.data.Dataset``.
-tf.data.Dataset.from_lance = from_lance
-tf.data.Dataset.from_lance_batches = from_lance_batches
diff --git a/python/python/lance/torch/distance.py b/python/python/lance/torch/distance.py
index 81201027c87..ed5292cdb2f 100644
--- a/python/python/lance/torch/distance.py
+++ b/python/python/lance/torch/distance.py
@@ -150,8 +150,9 @@ def pairwise_l2(
     if x.dtype != y.dtype or (y2 is not None and x.dtype != y2.dtype):
         raise ValueError("pairwise_l2 data types do not match")
     origin_dtype = x.dtype
-    if x.device == torch.device("cpu") and x.dtype == torch.float16:
-        # Pytorch does not support `x @ y.T` for float16 on CPU
+    if x.device == torch.device("cpu") and x.dtype in (torch.float16, torch.bfloat16):
+        # Pytorch does not support `x @ y.T` for float16 on CPU, and bfloat16
+        # can hit backend-specific compiler paths. Accumulate in float32.
         x = x.type(torch.float32)
         y = y.type(torch.float32)
         if y2 is not None:
diff --git a/python/python/lance/udf.py b/python/python/lance/udf.py
index de6c7c4ff59..3a80349479e 100644
--- a/python/python/lance/udf.py
+++ b/python/python/lance/udf.py
@@ -205,7 +205,9 @@ def normalize_transform(
                     )
                 )
             )
-            if isinstance(sample_batch, pd.DataFrame):
+            if _check_for_pandas(sample_batch) and isinstance(
+                sample_batch, pd.DataFrame
+            ):
                 sample_batch = pa.RecordBatch.from_pandas(sample_batch)
             udf_like.output_schema = sample_batch.schema
 
@@ -233,7 +235,9 @@ def normalize_transform(
                     )
                 )
             )
-            if isinstance(sample_batch, pd.DataFrame):
+            if _check_for_pandas(sample_batch) and isinstance(
+                sample_batch, pd.DataFrame
+            ):
                 sample_batch = pa.RecordBatch.from_pandas(sample_batch)
             udf_like = BatchUDF(udf_like, output_schema=sample_batch.schema)
 
diff --git a/python/python/lance/util.py b/python/python/lance/util.py
index 5b94ad5c35a..180e2441b43 100644
--- a/python/python/lance/util.py
+++ b/python/python/lance/util.py
@@ -3,8 +3,18 @@
 
 from __future__ import annotations
 
+import uuid
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Iterator, Literal, Optional, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Iterable,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Union,
+    cast,
+)
 
 import pyarrow as pa
 
@@ -51,6 +61,29 @@ def td_to_micros(td: timedelta) -> int:
     return round(td / timedelta(microseconds=1))
 
 
+def _normalize_index_segment_ids(
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]],
+) -> Optional[List[str]]:
+    """Normalize a physical index segment selection to a list of UUID strings."""
+    if index_segments is None:
+        return None
+    if isinstance(index_segments, (str, uuid.UUID)):
+        raise TypeError(
+            "index_segments must be an iterable of str or uuid.UUID, "
+            f"not a single {type(index_segments)}."
+        )
+    segment_ids = []
+    for segment_id in index_segments:
+        if isinstance(segment_id, (str, uuid.UUID)):
+            segment_ids.append(str(segment_id))
+        else:
+            raise TypeError(
+                "index_segments must be an iterable of str or uuid.UUID. "
+                f"Got {type(segment_id)} instead."
+            )
+    return segment_ids
+
+
 class KMeans:
     """KMean model for clustering.
 
@@ -254,4 +287,4 @@ def _target_partition_size_to_num_partitions(
     if target_partition_size is None:
         target_partition_size = 8192
     num_partitions = num_rows // target_partition_size
-    return max(1, num_partitions, 4096)
+    return min(max(1, num_partitions), 4096)
diff --git a/python/python/lance/vector.py b/python/python/lance/vector.py
index 34a6154a321..75e6b5ba15b 100644
--- a/python/python/lance/vector.py
+++ b/python/python/lance/vector.py
@@ -19,9 +19,10 @@
 )
 from .dependencies import numpy as np
 from .log import LOGGER
-from .util import MetricType, _normalize_metric_type
+from .util import MetricType, _normalize_index_segment_ids, _normalize_metric_type
 
 if TYPE_CHECKING:
+    import uuid
     from pathlib import Path
 
     from . import LanceDataset
@@ -749,3 +750,174 @@ def _partition_and_pq_codes_assignment() -> Iterable[pa.RecordBatch]:
         data_file.path for frag in ds.get_fragments() for data_file in frag.data_files()
     ]
     return dst_dataset_uri, shuffle_buffers
+
+
+# =============================================================================
+# Hamming Distance Clustering
+# =============================================================================
+
+
+def hamming_clustering_for_ivf_partition(
+    dataset: "LanceDataset",
+    index_name: str,
+    partition_id: int,
+    hamming_threshold: int,
+    *,
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
+) -> pa.RecordBatchReader:
+    """
+    Perform hamming clustering on a partition of an IVF_FLAT index.
+
+    Loads a partition from every segment of an IVF_FLAT index on a hash
+    column, computes pairwise hamming distances between all hashes in the
+    combined partition, filters by threshold, and clusters the results using
+    union-find. All segments of the logical index must share the same global
+    IVF centroids; an error is raised if they do not.
+
+    Parameters
+    ----------
+    dataset : LanceDataset
+        The Lance dataset containing the hash column with an IVF_FLAT index.
+    index_name : str
+        Name of the IVF_FLAT index on the hash column
+    partition_id : int
+        The partition ID within the IVF_FLAT index
+    hamming_threshold : int
+        Maximum hamming distance to consider as similar
+    index_segments : iterable of str or uuid.UUID, optional
+        If specified, only these physical index segment UUIDs of the named
+        logical index contribute rows. Use
+        :meth:`LanceDataset.describe_indices` to obtain segment UUIDs from
+        ``IndexDescription.segments``. Defaults to all segments.
+
+    Returns
+    -------
+    pa.RecordBatchReader
+        A reader yielding batches with columns:
+
+        - 'representative': uint64 - The representative row ID for each cluster
+        - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    """
+    return dataset._ds.hamming_clustering_for_ivf_partition(
+        index_name,
+        partition_id,
+        hamming_threshold,
+        _normalize_index_segment_ids(index_segments),
+    )
+
+
+def get_ivf_partition_info(
+    dataset: "LanceDataset",
+    index_name: str,
+    *,
+    index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None,
+) -> List[dict]:
+    """
+    Get partition information for an IVF_FLAT index.
+
+    Partition sizes are aggregated across all segments of the logical index
+    unless a subset is selected via ``index_segments``.
+
+    Parameters
+    ----------
+    dataset : LanceDataset
+        The Lance dataset containing the hash column with an IVF_FLAT index.
+    index_name : str
+        Name of the IVF_FLAT index
+    index_segments : iterable of str or uuid.UUID, optional
+        If specified, only these physical index segment UUIDs of the named
+        logical index contribute to the sizes. Defaults to all segments.
+
+    Returns
+    -------
+    list[dict]
+        List of partition info dicts with 'partition_id' and 'size'
+    """
+    return dataset._ds.get_ivf_partition_info(
+        index_name, _normalize_index_segment_ids(index_segments)
+    )
+
+
+def hamming_clustering_for_sample(
+    dataset: "LanceDataset",
+    column: str,
+    sample_size: Optional[int] = None,
+    hamming_threshold: int = 10,
+) -> pa.RecordBatchReader:
+    """
+    Perform pairwise hamming distance clustering on a sample of the dataset.
+
+    Randomly samples rows from the dataset, computes pairwise hamming distances
+    between all hashes in the sample, filters by threshold, and clusters the
+    results using union-find.
+
+    Parameters
+    ----------
+    dataset : LanceDataset
+        The Lance dataset containing the hash column.
+    column : str
+        Name of the hash column (must be FixedSizeList<UInt8, N> where N is a
+        positive multiple of 8 bytes)
+    sample_size : int, optional
+        Number of rows to sample. If None, uses all rows.
+    hamming_threshold : int, default 10
+        Maximum hamming distance to consider as similar
+
+    Returns
+    -------
+    pa.RecordBatchReader
+        A reader yielding batches with columns:
+
+        - 'representative': uint64 - The representative row ID for each cluster
+        - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    """
+    return dataset._ds.hamming_clustering_for_sample(
+        column, sample_size, hamming_threshold
+    )
+
+
+def hamming_clustering_for_range(
+    dataset: "LanceDataset",
+    column: str,
+    fragment_id: int,
+    start_row: int,
+    num_rows: int,
+    hamming_threshold: int = 10,
+) -> pa.RecordBatchReader:
+    """
+    Perform pairwise hamming distance clustering on a contiguous range of rows.
+
+    Reads a contiguous range of rows from a specific fragment, computes pairwise
+    hamming distances between all hashes in the range, filters by threshold,
+    and clusters the results using union-find.
+
+    Unlike sampling, this reads sequential rows which is useful for distributed
+    processing where each worker handles a specific range of a fragment.
+
+    Parameters
+    ----------
+    dataset : LanceDataset
+        The Lance dataset containing the hash column.
+    column : str
+        Name of the hash column (must be FixedSizeList<UInt8, N> where N is a
+        positive multiple of 8 bytes)
+    fragment_id : int
+        The fragment ID to read from
+    start_row : int
+        The starting row offset within the fragment
+    num_rows : int
+        Number of rows to read from the start position
+    hamming_threshold : int, default 10
+        Maximum hamming distance to consider as similar
+
+    Returns
+    -------
+    pa.RecordBatchReader
+        A reader yielding batches with columns:
+
+        - 'representative': uint64 - The representative row ID for each cluster
+        - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    """
+    return dataset._ds.hamming_clustering_for_range(
+        column, fragment_id, start_row, num_rows, hamming_threshold
+    )
diff --git a/python/python/tests/compat/compat_decorator.py b/python/python/tests/compat/compat_decorator.py
index 0ab35672410..fdfe09a6879 100644
--- a/python/python/tests/compat/compat_decorator.py
+++ b/python/python/tests/compat/compat_decorator.py
@@ -153,6 +153,10 @@ def skip_read_after_current_write(self, version: str) -> bool:
         """Return True to skip the old-version read after current-version writes."""
         return False
 
+    def skip_write_after_current_write(self, version: str) -> bool:
+        """Return True to skip the old-version write after current-version writes."""
+        return False
+
     def skip_downgrade(self, version: str) -> bool:
         """Return True to skip the current-write -> old-read downgrade test."""
         return False
@@ -333,8 +337,10 @@ def test_func({sig_params}):
         obj.create()
     # Old version: verify can read
     venv = venv_factory.get_venv(version)
-    venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
-    venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
+    if not obj.skip_read_after_current_write(version):
+        venv.execute_method(obj, "check_read", obj.compat_env(version, "check_read"))
+    if not obj.skip_write_after_current_write(version):
+        venv.execute_method(obj, "check_write", obj.compat_env(version, "check_write"))
 '''
     else:  # upgrade_downgrade
         func_body = f'''
diff --git a/python/python/tests/compat/compat_sequence.py b/python/python/tests/compat/compat_sequence.py
new file mode 100644
index 00000000000..48df46500ca
--- /dev/null
+++ b/python/python/tests/compat/compat_sequence.py
@@ -0,0 +1,301 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Cross-version index maintenance-sequence search.
+
+Runs on the same per-ref venv substrate as the rest of this package: venv_factory
+(venv_manager.py) provisions one venv per ref, so the *setup* half of a sequence runs
+under `from_ref` and the *exercise* half under `to_ref` (the version split). After each
+run an oracle checks that the reader did not panic and that an index query agrees with a
+full (unindexed) scan. This *discovers* cross-version regressions (e.g. ENT-1662)
+without hand-coding the triggering sequence.
+
+The scenario is parameterized by index *kind* so every scalar index type gets the same
+aged-lifecycle, cross-version treatment. The oracle runs the same predicate twice --
+normally and with use_scalar_index=False (lance ignores the index) -- and requires
+the results to match. If the two query plans are identical the index wasn't used, so the
+comparison is skipped rather than failed (uninformative, not a regression). FTS has no
+"ignore the index" mode to diff against, so its oracle reconstructs ground truth from a
+full scan: tokenize every live row, then require an FTS search for a spread of sampled
+terms to return exactly the rows that contain them. The FTS scenarios run under both
+on-disk format versions (1 and 2), which take different merge paths. New Lance versions
+pin this with the create-index `format_version` parameter; old Lance versions still use
+`LANCE_FTS_FORMAT_VERSION`.
+
+The op vocabulary and bounds are deliberately small so the search is runnable; this is
+exhaustive over the maintenance-lifecycle grammar up to the configured lengths, not over
+every op permutation.
+"""
+
+import itertools
+import os
+import shutil
+from pathlib import Path
+
+ROWS_PER_WRITE = 200
+
+SETUP_TAIL_OPS = ["D", "C", "W"]
+EXERCISE_OPS = ["W", "D", "C", "Oa", "Om", "Od"]
+
+OP_NAMES = {
+    "W": "write rows",
+    "I": "create index",
+    "D": "delete rows",
+    "C": "compact",
+    "Oa": "optimize (append)",
+    "Om": "optimize (merge)",
+    "Od": "optimize",
+}
+
+
+def describe(kind, from_ref, to_ref, setup_ops, exercise_ops, fts_version=None):
+    """A plain-English description of a scenario for failure output."""
+    writer = ", then ".join(OP_NAMES[o] for o in ["W", "I", *setup_ops])
+    reader = ", then ".join(OP_NAMES[o] for o in exercise_ops)
+    tag = f" (fts fmt v{fts_version})" if fts_version is not None else ""
+    return f"{kind}{tag} ({from_ref} -> {to_ref}): writer [{writer}]; reader [{reader}]"
+
+
+# Index kinds covered by the maintenance-sequence search.
+SCALAR_KINDS = ["BTREE", "BITMAP", "LABEL_LIST", "NGRAM", "ZONEMAP", "BLOOMFILTER"]
+ALL_KINDS = ["INVERTED", *SCALAR_KINDS]
+
+
+class IndexScenario:
+    """A picklable, kind-parameterized scenario run across a version split."""
+
+    def __init__(self, kind, path, setup_ops, exercise_ops, fts_version=None):
+        self.kind = kind
+        self.path = str(path)
+        self.setup_ops = list(setup_ops)
+        self.exercise_ops = list(exercise_ops)
+        self.fts_version = fts_version
+        self.next_idx = 0
+
+    # --- in-venv helpers (only lance + pyarrow available) ---
+    def _open(self):
+        import lance
+
+        session = lance.Session(index_cache_size_bytes=0, metadata_cache_size_bytes=0)
+        return lance.dataset(self.path, session=session)
+
+    def _batch(self, a, b):
+        import pyarrow as pa
+
+        idx = list(range(a, b))
+        if self.kind == "INVERTED":
+            # Each row's text mixes tokens of different frequency: a unique term, a
+            # mid-frequency bucket (~1/7 of rows), and one shared by every row. Sampling
+            # across that spread exercises postings of varied length.
+            return pa.table(
+                {"idx": idx, "key": [f"term{i} bucket{i % 7} shared" for i in idx]}
+            )
+        if self.kind == "LABEL_LIST":
+            return pa.table({"idx": idx, "key": [[f"l{i % 8}"] for i in idx]})
+        if self.kind == "NGRAM":
+            return pa.table({"idx": idx, "key": [f"w{i % 50}x" for i in idx]})
+        # BTREE / BITMAP / ZONEMAP / BLOOMFILTER: integer column
+        card = 8 if self.kind == "BITMAP" else 50
+        key = [i if self.kind == "ZONEMAP" else i % card for i in idx]
+        return pa.table({"idx": idx, "key": key})
+
+    def _index_type(self):
+        return "INVERTED" if self.kind == "INVERTED" else self.kind
+
+    def _oracle_pred(self):
+        if self.kind == "LABEL_LIST":
+            return "array_has_any(key, ['l3'])"
+        if self.kind == "NGRAM":
+            return "contains(key, 'w3x')"
+        if self.kind == "ZONEMAP":
+            return "key >= 100 AND key < 300"
+        return "key == 3"  # BTREE / BITMAP / BLOOMFILTER
+
+    # --- ops ---
+    def _op_W(self):
+        import lance
+
+        a, b = self.next_idx, self.next_idx + ROWS_PER_WRITE
+        self.next_idx = b
+        tbl = self._batch(a, b)
+        if not os.path.exists(self.path):
+            lance.write_dataset(tbl, self.path)  # single fragment
+        else:
+            self._open().insert(tbl)
+
+    def _op_I(self):
+        kwargs = {"with_position": True} if self.kind == "INVERTED" else {}
+        if self.kind == "INVERTED" and self.fts_version is not None:
+            kwargs["format_version"] = int(self.fts_version)
+        self._open().create_scalar_index("key", self._index_type(), **kwargs)
+
+    def _op_D(self):
+        # Partial-range delete inside the id space so compaction rewrites and remaps the
+        # index per-row.
+        if self.next_idx == 0:
+            return
+        lo, hi = self.next_idx // 4, self.next_idx // 2
+        if hi > lo:
+            self._open().delete(f"idx >= {lo} AND idx < {hi}")
+
+    def _op_C(self):
+        self._open().optimize.compact_files()
+
+    def _op_Oa(self):
+        self._open().optimize.optimize_indices(num_indices_to_merge=0)
+
+    def _op_Om(self):
+        self._open().optimize.optimize_indices(num_indices_to_merge=10)
+
+    def _op_Od(self):
+        self._open().optimize.optimize_indices()
+
+    def _run(self, ops):
+        for op in ops:
+            getattr(self, f"_op_{op}")()
+
+    # --- methods invoked across the version split ---
+    def setup(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+        self.next_idx = 0
+        self._run(["W", "I"] + self.setup_ops)
+        return self.next_idx
+
+    def exercise_and_check(self):
+        self._run(self.exercise_ops)
+        ds = self._open()
+        if self.kind == "INVERTED":
+            # Differential oracle: rebuild the token -> rows map from a full (unindexed)
+            # scan, then require an FTS search for a spread of sampled terms to return
+            # exactly those rows. Catches a merge that drops or misassigns postings, not
+            # just a row-count drift. (Tokens here are alphanumeric and space-separated,
+            # so a whitespace split reproduces lance's tokenization.)
+            rows = ds.to_table(columns=["idx", "key"])
+            idxs = rows.column("idx").to_pylist()
+            texts = rows.column("key").to_pylist()
+            truth = {}
+            for i, text in zip(idxs, texts):
+                for tok in text.split():
+                    truth.setdefault(tok, set()).add(i)
+            if not truth:
+                return  # everything deleted; nothing to search
+            vocab = sorted(truth)
+            # A spread across the vocabulary plus the most common term.
+            sample = set(vocab[:: max(1, len(vocab) // 6)])
+            sample.add(max(truth, key=lambda t: len(truth[t])))
+            for term in sorted(sample):
+                hit = ds.to_table(full_text_query={"query": term, "columns": ["key"]})
+                got = set(hit.column("idx").to_pylist())
+                want = truth[term]
+                assert got == want, (
+                    f"FTS('{term}'): index returned {len(got)} rows, corpus has "
+                    f"{len(want)} (missing {sorted(want - got)[:5]}, "
+                    f"extra {sorted(got - want)[:5]})"
+                )
+            return
+        # Same column/predicate, index on vs forced off: use_scalar_index=False makes
+        # lance ignore the index, so the plans differ iff the index is used. If they are
+        # identical the index wasn't consulted here (the planner chose a scan after
+        # deletes), so the comparison is vacuous -- skip rather than compare two scans.
+        pred = self._oracle_pred()
+        plan_index = ds.scanner(filter=pred).explain_plan(True)
+        plan_scan = ds.scanner(filter=pred, use_scalar_index=False).explain_plan(True)
+        if plan_index == plan_scan:
+            return
+        got = ds.to_table(filter=pred).num_rows
+        expected = ds.to_table(filter=pred, use_scalar_index=False).num_rows
+        assert got == expected, (
+            f"{self.kind}: index gave {got} rows, full scan {expected}, for '{pred}'"
+        )
+
+
+def generate(max_length):
+    """Yield every (setup_ops, exercise_ops) whose combined length is 1..max_length,
+    breadth-first by total length (shorter first). `max_length` is the number of
+    maintenance ops after the implicit write + create-index, split between the writer
+    (setup) and reader (exercise) at every position. The order is neutral, so finding a
+    bug is a real search, not a sorted shortcut. The space grows fast with max_length,
+    so deeper bugs (ENT-1662 needs length 5) cost more to reach."""
+    for total in range(1, max_length + 1):
+        for setup_len in range(total):  # exercise gets total - setup_len >= 1
+            for s in itertools.product(SETUP_TAIL_OPS, repeat=setup_len):
+                for e in itertools.product(EXERCISE_OPS, repeat=total - setup_len):
+                    yield list(s), list(e)
+
+
+def search(
+    venv_factory,
+    from_ref,
+    to_ref,
+    base_path,
+    kind,
+    max_length=4,
+    shard=0,
+    num_shards=1,
+    stop_on_first=True,
+    fts_version=None,
+):
+    """Search index-maintenance sequences up to `max_length` ops for one `kind`, across
+    (from_ref -> to_ref). Runs only scenarios in this shard (i % num_shards == shard) so
+    the space can be split across parallel workers. For INVERTED, `fts_version` ("1" or
+    "2") pins the on-disk FTS format on both sides. New Lance versions receive this
+    through the create-index parameter and old Lance versions receive it through
+    LANCE_FTS_FORMAT_VERSION. Both are Fst token sets and exercise distinct merge paths.
+    Returns failures; stops on the first when `stop_on_first`."""
+    from_venv = venv_factory.get_venv(from_ref)
+    to_venv = venv_factory.get_venv(to_ref)
+    env = {}
+    if kind == "INVERTED" and fts_version is not None:
+        env["LANCE_FTS_FORMAT_VERSION"] = str(fts_version)
+    base = Path(base_path)
+    failures = []
+    # Each setup's aged dataset is built once under from_ref and snapshotted; every
+    # exercise for that setup runs on a *copy* of it (a dir copy is far cheaper
+    # than rebuilding the index). Cached per shard, keyed by the setup ops.
+    snapshots = {}  # tuple(setup) -> (snapshot_path, next_idx), or None if setup failed
+    try:
+        for i, (setup_tail, exercise) in enumerate(generate(max_length)):
+            if i % num_shards != shard:
+                continue
+            key = tuple(setup_tail)
+            if key not in snapshots:
+                snap = base / f"snap_{kind}_{len(snapshots)}"
+                shutil.rmtree(snap, ignore_errors=True)
+                builder = IndexScenario(kind, snap, setup_tail, [], fts_version)
+                try:
+                    next_idx = from_venv.execute_method(builder, "setup", env)
+                    snapshots[key] = (snap, next_idx)
+                except Exception as e:
+                    label = describe(
+                        kind, from_ref, to_ref, setup_tail, [], fts_version
+                    )
+                    err = str(e).strip()
+                    failures.append({"run": i, "sequence": label, "error": err})
+                    snapshots[key] = None
+                    shutil.rmtree(snap, ignore_errors=True)
+                    if stop_on_first:
+                        break
+            entry = snapshots[key]
+            if entry is None:
+                continue  # setup failed; skip its exercises
+            snap, next_idx = entry
+            ex_path = base / f"ex_{kind}_{i}"
+            shutil.rmtree(ex_path, ignore_errors=True)
+            shutil.copytree(snap, ex_path)
+            scenario = IndexScenario(kind, ex_path, setup_tail, exercise, fts_version)
+            scenario.next_idx = next_idx
+            label = describe(kind, from_ref, to_ref, setup_tail, exercise, fts_version)
+            try:
+                to_venv.execute_method(scenario, "exercise_and_check", env)
+            except Exception as e:
+                error = str(e).strip()
+                failures.append({"run": i, "sequence": label, "error": error})
+                if stop_on_first:
+                    break
+            finally:
+                shutil.rmtree(ex_path, ignore_errors=True)
+    finally:
+        for entry in snapshots.values():
+            if entry is not None:
+                shutil.rmtree(entry[0], ignore_errors=True)
+    return failures
diff --git a/python/python/tests/compat/test_index_sequence.py b/python/python/tests/compat/test_index_sequence.py
new file mode 100644
index 00000000000..4d0db694064
--- /dev/null
+++ b/python/python/tests/compat/test_index_sequence.py
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Cross-version index maintenance-sequence search, wired as a compat test.
+
+For each index kind, generate maintenance-op sequences from a small grammar, run the
+setup half under an older writer ref and the exercise half under a newer reader, and
+check after each that the reader did not panic and that an index query matches a full
+scan. Discovers cross-version regressions (e.g. ENT-1662, under INVERTED) with no
+hand-coded sequence.
+
+Refs and max length are environment-driven so the suite can run between two refs
+(versions, commits, or branches): COMPAT_FROM_REF / COMPAT_TO_REF / COMPAT_MAX_LENGTH /
+COMPAT_KINDS (comma-separated subset of kinds) / COMPAT_SHARDS (split each kind's search
+into this many cases so pytest-xdist (`-n auto`) parallelizes them across cores).
+"""
+
+import os
+
+import pytest
+
+from .compat_decorator import pylance_stable_versions
+from .compat_sequence import ALL_KINDS, search
+
+
+def _default_refs():
+    """The two most recent published stable releases (older -> newer)."""
+    versions = pylance_stable_versions()
+    if len(versions) >= 2:
+        return str(versions[-2]), str(versions[-1])
+    return "6.0.1", "7.0.0"  # fallback if PyPI is unreachable
+
+
+_default_from, _default_to = _default_refs()
+FROM_REF = os.environ.get("COMPAT_FROM_REF") or _default_from
+TO_REF = os.environ.get("COMPAT_TO_REF") or _default_to
+MAX_LENGTH = int(os.environ.get("COMPAT_MAX_LENGTH", "4"))
+KINDS = os.environ.get("COMPAT_KINDS", ",".join(ALL_KINDS)).split(",")
+# Many small shards (default 4x cores) so xdist's dynamic scheduler keeps every worker
+# busy and an oversubscribed `-n` has work to overlap.
+NUM_SHARDS = int(os.environ.get("COMPAT_SHARDS", str((os.cpu_count() or 1) * 4)))
+
+
+def _cases():
+    """(kind, fts_version) cases. FTS runs under both on-disk formats (v1, v2); the
+    scalar kinds are format-agnostic and run once."""
+    cases = []
+    for kind in KINDS:
+        if kind == "INVERTED":
+            cases.extend([("INVERTED", "1"), ("INVERTED", "2")])
+        else:
+            cases.append((kind, None))
+    return cases
+
+
+CASES = _cases()
+CASE_IDS = [k if v is None else f"{k}-fmtv{v}" for k, v in CASES]
+
+
+@pytest.mark.compat
+@pytest.mark.parametrize("kind,fts_version", CASES, ids=CASE_IDS)
+@pytest.mark.parametrize("shard", range(NUM_SHARDS))
+def test_index_maintenance_sequence_search(
+    venv_factory, tmp_path, kind, fts_version, shard
+):
+    failures = search(
+        venv_factory,
+        FROM_REF,
+        TO_REF,
+        tmp_path,
+        kind,
+        max_length=MAX_LENGTH,
+        shard=shard,
+        num_shards=NUM_SHARDS,
+        fts_version=fts_version,
+    )
+    # First line is the failure itself so it shows in pytest's bottom summary; the rest
+    # (if more than one) appears in the failure body.
+    assert not failures, "\n".join(
+        f"{f['sequence']}  ==>  {f['error']}" for f in failures
+    )
diff --git a/python/python/tests/compat/test_scalar_indices.py b/python/python/tests/compat/test_scalar_indices.py
index 35022df3b12..4f54e9d31c2 100644
--- a/python/python/tests/compat/test_scalar_indices.py
+++ b/python/python/tests/compat/test_scalar_indices.py
@@ -9,6 +9,7 @@
 and written by other versions.
 """
 
+import os
 import shutil
 from pathlib import Path
 
@@ -320,7 +321,12 @@ def create(self):
             max_rows_per_file=100,
             data_storage_version=safe_data_storage_version(self.compat_version),
         )
-        dataset.create_scalar_index("text", "INVERTED", with_position=True)
+        kwargs = {"with_position": True}
+        # Downgrade reads use older wheels, so current-created FTS indexes must
+        # stay on the legacy posting block layout.
+        if os.environ.get("LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE") == "1":
+            kwargs["block_size"] = 128
+        dataset.create_scalar_index("text", "INVERTED", format_version=1, **kwargs)
 
     def check_read(self):
         """Verify FTS index can be queried."""
@@ -351,7 +357,15 @@ def skip_downgrade(self, version: str) -> bool:
 
     def current_env(self, method_name: str) -> dict[str, str]:
         if method_name == "create":
-            return {"LANCE_FTS_FORMAT_VERSION": "1"}
+            return {
+                "LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE": "1",
+                "LANCE_FTS_FORMAT_VERSION": "1",
+            }
         if method_name == "check_write":
             return {"LANCE_FTS_FORMAT_VERSION": "2"}
         return {}
+
+    def compat_env(self, version: str, method_name: str) -> dict[str, str]:
+        if method_name in {"create", "check_write"}:
+            return {"LANCE_FTS_FORMAT_VERSION": "1"}
+        return {}
diff --git a/python/python/tests/compat/test_vector_indices.py b/python/python/tests/compat/test_vector_indices.py
index b98ffdf63e3..e97d6be8bf6 100644
--- a/python/python/tests/compat/test_vector_indices.py
+++ b/python/python/tests/compat/test_vector_indices.py
@@ -8,6 +8,7 @@
 can be read and written by other versions.
 """
 
+import os
 import shutil
 from pathlib import Path
 
@@ -268,6 +269,21 @@ class IvfRqVectorIndex(UpgradeDowngradeTest):
     def __init__(self, path: Path):
         self.path = path
 
+    def current_env(self, method_name: str):
+        if method_name == "check_read":
+            return {"LANCE_COMPAT_CURRENT_RUNTIME": "1"}
+        return {}
+
+    def skip_read_after_current_write(self, version: str) -> bool:
+        # Newly written IVF_RQ indexes carry raw-query estimator metadata and
+        # split-code schema that older runtimes cannot query or optimize safely.
+        # The upgrade_downgrade variant still covers old 1-bit residual-query
+        # indexes being read and rewritten by the current runtime.
+        return True
+
+    def skip_write_after_current_write(self, version: str) -> bool:
+        return True
+
     def create(self):
         """Create dataset with IVF_RQ vector index."""
         shutil.rmtree(self.path, ignore_errors=True)
@@ -319,6 +335,12 @@ def check_read(self):
 
         stats = ds.stats.index_stats(name)
         assert stats["num_indexed_rows"] > 0
+        if os.environ.get("LANCE_COMPAT_CURRENT_RUNTIME") == "1":
+            # Old 1-bit IVF_RQ indexes do not have split ex-code columns.
+            # The successful query above verifies the current reader does not
+            # require them.
+            sub_index = stats["indices"][0]["sub_index"]
+            assert sub_index["num_bits"] == 1
 
     def check_write(self):
         """Verify can insert vectors and run optimize workflows."""
diff --git a/python/python/tests/compat/venv_manager.py b/python/python/tests/compat/venv_manager.py
index 9e16b7e2dc7..c4b23486cd3 100644
--- a/python/python/tests/compat/venv_manager.py
+++ b/python/python/tests/compat/venv_manager.py
@@ -8,15 +8,39 @@
 with specific Lance versions installed.
 """
 
+import contextlib
+import glob
 import os
 import pickle
+import re
+import shutil
 import struct
 import subprocess
 import sys
 from pathlib import Path
 from typing import Any, Optional
 
-from packaging.version import Version
+from packaging.version import InvalidVersion, Version
+
+try:
+    import fcntl
+except ImportError:  # pragma: no cover - non-POSIX
+    fcntl = None
+
+
+@contextlib.contextmanager
+def _venv_lock(lock_path: Path):
+    """Hold an exclusive lock so parallel workers don't race creating the same venv."""
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(lock_path, "w") as handle:
+        if fcntl is not None:
+            fcntl.flock(handle, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            if fcntl is not None:
+                fcntl.flock(handle, fcntl.LOCK_UN)
+
 
 NAMESPACE_0_6_DEPENDENCY = "lance-namespace<0.7"
 NAMESPACE_0_7_DEPENDENCY = "lance-namespace>=0.7.2,<0.8"
@@ -31,6 +55,47 @@ def _lance_namespace_dependency(pylance_version: str) -> str:
     return NAMESPACE_0_6_DEPENDENCY
 
 
+def _is_release_version(ref: str) -> bool:
+    """A ref is treated as a published release (install a wheel) if it parses as a
+    version; anything else (commit sha, branch, tag) is built from source."""
+    try:
+        Version(ref)
+        return True
+    except InvalidVersion:
+        return False
+
+
+def _prebuilt_wheel_for(ref: str) -> Optional[str]:
+    """A prebuilt wheel to install for `ref` instead of building it from source.
+
+    When CI has already built a ref (e.g. the PR head, built once by the Python build
+    job), COMPAT_PREBUILT_REF names that ref and COMPAT_PREBUILT_WHEEL points at the
+    wheel (a path or glob). Lets the PR workflow reuse that wheel rather than rebuilding
+    the reader. Returns None when no prebuilt wheel applies to `ref`.
+    """
+    if os.environ.get("COMPAT_PREBUILT_REF") != ref:
+        return None
+    pattern = os.environ.get("COMPAT_PREBUILT_WHEEL")
+    if not pattern:
+        return None
+    matches = sorted(glob.glob(pattern))
+    if not matches:
+        raise FileNotFoundError(
+            f"COMPAT_PREBUILT_WHEEL={pattern!r} matched no wheel for ref {ref!r}"
+        )
+    return matches[0]
+
+
+def _repo_root() -> Path:
+    """Lance source checkout holding this test file (used to build refs from source)."""
+    # .../python/python/tests/compat/venv_manager.py -> repo root is parents[4]
+    return Path(__file__).resolve().parents[4]
+
+
+def _safe(ref: str) -> str:
+    return re.sub(r"[^A-Za-z0-9._-]", "_", ref)
+
+
 class VenvExecutor:
     """Manages a virtual environment with a specific Lance version."""
 
@@ -52,6 +117,8 @@ def __init__(self, version: str, venv_path: Path, persistent: bool = False):
         self.persistent = persistent
         self._created = False
         self._subprocess: Optional[subprocess.Popen] = None
+        self._stderr_path: Optional[Path] = None
+        self._stderr_file = None
 
     @property
     def python_path(self) -> Path:
@@ -59,54 +126,61 @@ def python_path(self) -> Path:
             return self.venv_path / "Scripts" / "python.exe"
         return self.venv_path / "bin" / "python"
 
-    def _validate_venv(self) -> bool:
-        """Check if existing venv is valid and has correct Lance version."""
-        if not self.venv_path.exists():
-            return False
+    @property
+    def _marker_path(self) -> Path:
+        return self.venv_path / ".compat_ref"
 
+    def _validate_venv(self) -> bool:
+        """A cached venv is reusable if it exists and its recorded ref matches. A marker
+        file is used (not `pip show`) so source-built commit refs also validate."""
         if not self.python_path.exists():
             return False
-
-        # Check if pylance is installed with correct version
         try:
-            result = subprocess.run(
-                [str(self.python_path), "-m", "pip", "show", "pylance"],
-                capture_output=True,
-                text=True,
-                timeout=5,
-            )
-            if result.returncode != 0:
-                return False
-
-            # Parse version from output
-            for line in result.stdout.splitlines():
-                if line.startswith("Version:"):
-                    installed_version = line.split(":", 1)[1].strip()
-                    return installed_version == self.version
-
-        except Exception:
+            return self._marker_path.read_text().strip() == self.version
+        except OSError:
             return False
 
-        return False
-
     def create(self):
         """Create the virtual environment and install the specified Lance version."""
         if self._created:
             return
-
-        # Check if persistent venv already exists and is valid
         if self.persistent and self._validate_venv():
             self._created = True
             return
 
-        # Create virtual environment
+        # Lock so parallel workers don't build the same venv at once; re-check in the
+        # lock since another worker may have just finished it.
+        with _venv_lock(self.venv_path.parent / f".lock_{_safe(self.version)}"):
+            if not self._validate_venv():
+                if self.venv_path.exists():
+                    shutil.rmtree(self.venv_path)  # drop any partial build
+                subprocess.run(
+                    [sys.executable, "-m", "venv", str(self.venv_path)],
+                    check=True,
+                    capture_output=True,
+                )
+                # Prefer a wheel CI already built for this ref; else a published
+                # release installs its wheel; else build the ref (commit/branch/tag)
+                # from source -- so two arbitrary refs can be compared and only the
+                # ones without a wheel pay a build.
+                prebuilt = _prebuilt_wheel_for(self.version)
+                if prebuilt is not None:
+                    self._install_wheel(prebuilt)
+                elif _is_release_version(self.version):
+                    self._install_release_wheel()
+                else:
+                    self._build_from_source()
+                self._marker_path.write_text(self.version)
+        self._created = True
+
+    def _install_wheel(self, wheel: str):
         subprocess.run(
-            [sys.executable, "-m", "venv", str(self.venv_path)],
+            [str(self.python_path), "-m", "pip", "install", "--quiet", wheel, "pytest"],
             check=True,
             capture_output=True,
         )
 
-        # Install specific pylance version and pytest
+    def _install_release_wheel(self):
         subprocess.run(
             [
                 str(self.python_path),
@@ -131,7 +205,55 @@ def create(self):
             capture_output=True,
         )
 
-        self._created = True
+    def _build_from_source(self):
+        """Build a wheel for an arbitrary git ref via a worktree + maturin, then install
+        it. The worktree/build is cached by ref so it is paid at most once."""
+        py = str(self.python_path)
+        src = self.venv_path.parent / f"src_{_safe(self.version)}"
+        if not src.exists():
+            subprocess.run(
+                [
+                    "git",
+                    "-C",
+                    str(_repo_root()),
+                    "worktree",
+                    "add",
+                    "--detach",
+                    str(src),
+                    self.version,
+                ],
+                check=True,
+                capture_output=True,
+            )
+        subprocess.run(
+            [py, "-m", "pip", "install", "--quiet", "maturin", "pytest", "pyarrow"],
+            check=True,
+            capture_output=True,
+        )
+        wheels = src / "target" / "compat-wheels"
+        subprocess.run(
+            [
+                py,
+                "-m",
+                "maturin",
+                "build",
+                "--release",
+                "--interpreter",
+                py,
+                "-m",
+                str(src / "python" / "Cargo.toml"),
+                "--out",
+                str(wheels),
+            ],
+            check=True,
+            capture_output=True,
+        )
+        wheel = next(wheels.glob("pylance-*.whl"))
+        subprocess.run(
+            [py, "-m", "pip", "install", "--quiet", str(wheel)],
+            check=True,
+            capture_output=True,
+        )
 
     def _ensure_subprocess(self):
         """Ensure the persistent subprocess is running."""
@@ -147,14 +269,35 @@ def _ensure_subprocess(self):
         tests_dir = Path(__file__).parent.parent
         env["PYTHONPATH"] = str(tests_dir)
 
+        # Capture stderr to a file so a Rust panic (which crashes the runner) can be
+        # surfaced in the error instead of an opaque "broken pipe".
+        self._stderr_path = self.venv_path / ".runner_stderr.log"
+        self._stderr_file = open(self._stderr_path, "w")
         self._subprocess = subprocess.Popen(
             [str(self.python_path), "-u", str(runner_script)],
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
-            stderr=None,  # Inherit stderr to see timing messages
+            stderr=self._stderr_file,
             env=env,
         )
 
+    def _last_panic(self) -> str:
+        """Pull the panic message from the runner's captured stderr, if any."""
+        try:
+            text = self._stderr_path.read_text()
+        except (OSError, AttributeError):
+            return ""
+        lines = text.splitlines()
+        for i, line in enumerate(lines):
+            if "panicked at" in line:
+                # Compact the long path to just "builder.rs:962:57"
+                loc = line.split("panicked at", 1)[1].strip().rstrip(":")
+                loc = loc.rsplit("/", 1)[-1]
+                msg = lines[i + 1].strip() if i + 1 < len(lines) else ""
+                return f"panic at {loc}: {msg}" if msg else f"panic at {loc}"
+        tail = [line.strip() for line in lines if line.strip()]
+        return tail[-1] if tail else ""
+
     def _send_message(self, obj: Any):
         """Send a length-prefixed pickled message to subprocess."""
         data = pickle.dumps(obj)
@@ -165,18 +308,19 @@ def _send_message(self, obj: Any):
 
     def _receive_message(self) -> Any:
         """Receive a length-prefixed pickled message from subprocess."""
-        # Read 4-byte length header
+        # Short reads mean the subprocess closed stdout (usually a crash); raise
+        # EOFError so the caller can surface the panic from captured stderr.
         length_bytes = self._subprocess.stdout.read(4)
         if len(length_bytes) < 4:
-            raise RuntimeError("Failed to read message length from subprocess")
+            raise EOFError("subprocess closed stdout before sending a message length")
 
         length = struct.unpack(">I", length_bytes)[0]
 
         # Read message data
         data = self._subprocess.stdout.read(length)
         if len(data) < length:
-            raise RuntimeError(
-                f"Incomplete message: expected {length} bytes, got {len(data)}"
+            raise EOFError(
+                f"incomplete message: expected {length} bytes, got {len(data)}"
             )
 
         return pickle.loads(data)
@@ -234,11 +378,15 @@ def execute_method(
                 raise RuntimeError(error_msg)
 
         except (BrokenPipeError, EOFError, struct.error) as e:
-            # Subprocess died or communication failed
-            raise RuntimeError(
-                f"Communication with venv subprocess failed (Lance {self.version}):\n"
-                f"Error: {e}"
-            )
+            # Subprocess died (usually a Rust panic); flush it, then surface that.
+            if self._subprocess is not None:
+                try:
+                    self._subprocess.wait(timeout=2)
+                except Exception:
+                    pass
+            panic = self._last_panic()
+            detail = panic or f"subprocess communication failed: {e}"
+            raise RuntimeError(f"Lance {self.version}: {detail}")
 
     def cleanup(self):
         """Remove the virtual environment directory and terminate subprocess."""
@@ -295,7 +443,7 @@ def get_venv(self, version: str) -> VenvExecutor:
             Executor for the specified version
         """
         if version not in self.venvs:
-            venv_path = self.base_path / f"venv_{version}"
+            venv_path = self.base_path / f"venv_{_safe(version)}"
             executor = VenvExecutor(version, venv_path, persistent=self.persistent)
             executor.create()
             self.venvs[version] = executor
diff --git a/python/python/tests/test_arrow.py b/python/python/tests/test_arrow.py
index b6e6024e0e7..a0be09846e1 100644
--- a/python/python/tests/test_arrow.py
+++ b/python/python/tests/test_arrow.py
@@ -7,10 +7,11 @@
 from pathlib import Path
 
 import lance
+import lance.arrow
 import numpy as np
 import pandas as pd
 import pyarrow as pa
-import pytest
+import pytest  # pyright: ignore[reportMissingImports]
 from lance.arrow import (
     BFloat16,
     BFloat16Array,
@@ -272,8 +273,6 @@ def test_image_uri_arrays(tmp_path: Path, png_uris):
 
 
 def test_image_tensor_arrays(tmp_path: Path, png_uris):
-    tf = pytest.importorskip("tensorflow")
-
     n = 10
 
     encoded_image_array = ImageURIArray.from_uris(png_uris).read_uris()
@@ -296,22 +295,22 @@ def test_image_tensor_arrays(tmp_path: Path, png_uris):
     assert tensor_image_array.storage.type == pa.list_(pa.uint8(), 4)
     assert tensor_image_array[2].as_py() == [42, 42, 42, 255]
 
-    test_tensor = tf.constant(
-        np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4))
-    )
+    test_tensor = np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4))
 
     assert test_tensor.shape == (n, 1, 1, 4)
-    assert tf.math.reduce_all(
-        tf.convert_to_tensor(tensor_image_array.to_numpy()) == test_tensor
-    )
+    assert np.array_equal(tensor_image_array.to_numpy(), test_tensor)
     assert tensor_image_array.to_encoded().to_tensor() == tensor_image_array
 
     def png_encoder(images):
-        import tensorflow as tf
+        import io
 
-        encoded_images = (
-            tf.io.encode_png(x).numpy() for x in tf.convert_to_tensor(images)
-        )
+        from PIL import Image  # pyright: ignore[reportMissingImports]
+
+        encoded_images = []
+        for image in images:
+            with io.BytesIO() as buf:
+                Image.fromarray(image).save(buf, format="PNG")
+                encoded_images.append(buf.getvalue())
         return pa.array(encoded_images, type=pa.binary())
 
     assert tensor_image_array.to_encoded(png_encoder).to_tensor() == tensor_image_array
@@ -323,20 +322,18 @@ def png_encoder(images):
         uris = [str(Path(x)) for x in uris]
 
     encoded_image_array = ImageArray.from_array(uris).read_uris()
-    with pytest.raises(
-        tf.errors.InvalidArgumentError, match="Shapes of all inputs must match"
-    ):
+    with pytest.raises(ValueError, match="all input arrays must have the same shape"):
         encoded_image_array.to_tensor()
 
     pattern = r"(object at) 0x[\w\d]+(:?>)"
     repl = r"\1 0x..\2"
-    assert re.sub(pattern, repl, encoded_image_array.__repr__()) == (
-        "<lance.arrow.EncodedImageArray object at 0x..>\n"
-        "[<tf.Tensor: shape=(1, 1, 4), dtype=uint8, numpy=array([[[ 42,  42,  42, "
-        "255]]], dtype=uint8)>, ..]\n"
+    repr_ = re.sub(pattern, repl, encoded_image_array.__repr__())
+    assert repr_.startswith("<lance.arrow.EncodedImageArray object at 0x..>\n[<PIL.")
+    assert "image mode=RGBA size=1x1" in repr_
+    assert (
         "b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\x01\\x00\\x00"
         "\\x00\\x01\\x08\\x06\\x00\\x00\\x00\\x1f'"
-    )
+    ) in repr_
 
 
 def test_roundtrip_image_tensor(tmp_path: Path):
diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py
index 5a896d21c5d..e095d202bd7 100644
--- a/python/python/tests/test_blob.py
+++ b/python/python/tests/test_blob.py
@@ -1,18 +1,24 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The Lance Authors
 
+import gc
 import importlib
 import io
+import queue
 import subprocess
 import sys
 import tarfile
 import textwrap
+import threading
+import uuid
+from pathlib import Path
 
 import lance
 import pandas as pd
 import pyarrow as pa
 import pytest
 from lance import Blob, BlobColumn, BlobFile, DatasetBasePath
+from lance.file import LanceFileSession
 from lance.fragment import write_fragments
 
 lance_dataset_module = importlib.import_module("lance.dataset")
@@ -45,6 +51,61 @@ def _external_blob_table(blob_path, payload=b"hello"):
     return pa.table({"blob": lance.blob_array([blob_path.as_uri()])})
 
 
+def _blob_sidecar_path(data_dir, data_file_key, blob_id):
+    sidecar_name = f"{int(f'{blob_id:032b}'[::-1], 2):032b}.blob"
+    return data_dir / data_file_key / sidecar_name
+
+
+def _add_columns_blob_v2_values(tmp_path):
+    external_base = tmp_path / "external_base"
+    external_blob = external_base / "external_blob.bin"
+    external_blob.parent.mkdir(parents=True, exist_ok=True)
+    external_blob.write_bytes(b"external")
+
+    payloads = [
+        b"inline",
+        b"p" * (64 * 1024 + 1024),
+        b"d" * (4 * 1024 * 1024 + 1024),
+        b"external",
+    ]
+    values = [payloads[0], payloads[1], payloads[2], external_blob.as_uri()]
+    initial_bases = [DatasetBasePath(external_base.as_uri(), name="external", id=1)]
+    return values, payloads, initial_bases
+
+
+def _assert_blob_v2_add_columns_result(dataset, column, payloads):
+    desc = dataset.to_table(columns=[column]).column(column).chunk(0)
+
+    assert desc.field("kind").to_pylist() == [0, 1, 2, 3]
+    assert desc.field("blob_id").to_pylist()[3] == 1
+    assert desc.field("blob_uri").to_pylist()[3] == "external_blob.bin"
+
+    blobs = dataset.take_blobs(column, indices=range(len(payloads)))
+    assert [blob.readall() for blob in blobs] == payloads
+
+
+def _dataset_file_set(dataset_path):
+    return {
+        path.relative_to(dataset_path)
+        for path in dataset_path.rglob("*")
+        if path.is_file()
+    }
+
+
+def _write_two_fragment_blob_v2_seed_dataset(tmp_path, name):
+    values, payloads, initial_bases = _add_columns_blob_v2_values(tmp_path)
+    dataset_path = tmp_path / name
+    ds = lance.write_dataset(
+        pa.table({"id": range(8)}),
+        dataset_path,
+        data_storage_version="2.2",
+        initial_bases=initial_bases,
+        max_rows_per_file=4,
+        max_rows_per_group=4,
+    )
+    return ds, dataset_path, values, payloads
+
+
 def _out_of_order_blob_selection(dataset_with_blobs, selection_kind):
     addresses = _blob_row_addresses(dataset_with_blobs)
     expected = [(addresses[4], b"quux"), (addresses[0], b"foo")]
@@ -121,6 +182,40 @@ def test_scan_blob_as_binary(tmp_path):
     assert tbl.column("blobs").to_pylist() == values
 
 
+def test_v2_0_blob_descriptor_projection_and_reads(tmp_path):
+    values = [b"abc", b"defgh", b"ijklmnop"]
+    blob_field = pa.field(
+        "blob", pa.large_binary(), metadata={"lance-encoding:blob": "true"}
+    )
+    schema = pa.schema([pa.field("id", pa.int32()), blob_field])
+    table = pa.table(
+        {"id": [0, 1, 2], "blob": values},
+        schema=schema,
+    )
+    ds = lance.write_dataset(table, tmp_path / "test_ds", data_storage_version="2.0")
+
+    for blob_handling in [None, "all_descriptions", "blobs_descriptions"]:
+        kwargs = {} if blob_handling is None else {"blob_handling": blob_handling}
+        descriptions = ds.scanner(columns=["blob"], **kwargs).to_table().column("blob")
+        chunk = descriptions.chunk(0)
+        assert chunk.type == pa.struct(
+            [
+                pa.field("position", pa.uint64()),
+                pa.field("size", pa.uint64()),
+            ]
+        )
+        assert chunk.field("size").to_pylist() == [3, 5, 8]
+
+    tbl = ds.scanner(columns=["blob"], blob_handling="all_binary").to_table()
+    assert tbl.column("blob").to_pylist() == values
+    assert [blob.read() for blob in ds.take_blobs("blob", indices=[0, 1, 2])] == values
+    assert ds.read_blobs("blob", indices=[0, 1, 2]) == [
+        (0, b"abc"),
+        (1, b"defgh"),
+        (2, b"ijklmnop"),
+    ]
+
+
 def test_fragment_scan_blob_as_binary(tmp_path):
     values = [b"foo", b"bar", b"baz"]
     arr = pa.array(values, pa.large_binary())
@@ -533,6 +628,232 @@ def test_blob_extension_write_inline(tmp_path):
         assert f.read() == b"foo"
 
 
+def test_blob_field_threshold_metadata():
+    field = lance.blob_field(
+        "blob",
+        inline_size_threshold=16 * 1024,
+        dedicated_size_threshold=2 * 1024 * 1024,
+        pack_file_size_threshold=512 * 1024 * 1024,
+    )
+
+    assert field.metadata[b"lance-encoding:blob-inline-size-threshold"] == b"16384"
+    assert field.metadata[b"lance-encoding:blob-dedicated-size-threshold"] == b"2097152"
+    assert (
+        field.metadata[b"lance-encoding:blob-pack-file-size-threshold"] == b"536870912"
+    )
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "error", "message"),
+    [
+        pytest.param(
+            {"inline_size_threshold": -1},
+            ValueError,
+            "inline_size_threshold must be non-negative",
+            id="negative_inline",
+        ),
+        pytest.param(
+            {"dedicated_size_threshold": 0},
+            ValueError,
+            "dedicated_size_threshold must be positive",
+            id="zero_dedicated",
+        ),
+        pytest.param(
+            {"dedicated_size_threshold": -1},
+            ValueError,
+            "dedicated_size_threshold must be positive",
+            id="negative_dedicated",
+        ),
+        pytest.param(
+            {"inline_size_threshold": True},
+            TypeError,
+            "inline_size_threshold must be an int",
+            id="bool_inline",
+        ),
+        pytest.param(
+            {"dedicated_size_threshold": True},
+            TypeError,
+            "dedicated_size_threshold must be an int",
+            id="bool_dedicated",
+        ),
+        pytest.param(
+            {"inline_size_threshold": 1.5},
+            TypeError,
+            "inline_size_threshold must be an int",
+            id="float_inline",
+        ),
+        pytest.param(
+            {"inline_size_threshold": 2**100},
+            OverflowError,
+            "inline_size_threshold must fit in a Rust usize",
+            id="overflow_inline",
+        ),
+        pytest.param(
+            {"dedicated_size_threshold": 2**100},
+            OverflowError,
+            "dedicated_size_threshold must fit in a Rust usize",
+            id="overflow_dedicated",
+        ),
+        pytest.param(
+            {"pack_file_size_threshold": 0},
+            ValueError,
+            "pack_file_size_threshold must be positive",
+            id="zero_pack_file",
+        ),
+        pytest.param(
+            {"pack_file_size_threshold": -1},
+            ValueError,
+            "pack_file_size_threshold must be positive",
+            id="negative_pack_file",
+        ),
+        pytest.param(
+            {"pack_file_size_threshold": True},
+            TypeError,
+            "pack_file_size_threshold must be an int",
+            id="bool_pack_file",
+        ),
+        pytest.param(
+            {"pack_file_size_threshold": 2**100},
+            OverflowError,
+            "pack_file_size_threshold must fit in a Rust usize",
+            id="overflow_pack_file",
+        ),
+    ],
+)
+def test_blob_field_rejects_invalid_thresholds(kwargs, error, message):
+    with pytest.raises(error, match=message):
+        lance.blob_field("blob", **kwargs)
+
+
+def test_blob_extension_inline_threshold_per_column(tmp_path):
+    payload = b"x" * 2048
+    schema = pa.schema(
+        [
+            lance.blob_field("inline_blob", inline_size_threshold=4096),
+            lance.blob_field("packed_blob", inline_size_threshold=1024),
+        ]
+    )
+    table = pa.table(
+        {
+            "inline_blob": lance.blob_array([payload]),
+            "packed_blob": lance.blob_array([payload]),
+        },
+        schema=schema,
+    )
+    ds = lance.write_dataset(
+        table,
+        tmp_path / "test_ds_v2_inline_threshold_per_column",
+        data_storage_version="2.2",
+    )
+
+    desc = ds.to_table(columns=["inline_blob", "packed_blob"])
+    assert desc.column("inline_blob").chunk(0).field("kind").to_pylist() == [0]
+    assert desc.column("packed_blob").chunk(0).field("kind").to_pylist() == [1]
+
+
+def test_blob_extension_threshold_metadata_persists_after_reopen(tmp_path):
+    dataset_path = tmp_path / "test_ds_v2_threshold_metadata_persists"
+    schema = pa.schema([lance.blob_field("blob", inline_size_threshold=1024)])
+    table = pa.table({"blob": lance.blob_array([b"x"])}, schema=schema)
+
+    lance.write_dataset(table, dataset_path, data_storage_version="2.2")
+    reopened = lance.dataset(dataset_path)
+
+    assert (
+        reopened.schema.field("blob").metadata[
+            b"lance-encoding:blob-inline-size-threshold"
+        ]
+        == b"1024"
+    )
+
+
+def test_blob_extension_append_rejects_explicit_threshold_mismatch(tmp_path):
+    dataset_path = tmp_path / "test_ds_v2_append_threshold_mismatch"
+    initial_schema = pa.schema([lance.blob_field("blob", inline_size_threshold=4096)])
+    initial = pa.table(
+        {"blob": lance.blob_array([b"x" * 2048])},
+        schema=initial_schema,
+    )
+    lance.write_dataset(initial, dataset_path, data_storage_version="2.2")
+
+    append_schema = pa.schema([lance.blob_field("blob", inline_size_threshold=1024)])
+    append = pa.table(
+        {"blob": lance.blob_array([b"x" * 2048])},
+        schema=append_schema,
+    )
+
+    with pytest.raises(
+        OSError, match="Cannot append data with blob threshold metadata"
+    ):
+        lance.write_dataset(append, dataset_path, mode="append")
+
+
+def test_blob_extension_pack_file_threshold_metadata_persists_after_reopen(
+    tmp_path: Path,
+):
+    dataset_path = tmp_path / "test_ds_v2_pack_file_threshold_persists"
+    threshold = 512 * 1024 * 1024
+    schema = pa.schema([lance.blob_field("blob", pack_file_size_threshold=threshold)])
+    table = pa.table({"blob": lance.blob_array([b"x"])}, schema=schema)
+
+    lance.write_dataset(table, dataset_path, data_storage_version="2.2")
+    reopened = lance.dataset(dataset_path)
+
+    assert (
+        reopened.schema.field("blob").metadata[
+            b"lance-encoding:blob-pack-file-size-threshold"
+        ]
+        == str(threshold).encode()
+    )
+
+
+def test_blob_extension_append_rejects_pack_file_threshold_mismatch(tmp_path: Path):
+    dataset_path = tmp_path / "test_ds_v2_append_pack_file_mismatch"
+    initial_schema = pa.schema(
+        [lance.blob_field("blob", pack_file_size_threshold=512 * 1024 * 1024)]
+    )
+    initial = pa.table(
+        {"blob": lance.blob_array([b"x" * 2048])},
+        schema=initial_schema,
+    )
+    lance.write_dataset(initial, dataset_path, data_storage_version="2.2")
+
+    append_schema = pa.schema(
+        [lance.blob_field("blob", pack_file_size_threshold=256 * 1024 * 1024)]
+    )
+    append = pa.table(
+        {"blob": lance.blob_array([b"x" * 2048])},
+        schema=append_schema,
+    )
+
+    with pytest.raises(
+        OSError, match="Cannot append data with blob threshold metadata"
+    ):
+        lance.write_dataset(append, dataset_path, mode="append")
+
+
+def test_blob_extension_dedicated_threshold_precedes_inline_threshold(tmp_path):
+    payload = b"x" * 2048
+    schema = pa.schema(
+        [
+            lance.blob_field(
+                "blob",
+                inline_size_threshold=4096,
+                dedicated_size_threshold=1024,
+            )
+        ]
+    )
+    table = pa.table({"blob": lance.blob_array([payload])}, schema=schema)
+    ds = lance.write_dataset(
+        table,
+        tmp_path / "test_ds_v2_dedicated_precedes_inline",
+        data_storage_version="2.2",
+    )
+
+    desc = ds.to_table(columns=["blob"]).column("blob").chunk(0)
+    assert desc.field("kind").to_pylist() == [2]
+
+
 def test_blob_extension_write_external(tmp_path):
     blob_path = tmp_path / "external_blob.bin"
     blob_path.write_bytes(b"hello")
@@ -608,6 +929,497 @@ def test_blob_extension_write_external_ingest_rejects_reference_only_options(tmp
         )
 
 
+def test_blob_extension_add_columns_record_batch_reader_all_kinds(tmp_path):
+    values, payloads, initial_bases = _add_columns_blob_v2_values(tmp_path)
+    ds = lance.write_dataset(
+        pa.table({"id": range(4)}),
+        tmp_path / "test_add_columns_reader_blob_v2",
+        data_storage_version="2.2",
+        initial_bases=initial_bases,
+    )
+
+    ds.add_columns(pa.table({"blob": lance.blob_array(values)}).to_reader())
+
+    _assert_blob_v2_add_columns_result(ds, "blob", payloads)
+
+
+@pytest.mark.parametrize(
+    "failure_mode",
+    [
+        pytest.param("raises_after_first_fragment", id="reader_raises_mid_stream"),
+        pytest.param("wrong_schema", id="reader_yields_wrong_schema"),
+        pytest.param("too_many_rows", id="reader_produces_too_many_rows"),
+    ],
+)
+def test_blob_extension_add_columns_record_batch_reader_failure_cleans_files(
+    tmp_path,
+    failure_mode,
+):
+    ds, dataset_path, values, payloads = _write_two_fragment_blob_v2_seed_dataset(
+        tmp_path,
+        f"test_add_columns_reader_blob_v2_fail_cleanup_{failure_mode}",
+    )
+    external_blob_path = tmp_path / "external_base" / "external_blob.bin"
+    files_before = _dataset_file_set(dataset_path)
+
+    schema = pa.schema([lance.blob_field("blob")])
+    first_fragment_batch = pa.record_batch([lance.blob_array(values)], schema=schema)
+    second_fragment_batch = pa.record_batch([lance.blob_array(values)], schema=schema)
+
+    if failure_mode == "raises_after_first_fragment":
+        match = "reader failed after first fragment"
+
+        def failing_reader():
+            yield first_fragment_batch
+            raise RuntimeError("reader failed after first fragment")
+
+    elif failure_mode == "wrong_schema":
+        match = "field names"
+
+        def failing_reader():
+            yield first_fragment_batch
+            yield pa.record_batch([pa.array(range(4))], ["not_blob"])
+
+    else:
+        match = "Stream produced more values than expected for dataset"
+
+        def failing_reader():
+            yield first_fragment_batch
+            yield second_fragment_batch
+            yield pa.record_batch([lance.blob_array([payloads[0]])], schema=schema)
+
+    with pytest.raises(OSError, match=match):
+        ds.add_columns(failing_reader(), reader_schema=schema)
+
+    assert ds.version == 1
+    assert _dataset_file_set(dataset_path) == files_before
+    assert external_blob_path.exists()
+
+
+def test_blob_extension_add_columns_batch_udf_failure_cleans_files(tmp_path):
+    ds, dataset_path, values, _ = _write_two_fragment_blob_v2_seed_dataset(
+        tmp_path,
+        "test_add_columns_udf_blob_v2_fail_cleanup",
+    )
+    external_blob_path = tmp_path / "external_base" / "external_blob.bin"
+    files_before = _dataset_file_set(dataset_path)
+    call_count = 0
+
+    @lance.batch_udf(output_schema=pa.schema([lance.blob_field("blob")]))
+    def fail_on_second_fragment(batch):
+        nonlocal call_count
+        call_count += 1
+        if call_count == 2:
+            raise RuntimeError("udf failed after first fragment")
+        blob_values = [values[row.as_py() % len(values)] for row in batch["id"]]
+        return pa.record_batch(
+            [lance.blob_array(blob_values)],
+            ["blob"],
+        )
+
+    with pytest.raises(OSError, match="udf failed after first fragment"):
+        ds.add_columns(fail_on_second_fragment, read_columns=["id"], batch_size=4)
+
+    assert call_count == 2
+    assert ds.version == 1
+    assert _dataset_file_set(dataset_path) == files_before
+    assert external_blob_path.exists()
+
+
+def test_blob_extension_add_columns_batch_udf_all_kinds(tmp_path):
+    values, payloads, initial_bases = _add_columns_blob_v2_values(tmp_path)
+    ds = lance.write_dataset(
+        pa.table({"id": range(4)}),
+        tmp_path / "test_add_columns_udf_blob_v2",
+        data_storage_version="2.2",
+        initial_bases=initial_bases,
+    )
+
+    @lance.batch_udf(output_schema=pa.schema([lance.blob_field("blob")]))
+    def make_blob_column(batch):
+        return pa.record_batch(
+            [lance.blob_array([values[row.as_py()] for row in batch["id"]])],
+            ["blob"],
+        )
+
+    ds.add_columns(make_blob_column, read_columns=["id"])
+
+    _assert_blob_v2_add_columns_result(ds, "blob", payloads)
+
+
+def test_blob_extension_add_columns_all_nulls_blob_v2(tmp_path):
+    ds = lance.write_dataset(
+        pa.table({"id": range(4)}),
+        tmp_path / "test_add_columns_all_nulls_blob_v2",
+        data_storage_version="2.2",
+    )
+
+    ds.add_columns(lance.blob_field("blob"))
+
+    assert ds.to_table(columns=["blob"]).column("blob").to_pylist() == [None] * 4
+    assert ds.take_blobs("blob", indices=range(4)) == []
+
+
+def test_blob_descriptor_array_builder_writes_prepared_packed_blob_for_data_replacement(
+    tmp_path,
+):
+    dataset_uri = tmp_path / "test_blob_descriptor_array_builder_data_replacement"
+    logical_schema = pa.schema(
+        [
+            pa.field("id", pa.uint32(), nullable=False),
+            lance.blob_field("blob"),
+        ]
+    )
+    initial = pa.table(
+        [
+            pa.array([0], type=pa.uint32()),
+            lance.blob_array([b"initial"]),
+        ],
+        schema=logical_schema,
+    )
+    ds = lance.write_dataset(initial, dataset_uri, data_storage_version="2.2")
+
+    file_id = str(uuid.uuid4())
+    data_file_name = f"{file_id}.lance"
+    blob_id = 1
+    blob_path = _blob_sidecar_path(dataset_uri / "data", file_id, blob_id)
+
+    files = LanceFileSession(dataset_uri / "data")
+    blob_writer = lance.BlobDescriptorArrayBuilder("blob")
+    packed = files.open_packed_blob_writer(data_file_name, blob_id)
+    assert packed.path.endswith(blob_path.relative_to(dataset_uri).as_posix())
+    packed.write_blob(b"replacement")
+    blob_writer.extend(packed.finish())
+
+    physical_schema = pa.schema(
+        [
+            pa.field("id", pa.uint32(), nullable=False),
+            blob_writer.field,
+        ]
+    )
+    replacement = pa.record_batch(
+        [
+            pa.array([1], type=pa.uint32()),
+            blob_writer.finish(),
+        ],
+        schema=physical_schema,
+    )
+
+    with files.open_writer(
+        data_file_name, schema=physical_schema, version="2.2"
+    ) as file_writer:
+        file_writer.write_batch(replacement)
+
+    data_file = lance.fragment.DataFile.create(ds, data_file_name)
+    operation = lance.LanceOperation.DataReplacement(
+        [lance.LanceOperation.DataReplacementGroup(0, data_file)]
+    )
+    ds = lance.LanceDataset.commit(ds, operation, read_version=ds.version)
+
+    assert ds.to_table(columns=["id"]).column("id").to_pylist() == [1]
+    blobs = ds.take_blobs("blob", indices=[0])
+    assert len(blobs) == 1
+    assert blobs[0].readall() == b"replacement"
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        pytest.param(b"payload", id="bytes"),
+        pytest.param(bytearray(b"payload"), id="bytearray"),
+        pytest.param(memoryview(b"payload"), id="memoryview"),
+        pytest.param(list(b"payload"), id="integer_sequence"),
+    ],
+)
+def test_packed_blob_writer_scalar_buffer_inputs(tmp_path, payload):
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blob(payload)
+    descriptors = packed.finish()
+
+    assert [repr(descriptor) for descriptor in descriptors] == [
+        "Packed { blob_id: 7, offset: 0, size: 7 }"
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"payload"
+
+
+@pytest.mark.parametrize("array_type", [pa.binary(), pa.large_binary()])
+@pytest.mark.parametrize("as_chunked", [False, True], ids=["array", "chunked_array"])
+@pytest.mark.parametrize(
+    "values,slice_offset,slice_length,expected_values,expected_data",
+    [
+        pytest.param(
+            [b"prefix", b"a", None, b"", b"bc", b"suffix"],
+            1,
+            4,
+            [b"a", None, b"", b"bc"],
+            b"abc",
+            id="interleaved_null",
+        ),
+        pytest.param(
+            [b"prefix", b"a", b"", b"bc", b"suffix"],
+            1,
+            3,
+            [b"a", b"", b"bc"],
+            b"abc",
+            id="all_valid",
+        ),
+        pytest.param(
+            [b"prefix", None, None, b"suffix"],
+            1,
+            2,
+            [None, None],
+            b"",
+            id="all_null",
+        ),
+        pytest.param(
+            [b"prefix", b"suffix"],
+            1,
+            0,
+            [],
+            b"",
+            id="empty",
+        ),
+    ],
+)
+def test_packed_blob_writer_bulk_arrow_array(
+    tmp_path,
+    array_type,
+    as_chunked,
+    values,
+    slice_offset,
+    slice_length,
+    expected_values,
+    expected_data,
+):
+    file_id = str(uuid.uuid4())
+    data_file_name = f"{file_id}.lance"
+    blob_id = 7
+    payloads = pa.array(values, type=array_type).slice(slice_offset, slice_length)
+    if as_chunked:
+        split_at = max(1, len(payloads) // 2)
+        payloads = pa.chunked_array(
+            [payloads.slice(0, split_at), payloads.slice(split_at)]
+        )
+
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(data_file_name, blob_id)
+    with pytest.raises(ValueError, match="available after finish_array"):
+        packed.field
+    packed.write_blobs(payloads)
+    descriptors = packed.finish_array("image_bytes")
+    descriptor_field = packed.field
+
+    expected_descriptors = []
+    position = 0
+    for value in expected_values:
+        if value is None:
+            expected_descriptors.append(None)
+        else:
+            expected_descriptors.append(
+                {
+                    "kind": 1,
+                    "data": None,
+                    "uri": None,
+                    "blob_id": blob_id,
+                    "blob_size": len(value),
+                    "position": position,
+                }
+            )
+            position += len(value)
+
+    assert descriptors.to_pylist() == expected_descriptors
+    assert descriptor_field == lance.BlobDescriptorArrayBuilder("image_bytes").field
+    assert descriptor_field.metadata[b"ARROW:extension:name"] == b"lance.blob.v2"
+    assert pa.record_batch(
+        [descriptors], schema=pa.schema([descriptor_field])
+    ).num_rows == len(expected_values)
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == expected_data
+
+
+@pytest.mark.parametrize(
+    "array_type,offset_type",
+    [
+        pytest.param(pa.binary(), pa.int32(), id="binary"),
+        pytest.param(pa.large_binary(), pa.int64(), id="large_binary"),
+    ],
+)
+def test_packed_blob_writer_bulk_excludes_physical_null_bytes(
+    tmp_path, array_type, offset_type
+):
+    offsets = pa.array([0, 1, 5, 5, 7], type=offset_type).buffers()[1]
+    payloads = pa.Array.from_buffers(
+        array_type,
+        4,
+        [
+            pa.py_buffer(bytes([0b00001101])),
+            offsets,
+            pa.py_buffer(b"aJUNKbc"),
+        ],
+    )
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blobs(payloads)
+    descriptors = packed.finish_array("image_bytes")
+
+    assert descriptors.to_pylist() == [
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 1,
+            "position": 0,
+        },
+        None,
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 0,
+            "position": 1,
+        },
+        {
+            "kind": 1,
+            "data": None,
+            "uri": None,
+            "blob_id": blob_id,
+            "blob_size": 2,
+            "position": 1,
+        },
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"abc"
+
+
+@pytest.mark.parametrize(
+    "array_type,offset_type",
+    [
+        pytest.param(pa.binary(), pa.int32(), id="binary"),
+        pytest.param(pa.large_binary(), pa.int64(), id="large_binary"),
+    ],
+)
+@pytest.mark.parametrize("as_chunked", [False, True], ids=["array", "chunked_array"])
+def test_packed_blob_writer_bulk_rejects_non_monotonic_offsets(
+    tmp_path, array_type, offset_type, as_chunked
+):
+    offsets = pa.array([0, 2, 1, 2], type=offset_type).buffers()[1]
+    malformed = pa.Array.from_buffers(
+        array_type,
+        3,
+        [None, offsets, pa.py_buffer(b"ab")],
+    )
+    payloads = malformed
+    expected_context = "Packed blob payload array"
+    if as_chunked:
+        payloads = pa.chunked_array([pa.array([b"valid"], type=array_type), malformed])
+        expected_context = "Packed blob payload chunk 1"
+
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    with pytest.raises(ValueError, match="invalid Arrow data") as error:
+        packed.write_blobs(payloads)
+    assert expected_context in str(error.value)
+    assert "non-monotonic offset" in str(error.value)
+
+    packed.write_blob(b"still usable")
+    descriptors = packed.finish_array("blob")
+    assert len(descriptors) == 1
+    assert (
+        _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"still usable"
+    )
+
+
+def test_packed_blob_writer_mixed_calls_preserve_legacy_finish_alignment(tmp_path):
+    file_id = str(uuid.uuid4())
+    blob_id = 7
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer(f"{file_id}.lance", blob_id)
+
+    packed.write_blob(b"s")
+    packed.write_blobs(pa.array([b"a", None, b""], type=pa.binary()))
+    packed.write_blobs(pa.array([None, b"bc"], type=pa.large_binary()))
+    descriptors = packed.finish()
+
+    assert [repr(descriptor) for descriptor in descriptors] == [
+        "Packed { blob_id: 7, offset: 0, size: 1 }",
+        "Packed { blob_id: 7, offset: 1, size: 1 }",
+        "Null",
+        "Packed { blob_id: 7, offset: 2, size: 0 }",
+        "Null",
+        "Packed { blob_id: 7, offset: 2, size: 2 }",
+    ]
+    assert _blob_sidecar_path(tmp_path, file_id, blob_id).read_bytes() == b"sabc"
+
+
+@pytest.mark.parametrize(
+    "payloads",
+    [
+        pytest.param(pa.array([1, 2], type=pa.int32()), id="array"),
+        pytest.param(pa.chunked_array([[1], [2]], type=pa.int32()), id="chunked_array"),
+        pytest.param([b"not an Arrow array"], id="python_list"),
+    ],
+)
+def test_packed_blob_writer_bulk_rejects_non_binary_array(tmp_path, payloads):
+    files = LanceFileSession(tmp_path)
+    packed = files.open_packed_blob_writer("data-file.lance", 1)
+
+    with pytest.raises(ValueError, match="Binary") as error:
+        packed.write_blobs(payloads)
+    if isinstance(payloads, pa.Array):
+        assert "chunk" not in str(error.value)
+
+    packed.write_blob(b"still usable")
+    assert len(packed.finish_array("blob")) == 1
+
+
+@pytest.mark.parametrize(
+    "open_writer",
+    [
+        pytest.param("open_packed_blob_writer", id="packed"),
+        pytest.param("open_dedicated_blob_writer", id="dedicated"),
+    ],
+)
+def test_failed_blob_writer_traceback_can_be_released_on_another_thread(
+    tmp_path, monkeypatch, open_writer
+):
+    failures = queue.Queue()
+
+    def fail_with_live_writer():
+        files = LanceFileSession(tmp_path)
+        writer = getattr(files, open_writer)("data-file.lance", 1)
+        assert writer.blob_id == 1
+        try:
+            raise OSError("simulated object-store write failure")
+        except OSError as error:
+            # The traceback retains this frame and its local writer, matching a
+            # writer-thread failure handed to an owning thread for propagation.
+            failures.put(error)
+
+    writer_thread = threading.Thread(target=fail_with_live_writer)
+    writer_thread.start()
+    writer_thread.join(timeout=10)
+    assert not writer_thread.is_alive()
+
+    unraisable = []
+    monkeypatch.setattr(sys, "unraisablehook", unraisable.append)
+    error = failures.get_nowait()
+    assert str(error) == "simulated object-store write failure"
+    del error
+    gc.collect()
+
+    assert unraisable == []
+
+
 def test_blob_extension_write_fragments_external_denied_by_default(tmp_path):
     blob_path = tmp_path / "external_blob.bin"
 
@@ -1125,6 +1937,38 @@ def test_read_blobs_resolves_nested_field_path(dataset_with_nested_blobs):
     assert [data for _, data in results] == [b"foo", b"baz"]
 
 
+def test_write_nested_blob_v2_and_take_by_field_path(tmp_path):
+    packed = b"x" * (70 * 1024)
+    blob_field = lance.blob_field("blob")
+    info_fields = [pa.field("name", pa.string()), blob_field]
+    info_type = pa.struct(info_fields)
+    info_array = pa.StructArray.from_arrays(
+        [pa.array(["a", "b", "c"]), lance.blob_array([b"foo", packed, None])],
+        fields=info_fields,
+    )
+    table = pa.table(
+        [info_array],
+        schema=pa.schema([pa.field("info", info_type)]),
+    )
+
+    dataset = lance.write_dataset(
+        table,
+        tmp_path / "nested_blob_v2",
+        data_storage_version="2.2",
+    )
+
+    desc = dataset.to_table(columns=["info.blob"]).column("info.blob").chunk(0)
+    assert desc.field("kind").to_pylist()[:2] == [0, 1]
+
+    blobs = dataset.take_blobs("info.blob", indices=[0, 1])
+    with blobs[0] as f:
+        assert f.read() == b"foo"
+    with blobs[1] as f:
+        assert f.read() == packed
+
+    assert dataset.take_blobs("info.blob", indices=[2]) == []
+
+
 def test_to_pandas_returns_blob_files_for_projected_nested_fields(
     dataset_with_nested_blobs,
 ):
diff --git a/python/python/tests/test_coerce_query_vector.py b/python/python/tests/test_coerce_query_vector.py
new file mode 100644
index 00000000000..7a5d341f8e0
--- /dev/null
+++ b/python/python/tests/test_coerce_query_vector.py
@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""Tests for _coerce_query_vector to ensure invalid input types
+raise TypeError instead of falling into the numpy conversion branch."""
+
+import numpy as np
+import pyarrow as pa
+import pytest
+from lance.dataset import _coerce_query_vector
+
+
+class TestCoerceQueryVectorInvalidTypes:
+    """Non-vector inputs should raise TypeError, not numpy ValueError."""
+
+    def test_string_raises_typeerror(self):
+        with pytest.raises(TypeError, match="query vector must be list-like"):
+            _coerce_query_vector("not a vector")
+
+    def test_integer_raises_typeerror(self):
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(42)
+
+    def test_object_raises_typeerror(self):
+        """A random object that is not array-like should raise TypeError."""
+
+        class NotAVector:
+            pass
+
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(NotAVector())
+
+    def test_none_raises_typeerror(self):
+        with pytest.raises(
+            TypeError, match="Query vectors should be an array of floats"
+        ):
+            _coerce_query_vector(None)
+
+
+class TestCoerceQueryVectorValidTypes:
+    """Valid vector inputs should be coerced successfully."""
+
+    def test_list_of_floats(self):
+        result, dim = _coerce_query_vector([1.0, 2.0, 3.0])
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_tuple_of_floats(self):
+        result, dim = _coerce_query_vector((1.0, 2.0, 3.0))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_numpy_array(self):
+        result, dim = _coerce_query_vector(np.array([1.0, 2.0, 3.0]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_float_array(self):
+        result, dim = _coerce_query_vector(pa.array([1.0, 2.0, 3.0]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_int_array_cast_to_float(self):
+        result, dim = _coerce_query_vector(pa.array([1, 2, 3]))
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
+
+    def test_pa_chunked_array(self):
+        chunked = pa.chunked_array([[1.0, 2.0, 3.0]])
+        result, dim = _coerce_query_vector(chunked)
+        assert isinstance(result, pa.FloatingPointArray)
+        assert dim == 3
diff --git a/python/python/tests/test_column_names.py b/python/python/tests/test_column_names.py
index f7b5962b523..d402ba5bcbb 100644
--- a/python/python/tests/test_column_names.py
+++ b/python/python/tests/test_column_names.py
@@ -349,7 +349,7 @@ def test_scalar_index_with_special_chars(self, special_char_dataset):
 
         indices = special_char_dataset.describe_indices()
         assert len(indices) == 1
-        assert indices[0].field_names == ["user-id"]
+        assert indices[0].field_names == ["`user-id`"]
         assert indices[0].name == "user-id_idx"
 
         # Query using the indexed column (requires backticks in filter)
@@ -462,7 +462,7 @@ def test_scalar_index_with_nested_mixed_case(self, nested_mixed_case_dataset):
         indices = nested_mixed_case_dataset.describe_indices()
         assert len(indices) == 1
         assert indices[0].name == "MetaData.userId_idx"
-        assert indices[0].field_names == ["userId"]
+        assert indices[0].field_names == ["MetaData.userId"]
 
         # Query using the indexed column
         result = nested_mixed_case_dataset.to_table(filter="MetaData.userId = 50")
@@ -512,7 +512,7 @@ def test_scalar_index_with_lowercased_nested_path(self, nested_mixed_case_datase
         assert len(indices) == 1
         # Should store with correct case from schema
         assert indices[0].name == "MetaData.userId_idx"
-        assert indices[0].field_names == ["userId"]
+        assert indices[0].field_names == ["MetaData.userId"]
 
         # Query should also work with correct case
         result = nested_mixed_case_dataset.to_table(filter="MetaData.userId = 50")
@@ -576,7 +576,7 @@ def test_scalar_index_with_nested_special_chars(self, nested_special_char_datase
 
         indices = nested_special_char_dataset.describe_indices()
         assert len(indices) == 1
-        assert indices[0].field_names == ["user-id"]
+        assert indices[0].field_names == ["`meta-data`.`user-id`"]
         assert indices[0].name == "meta-data.user-id_idx"
 
         # Query using the indexed column (backticks required in filter)
@@ -600,7 +600,7 @@ def test_scalar_index_on_top_level_special_chars(self, nested_special_char_datas
 
         indices = nested_special_char_dataset.describe_indices()
         assert len(indices) == 1
-        assert indices[0].field_names == ["row-id"]
+        assert indices[0].field_names == ["`row-id`"]
 
         result = nested_special_char_dataset.to_table(filter="`row-id` = 50")
         assert result.num_rows == 1
diff --git a/python/python/tests/test_count_pushdown.py b/python/python/tests/test_count_pushdown.py
new file mode 100644
index 00000000000..896bd629fc9
--- /dev/null
+++ b/python/python/tests/test_count_pushdown.py
@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+"""End-to-end tests for count-from-mask pushdown.
+
+The optimizer rule under test (`CountPushdown`) rewrites
+``SELECT COUNT(*) ... WHERE indexed_col <op> v`` into
+``AggregateExec(Final) → CountFromMaskExec → ScalarIndexExec`` when the
+index covers every dataset fragment, or splits into a Union of a pushdown
+branch over the indexed fragments and a scan branch over the rest when
+coverage is partial. This is category 1 (count-from-mask) of the four
+aggregate-acceleration categories; the other three (mask-to-answer,
+zone-aware, dimension-keyed) are not implemented yet.
+
+Each test exercises a different state of the dataset (clean, with deletions,
+with updates that introduce unindexed fragments, with a fully-deleted indexed
+fragment) and asserts:
+
+  1. The returned count matches the ground truth (correctness), and
+  2. The plan routes through ``CountFromMaskExec`` (the rule fired).
+
+For the cases where the index covers the whole dataset, the tests also assert
+no ``LanceRead`` is present in the plan — proof that the count is being
+answered from index metadata, not by scanning column data. The happy-path
+test additionally re-runs the query and asserts the second call performs no
+I/O.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import lance
+import pyarrow as pa
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+# --------------------------------------------------------------------------
+# Helpers
+# --------------------------------------------------------------------------
+
+# 4 fragments × 25 rows = 100 rows; values 0..99 in `x`.
+NUM_FRAGMENTS = 4
+ROWS_PER_FRAGMENT = 25
+NUM_ROWS = NUM_FRAGMENTS * ROWS_PER_FRAGMENT  # 100
+
+
+def _make_dataset(tmp_path: Path) -> lance.LanceDataset:
+    """Build a 4-fragment dataset with a BTREE index on `x`."""
+    table = pa.table({"x": pa.array(range(NUM_ROWS), pa.int64())})
+    dataset = lance.write_dataset(
+        table,
+        tmp_path / "ds",
+        max_rows_per_file=ROWS_PER_FRAGMENT,
+    )
+    assert len(dataset.get_fragments()) == NUM_FRAGMENTS
+    dataset.create_scalar_index("x", "BTREE")
+    return dataset
+
+
+def _filtered_count_plan(dataset: lance.LanceDataset, filter: str) -> str:
+    """Return the ``analyze_plan(count_rows=True)`` output for a filtered
+    ``COUNT(*)`` — the same plan ``count_rows(filter=…)`` actually executes."""
+    return dataset.scanner(columns=[], with_row_id=True, filter=filter).analyze_plan(
+        count_rows=True
+    )
+
+
+def _assert_pushdown_fired(plan: str) -> None:
+    assert "CountFromMask" in plan, f"expected CountFromMaskExec in plan, got:\n{plan}"
+
+
+def _assert_no_column_scan(plan: str) -> None:
+    """Stricter: no LanceRead anywhere. Only applies when the index covers
+    every dataset fragment (no partial-coverage split branch)."""
+    assert "LanceRead" not in plan, (
+        f"unexpected LanceRead in plan — column data was scanned:\n{plan}"
+    )
+
+
+# --------------------------------------------------------------------------
+# Tests
+# --------------------------------------------------------------------------
+
+
+def test_filtered_count_with_scalar_index(tmp_path: Path):
+    """Happy path: filtered count on an indexed column, run twice.
+
+    The second call must perform zero I/O — proof the rule routed the count
+    through the index/deletion-mask metadata both times and the second call
+    re-used the cache. The check uses ``dataset.io_stats_incremental()``
+    rather than parsing the plan's ``bytes_read=…`` so we get a direct
+    accounting of every object-store read the dataset performed during the
+    second call, not just what the plan happens to surface.
+    """
+    dataset = _make_dataset(tmp_path)
+    filter = "x < 50"
+    expected = 50
+
+    # Verify the rule fires for this shape.
+    _assert_pushdown_fired(_filtered_count_plan(dataset, filter))
+    _assert_no_column_scan(_filtered_count_plan(dataset, filter))
+
+    # First call warms the index + deletion-mask caches.
+    assert dataset.count_rows(filter=filter) == expected
+    # Reset counters so the next snapshot only reflects the second call.
+    dataset.io_stats_incremental()
+
+    # Second call: must do zero I/O.
+    assert dataset.count_rows(filter=filter) == expected
+    stats = dataset.io_stats_incremental()
+    assert stats.read_iops == 0, f"expected 0 read_iops, got {stats.read_iops}"
+    assert stats.read_bytes == 0, f"expected 0 read_bytes, got {stats.read_bytes}"
+
+
+def test_filtered_count_with_deleted_rows(tmp_path: Path):
+    """Some matching rows are deleted — the count must reflect the deletions.
+
+    Deletions don't change fragment coverage, so the index still covers every
+    dataset fragment and the rule emits a single pushdown branch (no scan).
+    """
+    dataset = _make_dataset(tmp_path)
+    # Delete three rows that match the filter (x < 50).
+    dataset.delete("x = 10 OR x = 20 OR x = 30")
+    plan = _filtered_count_plan(dataset, "x < 50")
+    _assert_pushdown_fired(plan)
+    _assert_no_column_scan(plan)
+    assert dataset.count_rows(filter="x < 50") == 50 - 3
+
+
+def test_filtered_count_with_updated_rows(tmp_path: Path):
+    """Updates move rows in/out of the filter set.
+
+    Before:  x < 50 ⇒ 50 rows match (values 0..49).
+    After:
+      - x = 5 → x = 100        (one row leaves the matched set)
+      - x = 7 → x = 101        (another row leaves)
+      - x = 60 → x = 8         (a row joins the matched set)
+      - x = 70 → x = 9         (another joins)
+
+    Net change: −2 + 2 = 0, so the final count is still 50, but the
+    underlying row identities have shifted. Each update is materialized as
+    a delete + insert into a new fragment in Lance — the new fragments are
+    not in the index's coverage, so the optimizer rule emits a split plan:
+    pushdown for the originally-indexed fragments, plus a scan branch for
+    the rewritten fragments. The final count must still be correct.
+    """
+    dataset = _make_dataset(tmp_path)
+    dataset.update({"x": "100"}, where="x = 5")
+    dataset.update({"x": "101"}, where="x = 7")
+    dataset.update({"x": "8"}, where="x = 60")
+    dataset.update({"x": "9"}, where="x = 70")
+
+    plan = _filtered_count_plan(dataset, "x < 50")
+    _assert_pushdown_fired(plan)
+    # 50 originally matching − 2 that left + 2 that joined = 50.
+    assert dataset.count_rows(filter="x < 50") == 50
+
+
+def test_filtered_count_with_whole_fragment_deleted(tmp_path: Path):
+    """Delete every row in one indexed fragment.
+
+    Fragment 0 covers x ∈ [0, 25). Deleting all of those rows removes 25
+    matches of `x < 50`, dropping the count from 50 to 25.
+
+    Lance retires the now-empty fragment, so the dataset has 3 fragments
+    while the index still claims 4 — the index is a strict *superset* of
+    the dataset, which is safe (the extra index entries simply don't
+    apply). The rule emits a single pushdown branch (no scan needed).
+    """
+    dataset = _make_dataset(tmp_path)
+    dataset.delete("x < 25")
+    plan = _filtered_count_plan(dataset, "x < 50")
+    _assert_pushdown_fired(plan)
+    _assert_no_column_scan(plan)
+    assert dataset.count_rows(filter="x < 50") == 25
diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py
index b5c81669fa1..898bdf93069 100644
--- a/python/python/tests/test_dataset.py
+++ b/python/python/tests/test_dataset.py
@@ -31,7 +31,7 @@
 from lance.commit import CommitConflictError
 from lance.dataset import LANCE_COMMIT_MESSAGE_KEY, AutoCleanupConfig
 from lance.debug import format_fragment
-from lance.file import stable_version
+from lance.file import LanceFileWriter, stable_version
 from lance.schema import LanceSchema
 from lance.util import validate_vector_index
 
@@ -93,6 +93,25 @@ def test_roundtrip_types(tmp_path: Path):
     assert dataset.to_table() == table
 
 
+@pytest.mark.parametrize("data_storage_version", ["legacy", "stable", "2.1"])
+def test_write_zero_dimension_fixed_size_list(
+    tmp_path: Path, data_storage_version: str
+):
+    # Zero-dimension fixed-size lists must be rejected with a clean error
+    # instead of a divide-by-zero panic (#5102)
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("vec", pa.list_(pa.float32(), 0)),
+        ]
+    )
+    table = pa.table({"id": [1], "vec": [[]]}, schema=schema)
+    with pytest.raises(OSError, match="dimension must be a positive integer"):
+        lance.write_dataset(
+            table, tmp_path / "ds.lance", data_storage_version=data_storage_version
+        )
+
+
 def test_dataset_overwrite(tmp_path: Path):
     table1 = pa.Table.from_pylist([{"a": 1, "b": 2}, {"a": 10, "b": 20}])
     base_dir = tmp_path / "test"
@@ -424,16 +443,27 @@ def test_enable_stable_row_ids(tmp_path: Path):
     assert table_after["_rowaddr"][3].as_py() == (2 << 32) + 3
 
 
-def test_has_stable_row_ids_property(tmp_path: Path):
-    table = pa.Table.from_pylist([{"a": 1}, {"a": 2}])
+@pytest.mark.parametrize("enable_stable_row_ids", [True, False])
+@pytest.mark.parametrize(
+    "rows",
+    [[{"a": 1}, {"a": 2}], []],
+    ids=["non_empty", "empty"],
+)
+def test_has_stable_row_ids_property(tmp_path: Path, enable_stable_row_ids: bool, rows):
+    schema = pa.schema([pa.field("a", pa.int64())])
+    table = pa.Table.from_pylist(rows, schema=schema)
 
-    stable_path = tmp_path / "stable"
-    lance.write_dataset(table, stable_path, enable_stable_row_ids=True)
-    assert lance.dataset(stable_path).has_stable_row_ids is True
+    path = tmp_path / f"stable_row_ids_{enable_stable_row_ids}_{len(rows)}"
+    lance.write_dataset(
+        table,
+        path,
+        enable_stable_row_ids=enable_stable_row_ids,
+    )
+    ds = lance.dataset(path)
 
-    non_stable_path = tmp_path / "non_stable"
-    lance.write_dataset(table, non_stable_path, enable_stable_row_ids=False)
-    assert lance.dataset(non_stable_path).has_stable_row_ids is False
+    assert ds.count_rows() == len(rows)
+    assert len(ds.get_fragments()) == (0 if len(rows) == 0 else 1)
+    assert ds.has_stable_row_ids is enable_stable_row_ids
 
 
 def _list_manifests(versions_dir):
@@ -934,6 +964,49 @@ def test_take_with_projection(tmp_path: Path):
     assert table3 == table2
 
 
+def test_take_with_json_column(tmp_path: Path):
+    """Test that take/take_rows return JSON columns in Arrow JSON format (Utf8).
+
+    Previously, take would return lance.json (LargeBinary) instead of
+    arrow.json (Utf8), which is the user-facing format.
+    """
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "id": pa.array(range(10), type=pa.int64()),
+            "meta": pa.array(
+                [f'{{"val":{i}}}' for i in range(10)],
+                type=json_type,
+            ),
+        }
+    )
+    base_dir = tmp_path / "test_take_json"
+    lance.write_dataset(data, base_dir)
+    dataset = lance.dataset(base_dir)
+
+    # Dataset.take should return arrow.json type
+    result = dataset.take([2, 5, 8])
+    meta_field = result.schema.field("meta")
+    assert meta_field.type == pa.utf8() or meta_field.type == pa.json_(), (
+        f"Expected arrow.json (Utf8), got {meta_field.type}"
+    )
+    metas = result.column("meta").to_pylist()
+    assert metas[0] == '{"val":2}'
+    assert metas[1] == '{"val":5}'
+    assert metas[2] == '{"val":8}'
+
+    # Dataset._take_rows should also return arrow.json type
+    result = dataset._take_rows([0, 3, 9])
+    meta_field = result.schema.field("meta")
+    assert meta_field.type == pa.utf8() or meta_field.type == pa.json_(), (
+        f"Expected arrow.json (Utf8), got {meta_field.type}"
+    )
+    metas = result.column("meta").to_pylist()
+    assert metas[0] == '{"val":0}'
+    assert metas[1] == '{"val":3}'
+    assert metas[2] == '{"val":9}'
+
+
 def test_filter(tmp_path: Path):
     table = pa.Table.from_pydict({"a": range(100), "b": range(100)})
     base_dir = tmp_path / "test"
@@ -1377,6 +1450,45 @@ def test_cleanup_old_versions(tmp_path):
     assert stats.old_versions == 1
 
 
+def test_explain_cleanup_old_versions(tmp_path):
+    table = pa.Table.from_pydict({"a": range(100), "b": range(100)})
+    base_dir = tmp_path / "test"
+    lance.write_dataset(table, base_dir)
+    time.sleep(0.1)
+    moment = datetime.now()
+    lance.write_dataset(table, base_dir, mode="overwrite")
+
+    dataset = lance.dataset(base_dir)
+    before_versions = len(dataset.versions())
+
+    explanation = dataset.explain_cleanup_old_versions(
+        older_than=(datetime.now() - moment),
+        include_files=True,
+        max_files=1000,
+    )
+
+    assert explanation.read_version == dataset.version
+    assert explanation.stats.bytes_removed > 0
+    assert explanation.stats.old_versions == 1
+    assert explanation.candidate_files
+    assert not explanation.candidate_files_truncated
+    assert len(dataset.versions()) == before_versions
+
+    summary = dataset.explain_cleanup_old_versions(older_than=(datetime.now() - moment))
+    assert summary.stats.old_versions == explanation.stats.old_versions
+    assert summary.candidate_files == []
+
+    with pytest.raises(ValueError, match="max_files must be positive"):
+        dataset.explain_cleanup_old_versions(
+            older_than=(datetime.now() - moment),
+            max_files=0,
+        )
+
+    stats = dataset.cleanup_old_versions(older_than=(datetime.now() - moment))
+    assert stats.bytes_removed == explanation.stats.bytes_removed
+    assert stats.old_versions == explanation.stats.old_versions
+
+
 def test_cleanup_error_when_tagged_old_versions(tmp_path):
     table = pa.Table.from_pydict({"a": range(100), "b": range(100)})
     base_dir = tmp_path / "test"
@@ -1742,6 +1854,7 @@ def test_commit_batch_append():
     result = lance.LanceDataset.commit_batch(dataset, [txn2, txn3])
     dataset = result["dataset"]
     assert dataset.version == 2
+    assert dataset.checkout_version(1).version == 1
     assert len(dataset.get_fragments()) == 3
     assert dataset.to_table() == pa.concat_tables([data1, data2, data3])
     merged_txn = result["merged"]
@@ -2373,6 +2486,146 @@ def test_merge_insert_subcols(tmp_path: Path):
     assert dataset.to_table().sort_by("a") == expected
 
 
+def test_merge_insert_full_fragment_rewrite_json_e2e(tmp_path: Path):
+    """End-to-end test: merge_insert with JSON columns where ALL rows are updated.
+
+    This exercises the "all rows updated" fast path in handle_fragment which
+    bypasses the Updater and writes directly. Without proper JSON conversion,
+    Utf8 data (i32 offsets) is written with a LargeBinary schema (i64 offsets),
+    causing a decoder panic on subsequent reads:
+        "the offset of the new Buffer cannot exceed the existing Length:
+         slice offset=0 Length=N selfLen=N/2"
+
+    Conditions to trigger the fast path:
+    1. Subschema update (not all columns provided)
+    2. ALL rows in a fragment are matched/updated
+    3. data_storage_version = 2.2
+    """
+    import json
+
+    json_type = pa.json_()
+
+    # Create dataset with v2.2 storage format
+    initial_data = pa.table(
+        {
+            "unique_id": pa.array([f"id_{i}" for i in range(5)], type=pa.utf8()),
+            "coarse": pa.array(
+                [f'{{"original":{i}}}' for i in range(5)],
+                type=json_type,
+            ),
+            "score": pa.array(range(5), type=pa.int64()),
+        }
+    )
+    dataset = lance.write_dataset(
+        initial_data,
+        tmp_path / "e2e_json_rewrite",
+        data_storage_version="2.2",
+    )
+    assert dataset.count_rows() == 5
+    assert len(dataset.get_fragments()) == 1
+
+    # Subschema merge_insert: only provide [unique_id, coarse], update ALL rows
+    # This triggers: subschema → v1 path → all rows matched → fast path
+    update_data = pa.table(
+        {
+            "unique_id": pa.array([f"id_{i}" for i in range(5)], type=pa.utf8()),
+            "coarse": pa.array(
+                [f'{{"updated":true,"id":{i}}}' for i in range(5)],
+                type=json_type,
+            ),
+        }
+    )
+    dataset.merge_insert("unique_id").when_matched_update_all().execute(update_data)
+
+    # Critical: read back should NOT panic
+    result = dataset.to_table().sort_by("unique_id")
+    assert result.num_rows == 5
+
+    # Verify score column (not in update) is preserved
+    scores = result.column("score").to_pylist()
+    assert scores == [0, 1, 2, 3, 4]
+
+    # Verify JSON column was updated correctly
+    coarse_values = result.column("coarse").to_pylist()
+    for i, val in enumerate(coarse_values):
+        parsed = json.loads(val) if isinstance(val, str) else val
+        assert parsed.get("updated") is True, (
+            f"Row {i} should have updated coarse, got {val}"
+        )
+
+    # Verify take also works (different read path)
+    take_result = dataset.take([0, 2, 4])
+    assert take_result.num_rows == 3
+    take_coarse = take_result.column("coarse").to_pylist()
+    for val in take_coarse:
+        parsed = json.loads(val) if isinstance(val, str) else val
+        assert parsed.get("updated") is True
+
+    # Verify sample also works
+    sample_result = dataset.sample(3)
+    assert sample_result.num_rows == 3
+
+
+def test_merge_insert_subcols_with_json_column(tmp_path: Path):
+    """Test merge_insert with subschema update on a JSON extension type column.
+
+    Previously this would fail with:
+    'Incorrect datatype for StructArray field, expected Utf8 got LargeBinary'
+    because the update_fragments path didn't handle the Arrow JSON ↔ Lance JSON
+    type mismatch during interleave.
+    """
+    import json
+
+    json_type = pa.json_()
+    initial_data = pa.table(
+        {
+            "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()),
+            "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()),
+            "score": pa.array([10, 20, 30, 40, 50], type=pa.int64()),
+            "meta": pa.array(
+                ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'],
+                type=json_type,
+            ),
+        }
+    )
+    dataset = lance.write_dataset(initial_data, tmp_path / "merge_json_subcols")
+
+    # Subschema update: only provide id (key) + meta (JSON column to update)
+    new_values = pa.table(
+        {
+            "id": pa.array([2, 4], type=pa.int64()),
+            "meta": pa.array(
+                ['{"updated":true,"id":2}', '{"updated":true,"id":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    # This should NOT raise a type mismatch error
+    dataset.merge_insert("id").when_matched_update_all().execute(new_values)
+
+    # Verify results
+    result = dataset.to_table().sort_by("id")
+    ids = result.column("id").to_pylist()
+    scores = result.column("score").to_pylist()
+    metas = result.column("meta").to_pylist()
+
+    # Score column (not in update) should be preserved
+    assert scores == [10, 20, 30, 40, 50]
+
+    # Meta column should be updated for id=2 and id=4
+    for id_val, meta_val in zip(ids, metas):
+        parsed = json.loads(meta_val) if isinstance(meta_val, str) else meta_val
+        if id_val in (2, 4):
+            assert parsed.get("updated") is True, (
+                f"id={id_val} should have updated meta, got {meta_val}"
+            )
+        else:
+            assert "x" in str(parsed), (
+                f"id={id_val} should have original meta, got {meta_val}"
+            )
+
+
 def test_merge_insert_defaults_to_pk_when_on_omitted(tmp_path):
     base_dir = tmp_path / "merge_insert_pk_default"
 
@@ -4710,6 +4963,281 @@ def test_data_replacement(tmp_path: Path):
     assert tbl == expected
 
 
+def _write_overlay_file(
+    dataset, base_dir: Path, name: str, batch: pa.Table, fields: List[int]
+):
+    """Write an overlay value file (one value column per covered field, no key
+    column) and return a DataFile mapping its columns to the given dataset
+    `fields`. The file version is copied from a base data file."""
+    path = base_dir / "data" / name
+    with LanceFileWriter(str(path)) as writer:
+        writer.write_batch(batch)
+    base_df = dataset.get_fragments()[0].metadata.files[0]
+    return lance.fragment.DataFile(
+        path=name,
+        fields=fields,
+        column_indices=list(range(len(fields))),
+        file_major_version=base_df.file_major_version,
+        file_minor_version=base_df.file_minor_version,
+        file_size_bytes=os.path.getsize(path),
+    )
+
+
+def test_data_overlay_dense(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    # Overlay `val` at physical offsets {1, 4} with new values.
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    assert data_file.fields == [1]  # `val` is field id 1
+
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4])
+    op = lance.LanceOperation.DataOverlay(
+        [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+    )
+    dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version)
+
+    result = dataset.to_table()
+    assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90]
+    # The unrelated `id` column is untouched.
+    assert result.column("id").to_pylist() == list(range(10))
+
+
+def test_data_overlay_newest_wins(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    older = _write_overlay_file(
+        dataset,
+        base_dir,
+        "older.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [
+                lance.LanceOperation.DataOverlayGroup(
+                    0,
+                    [lance.LanceOperation.DataOverlayFile(older, offsets=[1, 4])],
+                )
+            ]
+        ),
+        read_version=dataset.version,
+    )
+    # A newer overlay re-covers offset 1; it must win there.
+    newer = _write_overlay_file(
+        dataset,
+        base_dir,
+        "newer.lance",
+        pa.table({"val": pa.array([999], pa.int32())}),
+        fields=[1],
+    )
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [
+                lance.LanceOperation.DataOverlayGroup(
+                    0, [lance.LanceOperation.DataOverlayFile(newer, offsets=[1])]
+                )
+            ]
+        ),
+        read_version=dataset.version,
+    )
+
+    val = dataset.to_table().column("val").to_pylist()
+    assert val[1] == 999  # newest overlay wins
+    assert val[4] == 444  # only the older overlay covers offset 4
+
+
+def test_data_overlay_sparse_per_field(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    # Sparse overlay: `id` covers offset {2}, `val` covers offset {3}. The value
+    # file carries one value per field (rank 0 of each field's coverage).
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "sparse.lance",
+        pa.table(
+            {
+                "id": pa.array([777], pa.int32()),
+                "val": pa.array([330], pa.int32()),
+            }
+        ),
+        fields=[0, 1],
+    )
+    assert data_file.fields == [0, 1]
+
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[[2], [3]])
+    op = lance.LanceOperation.DataOverlay(
+        [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+    )
+    dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version)
+
+    result = dataset.to_table()
+    assert result.column("id").to_pylist()[2] == 777
+    assert result.column("val").to_pylist()[3] == 330
+    # Fields resolve independently: id at offset 3 and val at offset 2 fall through.
+    assert result.column("id").to_pylist()[3] == 3
+    assert result.column("val").to_pylist()[2] == 20
+
+
+def test_data_overlay_round_trips_through_fragment_metadata(tmp_path: Path):
+    import json
+
+    base_dir = tmp_path / "test"
+    table = pa.table(
+        {
+            "id": pa.array(range(10), pa.int32()),
+            "val": pa.array([i * 10 for i in range(10)], pa.int32()),
+        }
+    )
+    dataset = lance.write_dataset(table, base_dir)
+
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([111, 444], pa.int32())}),
+        fields=[1],
+    )
+    overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4])
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.DataOverlay(
+            [lance.LanceOperation.DataOverlayGroup(0, [overlay])]
+        ),
+        read_version=dataset.version,
+    )
+    overlay_version = dataset.version
+
+    # Reading the fragment surfaces its overlays, stamped with the commit version.
+    metadata = dataset.get_fragments()[0].metadata
+    assert len(metadata.overlays) == 1
+    assert metadata.overlays[0].offsets == [1, 4]
+    assert metadata.overlays[0].committed_version == overlay_version
+
+    # The overlays survive a JSON round-trip of the metadata.
+    restored = lance.fragment.FragmentMetadata.from_json(json.dumps(metadata.to_json()))
+    assert len(restored.overlays) == 1
+    assert restored.overlays[0].offsets == [1, 4]
+    assert restored.overlays[0].committed_version == overlay_version
+
+    # A commit that round-trips the fragment (here an Overwrite) must keep the
+    # overlays, so the overlay still resolves on read instead of being dropped.
+    dataset = lance.LanceDataset.commit(
+        dataset,
+        lance.LanceOperation.Overwrite(dataset.schema, [restored]),
+        read_version=dataset.version,
+    )
+    result = dataset.to_table()
+    assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90]
+    assert result.column("id").to_pylist() == list(range(10))
+
+
+def test_data_overlay_rejects_invalid_offsets(tmp_path: Path):
+    base_dir = tmp_path / "test"
+    table = pa.table({"val": pa.array([0, 1, 2], pa.int32())})
+    dataset = lance.write_dataset(table, base_dir)
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([9], pa.int32())}),
+        fields=[0],
+    )
+
+    # offsets is neither a flat list of ints (dense) nor a list of per-field int
+    # lists (sparse), so the coverage shape can't be resolved.
+    with pytest.raises(ValueError, match="offsets must be a list"):
+        lance.LanceDataset.commit(
+            dataset,
+            lance.LanceOperation.DataOverlay(
+                [
+                    lance.LanceOperation.DataOverlayGroup(
+                        0,
+                        [
+                            lance.LanceOperation.DataOverlayFile(
+                                data_file, offsets=[0, [1]]
+                            )
+                        ],
+                    )
+                ]
+            ),
+            read_version=dataset.version,
+        )
+
+
+@pytest.mark.parametrize(
+    "offsets",
+    [
+        [2, 1],  # dense, descending
+        [1, 1],  # dense, duplicate
+        [[2, 1]],  # sparse, descending
+        [[1, 1]],  # sparse, duplicate
+    ],
+)
+def test_data_overlay_rejects_unsorted_offsets(tmp_path: Path, offsets):
+    # Offsets map positionally to value rows in data_file. A RoaringBitmap would
+    # silently reorder/dedup them, so a non-ascending list must be rejected up
+    # front rather than corrupting the row mapping.
+    base_dir = tmp_path / "test"
+    table = pa.table({"val": pa.array([0, 1, 2], pa.int32())})
+    dataset = lance.write_dataset(table, base_dir)
+    data_file = _write_overlay_file(
+        dataset,
+        base_dir,
+        "ov.lance",
+        pa.table({"val": pa.array([9, 9], pa.int32())}),
+        fields=[0],
+    )
+
+    with pytest.raises(ValueError, match="strictly ascending"):
+        lance.LanceDataset.commit(
+            dataset,
+            lance.LanceOperation.DataOverlay(
+                [
+                    lance.LanceOperation.DataOverlayGroup(
+                        0,
+                        [
+                            lance.LanceOperation.DataOverlayFile(
+                                data_file, offsets=offsets
+                            )
+                        ],
+                    )
+                ]
+            ),
+            read_version=dataset.version,
+        )
+
+
 def test_schema_project_drop_column(tmp_path: Path):
     table = pa.Table.from_pydict({"a": range(100, 200), "b": range(300, 400)})
     base_dir = tmp_path / "test"
@@ -5538,6 +6066,8 @@ def test_branches(tmp_path: Path):
     branch1 = ds_main.create_branch("branch1")
     ds_main.branches.replace_metadata("branch1", {"description": "branch one"})
     assert branch1.version == 1
+    # The dataset returned by create_branch must be fully constructed
+    assert branch1.checkout_version(("main", None)).version == 1
     branch1_append = pa.Table.from_pydict({"a": [7, 8], "b": [9, 10]})
     branch1 = lance.write_dataset(branch1_append, branch1, mode="append")
     assert branch1.version == 2
@@ -5657,4 +6187,33 @@ def test_default_scan_options_nearest(tmp_path: Path) -> None:
     distances = result["_distance"].to_pylist()
     assert distances == sorted(distances)
 
-    assert "id" in result.column_names
+
+def test_tracked_files(tmp_path):
+    table = pa.table({"x": [1, 2, 3]})
+    ds = lance.write_dataset(table, tmp_path / "ds")
+    ds.delete("x = 2")  # adds a deletion file
+
+    reader = ds.tracked_files()
+    assert isinstance(reader, pa.RecordBatchReader)
+
+    result = reader.read_all()
+    assert result.schema.field("version").type == pa.int64()
+    assert result.num_rows >= 2  # at least manifest + data file
+
+    types = set(result.column("type").to_pylist())
+    assert "manifest" in types
+    assert "data file" in types
+    assert "deletion file" in types
+
+
+def test_all_files(tmp_path):
+    table = pa.table({"x": [1, 2, 3]})
+    ds = lance.write_dataset(table, tmp_path / "ds")
+
+    reader = ds.all_files()
+    assert isinstance(reader, pa.RecordBatchReader)
+
+    result = reader.read_all()
+    assert result.schema.field("size_bytes").type == pa.int64()
+    assert result.num_rows >= 2  # at least manifest + data file
+    assert all(s > 0 for s in result.column("size_bytes").to_pylist())
diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py
index c71654769b8..d0f0fb0b8b4 100644
--- a/python/python/tests/test_file.py
+++ b/python/python/tests/test_file.py
@@ -736,6 +736,44 @@ def test_session_list_with_trailing_slash(tmp_path):
     assert files_no_slash == ["dir/file.lance"]
 
 
+def test_session_list_with_delimiter(tmp_path):
+    """Test that LanceFileSession.list_with_delimiter() is non-recursive."""
+    session = LanceFileSession(str(tmp_path))
+    schema = pa.schema([pa.field("x", pa.int64())])
+
+    # Two top-level files and two nested subtrees.
+    with session.open_writer("file1.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [1]}))
+    with session.open_writer("file2.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [2]}))
+    with session.open_writer("subdir/file3.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [3]}))
+    with session.open_writer("subdir/nested/file4.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [4]}))
+    with session.open_writer("other/file5.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [5]}))
+
+    # Listing the base path returns only the immediate children: the two
+    # top-level files and the two child directories (not their contents).
+    result = session.list_with_delimiter()
+    assert sorted(result.common_prefixes) == ["other", "subdir"]
+    assert sorted(result.objects) == ["file1.lance", "file2.lance"]
+
+    # Listing a subdirectory descends exactly one level: the direct file and
+    # the nested directory, but not the file inside the nested directory.
+    subdir = session.list_with_delimiter("subdir")
+    assert subdir.common_prefixes == ["subdir/nested"]
+    assert subdir.objects == ["subdir/file3.lance"]
+
+    # Trailing slash behaves the same as no trailing slash.
+    assert session.list_with_delimiter("subdir/") == subdir
+
+    # A non-existent prefix yields empty results rather than erroring.
+    empty = session.list_with_delimiter("nonexistent")
+    assert empty.common_prefixes == []
+    assert empty.objects == []
+
+
 def test_session_contains(tmp_path):
     """Test that LanceFileSession.contains() works correctly"""
     session = LanceFileSession(str(tmp_path))
@@ -759,6 +797,56 @@ def test_session_contains(tmp_path):
     assert not session.contains("subdir/nonexistent.lance")
 
 
+def test_session_read_range(tmp_path):
+    """Test that LanceFileSession.read_range() returns the requested bytes."""
+    session = LanceFileSession(str(tmp_path))
+
+    payload = bytes(range(256))
+    local = tmp_path / "src.bin"
+    local.write_bytes(payload)
+    session.upload_file(str(local), "data/file.bin")
+
+    # A range in the middle of the file.
+    assert session.read_range("data/file.bin", 10, 5) == payload[10:15]
+    # From the start.
+    assert session.read_range("data/file.bin", 0, 4) == payload[0:4]
+    # Up to the end.
+    assert session.read_range("data/file.bin", 250, 6) == payload[250:256]
+    # A zero-length read yields empty bytes.
+    assert session.read_range("data/file.bin", 100, 0) == b""
+
+    # Reading a missing object raises OSError (consistent with download_file).
+    with pytest.raises(OSError):
+        session.read_range("data/missing.bin", 0, 4)
+
+
+def test_session_delete_file(tmp_path):
+    """Test that LanceFileSession.delete_file() removes files and is idempotent."""
+    session = LanceFileSession(str(tmp_path))
+    schema = pa.schema([pa.field("x", pa.int64())])
+
+    with session.open_writer("test.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [1]}))
+    with session.open_writer("subdir/nested.lance", schema=schema) as writer:
+        writer.write_batch(pa.table({"x": [2]}))
+
+    # Deleting an existing file removes it.
+    assert session.contains("test.lance")
+    session.delete_file("test.lance")
+    assert not session.contains("test.lance")
+
+    # Nested paths work too.
+    assert session.contains("subdir/nested.lance")
+    session.delete_file("subdir/nested.lance")
+    assert not session.contains("subdir/nested.lance")
+
+    # Deleting a missing path raises OSError (consistent with download_file).
+    with pytest.raises(OSError):
+        session.delete_file("test.lance")
+    with pytest.raises(OSError):
+        session.delete_file("never_existed.lance")
+
+
 def test_struct_null_regression():
     import lance
 
diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py
index c6cee711047..11276a2213d 100644
--- a/python/python/tests/test_fragment.py
+++ b/python/python/tests/test_fragment.py
@@ -279,7 +279,7 @@ def test_fragment_meta():
         "file_size_bytes=100), DataFile(path='1.lance', fields=[1], column_indices=[], "
         "file_major_version=0, file_minor_version=0, file_size_bytes=None)], "
         "physical_rows=100, deletion_file=None, row_id_meta=None, "
-        "created_at_version_meta=None, last_updated_at_version_meta=None)"
+        "created_at_version_meta=None, last_updated_at_version_meta=None, overlays=[])"
     )
 
 
@@ -800,3 +800,165 @@ def test_fragment_update_columns_error_on_metadata_column(tmp_path):
         "metadata column" in str(exc_info.value).lower()
         or "cannot be updated" in str(exc_info.value).lower()
     )
+
+
+def test_fragment_delete_rows(tmp_path: Path):
+    # LanceFragment.delete_rows deletes by local row offset (not a predicate):
+    # exactly the given rows are removed, the rest survive, and the fragment is
+    # gone when every row is deleted.
+    data = pa.table({"a": range(100), "b": range(100)})
+    dataset = lance.write_dataset(data, tmp_path)
+    frag = dataset.get_fragment(0)
+
+    new_meta = frag.delete_rows([0, 5, 99])
+    assert new_meta is not None
+    op = LanceOperation.Delete(
+        updated_fragments=[new_meta],
+        deleted_fragment_ids=[],
+        predicate="delete_rows([0, 5, 99])",
+    )
+    dataset = lance.LanceDataset.commit(dataset.uri, op, read_version=dataset.version)
+
+    assert dataset.count_rows() == 97
+    remaining = set(dataset.to_table(columns=["a"])["a"].to_pylist())
+    assert {0, 5, 99}.isdisjoint(remaining)
+    assert {1, 98}.issubset(remaining)
+
+    # Deleting every row removes the fragment entirely (returns None).
+    frag = lance.dataset(tmp_path).get_fragment(0)
+    assert frag.delete_rows(range(100)) is None
+
+    # Offsets outside the fragment's physical rows are rejected, not silently
+    # written to the deletion file.
+    frag = lance.dataset(tmp_path).get_fragment(0)
+    with pytest.raises(ValueError, match="out of range"):
+        frag.delete_rows([100])
+    with pytest.raises(ValueError, match="out of range"):
+        frag.delete_rows([0, 50, 1000])
+
+
+def test_fragment_take_with_json_column(tmp_path):
+    """Test that FileFragment.take returns JSON columns in Arrow JSON format."""
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "id": pa.array(range(10), type=pa.int64()),
+            "meta": pa.array(
+                [f'{{"val":{i}}}' for i in range(10)],
+                type=json_type,
+            ),
+        }
+    )
+    dataset_uri = tmp_path / "test_frag_take_json"
+    dataset = lance.write_dataset(data, dataset_uri)
+
+    fragment = dataset.get_fragment(0)
+    result = fragment.take([1, 4, 7])
+
+    # Should return arrow.json type (Utf8), not lance.json (LargeBinary)
+    meta_field = result.schema.field("meta")
+    assert meta_field.type == pa.utf8() or meta_field.type == pa.json_(), (
+        f"Expected arrow.json (Utf8), got {meta_field.type}"
+    )
+
+    metas = result.column("meta").to_pylist()
+    assert metas[0] == '{"val":1}'
+    assert metas[1] == '{"val":4}'
+    assert metas[2] == '{"val":7}'
+
+
+def test_fragment_create_with_json_column(tmp_path):
+    """Test that LanceFragment.create works with Arrow JSON extension type.
+
+    Previously the single-fragment create path skipped the Arrow JSON (Utf8) ->
+    Lance JSON (JSONB LargeBinary) conversion that write_dataset/write_fragments
+    perform, so the raw UTF-8 string bytes were written into a column whose schema
+    declared JSONB. Reads then miss-decoded the bytes and returned garbage.
+    """
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "uid": pa.array(["a", "b", "c", "d"], type=pa.utf8()),
+            "payload": pa.array(
+                ['{"x":1}', '{"x":2}', '{"y":3}', '{"y":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    frag = LanceFragment.create(tmp_path, data)
+    operation = LanceOperation.Overwrite(data.schema, [frag])
+    dataset = LanceDataset.commit(tmp_path, operation)
+
+    result = dataset.to_table()
+    assert result.column("uid").to_pylist() == ["a", "b", "c", "d"]
+    payloads = result.column("payload").to_pylist()
+    assert [json.loads(p) for p in payloads] == [
+        {"x": 1},
+        {"x": 2},
+        {"y": 3},
+        {"y": 4},
+    ]
+
+
+def test_fragment_update_columns_with_json_column(tmp_path):
+    """Test that fragment update_columns works with Arrow JSON extension type.
+
+    Previously this would fail with a type mismatch error because the
+    HashJoiner didn't convert Arrow JSON (Utf8) to Lance JSON (LargeBinary).
+    """
+    # Create initial dataset with a JSON extension type column
+    json_type = pa.json_()
+    data = pa.table(
+        {
+            "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()),
+            "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()),
+            "meta": pa.array(
+                ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'],
+                type=json_type,
+            ),
+        }
+    )
+    dataset_uri = tmp_path / "test_update_cols_json"
+    dataset = lance.write_dataset(data, dataset_uri)
+
+    # Prepare update data: update the JSON column for some rows
+    update_data = pa.table(
+        {
+            "_rowid": pa.array([1, 3], type=pa.uint64()),
+            "meta": pa.array(
+                ['{"updated":true,"id":2}', '{"updated":true,"id":4}'],
+                type=json_type,
+            ),
+        }
+    )
+
+    # This should NOT raise a type mismatch error
+    fragment = dataset.get_fragment(0)
+    updated_fragment, fields_modified = fragment.update_columns(update_data)
+
+    assert len(fields_modified) > 0
+
+    # Commit and verify
+    op = LanceOperation.Update(
+        updated_fragments=[updated_fragment],
+        fields_modified=fields_modified,
+    )
+    updated_dataset = lance.LanceDataset.commit(
+        str(dataset_uri), op, read_version=dataset.version
+    )
+
+    result = updated_dataset.to_table()
+    ids = result.column("id").to_pylist()
+    metas = result.column("meta").to_pylist()
+
+    for i, (id_val, meta_val) in enumerate(zip(ids, metas)):
+        meta = json.loads(meta_val) if isinstance(meta_val, str) else meta_val
+        if id_val == 2 or id_val == 4:
+            assert "updated" in meta_val or meta.get("updated") is True, (
+                f"id={id_val} should be updated, got {meta_val}"
+            )
+        else:
+            assert "x" in meta_val or "x" in str(meta), (
+                f"id={id_val} should have original value, got {meta_val}"
+            )
diff --git a/python/python/tests/test_indices.py b/python/python/tests/test_indices.py
index 7f6595f2ecc..ae51e6ddb0d 100644
--- a/python/python/tests/test_indices.py
+++ b/python/python/tests/test_indices.py
@@ -8,7 +8,7 @@
 import numpy as np
 import pyarrow as pa
 import pytest
-from lance.file import LanceFileReader
+from lance.file import LanceFileReader, LanceFileWriter
 from lance.indices import IndicesBuilder, IvfModel, PqModel
 
 NUM_ROWS_PER_FRAGMENT = 10000
@@ -25,7 +25,7 @@
 
 def make_ds(num_rows: int, rows_per_frag: int, tmpdir: pathlib.Path, dtype: str):
     vectors = np.random.randn(num_rows, DIMENSION).astype(dtype)
-    vectors.shape = -1
+    vectors = vectors.reshape(-1)
     vectors = pa.FixedSizeListArray.from_arrays(vectors, DIMENSION)
     table = pa.Table.from_arrays([vectors], names=["vectors"])
     uri = str(tmpdir / "dataset")
@@ -53,7 +53,7 @@ def small_rand_dataset(tmpdir, request):
 @pytest.fixture
 def mostly_null_dataset(tmpdir, request):
     vectors = np.random.randn(NUM_ROWS, DIMENSION).astype(np.float32)
-    vectors.shape = -1
+    vectors = vectors.reshape(-1)
     vectors = pa.FixedSizeListArray.from_arrays(vectors, DIMENSION)
     vectors = vectors.to_pylist()
     vectors = [vec if i % 10 == 0 else None for i, vec in enumerate(vectors)]
@@ -209,6 +209,29 @@ def test_gen_pq(tmpdir, rand_dataset, rand_ivf):
     assert pq.dimension == reloaded.dimension
     assert pq.codebook == reloaded.codebook
 
+    pq_4bit = IndicesBuilder(rand_dataset, "vectors").train_pq(
+        rand_ivf,
+        sample_rate=2,
+        num_bits=4,
+    )
+    assert pq_4bit.num_bits == 4
+    assert len(pq_4bit.codebook) == 16
+
+    pq_4bit.save(str(tmpdir / "pq_4bit"))
+    reloaded = PqModel.load(str(tmpdir / "pq_4bit"))
+    assert reloaded.num_bits == 4
+
+    legacy_pq_uri = str(tmpdir / "legacy_pq")
+    with LanceFileWriter(
+        legacy_pq_uri,
+        pa.schema(
+            [pa.field("codebook", pq.codebook.type)],
+            metadata={b"num_subvectors": str(pq.num_subvectors).encode()},
+        ),
+    ) as writer:
+        writer.write_batch(pa.table([pq.codebook], names=["codebook"]))
+    assert PqModel.load(legacy_pq_uri).num_bits == 8
+
 
 def test_ivf_centroids_fragment_ids(tmpdir):
     rows_per_fragment = 32
@@ -219,7 +242,7 @@ def test_ivf_centroids_fragment_ids(tmpdir):
         ],
         axis=0,
     )
-    vectors.shape = -1
+    vectors = vectors.reshape(-1)
     table = pa.Table.from_arrays(
         [pa.FixedSizeListArray.from_arrays(vectors, DIMENSION)], names=["vectors"]
     )
@@ -300,7 +323,7 @@ def test_indices_builder_multivector_distributed_dimensions(tmpdir, monkeypatch)
 
     captured_dimensions = {}
 
-    def train_pq_model(*args):
+    def train_pq_model(*args, **kwargs):
         captured_dimensions["train_pq"] = args[2]
         return codebook
 
diff --git a/python/python/tests/test_json.py b/python/python/tests/test_json.py
index ad2ab6c3ada..7a6a11883dd 100644
--- a/python/python/tests/test_json.py
+++ b/python/python/tests/test_json.py
@@ -861,3 +861,34 @@ def test_json_merge_insert(tmp_path: Path):
 
     result = dataset.to_table(filter="json_get_int(data, 'score') >= 35")
     assert result.num_rows == 2
+
+
+def test_json_fragment_session_take(tmp_path: Path):
+    """FragmentSession.take must return JSON columns as pa.json_(), not raw JSONB."""
+
+    dataset_path = tmp_path / "json_fragment_session.lance"
+
+    table = pa.table(
+        {
+            "id": pa.array([1, 2, 3, 4, 5], type=pa.int32()),
+            "data": pa.array(
+                [json.dumps({"name": "row", "score": v}) for v in range(5)],
+                type=pa.json_(),
+            ),
+        }
+    )
+    lance.write_dataset(table, dataset_path)
+
+    dataset = lance.dataset(dataset_path)
+    fragment = dataset.get_fragments()[0]
+    session = fragment.open_session(columns=["id", "data"])
+    result = session.take([0, 2, 4])
+
+    # JSON column must be the logical pa.json_() type, not raw LargeBinary JSONB.
+    assert result.schema.field("data").type == pa.json_()
+    assert result.column("id").to_pylist() == [1, 3, 5]
+    assert [json.loads(v) for v in result.column("data").to_pylist()] == [
+        {"name": "row", "score": 0},
+        {"name": "row", "score": 2},
+        {"name": "row", "score": 4},
+    ]
diff --git a/python/python/tests/test_lance.py b/python/python/tests/test_lance.py
index 0162e370665..2ad3af2e029 100644
--- a/python/python/tests/test_lance.py
+++ b/python/python/tests/test_lance.py
@@ -248,6 +248,28 @@ def test_io_counters(tmp_path):
     assert lance.bytes_read_counter() > starting_bytes
 
 
+def test_simd_info():
+    info = lance.simd_info()
+    assert info["tier"] in (
+        "none",
+        "sse",
+        "avx",
+        "avx_fma",
+        "avx2",
+        "avx512",
+        "avx512_fp16",
+        "neon",
+        "lsx",
+        "lasx",
+    )
+    assert isinstance(info["target_arch"], str) and info["target_arch"]
+    if info["target_arch"] == "x86_64":
+        # The x86_64 ABI mandates SSE2.
+        assert "sse2" in info["host_features"]
+    else:
+        assert info["host_features"] == []
+
+
 @pytest.mark.parametrize(
     "row_param, column_name",
     [("with_row_id", "_rowid"), ("with_row_address", "_rowaddr")],
diff --git a/python/python/tests/test_mem_wal.py b/python/python/tests/test_mem_wal.py
index b8c859cb637..2871baad986 100644
--- a/python/python/tests/test_mem_wal.py
+++ b/python/python/tests/test_mem_wal.py
@@ -11,6 +11,7 @@
 from lance.mem_wal import (
     LsmPointLookupPlanner,
     LsmScanner,
+    LsmVectorSearchPlanner,
     ShardingField,
     ShardingSpec,
     ShardSnapshot,
@@ -60,9 +61,16 @@ def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa.
 
     The collector resolves flushed generation paths as:
         {base_dataset_path}/_mem_wal/{shard_id}/{gen_folder}
+
+    Production flush also writes a primary-key dedup sidecar (`_pk_index/`) that
+    the LSM scanner opens to dedup across generations; stage it here too so the
+    flushed generation faithfully matches what flush produces.
     """
+    from lance.lance import _write_pk_sidecar
+
     gen_path = os.path.join(base_path, "_mem_wal", shard_id, gen_folder)
     lance.write_dataset(data, gen_path, schema=_LOOKUP_SCHEMA)
+    _write_pk_sidecar(gen_path, data, ["id"])
 
 
 def test_point_lookup_with_memtables(tmp_path):
@@ -156,6 +164,11 @@ def test_lsm_scanner_with_memtables(tmp_path):
     assert name_by_id[2] == "gen1_2", "Flushed gen must overwrite base for id=2"
     assert name_by_id[3] == "base_3"
 
+    offset_table = (
+        LsmScanner.from_snapshots(base_ds, [snap]).limit(None, offset=1).to_table()
+    )
+    assert len(offset_table) == 2, "Offset-only LSM scan should not require a limit"
+
 
 def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path):
     ds_path = str(tmp_path / "base")
@@ -183,6 +196,34 @@ def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path):
             time.sleep(0.05)
 
 
+def test_shard_writer_delete_binding_masks_base_row(tmp_path):
+    ds_path = str(tmp_path / "base")
+    shard_id = str(uuid.uuid4())
+    ds = lance.write_dataset(
+        _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA
+    )
+    ds.initialize_mem_wal()
+
+    delete_keys = pa.table({"id": pa.array([2], type=pa.int64())})
+
+    with ds.mem_wal_writer(
+        shard_id,
+        durable_write=True,
+        sync_indexed_write=True,
+        max_wal_buffer_size=1,
+        max_wal_flush_interval_ms=10,
+    ) as writer:
+        writer.put(_lookup_table([4], "writer"))
+        writer.delete(delete_keys)
+        table = writer.lsm_scanner().to_table()
+
+    rows = {row["id"]: row["name"] for row in table.to_pylist()}
+    assert rows[1] == "base_1"
+    assert 2 not in rows, "deleted base row should be masked by the tombstone"
+    assert rows[3] == "base_3"
+    assert rows[4] == "writer_4"
+
+
 _VDIM = 4  # matches Rust test fixture dimension
 
 
@@ -214,6 +255,40 @@ def _vector_search_table(ids):
     )
 
 
+def test_lsm_vector_search_filter_binding(tmp_path):
+    ds_path = str(tmp_path / "vec")
+    ds = lance.write_dataset(
+        _vector_search_table(range(400)), ds_path, schema=_vector_search_schema()
+    )
+    ds.create_index(
+        "vector",
+        index_type="IVF_PQ",
+        name="vector_idx",
+        num_partitions=2,
+        num_sub_vectors=2,
+    )
+    ds.initialize_mem_wal(maintained_indexes=["vector_idx"])
+
+    planner = LsmVectorSearchPlanner(ds, [], "vector", filter="id >= 200")
+    query_id = 250
+    query = pa.array([25.0, 25.1, 25.2, 25.3], type=pa.float32())
+    table = planner.plan_search(query, k=20, nprobes=2, columns=["id"]).to_table()
+
+    ids = table.column("id").to_pylist()
+    assert ids, "filtered vector search should return at least one row"
+    assert min(ids) >= 200
+    assert query_id in ids, f"filtered vector search recall missed id={query_id}: {ids}"
+
+    with pytest.raises(ValueError, match="k must be positive"):
+        planner.plan_search(query, k=0, nprobes=2, columns=["id"])
+    with pytest.raises(ValueError, match="nprobes must be positive"):
+        planner.plan_search(query, k=20, nprobes=0, columns=["id"])
+    with pytest.raises(ValueError, match="overfetch_factor must be finite"):
+        planner.plan_search(
+            query, k=20, nprobes=2, columns=["id"], overfetch_factor=math.inf
+        )
+
+
 VECTOR_DIM = 32
 ROWS_PER_BATCH = 50
 NUM_WRITE_ROUNDS = 3
diff --git a/python/python/tests/test_multi_base.py b/python/python/tests/test_multi_base.py
index 113f3224d14..128a8905842 100644
--- a/python/python/tests/test_multi_base.py
+++ b/python/python/tests/test_multi_base.py
@@ -82,6 +82,60 @@ def test_multi_base_create_and_read(self):
             data.sort_values("id").reset_index(drop=True),
         )
 
+    def test_write_target_all_bases(self):
+        """target_all_bases=True rotates across primary and all initial bases."""
+        data = self.create_test_data(300)
+
+        dataset = lance.write_dataset(
+            data,
+            self.primary_uri,
+            mode="create",
+            initial_bases=[
+                DatasetBasePath(self.path1_uri, name="path1"),
+                DatasetBasePath(self.path2_uri, name="path2"),
+            ],
+            target_all_bases=True,
+            max_rows_per_file=100,
+        )
+
+        base_ids = [f.data_files()[0].base_id for f in dataset.get_fragments()]
+        assert base_ids == [None, 1, 2]
+        assert len(dataset.to_table()) == 300
+
+    def test_base_scoped_storage_options(self):
+        """base_<id>.<key> storage options flow through write and read."""
+        data = self.create_test_data(200)
+
+        # Local stores ignore these options; this verifies base-scoped entries
+        # are resolved per base without breaking the write or read path.
+        storage_options = {
+            "shared_option": "shared",
+            "base_1.scoped_option": "base1-value",
+        }
+
+        dataset = lance.write_dataset(
+            data,
+            self.primary_uri,
+            mode="create",
+            initial_bases=[DatasetBasePath(self.path1_uri, name="path1")],
+            target_bases=["path1"],
+            max_rows_per_file=100,
+            storage_options=storage_options,
+        )
+        assert dataset.count_rows() == 200
+
+        # Data files land in the scoped base, not the primary path
+        assert list(Path(self.path1_uri).glob("**/*.lance"))
+        assert not list((Path(self.primary_uri) / "data").glob("*.lance"))
+
+        # Reopen with the same flat options and read through the base store
+        dataset = lance.dataset(self.primary_uri, storage_options=storage_options)
+        result = dataset.to_table().to_pandas()
+        pd.testing.assert_frame_equal(
+            result.sort_values("id").reset_index(drop=True),
+            data.sort_values("id").reset_index(drop=True),
+        )
+
     def test_multi_base_append_mode(self):
         """Test appending data to a multi-base dataset."""
         # Create initial dataset
@@ -1248,3 +1302,307 @@ def test_write_fragments_create_mode_with_initial_bases(self):
         dataset_root = Path(dataset_uri)
         data_files_root = list(dataset_root.glob("*.lance"))
         assert len(data_files_root) == 0, "Should not have data files in root"
+
+
+class TestDataReplacementWithBases:
+    """DataReplacement must preserve DataFile.base_id so replacement files can
+    live in (and resolve against) a storage base other than the dataset root."""
+
+    def setup_method(self):
+        self.test_dir = tempfile.mkdtemp()
+        self.primary_uri = str(Path(self.test_dir) / "primary")
+        self.base1_uri = str(Path(self.test_dir) / "base1")
+        Path(self.base1_uri).mkdir(parents=True, exist_ok=True)
+
+    def teardown_method(self):
+        if hasattr(self, "test_dir"):
+            shutil.rmtree(self.test_dir, ignore_errors=True)
+
+    def _make_two_base_dataset(self) -> "lance.LanceDataset":
+        """Fragment 0 at the dataset root, fragment 1 in base1; column a."""
+        ds = lance.write_dataset(
+            pa.table({"a": list(range(8))}),
+            self.primary_uri,
+            max_rows_per_file=8,
+        )
+        ds.add_bases([DatasetBasePath(self.base1_uri, name="b1", is_dataset_root=True)])
+        return lance.write_dataset(
+            pa.table({"a": list(range(8, 16))}),
+            self.primary_uri,
+            mode="append",
+            max_rows_per_file=8,
+            target_bases=["b1"],
+        )
+
+    def _write_bare_file(self, data_dir: str, data: pa.Table) -> str:
+        from lance.file import LanceFileWriter
+
+        name = f"{uuid.uuid4()}.lance"
+        Path(data_dir).mkdir(parents=True, exist_ok=True)
+        with LanceFileWriter(f"{data_dir}/{name}") as writer:
+            writer.write_batch(data)
+        return name
+
+    def _b_data_file(self, name: str, base_id=None) -> "lance.fragment.DataFile":
+        from lance.file import stable_version
+        from lance.fragment import DataFile
+
+        return DataFile(
+            path=name,
+            fields=[1],  # field id of column "b" (a=0, b=1)
+            column_indices=[0],
+            file_major_version=int(stable_version().split(".")[0]),
+            file_minor_version=int(stable_version().split(".")[1]),
+            base_id=base_id,
+        )
+
+    def test_data_replacement_new_column_into_base(self):
+        """The all-NULL-column special case: the new column's data file for a
+        base fragment is written into that base and must keep its base_id."""
+        ds = self._make_two_base_dataset()
+        ds.add_columns(pa.field("b", pa.int32()))
+        ds = lance.dataset(self.primary_uri)
+
+        root_name = self._write_bare_file(
+            f"{self.primary_uri}/data",
+            pa.table({"b": pa.array([x * 10 for x in range(8)], pa.int32())}),
+        )
+        base_name = self._write_bare_file(
+            f"{self.base1_uri}/data",
+            pa.table({"b": pa.array([x * 10 for x in range(8, 16)], pa.int32())}),
+        )
+
+        op = lance.LanceOperation.DataReplacement(
+            [
+                lance.LanceOperation.DataReplacementGroup(
+                    0, self._b_data_file(root_name)
+                ),
+                lance.LanceOperation.DataReplacementGroup(
+                    1, self._b_data_file(base_name, base_id=1)
+                ),
+            ]
+        )
+        ds = lance.LanceDataset.commit(self.primary_uri, op, read_version=ds.version)
+
+        frags = ds.get_fragments()
+        root_files = {f.path: f.base_id for f in frags[0].data_files()}
+        base_files = {f.path: f.base_id for f in frags[1].data_files()}
+        assert root_files[root_name] is None
+        assert base_files[base_name] == 1
+
+        table = ds.to_table()
+        assert table.column("b").to_pylist() == [x * 10 for x in range(16)]
+
+    def test_data_replacement_replace_existing_file_in_base(self):
+        """The replace-existing-file branch must take the new file's base_id."""
+        ds = self._make_two_base_dataset()
+        ds.add_columns(pa.field("b", pa.int32()))
+        ds = lance.dataset(self.primary_uri)
+
+        first = self._write_bare_file(
+            f"{self.base1_uri}/data",
+            pa.table({"b": pa.array([0] * 8, pa.int32())}),
+        )
+        op = lance.LanceOperation.DataReplacement(
+            [lance.LanceOperation.DataReplacementGroup(1, self._b_data_file(first, 1))]
+        )
+        ds = lance.LanceDataset.commit(self.primary_uri, op, read_version=ds.version)
+
+        # Replace the same column file again, still in base1.
+        second = self._write_bare_file(
+            f"{self.base1_uri}/data",
+            pa.table({"b": pa.array([x * 10 for x in range(8, 16)], pa.int32())}),
+        )
+        op = lance.LanceOperation.DataReplacement(
+            [lance.LanceOperation.DataReplacementGroup(1, self._b_data_file(second, 1))]
+        )
+        ds = lance.LanceDataset.commit(self.primary_uri, op, read_version=ds.version)
+
+        base_files = {f.path: f.base_id for f in ds.get_fragments()[1].data_files()}
+        assert base_files[second] == 1
+        assert first not in base_files
+        assert ds.to_table().column("b").to_pylist()[8:] == [
+            x * 10 for x in range(8, 16)
+        ]
+
+
+class TestMergeInsertMultiBase:
+    """Test merge insert on multi-base datasets."""
+
+    def setup_method(self):
+        """Set up test directories for each test."""
+        self.test_dir = tempfile.mkdtemp()
+        self.primary_uri = str(Path(self.test_dir) / "primary")
+        self.base1_uri = str(Path(self.test_dir) / "base1")
+        self.base2_uri = str(Path(self.test_dir) / "base2")
+        for uri in [self.primary_uri, self.base1_uri, self.base2_uri]:
+            Path(uri).mkdir(parents=True, exist_ok=True)
+
+    def teardown_method(self):
+        """Clean up test directories after each test."""
+        if hasattr(self, "test_dir"):
+            shutil.rmtree(self.test_dir, ignore_errors=True)
+
+    def create_dataset(self):
+        """Dataset with two registered bases and initial data in base1."""
+        initial_data = pd.DataFrame(
+            {
+                "id": range(100),
+                "value": [f"initial_{i}" for i in range(100)],
+            }
+        )
+        return lance.write_dataset(
+            initial_data,
+            self.primary_uri,
+            mode="create",
+            initial_bases=[
+                DatasetBasePath(self.base1_uri, name="base1"),
+                DatasetBasePath(self.base2_uri, name="base2"),
+            ],
+            target_bases=["base1"],
+            max_rows_per_file=50,
+        )
+
+    def base_name_of(self, dataset, data_file):
+        base_paths = dataset._ds.base_paths()
+        if data_file.base_id is None:
+            return None
+        return base_paths[data_file.base_id].name
+
+    def test_merge_insert_without_target_bases(self):
+        """Merge insert on a multi-base dataset writes to primary by default."""
+        dataset = self.create_dataset()
+
+        new_data = pd.DataFrame(
+            {
+                "id": range(50, 150),
+                "value": [f"updated_{i}" for i in range(50, 150)],
+            }
+        )
+        stats = (
+            dataset.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(new_data)
+        )
+        assert stats["num_updated_rows"] == 50
+        assert stats["num_inserted_rows"] == 50
+
+        result = dataset.to_table().to_pandas().sort_values("id")
+        assert len(result) == 150
+        assert list(result[result["id"] >= 50]["value"]) == [
+            f"updated_{i}" for i in range(50, 150)
+        ]
+
+        # New fragments (merge output) are in primary storage.
+        max_initial_fragment_id = 1  # 100 rows / 50 per file -> fragments 0, 1
+        for fragment in dataset.get_fragments():
+            is_initial = fragment.fragment_id <= max_initial_fragment_id
+            for data_file in fragment.data_files():
+                expected = "base1" if is_initial else None
+                assert self.base_name_of(dataset, data_file) == expected
+
+    def test_merge_insert_with_target_bases(self):
+        """Merge insert routes new fragments to the requested base."""
+        dataset = self.create_dataset()
+
+        new_data = pd.DataFrame(
+            {
+                "id": range(50, 150),
+                "value": [f"updated_{i}" for i in range(50, 150)],
+            }
+        )
+        stats = (
+            dataset.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .target_bases(["base2"])
+            .execute(new_data)
+        )
+        assert stats["num_updated_rows"] == 50
+        assert stats["num_inserted_rows"] == 50
+
+        result = dataset.to_table().to_pandas().sort_values("id")
+        assert len(result) == 150
+        assert list(result[result["id"] >= 50]["value"]) == [
+            f"updated_{i}" for i in range(50, 150)
+        ]
+
+        merge_files = 0
+        for fragment in dataset.get_fragments():
+            if fragment.fragment_id > 1:
+                for data_file in fragment.data_files():
+                    assert self.base_name_of(dataset, data_file) == "base2"
+                    merge_files += 1
+        assert merge_files > 0
+        assert list(Path(self.base2_uri).glob("**/*.lance"))
+
+        # The routed dataset stays readable from a fresh instance.
+        reloaded = lance.dataset(self.primary_uri)
+        assert reloaded.count_rows() == 150
+
+    def test_merge_insert_with_unknown_target_base(self):
+        """Merge insert referencing an unregistered base fails."""
+        dataset = self.create_dataset()
+
+        new_data = pd.DataFrame({"id": [1], "value": ["updated_1"]})
+        with pytest.raises(Exception, match="not found in available bases"):
+            (
+                dataset.merge_insert("id")
+                .when_matched_update_all()
+                .when_not_matched_insert_all()
+                .target_bases(["nonexistent"])
+                .execute(new_data)
+            )
+
+    def test_merge_insert_target_primary_via_uri(self):
+        """The dataset URI in target_bases selects primary storage."""
+        dataset = self.create_dataset()
+
+        new_data = pd.DataFrame({"id": [200], "value": ["inserted_200"]})
+        (
+            dataset.merge_insert("id")
+            .when_not_matched_insert_all()
+            .target_bases([dataset.uri, "base2"])
+            .execute(new_data)
+        )
+        assert dataset.count_rows() == 101
+
+        # The single new file lands in the first slot: primary storage.
+        for fragment in dataset.get_fragments():
+            if fragment.fragment_id > 1:
+                for data_file in fragment.data_files():
+                    assert self.base_name_of(dataset, data_file) is None
+
+    def test_merge_insert_target_all_bases(self):
+        """target_all_bases spreads new files across all bases, primary first."""
+        dataset = self.create_dataset()
+
+        new_data = pd.DataFrame({"id": [300], "value": ["inserted_300"]})
+        (
+            dataset.merge_insert("id")
+            .when_not_matched_insert_all()
+            .target_all_bases()
+            .execute(new_data)
+        )
+        assert dataset.count_rows() == 101
+        # A single new file lands in the first slot: primary storage.
+        newest = max(f.fragment_id for f in dataset.get_fragments())
+        for fragment in dataset.get_fragments():
+            if fragment.fragment_id == newest:
+                for data_file in fragment.data_files():
+                    assert self.base_name_of(dataset, data_file) is None
+
+        new_data = pd.DataFrame({"id": [301], "value": ["inserted_301"]})
+        (
+            dataset.merge_insert("id")
+            .when_not_matched_insert_all()
+            .target_all_bases(include_primary=False)
+            .execute(new_data)
+        )
+        assert dataset.count_rows() == 102
+        newest = max(f.fragment_id for f in dataset.get_fragments())
+        for fragment in dataset.get_fragments():
+            if fragment.fragment_id == newest:
+                for data_file in fragment.data_files():
+                    assert self.base_name_of(dataset, data_file) == "base1"
diff --git a/python/python/tests/test_namespace_dir.py b/python/python/tests/test_namespace_dir.py
index 1991b82946e..fa1bc93b422 100644
--- a/python/python/tests/test_namespace_dir.py
+++ b/python/python/tests/test_namespace_dir.py
@@ -29,6 +29,8 @@
     CountTableRowsRequest,
     CreateNamespaceRequest,
     CreateNamespaceResponse,
+    CreateTableBranchRequest,
+    CreateTableBranchResponse,
     CreateTableIndexRequest,
     CreateTableIndexResponse,
     CreateTableRequest,
@@ -37,6 +39,8 @@
     CreateTableVersionResponse,
     DeclareTableRequest,
     DeclareTableResponse,
+    DeleteTableBranchRequest,
+    DeleteTableBranchResponse,
     DeregisterTableRequest,
     DeregisterTableResponse,
     DescribeNamespaceRequest,
@@ -54,6 +58,8 @@
     InsertIntoTableResponse,
     ListNamespacesRequest,
     ListNamespacesResponse,
+    ListTableBranchesRequest,
+    ListTableBranchesResponse,
     ListTableIndicesRequest,
     ListTableIndicesResponse,
     ListTablesRequest,
@@ -71,6 +77,8 @@
     InvalidInputError,
     NamespaceNotEmptyError,
     NamespaceNotFoundError,
+    TableBranchAlreadyExistsError,
+    TableBranchNotFoundError,
     TableNotFoundError,
 )
 
@@ -151,6 +159,21 @@ def create_table_version(
     ) -> CreateTableVersionResponse:
         return self._inner.create_table_version(request)
 
+    def create_table_branch(
+        self, request: CreateTableBranchRequest
+    ) -> CreateTableBranchResponse:
+        return self._inner.create_table_branch(request)
+
+    def list_table_branches(
+        self, request: ListTableBranchesRequest
+    ) -> ListTableBranchesResponse:
+        return self._inner.list_table_branches(request)
+
+    def delete_table_branch(
+        self, request: DeleteTableBranchRequest
+    ) -> DeleteTableBranchResponse:
+        return self._inner.delete_table_branch(request)
+
     def create_table_index(
         self, request: CreateTableIndexRequest
     ) -> CreateTableIndexResponse:
@@ -564,6 +587,110 @@ def test_register_table_rejects_path_traversal(self, temp_ns_client):
         assert "Path traversal is not allowed" in str(exc_info.value)
 
 
+class TestTableBranchOperations:
+    """Branch CRUD through the python bindings - mirrors the Rust branch
+    CRUD tests."""
+
+    def test_branch_crud_round_trip(self, temp_ns_client):
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+        ipc_data = table_to_ipc_bytes(create_test_data())
+        table_id = ["workspace", "branched_table"]
+        temp_ns_client.create_table(CreateTableRequest(id=table_id), ipc_data)
+
+        temp_ns_client.create_table_branch(
+            CreateTableBranchRequest(id=table_id, name="dev")
+        )
+        listed = temp_ns_client.list_table_branches(
+            ListTableBranchesRequest(id=table_id)
+        )
+        assert "dev" in listed.branches
+        assert listed.branches["dev"].parent_version == 1
+
+        # Duplicate creation and deleting a missing branch surface the typed
+        # branch errors (codes 23 and 22), not InternalError.
+        temp_ns_client.create_table_branch(
+            CreateTableBranchRequest(id=table_id, name="dev2")
+        )
+        with pytest.raises(TableBranchAlreadyExistsError):
+            temp_ns_client.create_table_branch(
+                CreateTableBranchRequest(id=table_id, name="dev2")
+            )
+
+        temp_ns_client.delete_table_branch(
+            DeleteTableBranchRequest(id=table_id, name="dev")
+        )
+        listed = temp_ns_client.list_table_branches(
+            ListTableBranchesRequest(id=table_id)
+        )
+        assert "dev" not in listed.branches
+        with pytest.raises(TableBranchNotFoundError):
+            temp_ns_client.delete_table_branch(
+                DeleteTableBranchRequest(id=table_id, name="dev")
+            )
+
+    def test_create_branch_from_other_branch(self, temp_ns_client):
+        """Forking from a non-main source branch records the right parent."""
+        create_ns_req = CreateNamespaceRequest(id=["workspace"])
+        temp_ns_client.create_namespace(create_ns_req)
+        ipc_data = table_to_ipc_bytes(create_test_data())
+        table_id = ["workspace", "fork_table"]
+        temp_ns_client.create_table(CreateTableRequest(id=table_id), ipc_data)
+
+        temp_ns_client.create_table_branch(
+            CreateTableBranchRequest(id=table_id, name="dev")
+        )
+        temp_ns_client.create_table_branch(
+            CreateTableBranchRequest(id=table_id, name="child", from_branch="dev")
+        )
+        listed = temp_ns_client.list_table_branches(
+            ListTableBranchesRequest(id=table_id)
+        )
+        assert listed.branches["child"].parent_branch == "dev"
+
+
+class _ForeignCodeError(Exception):
+    """Not a LanceNamespaceError, but carries the same integer code as
+    TABLE_NOT_FOUND."""
+
+    code = 4
+
+
+class _RaisingNamespace(LanceNamespace):
+    """A namespace whose describe_table raises the configured exception."""
+
+    def __init__(self, exc: Exception):
+        self._exc = exc
+
+    def namespace_id(self) -> str:
+        return "raising"
+
+    def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse:
+        raise self._exc
+
+
+class TestPythonNamespaceErrorMapping:
+    """The Rust adapter must trust the `code` attribute only on the
+    lance_namespace exception hierarchy."""
+
+    def test_namespace_error_identity_preserved(self):
+        ns = _RaisingNamespace(TableNotFoundError("no such table"))
+        with pytest.raises(TableNotFoundError, match="no such table"):
+            lance.dataset(namespace_client=ns, table_id=["t"])
+
+        # Branch error codes (22/23) survive the round trip too.
+        ns = _RaisingNamespace(TableBranchNotFoundError("no such branch"))
+        with pytest.raises(TableBranchNotFoundError, match="no such branch"):
+            lance.dataset(namespace_client=ns, table_id=["t"])
+
+    def test_foreign_code_attribute_not_trusted(self):
+        # The foreign exception must surface as itself, not be reinterpreted
+        # as a namespace error via its `code` attribute.
+        ns = _RaisingNamespace(_ForeignCodeError("boom"))
+        with pytest.raises(_ForeignCodeError, match="boom"):
+            lance.dataset(namespace_client=ns, table_id=["t"])
+
+
 class TestChildNamespaceOperations:
     """Tests for operations in child namespaces - mirrors Rust tests."""
 
@@ -979,6 +1106,49 @@ def test_external_manifest_store_invokes_namespace_apis(use_custom):
         ), "describe_table_version should be called once when opening version 1"
 
 
+def test_dataset_namespace_open_does_not_pass_version_to_describe_table():
+    """Dataset versions are applied to dataset open, not namespace describe_table."""
+
+    class VersionRejectingNamespace(CustomNamespace):
+        def __init__(self, inner: lance.namespace.DirectoryNamespace):
+            super().__init__(inner)
+            self.describe_versions = []
+
+        def describe_table(
+            self, request: DescribeTableRequest
+        ) -> DescribeTableResponse:
+            self.describe_versions.append(request.version)
+            assert request.version is None
+            return super().describe_table(request)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        inner_ns_client = lance.namespace.DirectoryNamespace(root=tmpdir)
+        ns_client = VersionRejectingNamespace(inner_ns_client)
+        table_id = ["test_table"]
+
+        table1 = pa.Table.from_pylist([{"a": 1}, {"a": 2}])
+        ds = lance.write_dataset(
+            table1, namespace_client=ns_client, table_id=table_id, mode="create"
+        )
+        assert ds.count_rows() == 2
+        assert ds.version == 1
+
+        table2 = pa.Table.from_pylist([{"a": 3}])
+        ds = lance.write_dataset(
+            table2, namespace_client=ns_client, table_id=table_id, mode="append"
+        )
+        assert ds.count_rows() == 3
+        assert ds.version == 2
+
+        version_one = lance.dataset(
+            namespace_client=ns_client, table_id=table_id, version=1
+        )
+        assert version_one.count_rows() == 2
+        assert version_one.version == 1
+        assert ns_client.describe_versions
+        assert all(version is None for version in ns_client.describe_versions)
+
+
 @pytest.mark.skipif(
     sys.platform == "win32",
     reason="Windows file locking prevents reliable concurrent filesystem operations",
diff --git a/python/python/tests/test_namespace_integration.py b/python/python/tests/test_namespace_integration.py
index 4605b755816..fc08370d247 100644
--- a/python/python/tests/test_namespace_integration.py
+++ b/python/python/tests/test_namespace_integration.py
@@ -31,6 +31,8 @@
 from lance_namespace import (
     CreateNamespaceRequest,
     CreateNamespaceResponse,
+    CreateTableBranchRequest,
+    CreateTableBranchResponse,
     CreateTableRequest,
     CreateTableResponse,
     CreateTableVersionRequest,
@@ -136,6 +138,11 @@ def create_table_version(
     ) -> CreateTableVersionResponse:
         return self._inner.create_table_version(request)
 
+    def create_table_branch(
+        self, request: CreateTableBranchRequest
+    ) -> CreateTableBranchResponse:
+        return self._inner.create_table_branch(request)
+
     def retrieve_ops_metrics(self) -> Optional[Dict[str, int]]:
         return self._inner.retrieve_ops_metrics()
 
@@ -199,6 +206,7 @@ def create_tracking_namespace(
     storage_options: dict,
     credential_expires_in_seconds: int = 60,
     use_custom: bool = False,
+    managed_versioning: bool = False,
 ):
     """Create a DirectoryNamespace with ops metrics and credential vending enabled.
 
@@ -212,6 +220,9 @@ def create_tracking_namespace(
         storage_options: Storage options to pass through (credentials, endpoint, etc.)
         credential_expires_in_seconds: Interval in seconds for credential expiration
         use_custom: If True, wrap in CustomNamespace for testing custom implementations
+        managed_versioning: If True, enable the manifest catalog so table versions
+            are tracked by the namespace and commits route through
+            create_table_version
 
     Returns:
         Tuple of (namespace_client, inner_namespace_client) where inner is always
@@ -238,6 +249,10 @@ def create_tracking_namespace(
     dir_props["vend_input_storage_options_refresh_interval_millis"] = str(
         credential_expires_in_seconds * 1000
     )
+    if managed_versioning:
+        dir_props["manifest_enabled"] = "true"
+        dir_props["table_version_tracking_enabled"] = "true"
+        dir_props["table_version_storage_enabled"] = "true"
 
     inner_ns_client = DirectoryNamespace(**dir_props)
     ns_client = _wrap_if_custom(inner_ns_client, use_custom)
@@ -558,6 +573,87 @@ def test_namespace_write_overwrite_mode(s3_bucket: str, use_custom: bool):
     assert get_describe_call_count(inner_ns_client) == call_count_before_reads
 
 
+@pytest.mark.integration
+@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
+def test_namespace_managed_branches(s3_bucket: str, use_custom: bool):
+    """Branches on a managed-versioning table over S3.
+
+    Branch commits must route through the catalog (create_table_version) and
+    leave main's chain untouched. A cross-branch checkout at an overlapping
+    version number must resolve the requested chain: branch version numbers
+    continue from the fork point, so the same number exists on both chains
+    with different data.
+    """
+    storage_options = copy.deepcopy(CONFIG)
+
+    ns_client, inner_ns_client = create_tracking_namespace(
+        bucket_name=s3_bucket,
+        storage_options=storage_options,
+        credential_expires_in_seconds=3600,
+        use_custom=use_custom,
+        managed_versioning=True,
+    )
+
+    table_name = uuid.uuid4().hex
+    table_id = ["test_ns", table_name]
+
+    def commit_count() -> int:
+        return inner_ns_client.retrieve_ops_metrics().get("create_table_version", 0)
+
+    lance.write_dataset(
+        pa.Table.from_pylist([{"a": 1}]),
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="create",
+        storage_options=storage_options,
+    )
+    ds = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 2}]),
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="append",
+        storage_options=storage_options,
+    )
+    assert commit_count() >= 2
+
+    ns_client.create_table_branch(
+        CreateTableBranchRequest(id=table_id, name="dev", from_version=2)
+    )
+
+    dev = ds.checkout_version(("dev", None))
+    commits_before_branch_append = commit_count()
+    dev = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 3}]),
+        dev,
+        mode="append",
+        storage_options=storage_options,
+    )
+    assert commit_count() == commits_before_branch_append + 1
+    assert sorted(dev.to_table()["a"].to_pylist()) == [1, 2, 3]
+
+    # Diverge main to the same version number as dev's tip.
+    ds = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 100}]),
+        namespace_client=ns_client,
+        table_id=table_id,
+        mode="append",
+        storage_options=storage_options,
+    )
+    assert sorted(ds.to_table()["a"].to_pylist()) == [1, 2, 100]
+
+    on_dev = ds.checkout_version(("dev", 3))
+    assert sorted(on_dev.to_table()["a"].to_pylist()) == [1, 2, 3]
+    back_on_main = dev.checkout_version(("main", None))
+    assert sorted(back_on_main.to_table()["a"].to_pylist()) == [1, 2, 100]
+
+    fresh = lance.dataset(
+        namespace_client=ns_client,
+        table_id=table_id,
+        storage_options=storage_options,
+    )
+    assert sorted(fresh.to_table()["a"].to_pylist()) == [1, 2, 100]
+
+
 @pytest.mark.integration
 @pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
 def test_namespace_distributed_write(s3_bucket: str, use_custom: bool):
diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py
index 801efcbd4f2..5d88af2566c 100644
--- a/python/python/tests/test_optimize.py
+++ b/python/python/tests/test_optimize.py
@@ -324,17 +324,56 @@ def test_defer_index_remap(tmp_path: Path):
     assert any(idx.name == "__lance_frag_reuse" for idx in indices)
 
 
+@pytest.mark.parametrize("use_commit_options", [True, False])
+def test_defer_index_remap_via_commit_options(tmp_path: Path, use_commit_options: bool):
+    """Compaction.commit respects defer_index_remap passed in options.
+
+    When options={"defer_index_remap": True} is supplied to Compaction.commit
+    the __lance_frag_reuse system index must appear in describe_indices().
+    When the option is omitted (default) no such system index is written.
+    """
+    base_dir = tmp_path / f"dataset_commit_opts_{use_commit_options}"
+    data = pa.table({"i": range(6_000), "val": range(6_000)})
+    dataset = lance.write_dataset(data, base_dir, max_rows_per_file=1_000)
+    dataset.create_scalar_index("i", "BTREE")
+    dataset.delete("i < 500")
+
+    plan = Compaction.plan(
+        dataset,
+        options=dict(target_rows_per_fragment=2_000, num_threads=1),
+    )
+    rewrites = [task.execute(dataset) for task in plan.tasks]
+
+    if use_commit_options:
+        Compaction.commit(dataset, rewrites, options={"defer_index_remap": True})
+    else:
+        Compaction.commit(dataset, rewrites)
+
+    dataset = lance.dataset(base_dir)
+    indices = dataset.describe_indices()
+    has_frag_reuse = any(idx.name == "__lance_frag_reuse" for idx in indices)
+
+    if use_commit_options:
+        assert has_frag_reuse, (
+            "expected __lance_frag_reuse system index when defer_index_remap=True "
+            "is passed to Compaction.commit"
+        )
+    else:
+        assert not has_frag_reuse, (
+            "did not expect __lance_frag_reuse system index when options is omitted "
+            "from Compaction.commit"
+        )
+
+
 @pytest.mark.filterwarnings("ignore::DeprecationWarning")
 def test_describe_indices_matches_list_indices_for_frag_reuse(tmp_path: Path):
     """describe_indices() and list_indices() must agree on the index_type
     string for every index, including the __lance_frag_reuse system index
     that defer_index_remap produces.
 
-    list_indices() special-cases system indices via infer_system_index_type()
-    in python/src/dataset.rs. describe_indices() in
-    rust/lance/src/index.rs::IndexDescriptionImpl::try_new does not, so it
-    falls through to a plugin lookup that has no entry for
-    FragmentReuseIndexDetails and reports 'Unknown' instead.
+    list_indices() is a wrapper over describe_indices(), so the two must stay
+    in sync. System indices are identified by name via infer_system_index_type()
+    in rust/lance/src/index.rs::IndexDescriptionImpl::try_new.
     """
     base_dir = tmp_path / "dataset"
     data = pa.table({"i": range(6_000), "val": range(6_000)})
@@ -535,3 +574,38 @@ def test_compaction_generates_rewrite_transaction(tmp_path: Path):
         t is not None and t.operation.__class__.__name__ == "Rewrite"
         for t in transactions
     )
+
+
+def test_remap_row_addrs(tmp_path: Path):
+    # Dataset.remap_row_addrs follows rows across a compaction via the
+    # fragment-reuse index: an address valid before the compaction maps to the
+    # row's new address after it. None when there is no fragment-reuse index.
+    base_dir = tmp_path / "dataset"
+    data = pa.table({"id": range(1_000), "v": range(1_000)})
+    ds = lance.write_dataset(data, base_dir, max_rows_per_file=100)  # 10 fragments
+
+    # No fragment-reuse index yet -> None (nothing to remap against).
+    addrs = pa.array([0, 1 << 32, (5 << 32) | 7], pa.uint64())
+    assert ds.remap_row_addrs(addrs) is None
+
+    before = ds.scanner(columns=["id"], with_row_address=True).to_table()
+    old = dict(zip(before["id"].to_pylist(), before["_rowaddr"].to_pylist()))
+
+    # A deferred-remap compaction records a fragment-reuse index only when it
+    # rewrites data an index covers, so index a column first.
+    ds.create_scalar_index("id", "BTREE")
+
+    ds.optimize.compact_files(
+        target_rows_per_fragment=1_000, defer_index_remap=True, num_threads=1
+    )
+    ds = lance.dataset(base_dir)
+    assert any(idx.name == "__lance_frag_reuse" for idx in ds.describe_indices())
+
+    after = ds.scanner(columns=["id"], with_row_address=True).to_table()
+    new = dict(zip(after["id"].to_pylist(), after["_rowaddr"].to_pylist()))
+
+    sample = [0, 137, 999]
+    remapped = ds.remap_row_addrs(
+        pa.array([old[i] for i in sample], pa.uint64())
+    ).to_pylist()
+    assert remapped == [new[i] for i in sample]
diff --git a/python/python/tests/test_otel.py b/python/python/tests/test_otel.py
new file mode 100644
index 00000000000..a96e0125f56
--- /dev/null
+++ b/python/python/tests/test_otel.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The Lance Authors
+
+import lance
+import pyarrow as pa
+import pytest
+
+# The metrics recorder is process-global and installed once, so the whole
+# bridge is exercised in a single test to avoid cross-test global-state coupling.
+
+
+def _metrics_by_name(reader):
+    data = reader.get_metrics_data()
+    result = {}
+    for resource_metrics in data.resource_metrics:
+        for scope_metrics in resource_metrics.scope_metrics:
+            for metric in scope_metrics.metrics:
+                result[metric.name] = metric
+    return result
+
+
+def test_instrument_lance_metrics_exports_object_store_metrics(tmp_path):
+    pytest.importorskip("opentelemetry.sdk.metrics")
+    from lance.otel import instrument_lance_metrics
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+
+    reader = InMemoryMetricReader()
+    provider = MeterProvider(metric_readers=[reader])
+    assert instrument_lance_metrics(provider)
+
+    # The catalog is populated once the recorder is installed.
+    from lance.lance import lance_metrics_catalog
+
+    catalog = {desc.name: desc for desc in lance_metrics_catalog()}
+    assert "lance_object_store_requests_total" in catalog
+    assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram"
+    # Gauges and the retryable counter are described too, so they surface in the
+    # export even when a plain write doesn't happen to emit them.
+    assert catalog["lance_object_store_retryable_responses_total"].kind == "counter"
+    assert catalog["lance_object_store_in_flight_requests"].kind == "gauge"
+
+    # Generate object store activity on the local filesystem (scheme "file").
+    table = pa.table({"id": pa.array(range(256))})
+    dataset = lance.write_dataset(table, str(tmp_path / "ds.lance"))
+    assert dataset.to_table().num_rows == 256
+
+    metrics = _metrics_by_name(reader)
+
+    requests = metrics["lance_object_store_requests_total"]
+    points = list(requests.data.data_points)
+    assert points, "expected at least one request data point"
+    # The `base` label carries the store scheme ("file") by default.
+    assert all("base" in p.attributes and "operation" in p.attributes for p in points)
+    assert sum(p.value for p in points) > 0
+
+    # Histograms are decomposed into bucket / count / sum observable counters.
+    bucket = metrics["lance_object_store_request_duration_seconds_bucket"]
+    bucket_points = list(bucket.data.data_points)
+    assert bucket_points
+    assert all("le" in p.attributes for p in bucket_points)
+    # The implicit +Inf bucket must be present and is the cumulative maximum.
+    assert any(p.attributes["le"] == "+Inf" for p in bucket_points)
+
+    count = metrics["lance_object_store_request_duration_seconds_count"]
+    assert sum(p.value for p in count.data.data_points) > 0
+
+    # The `_sum` instrument must also be wired and report positive latency.
+    duration_sum = metrics["lance_object_store_request_duration_seconds_sum"]
+    assert sum(p.value for p in duration_sum.data.data_points) > 0
+
+
+def test_snapshot_empty_before_install_is_safe():
+    # snapshot is callable regardless of installation state and never raises.
+    from lance.lance import snapshot_lance_metrics
+
+    assert isinstance(snapshot_lance_metrics(), list)
+
+
+def test_instrument_warns_when_recorder_unavailable(monkeypatch):
+    # A foreign `metrics` recorder already installed -> register returns False;
+    # instrument_lance_metrics must warn and return False without instrumenting.
+    pytest.importorskip("opentelemetry.sdk.metrics")
+    import lance.otel as otel
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+
+    monkeypatch.setattr(otel, "register_lance_metrics_recorder", lambda: False)
+
+    reader = InMemoryMetricReader()
+    provider = MeterProvider(metric_readers=[reader])
+    with pytest.warns(UserWarning, match="recorder"):
+        assert otel.instrument_lance_metrics(provider) is False
diff --git a/python/python/tests/test_s3_ddb.py b/python/python/tests/test_s3_ddb.py
index b9c9e4be6c0..dc9744115e2 100644
--- a/python/python/tests/test_s3_ddb.py
+++ b/python/python/tests/test_s3_ddb.py
@@ -212,6 +212,58 @@ def writh_dataset_with_start_barrier():
     assert lance.dataset(table_dir).count_rows() == expected_version * 2
 
 
+@pytest.mark.integration
+def test_s3_ddb_branches(s3_bucket: str, ddb_table: str):
+    """Branches on a table committed through the DynamoDB external manifest
+    store.
+
+    The DDB store keys version chains by base uri, so each branch chain must
+    get its own entries via its branch-qualified path. Both chains are given
+    the same version number with diverged data so a wrong-chain resolution
+    cannot pass silently.
+    """
+    storage_options = copy.deepcopy(CONFIG)
+    table_name = uuid.uuid4().hex
+    table_dir = f"s3+ddb://{s3_bucket}/{table_name}?ddbTableName={ddb_table}"
+
+    # main: v1 (a=1), v2 (a=2)
+    lance.write_dataset(
+        pa.Table.from_pylist([{"a": 1}]), table_dir, storage_options=storage_options
+    )
+    ds = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 2}]),
+        table_dir,
+        mode="append",
+        storage_options=storage_options,
+    )
+
+    # Fork "dev" at v2 and commit on it, then diverge main to the same
+    # version number.
+    dev = ds.create_branch("dev", 2)
+    dev = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 3}]),
+        dev,
+        mode="append",
+        storage_options=storage_options,
+    )
+    ds = lance.write_dataset(
+        pa.Table.from_pylist([{"a": 100}]),
+        table_dir,
+        mode="append",
+        storage_options=storage_options,
+    )
+
+    assert sorted(dev.to_table()["a"].to_pylist()) == [1, 2, 3]
+    assert sorted(ds.to_table()["a"].to_pylist()) == [1, 2, 100]
+
+    # Cross-branch checkout at the overlapping version number resolves each
+    # chain's own data.
+    on_dev = ds.checkout_version(("dev", 3))
+    assert sorted(on_dev.to_table()["a"].to_pylist()) == [1, 2, 3]
+    back_on_main = dev.checkout_version(("main", None))
+    assert sorted(back_on_main.to_table()["a"].to_pylist()) == [1, 2, 100]
+
+
 @pytest.mark.integration
 def test_s3_unsafe(s3_bucket: str):
     storage_options = copy.deepcopy(CONFIG)
diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py
index d173ef807ba..985072fe5de 100644
--- a/python/python/tests/test_scalar_index.py
+++ b/python/python/tests/test_scalar_index.py
@@ -108,6 +108,23 @@ def _commit_segmented_btree_index(dataset, column, index_name):
     return dataset.commit_existing_index_segments(index_name, column, segments)
 
 
+def test_create_scalar_index_rejects_invalid_uuid(tmp_path):
+    """Invalid UUID strings passed to create_scalar_index and merge_index_metadata
+    must surface as a Python ValueError at the FFI boundary."""
+    data = pa.table({"id": pa.array(range(100), type=pa.int64())})
+    dataset = lance.write_dataset(data, tmp_path / "ds")
+
+    with pytest.raises(ValueError, match="Invalid UUID"):
+        dataset.create_scalar_index(
+            column="id",
+            index_type="BTREE",
+            index_uuid="not-a-uuid",
+        )
+
+    with pytest.raises(ValueError, match="Invalid UUID"):
+        dataset.merge_index_metadata("also-not-a-uuid", index_type="BTREE")
+
+
 @pytest.fixture
 def btree_comparison_datasets(tmp_path):
     """Setup datasets for B-tree comparison tests"""
@@ -147,7 +164,7 @@ def btree_comparison_datasets(tmp_path):
     }
 
 
-def test_load_indices(indexed_dataset: lance.LanceDataset):
+def test_describe_indices_vector_and_scalar(indexed_dataset: lance.LanceDataset):
     indices = indexed_dataset.describe_indices()
     vec_idx = next(idx for idx in indices if "VectorIndex" in idx.type_url)
     scalar_idx = next(idx for idx in indices if idx.index_type == "BTree")
@@ -155,6 +172,154 @@ def test_load_indices(indexed_dataset: lance.LanceDataset):
     assert scalar_idx is not None
 
 
+def test_list_indices_characterization(indexed_dataset: lance.LanceDataset):
+    """Lock down the backwards-compatible shape of the deprecated list_indices().
+
+    list_indices() returns a list of plain dicts (one per index segment), not
+    Index dataclasses. This characterization test guards the dict keys and
+    values so the deprecated method stays backwards compatible.
+    """
+    with pytest.warns(DeprecationWarning):
+        indices = indexed_dataset.list_indices()
+
+    assert len(indices) == 2
+    by_name = {idx["name"]: idx for idx in indices}
+    assert set(by_name) == {"vector_idx", "meta_idx"}
+
+    expected_keys = {
+        "name",
+        "type",
+        "uuid",
+        "fields",
+        "version",
+        "fragment_ids",
+        "base_id",
+    }
+    for idx in indices:
+        assert set(idx) == expected_keys
+        assert isinstance(idx["uuid"], str) and len(idx["uuid"]) > 0
+        assert isinstance(idx["fields"], list)
+        assert isinstance(idx["fragment_ids"], set)
+        assert isinstance(idx["version"], int)
+        assert idx["type"] != "Unknown"
+        assert idx["base_id"] is None
+
+    vector_idx = by_name["vector_idx"]
+    assert vector_idx["type"] == "IVF_PQ"
+    assert vector_idx["fields"] == ["vector"]
+    assert vector_idx["fragment_ids"] == {0}
+
+    meta_idx = by_name["meta_idx"]
+    assert meta_idx["type"] == "BTree"
+    assert meta_idx["fields"] == ["meta"]
+    assert meta_idx["fragment_ids"] == {0}
+
+
+def test_list_indices_nested_field_path(tmp_path):
+    """list_indices() reports nested fields as full dotted paths."""
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("meta", pa.struct([pa.field("lang", pa.string())])),
+        ]
+    )
+    data = pa.table(
+        {
+            "id": [1, 2, 3],
+            "meta": [{"lang": "en"}, {"lang": "fr"}, {"lang": "en"}],
+        },
+        schema=schema,
+    )
+    ds = lance.write_dataset(data, tmp_path)
+    ds.create_scalar_index(column="meta.lang", index_type="BTREE")
+
+    with pytest.warns(DeprecationWarning):
+        indices = ds.list_indices()
+
+    assert len(indices) == 1
+    assert indices[0]["fields"] == ["meta.lang"]
+
+
+def _commit_index(ds, index):
+    """Commit a single raw Index entry via the CreateIndex operation."""
+    return lance.LanceDataset.commit(
+        ds.uri,
+        lance.LanceOperation.CreateIndex(new_indices=[index], removed_indices=[]),
+        read_version=ds.version,
+    )
+
+
+def test_list_indices_index_without_details(tmp_path):
+    """An index whose manifest entry has no index details (e.g. committed by an
+    older writer) is still reported on a best-effort basis: describe_indices()
+    does not error, and the type is reported as "Unknown"."""
+    from lance.dataset import Index
+
+    data = pa.table({"id": range(100), "val": range(100)})
+    ds = lance.write_dataset(data, tmp_path)
+
+    field_id = ds.schema.get_field_index("id")
+    fragment_ids = {f.fragment_id for f in ds.get_fragments()}
+    ds = _commit_index(
+        ds,
+        Index(
+            uuid=str(uuid.uuid4()),
+            name="legacy_idx",
+            fields=[field_id],
+            dataset_version=ds.version,
+            fragment_ids=fragment_ids,
+            index_version=0,
+        ),
+    )
+
+    described = ds.describe_indices()
+    assert len(described) == 1
+    assert described[0].name == "legacy_idx"
+    assert described[0].index_type == "Unknown"
+    assert described[0].type_url == ""
+
+    with pytest.warns(DeprecationWarning):
+        listed = ds.list_indices()
+    assert len(listed) == 1
+    assert listed[0]["name"] == "legacy_idx"
+    assert listed[0]["type"] == "Unknown"
+
+
+def test_list_indices_legacy_vector_index_without_details(tmp_path):
+    """A legacy vector index predates VectorIndexDetails: it has no index
+    details but stores a monolithic index file. Its type is recognized as
+    "Vector" from the index file rather than reported as "Unknown"."""
+    from lance.dataset import Index, IndexFile
+
+    data = pa.table({"id": range(100), "val": range(100)})
+    ds = lance.write_dataset(data, tmp_path)
+
+    field_id = ds.schema.get_field_index("id")
+    fragment_ids = {f.fragment_id for f in ds.get_fragments()}
+    ds = _commit_index(
+        ds,
+        Index(
+            uuid=str(uuid.uuid4()),
+            name="legacy_vector_idx",
+            fields=[field_id],
+            dataset_version=ds.version,
+            fragment_ids=fragment_ids,
+            index_version=0,
+            # "index.idx" is the legacy monolithic index file name; its presence
+            # is how a pre-details vector index is recognized.
+            files=[IndexFile(path="index.idx", size_bytes=0)],
+        ),
+    )
+
+    described = ds.describe_indices()
+    assert len(described) == 1
+    assert described[0].index_type == "Vector"
+
+    with pytest.warns(DeprecationWarning):
+        listed = ds.list_indices()
+    assert listed[0]["type"] == "Vector"
+
+
 def test_indexed_scalar_scan(indexed_dataset: lance.LanceDataset, data_table: pa.Table):
     sample_meta = data_table["meta"][50]
     expected_price = data_table["price"][50]
@@ -483,7 +648,10 @@ def make_fts_search(ds):
     assert "ScalarIndexQuery" in plan
     assert "MaterializeIndex" not in plan
     assert "FlatMatchQuery" in plan
-    assert "LanceScan" in plan
+    # Flat FTS now reads via FilteredReadExec (prints as `LanceRead`) so the
+    # BTree on `id` pushes into the unindexed-fragment scan too.
+    assert "LanceRead" in plan
+    assert "LanceScan" not in plan
     assert make_fts_search(ds).to_table().num_rows == 12
 
     # Update vector index but NOT scalar index
@@ -703,6 +871,51 @@ def test_fts_custom_stop_words(tmp_path):
     assert len(results["_rowid"].to_pylist()) == 1
 
 
+def test_fts_stop_words_respect_language_for_simple_tokenizer(tmp_path):
+    data = pa.table({"text": ["the lance data", "的 lance data"]})
+    ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+    ds.create_scalar_index(
+        "text",
+        "INVERTED",
+        base_tokenizer="simple",
+        stem=False,
+    )
+
+    results = ds.to_table(full_text_query="the", with_row_id=True)
+    assert results.num_rows == 0
+
+    results = ds.to_table(full_text_query="的", with_row_id=True)
+    assert results["text"].to_pylist() == ["的 lance data"]
+
+
+def test_fts_icu_stop_words_are_all_or_none(tmp_path):
+    data = pa.table({"text": ["the 的 lance data", "useful data"]})
+    ds = lance.write_dataset(data, tmp_path / "enabled", mode="overwrite")
+    ds.create_scalar_index(
+        "text",
+        "INVERTED",
+        base_tokenizer="icu",
+        stem=False,
+        remove_stop_words=True,
+    )
+
+    assert ds.to_table(full_text_query="the", with_row_id=True).num_rows == 0
+    assert ds.to_table(full_text_query="的", with_row_id=True).num_rows == 0
+    assert ds.to_table(full_text_query="lance", with_row_id=True).num_rows == 1
+
+    ds = lance.write_dataset(data, tmp_path / "disabled", mode="overwrite")
+    ds.create_scalar_index(
+        "text",
+        "INVERTED",
+        base_tokenizer="icu",
+        stem=False,
+        remove_stop_words=False,
+    )
+
+    assert ds.to_table(full_text_query="the", with_row_id=True).num_rows == 1
+    assert ds.to_table(full_text_query="的", with_row_id=True).num_rows == 1
+
+
 def test_rowid_order(dataset):
     dataset.create_scalar_index("doc", index_type="INVERTED", with_position=False)
     results = dataset.scanner(
@@ -736,6 +949,39 @@ def test_create_scalar_index_fts_alias(dataset):
     assert any(idx.index_type == "Inverted" for idx in dataset.describe_indices())
 
 
+def test_create_scalar_index_fts_block_size(dataset):
+    dataset.create_scalar_index(
+        "doc", index_type="INVERTED", with_position=False, block_size=256
+    )
+    indices = dataset.describe_indices()
+    doc_index = next(index for index in indices if index.name == "doc_idx")
+    assert doc_index.segments[0].index_version == 3
+
+    row = dataset.take(indices=[0], columns=["doc"])
+    query = row.column(0)[0].as_py().split(" ")[0]
+    results = dataset.scanner(columns=["doc"], full_text_query=query).to_table()
+    assert results.num_rows > 0
+
+    with pytest.raises(ValueError, match="block_size"):
+        dataset.create_scalar_index(
+            "doc", index_type="INVERTED", name="doc_invalid_129", block_size=129
+        )
+
+    with pytest.raises(ValueError, match="block_size"):
+        dataset.create_scalar_index(
+            "doc", index_type="INVERTED", name="doc_invalid_512", block_size=512
+        )
+
+    with pytest.raises(ValueError, match="block_size=256"):
+        dataset.create_scalar_index(
+            "doc",
+            index_type="INVERTED",
+            name="doc_invalid_v2_256",
+            block_size=256,
+            format_version=2,
+        )
+
+
 def test_multi_index_create(tmp_path):
     dataset = lance.write_dataset(
         pa.table({"ints": range(1024)}), tmp_path, max_rows_per_file=100
@@ -1656,6 +1902,55 @@ def test_icu_tokenizer(tmp_path):
     assert results["_rowid"].to_pylist() == [0]
 
 
+def test_icu_tokenizer_split_on_non_alphanumeric_default(tmp_path):
+    data = pa.table({"text": ["hello_world"]})
+    ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+    ds.create_scalar_index(
+        "text",
+        "INVERTED",
+        base_tokenizer="icu",
+        stem=False,
+        remove_stop_words=False,
+    )
+
+    results = ds.to_table(full_text_query="hello", prefilter=True, with_row_id=True)
+    assert results.num_rows == 0
+
+    results = ds.to_table(
+        full_text_query="hello_world", prefilter=True, with_row_id=True
+    )
+    assert results["_rowid"].to_pylist() == [0]
+
+
+def test_icu_tokenizer_split_on_non_alphanumeric(tmp_path):
+    data = pa.table(
+        {
+            "text": [
+                "hello_world こんにちは世界",
+                "alpha.beta",
+            ],
+        }
+    )
+    ds = lance.write_dataset(data, tmp_path, mode="overwrite")
+    ds.create_scalar_index(
+        "text",
+        "INVERTED",
+        base_tokenizer="icu/split",
+        stem=False,
+        remove_stop_words=False,
+    )
+
+    for query, expected_row_ids in [
+        ("hello", [0]),
+        ("world", [0]),
+        ("世界", [0]),
+        ("alpha", [1]),
+        ("beta", [1]),
+    ]:
+        results = ds.to_table(full_text_query=query, prefilter=True, with_row_id=True)
+        assert results["_rowid"].to_pylist() == expected_row_ids
+
+
 def test_jieba_invalid_user_dict_tokenizer(tmp_path):
     set_language_model_path()
     data = pa.table(
@@ -1903,6 +2198,87 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     assert small_bytes_read < large_bytes_read
 
 
+@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"])
+def test_address_domain_index_with_stable_row_ids(tmp_path: Path, index_type):
+    """Regression test for issue #7434.
+
+    Address-domain scalar indices (zonemap, bloom filter) report matches as
+    physical row addresses. On a stable-row-id dataset a row's stable id differs
+    from its physical address in every fragment except fragment 0, so the index
+    result must be translated back to the row-id domain before it prefilters the
+    scan. Without that translation the index silently drops matching rows in
+    fragments other than fragment 0 (often returning an empty result).
+    """
+
+    # A single value "a" is placed in non-adjacent fragments (0, 2, 4) so its
+    # matching rows span multiple fragments and diverge from fragment 0.
+    def block(v, n):
+        return [v] * n
+
+    vals = (
+        block("a", 5_000)
+        + block("b", 5_000)
+        + block("a", 5_000)
+        + block("c", 5_000)
+        + block("a", 5_000)
+    )
+    tbl = pa.table({"category": pa.array(vals, pa.string()), "id": range(len(vals))})
+    ds = lance.write_dataset(
+        tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True
+    )
+    assert ds.has_stable_row_ids
+    assert len(ds.get_fragments()) == 5
+
+    true_count = ds.scanner(
+        filter="category = 'a'", use_scalar_index=False
+    ).count_rows()
+    assert true_count == 15_000
+
+    ds.create_scalar_index("category", index_type=index_type, replace=True)
+
+    # The index must be consulted and must return the same rows as a full scan.
+    assert "ScalarIndexQuery" in ds.scanner(filter="category = 'a'").explain_plan()
+    indexed = ds.to_table(filter="category = 'a'", columns=["id"])
+    assert indexed.num_rows == true_count
+    assert sorted(indexed["id"].to_pylist()) == sorted(
+        ds.to_table(filter="category = 'a'", columns=["id"], use_scalar_index=False)[
+            "id"
+        ].to_pylist()
+    )
+
+
+def test_zonemap_with_stable_row_ids_after_compaction(tmp_path: Path):
+    """Zonemap results stay correct after a compaction relocates rows under
+    stable row ids (physical address != stable id for the surviving rows)."""
+    ds = lance.write_dataset(
+        pa.table({"x": range(0, 5_000)}),
+        tmp_path,
+        max_rows_per_file=5_000,
+        enable_stable_row_ids=True,
+    )
+    ds = lance.write_dataset(
+        pa.table({"x": range(5_000, 10_000)}),
+        tmp_path,
+        mode="append",
+        max_rows_per_file=5_000,
+        enable_stable_row_ids=True,
+    )
+    # Delete part of the first fragment, then compact so surviving rows keep
+    # their small stable ids but move to a freshly numbered fragment.
+    ds.delete("x >= 1000 AND x < 2000")
+    ds.optimize.compact_files(target_rows_per_fragment=100_000)
+    ds = lance.dataset(tmp_path)
+    assert len(ds.get_fragments()) == 1
+
+    ds.create_scalar_index("x", index_type="ZONEMAP")
+
+    filter_expr = "x >= 6000 AND x <= 6500"
+    assert "ScalarIndexQuery" in ds.scanner(filter=filter_expr).explain_plan()
+    expected = ds.to_table(filter=filter_expr, use_scalar_index=False)["x"].to_pylist()
+    actual = ds.to_table(filter=filter_expr)["x"].to_pylist()
+    assert sorted(actual) == sorted(expected) == list(range(6000, 6501))
+
+
 def test_zonemap_deletion_handling(tmp_path: Path):
     """Test zonemap deletion handling"""
     data = pa.table(
@@ -1932,6 +2308,33 @@ def test_zonemap_deletion_handling(tmp_path: Path):
     assert ids == [0, 2, 4, 6, 8]
 
 
+@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"])
+def test_address_domain_index_not_query_with_stable_row_ids(tmp_path: Path, index_type):
+    """Regression test: != queries return correct results on stable-row-id datasets.
+
+    Address-domain indices (zonemap, bloom filter) search returns physical row
+    addresses. Translation to row IDs happens at the Query leaf before the NOT
+    node is evaluated, so the NOT operates on a correctly translated AllowList.
+    Without the address-to-row-id translation the AllowList contains wrong IDs,
+    and the subsequent NOT excludes the wrong rows.
+    """
+    vals = list(range(5_000)) + list(range(5_000, 10_000))
+    tbl = pa.table({"x": vals})
+    ds = lance.write_dataset(
+        tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True
+    )
+    assert ds.has_stable_row_ids
+
+    ds.create_scalar_index("x", index_type=index_type, replace=True)
+
+    # Without address translation the NOT excludes the wrong rows, producing an
+    # incorrect result set rather than crashing.
+    expected = ds.to_table(filter="x != 42", use_scalar_index=False)["x"].to_pylist()
+    actual = ds.to_table(filter="x != 42")["x"].to_pylist()
+    assert sorted(actual) == sorted(expected)
+    assert len(actual) == 9_999
+
+
 def test_zonemap_index_remapping(tmp_path: Path):
     """Test zonemap index remapping after compaction and optimization"""
     # Create a dataset with 5 fragments by writing data in chunks
@@ -2531,6 +2934,15 @@ def scan_stats_callback(stats: lance.ScanStatistics):
     cache_entries_after_query = ds._ds.index_cache_entry_count()
     assert cache_entries_after_query == cache_entries_after_prewarm
 
+    segment_uuid = ds.describe_indices()[0].segments[0].uuid
+    ds = lance.dataset(phrase_path)
+    ds.prewarm_index("fts_idx", with_position=True, index_segments=[segment_uuid])
+    cache_entries_after_prewarm = ds._ds.index_cache_entry_count()
+    results = ds.to_table(full_text_query=PhraseQuery("word word", "fts"))
+    assert results.num_rows == test_table_size
+    cache_entries_after_query = ds._ds.index_cache_entry_count()
+    assert cache_entries_after_query == cache_entries_after_prewarm
+
     with pytest.raises(
         TypeError,
         match="takes 2 positional arguments",
@@ -3212,6 +3624,52 @@ def test_build_distributed_fts_index_basic(tmp_path):
     assert results.num_rows > 0, "No results found for search term 'frodo'"
 
 
+@pytest.mark.parametrize("index_type", ["INVERTED", "FTS"])
+def test_segment_fts(tmp_path, index_type):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=3, rows_per_fragment=100
+    )
+
+    index_name = f"text_{index_type.lower()}_segment_idx"
+    segments = [
+        ds.create_index_uncommitted(
+            column="text",
+            index_type=index_type,
+            name=index_name,
+            fragment_ids=[fragment.fragment_id],
+            with_position=False,
+            remove_stop_words=False,
+        )
+        for fragment in ds.get_fragments()
+    ]
+    committed_ds = ds.commit_existing_index_segments(index_name, "text", segments)
+
+    query = MatchQuery("frodo", "text")
+    results_without_index = committed_ds.scanner(
+        full_text_query=query,
+        columns=["id", "text"],
+        use_scalar_index=False,
+    ).to_table()
+    results_with_index = committed_ds.scanner(
+        full_text_query=query,
+        columns=["id", "text"],
+        use_scalar_index=True,
+    ).to_table()
+
+    compare_fts_results(results_without_index, results_with_index)
+    assert any(
+        idx.name == index_name and idx.index_type == "Inverted"
+        for idx in committed_ds.describe_indices()
+    )
+    assert (
+        "FlatMatchQuery"
+        not in committed_ds.scanner(
+            full_text_query=query,
+            use_scalar_index=True,
+        ).explain_plan()
+    )
+
+
 def test_compare_fts_results_identical(tmp_path):
     """
     Test compare_fts_results function with identical results.
@@ -3783,41 +4241,52 @@ def test_distribute_btree_index_build(tmp_path):
     )
 
 
-def _assert_committed_distributed_bitmap_index(ds, index_id, index_name, fragment_ids):
-    ds.merge_index_metadata(index_id, index_type="BITMAP")
+def test_bitmap_uncommitted_segments_can_be_committed_from_python(tmp_path):
+    dataset_path = tmp_path / "bitmap_segments.lance"
+    ds = generate_multi_fragment_bitmap_dataset(
+        dataset_path, num_fragments=4, rows_per_fragment=40
+    )
 
-    from lance.dataset import Index
+    index_name = "bitmap_segment_idx"
+    fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()]
+    fragment_groups = [
+        fragment_ids[idx : idx + 2] for idx in range(0, len(fragment_ids), 2)
+    ]
+    assert len(fragment_groups) >= 2
 
-    field_id = ds.schema.get_field_index("category")
-    index = Index(
-        uuid=index_id,
-        name=index_name,
-        fields=[field_id],
-        dataset_version=ds.version,
-        fragment_ids=set(fragment_ids),
-        index_version=0,
-    )
-    create_index_op = lance.LanceOperation.CreateIndex(
-        new_indices=[index],
-        removed_indices=[],
-    )
-    lance.LanceDataset.commit(
-        ds.uri,
-        create_index_op,
-        read_version=ds.version,
-    )
-    reopened_ds = lance.dataset(ds.uri)
+    staged_segments = [
+        ds.create_index_uncommitted(
+            column="category",
+            index_type="BITMAP",
+            name=index_name,
+            fragment_ids=fragment_group,
+        )
+        for fragment_group in fragment_groups
+    ]
+
+    assert len({segment.uuid for segment in staged_segments}) == len(staged_segments)
+    for segment, fragment_group in zip(staged_segments, fragment_groups):
+        assert segment.fragment_ids == set(fragment_group)
+        assert any(file.path == "bitmap_page_lookup.lance" for file in segment.files)
+        assert all(not file.path.startswith("part_") for file in segment.files)
 
-    stats = reopened_ds.stats.index_stats(index_name)
-    assert stats["index_type"] == "Bitmap"
+    merged_segment = ds.merge_existing_index_segments(staged_segments)
+    assert merged_segment.uuid not in {segment.uuid for segment in staged_segments}
+    assert merged_segment.fragment_ids == set(fragment_ids)
+    assert any(file.path == "bitmap_page_lookup.lance" for file in merged_segment.files)
+    assert all(not file.path.startswith("part_") for file in merged_segment.files)
+
+    ds = ds.commit_existing_index_segments(index_name, "category", [merged_segment])
+    descriptions = {index.name: index for index in ds.describe_indices()}
+    assert len(descriptions[index_name].segments) == 1
 
     filter_expr = "category = 3"
-    without_index = reopened_ds.scanner(
+    without_index = ds.scanner(
         filter=filter_expr,
         columns=["id", "category"],
         use_scalar_index=False,
     ).to_table()
-    with_index = reopened_ds.scanner(
+    with_index = ds.scanner(
         filter=filter_expr,
         columns=["id", "category"],
         use_scalar_index=True,
@@ -3826,77 +4295,83 @@ def _assert_committed_distributed_bitmap_index(ds, index_id, index_name, fragmen
     assert with_index.num_rows == without_index.num_rows
     assert with_index["id"].to_pylist() == without_index["id"].to_pylist()
     assert set(with_index["category"].to_pylist()) == {3}
-
-    explain = reopened_ds.scanner(
-        filter=filter_expr,
-        use_scalar_index=True,
-    ).explain_plan()
-    assert "ScalarIndexQuery" in explain
-
-    empty_without_index = reopened_ds.scanner(
-        filter="category = 99",
-        use_scalar_index=False,
-    ).to_table()
-    empty_with_index = reopened_ds.scanner(
-        filter="category = 99",
-        use_scalar_index=True,
-    ).to_table()
-    assert empty_with_index.num_rows == empty_without_index.num_rows == 0
+    assert (
+        "ScalarIndexQuery"
+        in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan()
+    )
 
 
-def test_distributed_bitmap_index_build(tmp_path):
-    ds = generate_multi_fragment_bitmap_dataset(
-        tmp_path / "bitmap_dist.lance", num_fragments=4, rows_per_fragment=40
+def test_zonemap_fragment_ids_parameter_validation(tmp_path):
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=2, rows_per_fragment=100
     )
 
-    index_id = str(uuid.uuid4())
-    index_name = "bitmap_multiple_fragment_idx"
-    fragments = ds.get_fragments()
-    fragment_ids = [fragment.fragment_id for fragment in fragments]
-    fragment_groups = [
-        fragment_ids[idx : idx + 2] for idx in range(0, len(fragment_ids), 2)
-    ]
-    assert len(fragment_groups) >= 2
-
-    for shard_id, fragment_group in enumerate(fragment_groups):
+    fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()]
+    with pytest.raises(ValueError, match="create_index_uncommitted"):
         ds.create_scalar_index(
-            column="category",
-            index_type=IndexConfig(
-                index_type="bitmap",
-                parameters={"shard_id": shard_id},
-            ),
-            name=index_name,
-            replace=False,
-            index_uuid=index_id,
-            fragment_ids=fragment_group,
+            column="id",
+            index_type="ZONEMAP",
+            fragment_ids=[fragment_ids[0]],
         )
 
-    _assert_committed_distributed_bitmap_index(ds, index_id, index_name, fragment_ids)
 
-
-def test_distributed_bitmap_index_build_single_fragment_shards(tmp_path):
-    ds = generate_multi_fragment_bitmap_dataset(
-        tmp_path / "bitmap_single_fragment_dist.lance",
-        num_fragments=4,
-        rows_per_fragment=40,
+def test_zonemap_segment_merge_and_commit_from_python(tmp_path):
+    rows_per_fragment = 20_000
+    ds = generate_multi_fragment_dataset(
+        tmp_path, num_fragments=4, rows_per_fragment=rows_per_fragment
     )
 
-    index_id = str(uuid.uuid4())
-    index_name = "bitmap_single_fragment_idx"
+    index_name = "id_zonemap_segments"
     fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()]
-    assert len(fragment_ids) >= 2
-
-    for fragment_id in fragment_ids:
-        ds.create_scalar_index(
-            column="category",
-            index_type="BITMAP",
+    staged_segments = [
+        ds.create_index_uncommitted(
+            column="id",
+            index_type="ZONEMAP",
             name=index_name,
-            replace=False,
-            index_uuid=index_id,
             fragment_ids=[fragment_id],
         )
+        for fragment_id in fragment_ids
+    ]
+
+    assert len({segment.uuid for segment in staged_segments}) == len(staged_segments)
+    for segment, fragment_id in zip(staged_segments, fragment_ids):
+        files = segment.files
+        assert files is not None
+        assert segment.fragment_ids == {fragment_id}
+        assert any(file.path == "zonemap.lance" for file in files)
+        assert all(not file.path.startswith("part_") for file in files)
+
+    merged_segment = ds.merge_existing_index_segments(staged_segments)
+    merged_files = merged_segment.files
+    assert merged_files is not None
+    assert merged_segment.uuid not in {segment.uuid for segment in staged_segments}
+    assert merged_segment.fragment_ids == set(fragment_ids)
+    assert any(file.path == "zonemap.lance" for file in merged_files)
+    assert all(not file.path.startswith("part_") for file in merged_files)
+
+    ds = ds.commit_existing_index_segments(index_name, "id", [merged_segment])
+    descriptions = {index.name: index for index in ds.describe_indices()}
+    assert descriptions[index_name].index_type == "ZoneMap"
+    assert len(descriptions[index_name].segments) == 1
+
+    filter_expr = "id >= 8200 AND id < 8300"
+    without_index = ds.scanner(
+        filter=filter_expr,
+        columns=["id", "text"],
+        use_scalar_index=False,
+    ).to_table()
+    with_index = ds.scanner(
+        filter=filter_expr,
+        columns=["id", "text"],
+        use_scalar_index=True,
+    ).to_table()
 
-    _assert_committed_distributed_bitmap_index(ds, index_id, index_name, fragment_ids)
+    assert with_index.num_rows == without_index.num_rows
+    assert with_index["id"].to_pylist() == without_index["id"].to_pylist()
+    assert (
+        "ScalarIndexQuery"
+        in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan()
+    )
 
 
 def test_merge_index_metadata_btree_soft_break(tmp_path):
@@ -4243,7 +4718,7 @@ def test_nested_field_btree_index(tmp_path):
     # Verify index was created
     indices = dataset.describe_indices()
     assert len(indices) == 1
-    assert indices[0].field_names == ["lang"]
+    assert indices[0].field_names == ["meta.lang"]
     assert indices[0].index_type == "BTree"
 
     # Test query using the index - filter for English language
@@ -4344,7 +4819,7 @@ def test_nested_field_fts_index(tmp_path):
     # Verify index was created
     indices = ds.describe_indices()
     assert len(indices) == 1
-    assert indices[0].field_names == ["text"]
+    assert indices[0].field_names == ["data.text"]
     assert indices[0].index_type == "Inverted"
 
     # Test full text search on nested field
@@ -4394,6 +4869,77 @@ def test_nested_field_fts_index(tmp_path):
     assert results.num_rows == 50
 
 
+def test_multiple_nested_field_fts_indices_e2e(tmp_path):
+    """Test FTS queries against multiple indexed nested string fields."""
+
+    def make_table(ids, text_values, summary_values):
+        return pa.table(
+            {
+                "id": ids,
+                "data": pa.StructArray.from_arrays(
+                    [
+                        pa.array(text_values, type=pa.string()),
+                        pa.array(summary_values, type=pa.string()),
+                    ],
+                    names=["text", "summary"],
+                ),
+            }
+        )
+
+    def result_ids(query):
+        return sorted(ds.to_table(full_text_query=query)["id"].to_pylist())
+
+    ds = lance.write_dataset(
+        make_table(
+            [0, 1, 2, 3],
+            [
+                "lance nested alpha",
+                "plain text",
+                None,
+                "phrase target here",
+            ],
+            [
+                "metadata only",
+                "database nested beta",
+                "lance beta",
+                "other",
+            ],
+        ),
+        tmp_path,
+    )
+
+    ds.create_scalar_index("data.text", index_type="INVERTED", with_position=True)
+    ds.create_scalar_index("data.summary", index_type="INVERTED", with_position=False)
+
+    indexed_fields = {
+        tuple(index.field_names)
+        for index in ds.describe_indices()
+        if index.index_type == "Inverted"
+    }
+    assert indexed_fields == {("data.text",), ("data.summary",)}
+
+    assert result_ids(MatchQuery("alpha", "data.text")) == [0]
+    assert result_ids(MatchQuery("beta", "data.summary")) == [1, 2]
+    assert result_ids("lance") == [0, 2]
+    assert result_ids(MultiMatchQuery("nested", ["data.text", "data.summary"])) == [
+        0,
+        1,
+    ]
+    assert result_ids(PhraseQuery("phrase target", "data.text")) == [3]
+
+    ds = lance.write_dataset(
+        make_table(
+            [4, 5],
+            ["fresh lance append", "plain append"],
+            ["other", "fresh beta append"],
+        ),
+        tmp_path,
+        mode="append",
+    )
+
+    assert result_ids("fresh") == [4, 5]
+
+
 def test_nested_field_bitmap_index(tmp_path):
     """Test BITMAP index creation and querying on nested fields"""
     # Create dataset with nested categorical field
@@ -4418,7 +4964,7 @@ def test_nested_field_bitmap_index(tmp_path):
     # Verify index was created
     indices = ds.describe_indices()
     assert len(indices) == 1
-    assert indices[0].field_names == ["color"]
+    assert indices[0].field_names == ["attributes.color"]
     assert indices[0].index_type == "Bitmap"
 
     # Test equality query
@@ -4526,9 +5072,11 @@ def test_json_inverted_match_query(tmp_path):
     assert results.num_rows == 1
 
 
-@pytest.mark.parametrize("fts_format_version", ["1", "2"])
-def test_describe_indices(tmp_path, monkeypatch, fts_format_version):
-    monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", fts_format_version)
+@pytest.mark.parametrize(
+    ("format_version", "expected_format_version"),
+    [(1, 1), (2, 2), ("v1", 1), ("v2", 2)],
+)
+def test_describe_indices(tmp_path, format_version, expected_format_version):
     data = pa.table(
         {
             "id": range(100),
@@ -4544,7 +5092,7 @@ def test_describe_indices(tmp_path, monkeypatch, fts_format_version):
         }
     )
     ds = lance.write_dataset(data, tmp_path)
-    ds.create_scalar_index("text", index_type="INVERTED")
+    ds.create_scalar_index("text", index_type="INVERTED", format_version=format_version)
     indices = ds.describe_indices()
     assert len(indices) == 1
 
@@ -4558,7 +5106,7 @@ def test_describe_indices(tmp_path, monkeypatch, fts_format_version):
     assert indices[0].segments[0].uuid is not None
     assert indices[0].segments[0].fragment_ids == {0}
     assert indices[0].segments[0].dataset_version_at_last_update == 1
-    assert indices[0].segments[0].index_version == int(fts_format_version)
+    assert indices[0].segments[0].index_version == expected_format_version
     assert indices[0].segments[0].created_at is not None
     assert isinstance(indices[0].segments[0].created_at, datetime)
     assert indices[0].segments[0].size_bytes is not None
@@ -4657,6 +5205,28 @@ def test_describe_indices(tmp_path, monkeypatch, fts_format_version):
         assert index.num_rows_indexed == 50
 
 
+def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypatch):
+    monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", "1")
+    data = pa.table({"text": ["document about lance database"]})
+    ds = lance.write_dataset(data, tmp_path)
+
+    ds.create_scalar_index("text", index_type="INVERTED")
+
+    indices = ds.describe_indices()
+    assert indices[0].segments[0].index_version == 2
+
+
+def test_create_inverted_index_rejects_invalid_format_version(tmp_path):
+    data = pa.table({"text": ["document about lance database"]})
+    ds = lance.write_dataset(data, tmp_path)
+
+    with pytest.raises(ValueError, match="unsupported FTS format version"):
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v4")
+
+    with pytest.raises(ValueError, match="format_version=3"):
+        ds.create_scalar_index("text", index_type="INVERTED", format_version="v3")
+
+
 def test_vector_filter_fts_search(tmp_path):
     # Create test data
     ids = list(range(1, 301))
@@ -4774,3 +5344,28 @@ def test_vector_filter_fts_search(tmp_path):
     )
     with pytest.raises(ValueError):
         scanner.to_table()
+
+
+@pytest.mark.parametrize("index_type", ["BTREE", "BITMAP", "ZONEMAP"])
+def test_large_string_scalar_index(tmp_path, index_type):
+    """large_string (LargeUtf8) must be accepted by BTREE, BITMAP, and ZONEMAP."""
+    table = pa.table(
+        {
+            "id": pa.array([1, 2, 3], pa.int32()),
+            "category": pa.array(["alpha", "beta", "alpha"], pa.large_string()),
+        }
+    )
+    ds = lance.write_dataset(table, tmp_path)
+    assert ds.schema.field("category").type == pa.large_string()
+
+    # Must not raise TypeError
+    ds.create_scalar_index("category", index_type=index_type)
+
+    indices = ds.describe_indices()
+    assert any("category" in idx.field_names for idx in indices), (
+        f"{index_type} index for large_string column not found in describe_indices()"
+    )
+
+    result = ds.scanner(filter="category = 'alpha'").to_table()
+    assert result.num_rows == 2
+    assert set(result.column("id").to_pylist()) == {1, 3}
diff --git a/python/python/tests/test_schema.py b/python/python/tests/test_schema.py
index fcff283ebe2..c384466082f 100644
--- a/python/python/tests/test_schema.py
+++ b/python/python/tests/test_schema.py
@@ -6,6 +6,7 @@
 
 import lance
 import pyarrow as pa
+import pytest
 from lance.schema import LanceSchema
 
 
@@ -60,3 +61,15 @@ def test_lance_schema(tmp_path: Path):
     s_fields = fields[1].children()
     assert s_fields[0].name() == "new_name"
     assert s_fields[0].id() == 2
+
+
+def test_lance_schema_from_protos_rejects_missing_parent():
+    # name (field 2): child; id (field 3): 7; parent_id (field 4): 42;
+    # logical_type (field 5): int32.
+    field_proto = b"\x12\x05child\x18\x07\x20\x2a\x2a\x05int32"
+
+    with pytest.raises(
+        ValueError,
+        match="Field 'child' \\(id=7\\) references parent id 42",
+    ):
+        LanceSchema._from_protos("{}", field_proto)
diff --git a/python/python/tests/test_schema_evolution.py b/python/python/tests/test_schema_evolution.py
index 205aaa4fa66..7df6962789e 100644
--- a/python/python/tests/test_schema_evolution.py
+++ b/python/python/tests/test_schema_evolution.py
@@ -6,6 +6,8 @@
 from pathlib import Path
 
 import lance
+import lance.dependencies as dependencies
+import lance.udf as udf_module
 import numpy as np
 import pandas as pd
 import pyarrow as pa
@@ -287,6 +289,37 @@ def mapper(batch: pa.RecordBatch):
     check_add_columns(dataset, expected, use_fragments, mapper)
 
 
+@pytest.mark.parametrize("use_batch_udf", [False, True])
+def test_add_columns_arrow_udf_without_pandas_dependency(
+    tmp_path: Path, monkeypatch, use_batch_udf
+):
+    table = pa.table({"caption": ["a Shutterstock photo", "clean"]})
+    dataset = lance.write_dataset(table, tmp_path)
+
+    def mapper(batch: pa.RecordBatch) -> pa.RecordBatch:
+        flags = pc.match_substring_regex(
+            pc.utf8_lower(batch["caption"]), "shutterstock"
+        )
+        return pa.record_batch([flags], names=["wm"])
+
+    if use_batch_udf:
+        mapper = lance.batch_udf()(mapper)
+
+    # CI has pandas installed, so simulate Lance's no-pandas lazy proxy state.
+    # Without the guard, accessing pd.DataFrame raises ModuleNotFoundError.
+    monkeypatch.setattr(dependencies, "_PANDAS_AVAILABLE", False)
+    monkeypatch.setattr(
+        udf_module,
+        "pd",
+        dependencies._LazyModule("pandas", module_available=False),
+    )
+
+    dataset.add_columns(mapper, read_columns=["caption"], batch_size=64)
+
+    expected = table.append_column("wm", pa.array([True, False]))
+    assert dataset.to_table() == expected
+
+
 def test_query_after_merge(tmp_path):
     # https://github.com/lancedb/lance/issues/1905
     tab = pa.table(
diff --git a/python/python/tests/test_tf.py b/python/python/tests/test_tf.py
deleted file mode 100644
index 3652df0e938..00000000000
--- a/python/python/tests/test_tf.py
+++ /dev/null
@@ -1,351 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The Lance Authors
-
-import os
-import warnings
-
-import lance
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-import pytest
-from lance.arrow import ImageArray
-from lance.fragment import LanceFragment
-
-pytest.skip("Skip tensorflow tests", allow_module_level=True)
-
-try:
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", category=DeprecationWarning)
-        import tensorflow as tf  # noqa: F401
-except ImportError:
-    pytest.skip(
-        "Tensorflow is not installed. Please install tensorflow to "
-        + "test lance.tf module.",
-        allow_module_level=True,
-    )
-
-from lance.tf.data import (  # noqa: E402
-    from_lance,
-    from_lance_batches,
-    lance_fragments,
-    lance_take_batches,
-)
-
-
-@pytest.fixture
-def tf_dataset(tmp_path):
-    df = pd.DataFrame(
-        {
-            "a": range(10000),
-            "s": [f"val-{i}" for i in range(10000)],
-            "vec": [[i * 0.2] * 128 for i in range(10000)],
-        }
-    )
-
-    schema = pa.schema(
-        [
-            pa.field("a", pa.int64()),
-            pa.field("s", pa.string()),
-            pa.field("vec", pa.list_(pa.float32(), 128)),
-        ]
-    )
-    tbl = pa.Table.from_pandas(df, schema=schema)
-    uri = tmp_path / "dataset.lance"
-    lance.write_dataset(
-        tbl,
-        uri,
-        schema=tbl.schema,
-        max_rows_per_group=100,
-        max_rows_per_file=1000,
-    )
-    return uri
-
-
-def test_fragment_dataset(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8")
-        assert batch["a"].shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_projection(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100, columns=["a"])
-
-    for idx, batch in enumerate(ds):
-        assert list(batch.keys()) == ["a"]
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["a"].shape == (100,)
-
-
-def test_filter(tf_dataset):
-    ds = from_lance(tf_dataset, batch_size=100, filter="a >= 5000")
-
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100 + 5000
-        assert batch["a"].shape == (100,)
-
-
-def test_namespace_table_id(monkeypatch):
-    calls = {}
-
-    class DummyScanner:
-        def __init__(self):
-            self._batch = pa.record_batch([pa.array([1, 2])], names=["a"])
-            self.projected_schema = self._batch.schema
-
-        def to_batches(self):
-            yield self._batch
-
-    class DummyDataset:
-        def scanner(self, **kwargs):
-            return DummyScanner()
-
-    def fake_dataset(uri=None, **kwargs):
-        calls["uri"] = uri
-        calls["kwargs"] = kwargs
-        return DummyDataset()
-
-    monkeypatch.setattr(lance, "dataset", fake_dataset)
-
-    ns = object()
-    ds = from_lance(
-        None,
-        namespace_client=ns,
-        table_id=["tbl"],
-        ignore_namespace_table_storage_options=True,
-    )
-
-    assert calls["kwargs"]["namespace_client"] is ns
-    assert calls["kwargs"]["table_id"] == ["tbl"]
-    assert calls["kwargs"]["ignore_namespace_table_storage_options"] is True
-
-    batches = list(ds)
-    assert [b["a"].numpy().tolist() for b in batches] == [[1, 2]]
-
-
-def test_scan_use_tf_data(tf_dataset):
-    ds = tf.data.Dataset.from_lance(tf_dataset)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8")
-        assert batch["a"].shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_pass_fragments(tf_dataset):
-    # Can pass fragments directly to from_lance
-    dataset = lance.dataset(tf_dataset)
-    ds = from_lance(tf_dataset, fragments=dataset.get_fragments(), batch_size=100)
-    ds_default = from_lance(tf_dataset, batch_size=100)
-    for batch, batch_default in zip(ds, ds_default):
-        assert batch["a"].numpy()[0] == batch_default["a"].numpy()[0]
-        assert batch["a"].numpy().shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )
-
-    # Can pass ids directly to from_lance
-    ds = from_lance(tf_dataset, fragments=[0, 1, 2], batch_size=100)
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 100
-        assert batch["a"].numpy().shape == (100,)
-
-
-def test_shuffle(tf_dataset):
-    fragments = lance_fragments(tf_dataset).shuffle(4, seed=20).take(3)
-
-    ds = from_lance(tf_dataset, fragments=fragments, batch_size=100)
-    raw_ds = lance.dataset(tf_dataset)
-    scanner = raw_ds.scanner(
-        fragments=[LanceFragment(raw_ds, fid) for fid in [0, 3, 1]], batch_size=100
-    )
-
-    for batch, raw_batch in zip(ds, scanner.to_batches()):
-        assert batch["a"].numpy()[0] == raw_batch.to_pydict()["a"][0]
-        assert batch["a"].numpy().shape == (100,)
-        assert batch["vec"].shape == (
-            100,
-            128,
-        )  # Fixed size list
-
-
-def test_dataset_batches(tf_dataset):
-    tf_dataset = lance.dataset(tf_dataset)
-    batch_size = 300
-    batches = list(
-        from_lance_batches(tf_dataset, batch_size=batch_size).as_numpy_iterator()
-    )
-    assert tf_dataset.count_rows() // batch_size + 1 == len(batches)
-    assert all(end - start == batch_size for start, end in batches[:-2])
-    assert batches[-1][1] - batches[-1][0] == tf_dataset.count_rows() % batch_size
-
-    skip = 5
-    batches_skipped = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, skip=skip
-        ).as_numpy_iterator()
-    )
-    assert batches_skipped == batches[skip:]
-
-    batches_shuffled = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, shuffle=True, seed=42
-        ).as_numpy_iterator()
-    )
-    # make sure it does a shuffle
-    assert batches_shuffled != batches
-    batches_shuffled2 = list(
-        from_lance_batches(
-            tf_dataset, batch_size=batch_size, shuffle=True, seed=42
-        ).as_numpy_iterator()
-    )
-    # make sure the shuffle can be deterministic
-    assert batches_shuffled == batches_shuffled2
-
-
-def test_take_dataset(tf_dataset):
-    tf_dataset = lance.dataset(tf_dataset)
-    batch_ds = from_lance_batches(
-        tf_dataset, batch_size=100, shuffle=True, seed=42
-    ).as_numpy_iterator()
-    lance_ds = lance_take_batches(tf_dataset, batch_ds)
-    lance_ds = lance_ds.unbatch().shuffle(400, seed=42).batch(100)
-
-    for batch in lance_ds:
-        assert batch["a"].numpy().shape == (100,)
-
-    batches = [(0, 200), (100, 200)]
-    lance_ds = lance_take_batches(tf_dataset, batches, columns=["a"])
-    for (start, end), batch in zip(batches, lance_ds):
-        assert batch["a"].numpy().tolist() == np.arange(start, end).tolist()
-        assert batch.keys() == {"a"}
-
-
-def test_var_length_list(tmp_path):
-    """Treat var length list as RaggedTensor."""
-    df = pd.DataFrame(
-        {
-            "a": range(200),
-            "l": [[i] * (i % 5 + 1) for i in range(200)],
-        }
-    )
-
-    schema = pa.schema(
-        [
-            pa.field("a", pa.int64()),
-            pa.field("l", pa.list_(pa.int32())),
-        ]
-    )
-    tbl = pa.Table.from_pandas(df, schema=schema)
-
-    uri = tmp_path / "dataset.lance"
-    lance.write_dataset(
-        tbl,
-        uri,
-        schema=tbl.schema,
-    )
-
-    output_signature = {
-        "a": tf.TensorSpec(shape=(None,), dtype=tf.int64),
-        "l": tf.RaggedTensorSpec(dtype=tf.dtypes.int32, shape=(8, None), ragged_rank=1),
-    }
-
-    ds = tf.data.Dataset.from_lance(
-        uri,
-        batch_size=8,
-        output_signature=output_signature,
-    )
-    for idx, batch in enumerate(ds):
-        assert batch["a"].numpy()[0] == idx * 8
-        assert batch["l"].shape == (8, None)
-        assert isinstance(batch["l"], tf.RaggedTensor)
-
-
-def test_nested_struct(tmp_path):
-    table = pa.table(
-        {
-            "x": pa.array(
-                [
-                    {
-                        "a": 1,
-                        "json": {"b": "hello", "x": b"abc"},
-                    },
-                    {
-                        "a": 24,
-                        "json": {"b": "world", "x": b"def"},
-                    },
-                ]
-            )
-        }
-    )
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-
-    ds = tf.data.Dataset.from_lance(
-        dataset,
-        batch_size=8,
-    )
-
-    for batch in ds:
-        tf.debugging.assert_equal(batch["x"]["a"], tf.constant([1, 24], dtype=tf.int64))
-        tf.debugging.assert_equal(
-            batch["x"]["json"]["b"], tf.constant(["hello", "world"])
-        )
-        tf.debugging.assert_equal(
-            batch["x"]["json"]["x"], tf.constant([b"abc", b"def"])
-        )
-
-
-def test_tensor(tmp_path):
-    arr = np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]], dtype=np.float32)
-    table = pa.table({"x": pa.FixedShapeTensorArray.from_numpy_ndarray(arr)})
-
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-    ds = tf.data.Dataset.from_lance(dataset)
-
-    for batch in ds:
-        assert batch["x"].shape == (2, 2, 3)
-        assert batch["x"].dtype == tf.float32
-        assert batch["x"].numpy().tolist() == arr.tolist()
-
-
-def test_image_types(tmp_path):
-    path = [os.path.join(os.path.dirname(__file__), "images/1.png")]
-    uris = ImageArray.from_array(path * 3)
-    encoded_images = uris.read_uris()
-    tensors = encoded_images.to_tensor()
-    table = pa.table(
-        {
-            "uris": uris,
-            "encoded_images": encoded_images,
-            "tensor_images": tensors,
-        }
-    )
-
-    uri = tmp_path / "dataset.lance"
-    dataset = lance.write_dataset(table, uri)
-    ds = tf.data.Dataset.from_lance(dataset)
-
-    for batch in ds:
-        assert batch["uris"].shape == (3,)
-        assert batch["uris"].dtype == tf.string
-        assert batch["uris"].numpy().astype("str").tolist() == uris.tolist()
-
-        assert batch["encoded_images"].shape == (3,)
-        assert batch["encoded_images"].dtype == tf.string
-        assert batch["encoded_images"].numpy().tolist() == encoded_images.tolist()
-
-        assert batch["tensor_images"].shape == (3, 1, 1, 4)
-        assert batch["tensor_images"].dtype == tf.uint8
-        assert batch["tensor_images"].numpy().tolist() == tensors.to_numpy().tolist()
diff --git a/python/python/tests/test_vector.py b/python/python/tests/test_vector.py
index c02c8312f88..12caec5f0d5 100644
--- a/python/python/tests/test_vector.py
+++ b/python/python/tests/test_vector.py
@@ -5,7 +5,12 @@
 import numpy as np
 import pyarrow as pa
 import pytest
-from lance.vector import vec_to_table
+from lance.vector import (
+    get_ivf_partition_info,
+    hamming_clustering_for_ivf_partition,
+    hamming_clustering_for_sample,
+    vec_to_table,
+)
 
 
 def test_dict():
@@ -147,3 +152,129 @@ def test_binary_vectors_invalid_metric(tmp_path):
                 "metric": "l2",
             }
         ).to_table()
+
+
+def _hash_table(hashes):
+    """Build a table with a ``hash`` column of FixedSizeList<UInt8, N>.
+
+    ``hashes`` is a list of byte sequences, one per row. The byte width must
+    be a positive multiple of 8.
+    """
+    byte_width = len(hashes[0])
+    assert byte_width > 0 and byte_width % 8 == 0
+    assert all(len(row) == byte_width for row in hashes)
+    flat = [byte for row in hashes for byte in row]
+    values = pa.FixedSizeListArray.from_arrays(
+        pa.array(flat, type=pa.uint8()), list_size=byte_width
+    )
+    return pa.Table.from_arrays([values], names=["hash"])
+
+
+@pytest.mark.parametrize("byte_width", [8, 16])
+def test_hamming_clustering_for_sample(tmp_path, byte_width):
+    hash_a = [0] * byte_width
+    hash_b = [0] * (byte_width - 8) + [255] + [0] * 7  # 8 bits from hash_a
+    hash_c = list(range(1, byte_width + 1))  # far from both
+    # Rows 0,1,2 share hash_a; rows 3,4 share hash_b; row 5 is unique.
+    table = _hash_table([hash_a, hash_a, hash_a, hash_b, hash_b, hash_c])
+    dataset = lance.write_dataset(table, tmp_path / "hashes")
+
+    # threshold 0 => only exact-match hashes cluster together. Full scan
+    # (sample_size=None) yields deterministic row ids 0..5.
+    result = hamming_clustering_for_sample(dataset, "hash", None, 0).read_all()
+
+    clusters = {
+        rep: sorted(dups)
+        for rep, dups in zip(
+            result["representative"].to_pylist(),
+            result["duplicates"].to_pylist(),
+        )
+    }
+    # Singleton row 5 is not emitted as a cluster.
+    assert clusters == {0: [1, 2], 3: [4]}
+
+
+@pytest.mark.parametrize("byte_width", [8, 16])
+def test_hamming_clustering_multi_segment(tmp_path, byte_width):
+    mask = (1 << 64) - 1
+
+    def hash_bytes(value):
+        if byte_width == 8:
+            lanes = [(value * 0x9E3779B97F4A7C15) & mask]
+        else:
+            # Adjacent logical values share the first 64-bit lane and differ in
+            # later lanes, so threshold-0 clustering must compare every lane.
+            lanes = [
+                ((value // 2) * 0x9E3779B97F4A7C15) & mask,
+                ((value * 0xD6E8FEB86659FD93) ^ 0xA5A5A5A5A5A5A5A5) & mask,
+            ]
+        return [
+            byte for lane_value in lanes for byte in lane_value.to_bytes(8, "little")
+        ]
+
+    # 25 distinct hash values, two copies each; the same table is written to
+    # fragment 0 and appended as fragment 1.
+    values = [i // 2 for i in range(50)]
+    table = _hash_table([hash_bytes(value) for value in values])
+    dataset = lance.write_dataset(table, tmp_path / "hashes")
+    dataset.create_index(
+        "hash", index_type="IVF_FLAT", num_partitions=4, metric="hamming"
+    )
+    dataset = lance.write_dataset(table, tmp_path / "hashes", mode="append")
+    # Optimizing with merge disabled creates a delta segment for fragment 1.
+    dataset.optimize.optimize_indices(num_indices_to_merge=0)
+
+    index = dataset.describe_indices()[0]
+    assert len(index.segments) == 2
+
+    infos = get_ivf_partition_info(dataset, index.name)
+    assert sum(info["size"] for info in infos) == 100
+
+    # All four copies of each value cluster together across both fragments.
+    frag1_start = 1 << 32
+    clusters = []
+    for info in infos:
+        result = hamming_clustering_for_ivf_partition(
+            dataset, index.name, info["partition_id"], 0
+        ).read_all()
+        clusters.extend(
+            zip(
+                result["representative"].to_pylist(),
+                result["duplicates"].to_pylist(),
+            )
+        )
+    assert len(clusters) == 25
+    for representative, duplicates in clusters:
+        assert representative < frag1_start
+        assert len(duplicates) == 3
+        assert any(dup >= frag1_start for dup in duplicates)
+
+    # Selecting the fragment-0 segment reproduces the single-segment scope.
+    first_segment = next(
+        segment for segment in index.segments if segment.fragment_ids == {0}
+    )
+    infos = get_ivf_partition_info(
+        dataset, index.name, index_segments=[first_segment.uuid]
+    )
+    assert sum(info["size"] for info in infos) == 50
+    num_selected_clusters = 0
+    for info in infos:
+        result = hamming_clustering_for_ivf_partition(
+            dataset,
+            index.name,
+            info["partition_id"],
+            0,
+            index_segments=[first_segment.uuid],
+        ).read_all()
+        for duplicates in result["duplicates"].to_pylist():
+            num_selected_clusters += 1
+            assert duplicates == [dup for dup in duplicates if dup < frag1_start]
+            assert len(duplicates) == 1
+    assert num_selected_clusters == 25
+
+    with pytest.raises(ValueError, match="invalid index segment uuid"):
+        get_ivf_partition_info(dataset, index.name, index_segments=["not-a-uuid"])
+    with pytest.raises(TypeError, match="str or uuid.UUID"):
+        get_ivf_partition_info(dataset, index.name, index_segments=[123])
+    with pytest.raises(TypeError, match="not a single"):
+        get_ivf_partition_info(dataset, index.name, index_segments=first_segment.uuid)
diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py
index 9df41dd8300..0cf4290e7a6 100644
--- a/python/python/tests/test_vector_index.py
+++ b/python/python/tests/test_vector_index.py
@@ -23,7 +23,10 @@
 from lance.dataset import VectorIndexReader
 from lance.indices import IndexFileVersion, IndicesBuilder
 from lance.query import MatchQuery, PhraseQuery
-from lance.util import validate_vector_index  # noqa: E402
+from lance.util import (  # noqa: E402
+    _target_partition_size_to_num_partitions,
+    validate_vector_index,
+)
 from lance.vector import vec_to_table  # noqa: E402
 
 
@@ -573,6 +576,8 @@ def test_index_with_pq_codebook(tmp_path):
         ivf_centroids=np.random.randn(1, 128).astype(np.float32),
         pq_codebook=pq_codebook,
     )
+    index = dataset.stats.index_stats("vector_idx")
+    assert index["indices"][0]["sub_index"]["nbits"] == 8
     validate_vector_index(dataset, "vector", refine_factor=10, pass_threshold=0.99)
 
     pq_codebook = pa.FixedShapeTensorArray.from_numpy_ndarray(pq_codebook)
@@ -589,6 +594,54 @@ def test_index_with_pq_codebook(tmp_path):
     validate_vector_index(dataset, "vector", refine_factor=10, pass_threshold=0.99)
 
 
+def test_index_with_4bit_numpy_pq_codebook(tmp_path):
+    tbl = create_table(nvec=1024, ndim=128)
+    dataset = lance.write_dataset(tbl, tmp_path)
+    pq_codebook = np.random.randn(4, 16, 128 // 4).astype(np.float32)
+
+    dataset = dataset.create_index(
+        "vector",
+        index_type="IVF_PQ",
+        num_partitions=1,
+        num_sub_vectors=4,
+        num_bits=4,
+        ivf_centroids=np.random.randn(1, 128).astype(np.float32),
+        pq_codebook=pq_codebook,
+    )
+
+    index = dataset.stats.index_stats("vector_idx")
+    assert index["indices"][0]["sub_index"]["nbits"] == 4
+
+    result = dataset.to_table(
+        nearest={
+            "column": "vector",
+            "q": np.random.randn(128).astype(np.float32),
+            "k": 10,
+        }
+    )
+    assert result.num_rows == 10
+
+
+def test_index_with_pq_codebook_rejects_wrong_num_bits_shape(tmp_path):
+    tbl = create_table(nvec=8, ndim=128)
+    dataset = lance.write_dataset(tbl, tmp_path)
+    pq_codebook = np.random.randn(4, 256, 128 // 4).astype(np.float32)
+
+    with pytest.raises(
+        ValueError,
+        match=r"\(sub_vectors, 16, dim\) for num_bits=4, got \(4, 256, 32\)",
+    ):
+        dataset.create_index(
+            "vector",
+            index_type="IVF_PQ",
+            num_partitions=1,
+            num_sub_vectors=4,
+            num_bits=4,
+            ivf_centroids=np.random.randn(1, 128).astype(np.float32),
+            pq_codebook=pq_codebook,
+        )
+
+
 @pytest.mark.cuda
 @pytest.mark.parametrize("nullify", [False, True])
 def test_create_index_using_cuda(tmp_path, nullify):
@@ -856,6 +909,12 @@ def test_create_ivf_pq_with_target_partition_size(dataset, tmp_path):
     assert ann_ds.stats.index_stats("vector_idx")["indices"][0]["num_partitions"] == 2
 
 
+def test_target_partition_size_to_num_partitions_clamps():
+    assert _target_partition_size_to_num_partitions(1000, 1000) == 1
+    assert _target_partition_size_to_num_partitions(1000, 500) == 2
+    assert _target_partition_size_to_num_partitions(8192 * 5000, 8192) == 4096
+
+
 def test_index_size_stats(tmp_path: Path):
     num_rows = 512
     dims = 32
@@ -1058,16 +1117,57 @@ def test_create_ivf_rq_skip_transpose():
     assert stats["indices"][0]["sub_index"]["packed"] is False
 
 
-def test_create_ivf_rq_rejects_unsupported_num_bits():
-    ds = lance.write_dataset(create_table(), "memory://")
+def _assert_recall_at_least(ds, query, metric=None, k=10, recall_requirement=0.5):
+    nearest = {"column": "vector", "q": query, "k": k}
+    if metric is not None:
+        nearest["metric"] = metric
 
-    with pytest.raises(NotImplementedError, match="only num_bits=1 is supported"):
-        ds.create_index(
-            "vector",
-            index_type="IVF_RQ",
-            num_partitions=4,
-            num_bits=2,
+    gt_ids = ds.to_table(nearest=nearest, columns=["id"])["id"].to_numpy()
+    create_index_kwargs = {
+        "index_type": "IVF_RQ",
+        "num_partitions": 4,
+        "num_bits": 9,
+    }
+    if metric is not None:
+        create_index_kwargs["metric"] = metric
+    indexed = ds.create_index("vector", **create_index_kwargs)
+    result_ids = indexed.to_table(nearest=nearest, columns=["id"])["id"].to_numpy()
+
+    assert result_ids.shape[0] == k
+    recall = len(set(gt_ids) & set(result_ids)) / k
+    assert recall >= recall_requirement, (
+        f"recall={recall}, gt={gt_ids}, result={result_ids}"
+    )
+    return indexed
+
+
+def test_create_ivf_rq_multi_bit_searches_l2_and_cosine():
+    rng = np.random.default_rng(42)
+    mat = rng.standard_normal((1000, 128)).astype(np.float32)
+    tbl = vec_to_table(data=mat).append_column("id", pa.array(range(len(mat))))
+
+    ds = lance.write_dataset(tbl, "memory://")
+    ds = _assert_recall_at_least(ds, mat[0])
+    stats = ds.stats.index_stats("vector_idx")
+    assert stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
+    for approx_mode in ["fast", "normal", "accurate"]:
+        result = ds.to_table(
+            nearest={
+                "column": "vector",
+                "q": mat[0],
+                "k": 10,
+                "approx_mode": approx_mode,
+            },
+            columns=["id"],
         )
+        assert result.num_rows == 10
+
+    cosine_ds = lance.write_dataset(tbl, "memory://")
+    cosine_ds = _assert_recall_at_least(cosine_ds, mat[1], metric="cosine")
+    cosine_stats = cosine_ds.stats.index_stats("vector_idx")
+    assert cosine_stats["indices"][0]["sub_index"]["num_bits"] == 9
+    assert cosine_stats["indices"][0]["sub_index"]["query_estimator"] == "raw_query"
 
 
 def test_create_ivf_rq_requires_dim_divisible_by_8():
@@ -1722,6 +1822,8 @@ def test_index_cast_centroids(tmp_path):
     values = pa.array([x for arr in centroids for x in arr], pa.float32())
     centroids = pa.FixedSizeListArray.from_arrays(values, 128)
 
+    # Cast invalidates the attached index; drop it first per the new contract.
+    dataset.drop_index(index_name)
     dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), 128)))
 
     # centroids are f32, but the column is now f16
@@ -1797,7 +1899,7 @@ def test_fragment_scan_disallowed_on_ann_with_index_scan_prefilter(tmp_path):
     assert results == results_no_scalar_index
 
 
-def test_load_indices(dataset):
+def test_describe_indices(dataset):
     indices = dataset.describe_indices()
     assert len(indices) == 0
 
@@ -2058,6 +2160,33 @@ def test_vector_index_invalid_query_parallelism(indexed_dataset):
         )
 
 
+def test_vector_index_with_approx_mode(indexed_dataset):
+    q = np.random.randn(128)
+
+    for approx_mode in ["fast", "normal", "accurate"]:
+        result = indexed_dataset.to_table(
+            nearest={
+                "column": "vector",
+                "q": q,
+                "k": 10,
+                "approx_mode": approx_mode,
+            }
+        )
+        assert len(result) == 10
+
+
+def test_vector_index_invalid_approx_mode(indexed_dataset):
+    with pytest.raises(ValueError, match="approx_mode"):
+        indexed_dataset.scanner(
+            nearest={
+                "column": "vector",
+                "q": np.random.randn(128),
+                "k": 10,
+                "approx_mode": "hacc",
+            }
+        )
+
+
 def test_knn_deleted_rows(tmp_path):
     data = create_table()
     ds = lance.write_dataset(data, tmp_path)
@@ -2141,7 +2270,7 @@ def test_nested_field_vector_index(tmp_path):
     # Verify index was created
     indices = dataset.describe_indices()
     assert len(indices) == 1
-    assert indices[0].field_names == ["embedding"]
+    assert indices[0].field_names == ["data.embedding"]
 
     # Test querying with the index
     query_vec = vectors[0]
@@ -2339,7 +2468,7 @@ def test_vector_index_distance_range(tmp_path):
     assert np.all(index_distances >= distance_range[0]) and np.all(
         index_distances < distance_range[1]
     )
-    assert np.allclose(brute_distances, index_distances, rtol=0.0, atol=0.0)
+    assert np.allclose(brute_distances, index_distances, rtol=1e-5, atol=0.0)
 
 
 # =============================================================================
@@ -3000,6 +3129,51 @@ def test_commit_existing_index_segments_accepts_index_metadata(tmp_path):
     assert 0 < len(results) <= 5
 
 
+def test_distributed_ivf_rq_shared_rotation(tmp_path):
+    """Two IVF_RQ segments built on separate fragments with one shared RaBitQ rotation
+    merge into a single committed, queryable index. The shared ``rabitq_model`` (from
+    ``lance.lance.indices.build_rq_model``) is what makes the independently built
+    segments mergeable."""
+    from lance.lance import indices
+
+    dim = 32
+    ds = _make_sample_dataset_base(
+        tmp_path, "dist_rq_merge", n_rows=512, dim=dim, max_rows_per_file=256
+    )
+    frags = ds.get_fragments()
+    assert len(frags) == 2
+
+    ivf_model = IndicesBuilder(ds, "vector").train_ivf(
+        num_partitions=2,
+        distance_type="l2",
+        sample_rate=8,
+    )
+    rabitq_model = indices.build_rq_model(dimension=dim, num_bits=1)
+    base_kwargs = {
+        "column": "vector",
+        "index_type": "IVF_RQ",
+        "num_partitions": 2,
+        "num_bits": 1,
+        "ivf_centroids": ivf_model.centroids,
+        "rabitq_model": rabitq_model,
+    }
+    first = ds.create_index_uncommitted(
+        **base_kwargs,
+        fragment_ids=[frags[0].fragment_id],
+    )
+    second = ds.create_index_uncommitted(
+        **base_kwargs,
+        fragment_ids=[frags[1].fragment_id],
+    )
+
+    merged = ds.merge_existing_index_segments([first, second])
+    ds = ds.commit_existing_index_segments("vector_idx", "vector", [merged])
+
+    q = np.random.rand(dim).astype(np.float32)
+    results = ds.to_table(nearest={"column": "vector", "q": q, "k": 5})
+    assert 0 < len(results) <= 5
+
+
 def test_commit_existing_index_segments_accepts_uncommitted_vector_segments(tmp_path):
     ds = _make_sample_dataset_base(tmp_path, "segment_commit_ds", 2000, 128)
     frags = ds.get_fragments()
diff --git a/python/src/blob.rs b/python/src/blob.rs
new file mode 100644
index 00000000000..1f03cd9abe5
--- /dev/null
+++ b/python/src/blob.rs
@@ -0,0 +1,504 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use crate::{error::PythonErrorExt, rt};
+use arrow::{
+    array::{Array, ArrayRef, GenericBinaryArray, OffsetSizeTrait, cast::AsArray, make_array},
+    pyarrow::{FromPyArrow, ToPyArrow},
+};
+use arrow_data::ArrayData;
+use arrow_schema::{DataType, Field};
+use bytes::Bytes;
+use lance::{
+    BlobDescriptor, BlobDescriptorArrayBuilder, BlobRange, DedicatedBlobWriter, PackedBlobWriter,
+};
+use pyo3::{
+    Bound, PyErr, PyResult,
+    exceptions::{PyRuntimeError, PyValueError},
+    pyclass, pymethods,
+    types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule, PyTypeMethods},
+};
+use std::{
+    borrow::Cow,
+    sync::{Arc, Mutex},
+};
+
+fn with_writer<W, R>(
+    inner: &Mutex<Option<W>>,
+    writer_name: &str,
+    operation: impl FnOnce(&W) -> R,
+) -> PyResult<R> {
+    let guard = inner.lock().map_err(|_| poisoned_writer(writer_name))?;
+    let writer = guard.as_ref().ok_or_else(|| finished_writer(writer_name))?;
+    Ok(operation(writer))
+}
+
+fn writer_mut<'a, W>(inner: &'a mut Mutex<Option<W>>, writer_name: &str) -> PyResult<&'a mut W> {
+    inner
+        .get_mut()
+        .map_err(|_| poisoned_writer(writer_name))?
+        .as_mut()
+        .ok_or_else(|| finished_writer(writer_name))
+}
+
+fn take_writer<W>(inner: &mut Mutex<Option<W>>, writer_name: &str) -> PyResult<W> {
+    inner
+        .get_mut()
+        .map_err(|_| poisoned_writer(writer_name))?
+        .take()
+        .ok_or_else(|| finished_writer(writer_name))
+}
+
+fn finished_writer(writer_name: &str) -> PyErr {
+    PyValueError::new_err(format!("{writer_name} is already finished"))
+}
+
+fn poisoned_writer(writer_name: &str) -> PyErr {
+    PyRuntimeError::new_err(format!("{writer_name} lock is poisoned"))
+}
+
+/// Reconstruct the PyArrow equivalent of [`BlobDescriptorArrayBuilder::field`].
+///
+/// Arrow's array bridge does not carry the enclosing extension field, so this
+/// rebuilds the canonical six nullable blob-v2 children and
+/// `ARROW:extension:name = lance.blob.v2` metadata.
+fn descriptor_field_to_pyarrow<'py>(
+    field: &Field,
+    py: pyo3::Python<'py>,
+) -> PyResult<Bound<'py, PyAny>> {
+    let pyarrow = PyModule::import(py, "pyarrow")?;
+    let child_fields = PyList::empty(py);
+    for (name, type_fn) in [
+        ("kind", "uint8"),
+        ("data", "large_binary"),
+        ("uri", "utf8"),
+        ("blob_id", "uint32"),
+        ("blob_size", "uint64"),
+        ("position", "uint64"),
+    ] {
+        let data_type = pyarrow.getattr(type_fn)?.call0()?;
+        let child = pyarrow.call_method1("field", (name, data_type, true))?;
+        child_fields.append(child)?;
+    }
+    let data_type = pyarrow.call_method1("struct", (child_fields,))?;
+    let metadata = PyDict::new(py);
+    metadata.set_item("ARROW:extension:name", "lance.blob.v2")?;
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("nullable", field.is_nullable())?;
+    kwargs.set_item("metadata", metadata)?;
+    pyarrow.call_method("field", (field.name().as_str(), data_type), Some(&kwargs))
+}
+
+/// Normalize inputs accepted by [`PyPackedBlobWriter::write_blobs`] into Arrow arrays.
+///
+/// BinaryArray, LargeBinaryArray, and ChunkedArray values of either binary type
+/// are accepted. Chunk boundaries, nulls, and empty values remain in the arrays;
+/// each row is later passed to the core writer as an optional byte slice.
+fn extract_blob_payloads(payloads: &Bound<'_, PyAny>) -> PyResult<Vec<ArrayRef>> {
+    match ArrayData::from_pyarrow_bound(payloads) {
+        Ok(data) => Ok(vec![validated_blob_payload(data, None)?]),
+        Err(_) => {
+            let pyarrow = PyModule::import(payloads.py(), "pyarrow")?;
+            let chunked_array_type = pyarrow.getattr("ChunkedArray")?;
+            if !payloads.is_instance(&chunked_array_type)? {
+                return Err(PyValueError::new_err(format!(
+                    "payloads must be a pyarrow BinaryArray, LargeBinaryArray, or ChunkedArray, got {}",
+                    payloads.get_type().name()?
+                )));
+            }
+
+            let chunked_data_type = DataType::from_pyarrow_bound(&payloads.getattr("type")?)?;
+            if !matches!(chunked_data_type, DataType::Binary | DataType::LargeBinary) {
+                return Err(PyValueError::new_err(format!(
+                    "Packed blob payloads must have Arrow type Binary or LargeBinary, got {chunked_data_type}"
+                )));
+            }
+
+            let chunks = payloads.getattr("chunks")?;
+            let mut arrays = Vec::with_capacity(chunks.len()?);
+            for (chunk_index, chunk) in chunks.try_iter()?.enumerate() {
+                let data = ArrayData::from_pyarrow_bound(&chunk?)?;
+                arrays.push(validated_blob_payload(data, Some(chunk_index))?);
+            }
+            Ok(arrays)
+        }
+    }
+}
+
+fn validated_blob_payload(data: ArrayData, chunk_index: Option<usize>) -> PyResult<ArrayRef> {
+    let context = chunk_index
+        .map(|index| format!("Packed blob payload chunk {index}"))
+        .unwrap_or_else(|| "Packed blob payload array".to_string());
+    if !matches!(data.data_type(), DataType::Binary | DataType::LargeBinary) {
+        return Err(PyValueError::new_err(format!(
+            "{context} must have Arrow type Binary or LargeBinary, got {}",
+            data.data_type()
+        )));
+    }
+    if data.is_empty() {
+        // PyArrow may export an empty slice without the values preceding its
+        // nonzero first offset. Normalize it because an empty array never
+        // observes those buffers, and Arrow validation would reject the slice.
+        return Ok(make_array(ArrayData::new_empty(data.data_type())));
+    }
+    data.validate_full().map_err(|error| {
+        PyValueError::new_err(format!("{context} contains invalid Arrow data: {error}"))
+    })?;
+    Ok(make_array(data))
+}
+
+/// Stream one Arrow binary array into the core writer as zero-copy row slices.
+///
+/// Null rows become `None` so the core writer records null descriptors, keeping
+/// its output row-aligned with the input.
+async fn write_binary_payloads<O: OffsetSizeTrait>(
+    writer: &mut PackedBlobWriter,
+    payloads: &GenericBinaryArray<O>,
+) -> PyResult<()> {
+    writer
+        .write_packed_blobs(
+            (0..payloads.len()).map(|row| payloads.is_valid(row).then(|| payloads.value(row))),
+        )
+        .await
+        .infer_error()
+}
+
+#[pyclass(name = "BlobDescriptor", skip_from_py_object)]
+#[derive(Clone)]
+pub struct PyBlobDescriptor {
+    inner: BlobDescriptor,
+}
+
+impl From<BlobDescriptor> for PyBlobDescriptor {
+    fn from(inner: BlobDescriptor) -> Self {
+        Self { inner }
+    }
+}
+
+#[pymethods]
+impl PyBlobDescriptor {
+    fn __repr__(&self) -> String {
+        format!("{:?}", self.inner)
+    }
+}
+
+#[pyclass(name = "BlobDescriptorArrayBuilder", skip_from_py_object)]
+pub struct PyBlobDescriptorArrayBuilder {
+    field: arrow_schema::Field,
+    inner: Option<BlobDescriptorArrayBuilder>,
+}
+
+impl PyBlobDescriptorArrayBuilder {
+    fn inner_mut(&mut self) -> PyResult<&mut BlobDescriptorArrayBuilder> {
+        self.inner
+            .as_mut()
+            .ok_or_else(|| PyValueError::new_err("BlobDescriptorArrayBuilder is already finished"))
+    }
+}
+
+#[pymethods]
+impl PyBlobDescriptorArrayBuilder {
+    #[new]
+    pub fn new(column: String) -> Self {
+        let inner = BlobDescriptorArrayBuilder::new(column);
+        Self {
+            field: inner.field().clone(),
+            inner: Some(inner),
+        }
+    }
+
+    #[getter]
+    pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        descriptor_field_to_pyarrow(&self.field, py)
+    }
+
+    pub fn extend_packed(
+        &mut self,
+        blob_id: u32,
+        offsets: Vec<u64>,
+        sizes: Vec<u64>,
+    ) -> PyResult<()> {
+        if offsets.len() != sizes.len() {
+            return Err(PyValueError::new_err(format!(
+                "offsets and sizes must have the same length, got {} and {}",
+                offsets.len(),
+                sizes.len()
+            )));
+        }
+        let ranges = offsets
+            .into_iter()
+            .zip(sizes)
+            .map(|(offset, size)| BlobRange { offset, size })
+            .collect::<Vec<_>>();
+        self.inner_mut()?
+            .extend_packed(blob_id, ranges)
+            .infer_error()
+    }
+
+    pub fn append_dedicated(&mut self, blob_id: u32, size: u64) -> PyResult<()> {
+        self.inner_mut()?
+            .push_dedicated(blob_id, size)
+            .infer_error()
+    }
+
+    pub fn append(&mut self, value: &Bound<'_, PyAny>) -> PyResult<()> {
+        let value = value.extract::<pyo3::PyRef<'_, PyBlobDescriptor>>()?;
+        self.inner_mut()?.push(value.inner.clone()).infer_error()
+    }
+
+    pub fn extend(&mut self, values: &Bound<'_, PyAny>) -> PyResult<()> {
+        let iter = values.try_iter()?;
+        for value in iter {
+            let value = value?.extract::<pyo3::PyRef<'_, PyBlobDescriptor>>()?;
+            self.inner_mut()?.push(value.inner.clone()).infer_error()?;
+        }
+        Ok(())
+    }
+
+    pub fn append_inline(&mut self, data: Vec<u8>) -> PyResult<()> {
+        self.inner_mut()?
+            .push_inline(Bytes::from(data))
+            .infer_error()
+    }
+
+    pub fn append_null(&mut self) -> PyResult<()> {
+        self.inner_mut()?.push_null().infer_error()
+    }
+
+    pub fn finish<'py>(&mut self, py: pyo3::Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.inner.take().ok_or_else(|| {
+            PyValueError::new_err("BlobDescriptorArrayBuilder is already finished")
+        })?;
+        let column = inner.finish().infer_error()?;
+        column.array().to_data().to_pyarrow(py)
+    }
+}
+
+#[pyclass(name = "PackedBlobWriter", skip_from_py_object)]
+pub struct PyPackedBlobWriter {
+    field: Option<Field>,
+    inner: Mutex<Option<PackedBlobWriter>>,
+}
+
+impl PyPackedBlobWriter {
+    pub(crate) async fn try_new(
+        object_store: Arc<lance_io::object_store::ObjectStore>,
+        data_file_path: object_store::path::Path,
+        blob_id: u32,
+    ) -> PyResult<Self> {
+        let inner =
+            PackedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id)
+                .await
+                .infer_error()?;
+        Ok(Self {
+            field: None,
+            inner: Mutex::new(Some(inner)),
+        })
+    }
+
+    fn with_inner<R>(&self, operation: impl FnOnce(&PackedBlobWriter) -> R) -> PyResult<R> {
+        with_writer(&self.inner, "PackedBlobWriter", operation)
+    }
+
+    fn inner_mut(&mut self) -> PyResult<&mut PackedBlobWriter> {
+        writer_mut(&mut self.inner, "PackedBlobWriter")
+    }
+
+    fn take_inner(&mut self) -> PyResult<PackedBlobWriter> {
+        take_writer(&mut self.inner, "PackedBlobWriter")
+    }
+}
+
+#[pymethods]
+impl PyPackedBlobWriter {
+    #[getter]
+    pub fn blob_id(&self) -> PyResult<u32> {
+        self.with_inner(PackedBlobWriter::blob_id)
+    }
+
+    #[getter]
+    pub fn path(&self) -> PyResult<String> {
+        self.with_inner(|writer| writer.path().to_string())
+    }
+
+    /// The descriptor field associated with the array returned by
+    /// :meth:`finish_array`.
+    ///
+    /// The field uses the name passed to ``finish_array`` and carries the
+    /// ``lance.blob.v2`` extension metadata. It is available only after
+    /// ``finish_array`` succeeds; accessing it earlier raises ``ValueError``.
+    #[getter]
+    pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult<Bound<'py, PyAny>> {
+        let field = self.field.as_ref().ok_or_else(|| {
+            PyValueError::new_err("PackedBlobWriter field is available after finish_array")
+        })?;
+        descriptor_field_to_pyarrow(field, py)
+    }
+
+    /// Append one packed blob.
+    ///
+    /// Python ``bytes`` are borrowed without copying. Other compatible byte
+    /// sequences use owned storage for the duration of the write.
+    pub fn write_blob(&mut self, data: Cow<'_, [u8]>) -> PyResult<()> {
+        rt().block_on(None, self.inner_mut()?.write_blob(data.as_ref()))?
+            .infer_error()
+    }
+
+    /// Append a batch of packed blob payloads.
+    ///
+    /// Parameters
+    /// ----------
+    /// payloads : pyarrow.BinaryArray, pyarrow.LargeBinaryArray, or pyarrow.ChunkedArray
+    ///     A binary Arrow array. Every chunk of a chunked array must be binary.
+    ///     Each input row produces one descriptor row, in order, across chunks
+    ///     and repeated calls. Null rows produce null descriptors; empty but
+    ///     non-null byte strings produce valid zero-length blobs.
+    ///
+    /// Examples
+    /// --------
+    /// >>> import pyarrow as pa
+    /// >>> payloads = pa.array([b"first", None, b""], type=pa.large_binary())
+    /// >>> writer.write_blobs(payloads)
+    /// >>> descriptors = writer.finish_array("blob")
+    /// >>> len(descriptors)
+    /// 3
+    pub fn write_blobs(&mut self, payloads: &Bound<'_, PyAny>) -> PyResult<()> {
+        let payloads = extract_blob_payloads(payloads)?;
+        let result = {
+            let writer = self.inner_mut()?;
+            rt().block_on(None, async {
+                for payloads in payloads {
+                    match payloads.data_type() {
+                        DataType::Binary => {
+                            write_binary_payloads(writer, payloads.as_binary::<i32>()).await?
+                        }
+                        DataType::LargeBinary => {
+                            write_binary_payloads(writer, payloads.as_binary::<i64>()).await?
+                        }
+                        data_type => {
+                            return Err(PyValueError::new_err(format!(
+                                "Packed blob payloads must have Arrow type Binary or LargeBinary, got {data_type}"
+                            )));
+                        }
+                    }
+                }
+                Ok(())
+            })
+        };
+        match result {
+            Ok(result) => result,
+            Err(error) => {
+                // KeyboardInterrupt drops the async batch future. Remove the core
+                // writer as well so RAII cleanup runs and a completed prefix cannot
+                // be reused as a new batch.
+                self.take_inner()?;
+                Err(error)
+            }
+        }
+    }
+
+    pub fn finish(&mut self) -> PyResult<Vec<PyBlobDescriptor>> {
+        let inner = self.take_inner()?;
+        let values = rt().block_on(None, inner.finish())?.infer_error()?;
+        Ok(values.into_iter().map(Into::into).collect())
+    }
+
+    /// Finish the upload and return its blob descriptors as a PyArrow array.
+    ///
+    /// The returned ``pyarrow.StructArray`` has one row per payload previously
+    /// passed to :meth:`write_blob` or :meth:`write_blobs`. The writer is consumed
+    /// by this call. After it succeeds, :attr:`field` returns the matching
+    /// extension field with ``field_name`` as its name.
+    ///
+    /// Parameters
+    /// ----------
+    /// field_name : str
+    ///     Name for the descriptor field exposed by :attr:`field`.
+    ///
+    /// Returns
+    /// -------
+    /// pyarrow.StructArray
+    ///     Row-aligned blob descriptors, including null rows from bulk input.
+    ///
+    /// Examples
+    /// --------
+    /// >>> import pyarrow as pa
+    /// >>> writer.write_blobs(pa.array([b"value", None]))
+    /// >>> descriptors = writer.finish_array("payload")
+    /// >>> descriptors.is_null().to_pylist()
+    /// [False, True]
+    /// >>> writer.field.name
+    /// 'payload'
+    pub fn finish_array<'py>(
+        &mut self,
+        py: pyo3::Python<'py>,
+        field_name: String,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let inner = self.take_inner()?;
+        let values = rt().block_on(None, inner.finish())?.infer_error()?;
+        let mut builder = BlobDescriptorArrayBuilder::new(field_name);
+        builder.extend(values).infer_error()?;
+        let column = builder.finish().infer_error()?;
+        let (field, array) = column.into_parts();
+        let array = array.to_data().to_pyarrow(py)?;
+        self.field = Some(field);
+        Ok(array)
+    }
+}
+
+#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object)]
+pub struct PyDedicatedBlobWriter {
+    inner: Mutex<Option<DedicatedBlobWriter>>,
+}
+
+impl PyDedicatedBlobWriter {
+    pub(crate) async fn try_new(
+        object_store: Arc<lance_io::object_store::ObjectStore>,
+        data_file_path: object_store::path::Path,
+        blob_id: u32,
+    ) -> PyResult<Self> {
+        let inner =
+            DedicatedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id)
+                .await
+                .infer_error()?;
+        Ok(Self {
+            inner: Mutex::new(Some(inner)),
+        })
+    }
+
+    fn with_inner<R>(&self, operation: impl FnOnce(&DedicatedBlobWriter) -> R) -> PyResult<R> {
+        with_writer(&self.inner, "DedicatedBlobWriter", operation)
+    }
+
+    fn inner_mut(&mut self) -> PyResult<&mut DedicatedBlobWriter> {
+        writer_mut(&mut self.inner, "DedicatedBlobWriter")
+    }
+
+    fn take_inner(&mut self) -> PyResult<DedicatedBlobWriter> {
+        take_writer(&mut self.inner, "DedicatedBlobWriter")
+    }
+}
+
+#[pymethods]
+impl PyDedicatedBlobWriter {
+    #[getter]
+    pub fn blob_id(&self) -> PyResult<u32> {
+        self.with_inner(DedicatedBlobWriter::blob_id)
+    }
+
+    #[getter]
+    pub fn path(&self) -> PyResult<String> {
+        self.with_inner(|writer| writer.path().to_string())
+    }
+
+    pub fn write(&mut self, data: Vec<u8>) -> PyResult<()> {
+        rt().block_on(None, self.inner_mut()?.write(data))?
+            .infer_error()
+    }
+
+    pub fn finish(&mut self) -> PyResult<PyBlobDescriptor> {
+        let inner = self.take_inner()?;
+        let value = rt().block_on(None, inner.finish())?.infer_error()?;
+        Ok(value.into())
+    }
+}
diff --git a/python/src/dataset.rs b/python/src/dataset.rs
index f70a9c7b1fb..a361e0b63a8 100644
--- a/python/src/dataset.rs
+++ b/python/src/dataset.rs
@@ -20,10 +20,11 @@ use blob::LanceBlobFile;
 use chrono::{Duration, TimeDelta, Utc};
 use futures::{StreamExt, TryFutureExt};
 use lance_index::vector::bq::RQBuildParams;
+use lance_index::vector::bq::storage::RabitQuantizationMetadata;
 use log::error;
 use object_store::path::Path;
 use pyo3::exceptions::{PyStopIteration, PyTypeError};
-use pyo3::types::{PyBytes, PyInt, PyList, PySet, PyString, PyTuple};
+use pyo3::types::{PyBytes, PyInt, PyList, PyString, PyTuple};
 use pyo3::{IntoPyObjectExt, prelude::*};
 use pyo3::{
     PyResult,
@@ -36,7 +37,7 @@ use pyo3::{
 use uuid::Uuid;
 
 use lance::dataset::AutoCleanupParams;
-use lance::dataset::cleanup::CleanupPolicyBuilder;
+use lance::dataset::cleanup::{CleanupFileKind, CleanupPolicyBuilder};
 use lance::dataset::refs::{Ref, TagContents};
 use lance::dataset::scanner::{
     AggregateExpr, ColumnOrdering, DatasetRecordBatchStream, ExecutionStatsCallback,
@@ -69,6 +70,7 @@ use lance_core::datatypes::BlobHandling;
 use lance_datafusion::utils::reader_to_stream;
 use lance_encoding::decoder::DecoderConfig;
 use lance_file::reader::FileReaderOptions;
+use lance_index::scalar::inverted::query::Occur;
 use lance_index::scalar::inverted::query::{
     BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
 };
@@ -76,15 +78,14 @@ use lance_index::{
     FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions,
     optimize::OptimizeOptions,
     progress::{IndexBuildProgress, NoopIndexBuildProgress},
+    scalar::inverted::InvertedListFormatVersion,
     scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams},
     vector::{
-        DEFAULT_QUERY_PARALLELISM, Query as VectorQuery, hnsw::builder::HnswBuildParams,
-        ivf::IvfBuildParams, pq::PQBuildParams, sq::builder::SQBuildParams,
+        ApproxMode, DEFAULT_QUERY_PARALLELISM, Query as VectorQuery,
+        hnsw::builder::HnswBuildParams, ivf::IvfBuildParams, pq::PQBuildParams,
+        sq::builder::SQBuildParams,
     },
 };
-use lance_index::{
-    infer_system_index_type, metrics::NoOpMetricsCollector, scalar::inverted::query::Occur,
-};
 use lance_io::object_store::{
     LanceNamespaceStorageOptionsProvider, ObjectStoreParams, StorageOptionsAccessor,
 };
@@ -107,7 +108,9 @@ use crate::utils::PyLance;
 use crate::{LanceReader, Scanner};
 use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
 
-use self::cleanup::CleanupStats;
+use self::cleanup::{
+    CleanupCandidateFile, CleanupExplanation, CleanupReferencedBranch, CleanupStats,
+};
 use self::commit::PyCommitLock;
 use self::io_stats::IoStats;
 
@@ -393,6 +396,23 @@ impl MergeInsertBuilder {
         Ok(slf)
     }
 
+    pub fn target_bases(
+        mut slf: PyRefMut<'_, Self>,
+        bases: Vec<String>,
+    ) -> PyResult<PyRefMut<'_, Self>> {
+        slf.builder.target_base_names_or_paths(bases);
+        Ok(slf)
+    }
+
+    #[pyo3(signature = (include_primary = true))]
+    pub fn target_all_bases(
+        mut slf: PyRefMut<'_, Self>,
+        include_primary: bool,
+    ) -> PyResult<PyRefMut<'_, Self>> {
+        slf.builder.target_all_bases(include_primary);
+        Ok(slf)
+    }
+
     pub fn execute(&mut self, new_data: &Bound<PyAny>) -> PyResult<Py<PyAny>> {
         let py = new_data.py();
         let new_data = convert_reader(new_data)?;
@@ -529,6 +549,23 @@ fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult<Vec<IndexSegm
     Ok(extracted)
 }
 
+fn parse_index_segment_ids(index_segments: Option<Vec<String>>) -> PyResult<Option<Vec<Uuid>>> {
+    index_segments
+        .map(|segments| {
+            segments
+                .into_iter()
+                .map(|segment| {
+                    Uuid::parse_str(&segment).map_err(|err| {
+                        PyValueError::new_err(format!(
+                            "invalid index segment uuid '{segment}': {err}"
+                        ))
+                    })
+                })
+                .collect::<PyResult<Vec<_>>>()
+        })
+        .transpose()
+}
+
 impl MergeInsertBuilder {
     fn build_stats<'a>(stats: &MergeStats, py: Python<'a>) -> PyResult<Bound<'a, PyDict>> {
         let dict = PyDict::new(py);
@@ -684,6 +721,89 @@ pub struct Dataset {
     pub(crate) ds: Arc<LanceDataset>,
 }
 
+impl Dataset {
+    async fn cleanup_policy(
+        &self,
+        older_than_micros: Option<i64>,
+        retain_versions: Option<usize>,
+        delete_unverified: Option<bool>,
+        error_if_tagged_old_versions: Option<bool>,
+        delete_rate_limit: Option<u64>,
+    ) -> lance_core::Result<lance::dataset::cleanup::CleanupPolicy> {
+        let mut builder = CleanupPolicyBuilder::default();
+        if let Some(v) = older_than_micros {
+            let older_than = Duration::microseconds(v);
+            builder = builder.before_timestamp(Utc::now() - older_than);
+        }
+        if let Some(v) = retain_versions {
+            builder = builder.retain_n_versions(self.ds.as_ref(), v).await?;
+        }
+        if let Some(v) = delete_unverified {
+            builder = builder.delete_unverified(v);
+        }
+        if let Some(v) = error_if_tagged_old_versions {
+            builder = builder.error_if_tagged_old_versions(v);
+        }
+        if let Some(v) = delete_rate_limit {
+            builder = builder.delete_rate_limit(v)?;
+        }
+        Ok(builder.build())
+    }
+}
+
+fn cleanup_stats(stats: lance::dataset::cleanup::RemovalStats) -> CleanupStats {
+    CleanupStats {
+        bytes_removed: stats.bytes_removed,
+        old_versions: stats.old_versions,
+        data_files_removed: stats.data_files_removed,
+        transaction_files_removed: stats.transaction_files_removed,
+        index_files_removed: stats.index_files_removed,
+        deletion_files_removed: stats.deletion_files_removed,
+    }
+}
+
+fn cleanup_file_kind(kind: CleanupFileKind) -> &'static str {
+    match kind {
+        CleanupFileKind::Manifest => "manifest",
+        CleanupFileKind::Data => "data",
+        CleanupFileKind::Transaction => "transaction",
+        CleanupFileKind::Index => "index",
+        CleanupFileKind::Deletion => "deletion",
+        CleanupFileKind::TemporaryManifest => "temporary_manifest",
+    }
+}
+
+fn cleanup_explanation(
+    explanation: lance::dataset::cleanup::CleanupExplanation,
+) -> CleanupExplanation {
+    CleanupExplanation {
+        read_version: explanation.read_version,
+        stats: cleanup_stats(explanation.stats),
+        candidate_files: explanation
+            .candidate_files
+            .into_iter()
+            .map(|file| CleanupCandidateFile {
+                path: file.path,
+                kind: cleanup_file_kind(file.kind).to_string(),
+                unverified: file.unverified,
+                size_bytes: file.size_bytes,
+            })
+            .collect(),
+        candidate_files_truncated: explanation.candidate_files_truncated,
+        candidate_file_limit: explanation.candidate_file_limit,
+        referenced_branches: explanation
+            .referenced_branches
+            .into_iter()
+            .map(|branch| CleanupReferencedBranch {
+                name: branch.name,
+                referenced_version: branch.referenced_version,
+                cleanup_candidate: branch.cleanup_candidate,
+            })
+            .collect(),
+        warnings: explanation.warnings,
+    }
+}
+
 #[pymethods]
 impl Dataset {
     #[allow(clippy::too_many_arguments)]
@@ -820,8 +940,14 @@ impl Dataset {
 
             // Set up commit handler only if namespace manages versioning
             if namespace_client_managed_versioning {
-                let external_store =
-                    LanceNamespaceExternalManifestStore::new(ns_client, tid.clone());
+                // The store derives the branch a request targets from the base
+                // path it is handed, resolved against the table root.
+                let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
+                    ns_client,
+                    tid.clone(),
+                    &uri,
+                )
+                .infer_error()?;
                 let commit_handler: Arc<dyn CommitHandler> =
                     Arc::new(ExternalManifestCommitHandler {
                         external_manifest_store: Arc::new(external_store),
@@ -932,6 +1058,32 @@ impl Dataset {
             })
     }
 
+    /// Remap row addresses across compactions still recorded in the
+    /// fragment-reuse index. Rows a compaction dropped become null. The index
+    /// retains only recent rounds (older ones are pruned as index remap catches
+    /// up), so remap promptly: an address whose round was pruned is returned
+    /// unchanged, not remapped. Returns None when there is no fragment-reuse
+    /// index.
+    fn remap_row_addrs(
+        &self,
+        py: Python,
+        addrs: PyArrowType<ArrayData>,
+    ) -> PyResult<Option<PyArrowType<ArrayData>>> {
+        use lance_index::metrics::NoOpMetricsCollector;
+
+        let array = make_array(addrs.0);
+        let frag_reuse_index = rt()
+            .block_on(
+                Some(py),
+                self.ds.open_frag_reuse_index(&NoOpMetricsCollector),
+            )?
+            .map_err(|err| {
+                PyIOError::new_err(format!("failed to open fragment reuse index: {err}"))
+            })?;
+
+        Ok(frag_reuse_index.map(|fri| PyArrowType(fri.remap_row_ids_array(array).to_data())))
+    }
+
     fn serialized_manifest(&self, py: Python) -> Py<PyAny> {
         let manifest_bytes = self.ds.manifest().serialized();
         PyBytes::new(py, &manifest_bytes).into()
@@ -952,79 +1104,6 @@ impl Dataset {
         Ok(dict.into())
     }
 
-    /// Load index metadata.
-    ///
-    /// This call will open the index and return its concrete index type.
-    fn load_indices(self_: PyRef<'_, Self>) -> PyResult<Vec<Py<PyAny>>> {
-        let index_metadata = rt()
-            .block_on(Some(self_.py()), self_.ds.load_indices())?
-            .map_err(|err| PyValueError::new_err(err.to_string()))?;
-        let py = self_.py();
-        index_metadata
-            .iter()
-            .map(|idx| {
-                let dict = PyDict::new(py);
-                let schema = self_.ds.schema();
-                let field_paths = idx
-                    .fields
-                    .iter()
-                    .map(|field_id| schema.field_path(*field_id).unwrap())
-                    .collect::<Vec<_>>();
-
-                let ds = self_.ds.clone();
-                let idx_type = match rt().block_on(Some(self_.py()), async {
-                    if let Some(system_index_type) = infer_system_index_type(idx) {
-                        Ok::<_, lance::Error>(system_index_type.to_string())
-                    } else {
-                        let idx = ds
-                            .open_generic_index(
-                                &field_paths[0],
-                                &idx.uuid.to_string(),
-                                &NoOpMetricsCollector,
-                            )
-                            .await?;
-                        Ok::<_, lance::Error>(idx.index_type().to_string())
-                    }
-                })? {
-                    Ok(r) => r,
-                    Err(error) => {
-                        log::warn!(
-                            "Cannot derive index type for index {} (uuid={}, type_url={:?}, version={}) on dataset {}: {}",
-                            idx.name,
-                            idx.uuid,
-                            idx.index_details.as_ref().map(|d| d.type_url.as_str()),
-                            idx.index_version,
-                            self_.ds.uri(),
-                            error,
-                        );
-                        // mark the type as unknown for any new index type
-                        "Unknown".to_owned()
-                    }
-                };
-
-                let fragment_set = PySet::empty(py).unwrap();
-                if let Some(bitmap) = &idx.fragment_bitmap {
-                    for fragment_id in bitmap.iter() {
-                        fragment_set.add(fragment_id).unwrap();
-                    }
-                }
-
-                dict.set_item("name", idx.name.clone()).unwrap();
-                // TODO: once we add more than vector indices, we need to:
-                // 1. Change protos and write path to persist index type
-                // 2. Use the new field from idx instead of hard coding it to Vector
-                dict.set_item("type", idx_type).unwrap();
-                dict.set_item("uuid", idx.uuid.to_string()).unwrap();
-                dict.set_item("fields", field_paths).unwrap();
-                dict.set_item("version", idx.dataset_version).unwrap();
-                dict.set_item("fragment_ids", fragment_set).unwrap();
-                dict.set_item("base_id", idx.base_id.map(|id| id as i64))
-                    .unwrap();
-                dict.into_py_any(py)
-            })
-            .collect::<PyResult<Vec<_>>>()
-    }
-
     #[allow(clippy::too_many_arguments)]
     #[pyo3(signature=(columns=None, columns_with_transform=None, filter=None, search_filter=None, prefilter=None, limit=None, offset=None, nearest=None, batch_size=None, batch_size_bytes=None, io_buffer_size=None, batch_readahead=None, fragment_readahead=None, scan_in_order=None, fragments=None, index_segments=None, with_row_id=None, with_row_address=None, use_stats=None, substrait_filter=None, fast_search=None, full_text_query=None, late_materialization=None, blob_handling=None, use_scalar_index=None, include_deleted_rows=None, scan_stats_callback=None, strict_batch_size=None, order_by=None, disable_scoring_autoprojection=None, substrait_aggregate=None))]
     fn scanner(
@@ -1303,6 +1382,7 @@ impl Dataset {
                 use_index,
                 ef,
                 query_parallelism,
+                approx_mode,
             ) = vector_query_params_from_dict(nearest, default_k)?;
 
             let (_, element_type) = get_vector_type(self_.ds.schema(), &column)
@@ -1369,6 +1449,7 @@ impl Dataset {
                         s = s.ef(ef);
                     }
                     s = s.query_parallelism(query_parallelism);
+                    s = s.approx_mode(approx_mode);
                     s.use_index(use_index);
                     if let Some((lower, upper)) = distance_range {
                         s.distance_range(lower, upper);
@@ -1927,37 +2008,61 @@ impl Dataset {
         error_if_tagged_old_versions: Option<bool>,
         delete_rate_limit: Option<u64>,
     ) -> PyResult<CleanupStats> {
-        let cleanup_stats = rt()
+        let stats = rt()
             .block_on(None, async {
-                let mut builder = CleanupPolicyBuilder::default();
-                if let Some(v) = older_than_micros {
-                    let older_than = Duration::microseconds(v);
-                    builder = builder.before_timestamp(Utc::now() - older_than);
-                }
-                if let Some(v) = retain_versions {
-                    builder = builder.retain_n_versions(self.ds.as_ref(), v).await?;
-                }
-                if let Some(v) = delete_unverified {
-                    builder = builder.delete_unverified(v);
-                }
-                if let Some(v) = error_if_tagged_old_versions {
-                    builder = builder.error_if_tagged_old_versions(v);
-                }
-                if let Some(v) = delete_rate_limit {
-                    builder = builder.delete_rate_limit(v)?;
-                }
+                let policy = self
+                    .cleanup_policy(
+                        older_than_micros,
+                        retain_versions,
+                        delete_unverified,
+                        error_if_tagged_old_versions,
+                        delete_rate_limit,
+                    )
+                    .await?;
+                self.ds.cleanup_with_policy(policy).await
+            })?
+            .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?;
+        Ok(cleanup_stats(stats))
+    }
 
-                self.ds.cleanup_with_policy(builder.build()).await
+    /// Explain cleanup old versions from the dataset without deleting files
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (older_than_micros = None, retain_versions = None, delete_unverified = None, error_if_tagged_old_versions = None, delete_rate_limit = None, include_files = false, max_files = 1000))]
+    fn explain_cleanup_old_versions(
+        &self,
+        older_than_micros: Option<i64>,
+        retain_versions: Option<usize>,
+        delete_unverified: Option<bool>,
+        error_if_tagged_old_versions: Option<bool>,
+        delete_rate_limit: Option<u64>,
+        include_files: bool,
+        max_files: usize,
+    ) -> PyResult<CleanupExplanation> {
+        let explanation = rt()
+            .block_on(None, async {
+                let policy = self
+                    .cleanup_policy(
+                        older_than_micros,
+                        retain_versions,
+                        delete_unverified,
+                        error_if_tagged_old_versions,
+                        delete_rate_limit,
+                    )
+                    .await?;
+                self.ds
+                    .cleanup(policy)
+                    .with_max_candidate_files(max_files)
+                    .explain()
+                    .await
             })?
             .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?;
-        Ok(CleanupStats {
-            bytes_removed: cleanup_stats.bytes_removed,
-            old_versions: cleanup_stats.old_versions,
-            data_files_removed: cleanup_stats.data_files_removed,
-            transaction_files_removed: cleanup_stats.transaction_files_removed,
-            index_files_removed: cleanup_stats.index_files_removed,
-            deletion_files_removed: cleanup_stats.deletion_files_removed,
-        })
+        let mut explanation = cleanup_explanation(explanation);
+        if !include_files {
+            explanation.candidate_files.clear();
+            explanation.candidate_files_truncated = false;
+            explanation.warnings.clear();
+        }
+        Ok(explanation)
     }
 
     fn tags_ordered(self_: PyRef<'_, Self>, order: Option<String>) -> PyResult<Py<PyAny>> {
@@ -2235,6 +2340,7 @@ impl Dataset {
             "LABEL_LIST" => IndexType::LabelList,
             "RTREE" => IndexType::RTree,
             "INVERTED" | "FTS" => IndexType::Inverted,
+            "FM" => IndexType::Fm,
             "IVF_FLAT" | "IVF_PQ" | "IVF_SQ" | "IVF_RQ" | "IVF_HNSW_FLAT" | "IVF_HNSW_PQ"
             | "IVF_HNSW_SQ" => IndexType::Vector,
             _ => {
@@ -2274,6 +2380,27 @@ impl Dataset {
                 index_type: "rtree".to_string(),
                 params: None,
             }),
+            "FM" => {
+                let mut params_json = serde_json::Map::new();
+                if let Some(kwargs) = kwargs
+                    && let Some(num_segments) = kwargs.get_item("num_segments")?
+                {
+                    let n: u32 = num_segments.extract()?;
+                    params_json.insert(
+                        "num_segments".to_string(),
+                        serde_json::Value::Number(n.into()),
+                    );
+                }
+                let params = if params_json.is_empty() {
+                    None
+                } else {
+                    Some(serde_json::Value::Object(params_json).to_string())
+                };
+                Box::new(ScalarIndexParams {
+                    index_type: "fm".to_string(),
+                    params,
+                })
+            }
             "SCALAR" => {
                 let Some(kwargs) = kwargs else {
                     return Err(PyValueError::new_err(
@@ -2337,12 +2464,34 @@ impl Dataset {
                     if let Some(prefix_only) = kwargs.get_item("prefix_only")? {
                         params = params.ngram_prefix_only(prefix_only.extract()?);
                     }
+                    if let Some(block_size) = kwargs.get_item("block_size")? {
+                        params = params
+                            .block_size(block_size.extract()?)
+                            .map_err(|e| PyValueError::new_err(e.to_string()))?;
+                    }
                     if let Some(memory_limit) = kwargs.get_item("memory_limit")? {
                         params = params.memory_limit_mb(memory_limit.extract()?);
                     }
                     if let Some(num_workers) = kwargs.get_item("num_workers")? {
                         params = params.num_workers(num_workers.extract()?);
                     }
+                    if let Some(format_version) = kwargs.get_item("format_version")?
+                        && !format_version.is_none()
+                    {
+                        let value = if let Ok(value) = format_version.cast::<PyString>() {
+                            value.to_string_lossy().to_string()
+                        } else if let Ok(value) = format_version.extract::<u32>() {
+                            value.to_string()
+                        } else {
+                            return Err(PyValueError::new_err(
+                                "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'",
+                            ));
+                        };
+                        let format_version = value
+                            .parse::<InvertedListFormatVersion>()
+                            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+                        params = params.format_version(format_version);
+                    }
                 }
                 Box::new(params)
             }
@@ -2380,10 +2529,22 @@ impl Dataset {
             None
         };
 
-        let index_uuid: Option<String> = if let Some(kwargs) = kwargs {
+        let index_uuid: Option<Uuid> = if let Some(kwargs) = kwargs {
             kwargs
                 .get_item("index_uuid")?
-                .and_then(|v| if v.is_none() { None } else { Some(v.extract()) })
+                .and_then(|v| {
+                    if v.is_none() {
+                        None
+                    } else {
+                        Some(v.extract::<String>())
+                    }
+                })
+                .transpose()?
+                .map(|s| {
+                    Uuid::parse_str(&s).map_err(|e| {
+                        PyValueError::new_err(format!("Invalid UUID string for index_uuid: {e}"))
+                    })
+                })
                 .transpose()?
         } else {
             None
@@ -2461,18 +2622,31 @@ impl Dataset {
         Ok(())
     }
 
-    #[pyo3(signature = (name, *, with_position = false))]
-    fn prewarm_index(&self, name: &str, with_position: bool) -> PyResult<()> {
+    #[pyo3(signature = (name, *, with_position = false, index_segments = None))]
+    fn prewarm_index(
+        &self,
+        name: &str,
+        with_position: bool,
+        index_segments: Option<Vec<String>>,
+    ) -> PyResult<()> {
+        let index_segments = parse_index_segment_ids(index_segments)?;
+
         rt().block_on(None, async {
             if with_position {
-                self.ds
-                    .prewarm_index_with_options(
-                        name,
-                        &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)),
-                    )
-                    .await
+                let options = PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true));
+                if let Some(index_segments) = index_segments.as_deref() {
+                    self.ds
+                        .prewarm_index_segments_with_options(name, index_segments, &options)
+                        .await
+                } else {
+                    self.ds.prewarm_index_with_options(name, &options).await
+                }
             } else {
-                self.ds.prewarm_index(name).await
+                if let Some(index_segments) = index_segments.as_deref() {
+                    self.ds.prewarm_index_segments(name, index_segments).await
+                } else {
+                    self.ds.prewarm_index(name).await
+                }
             }
         })?
         .infer_error()
@@ -2486,6 +2660,9 @@ impl Dataset {
         batch_readhead: Option<usize>,
         progress_callback: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<()> {
+        let parsed_uuid = Uuid::parse_str(index_uuid).map_err(|e| {
+            PyValueError::new_err(format!("Invalid UUID string for index_uuid: {e}"))
+        })?;
         let mut progress_handler =
             Self::make_index_progress_handler_from_callback(progress_callback)?;
         let progress: Arc<dyn IndexBuildProgress> = progress_handler
@@ -2497,7 +2674,7 @@ impl Dataset {
             async {
                 self.ds
                     .merge_index_metadata(
-                        index_uuid,
+                        &parsed_uuid,
                         IndexType::try_from(index_type)?,
                         batch_readhead,
                         progress,
@@ -2691,9 +2868,16 @@ impl Dataset {
                 && let (Some(ns_client), Some(tid)) = (namespace_client, table_id)
             {
                 // Create ExternalManifestCommitHandler from namespace client and table_id
-                // only when namespace manages versioning
+                // only when namespace manages versioning. The store derives the
+                // branch a request targets from the base path it is handed,
+                // resolved against the table root.
                 let ns_client = extract_namespace_arc(ns_client.py(), ns_client)?;
-                let external_store = LanceNamespaceExternalManifestStore::new(ns_client, tid);
+                let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
+                    ns_client,
+                    tid,
+                    &dest.table_root_uri()?,
+                )
+                .infer_error()?;
                 Some(Arc::new(ExternalManifestCommitHandler {
                     external_manifest_store: Arc::new(external_store),
                 }) as Arc<dyn CommitHandler>)
@@ -2909,6 +3093,54 @@ impl Dataset {
         Ok(PyArrowType(reader))
     }
 
+    #[pyo3(signature = (*, min_version=None, progress=None))]
+    fn tracked_files(
+        &self,
+        min_version: Option<u64>,
+        progress: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
+        use lance::dataset::files::{TrackedFilesOptions, TrackedFilesProgress};
+
+        let progress_cb: Option<Box<dyn Fn(TrackedFilesProgress) + Send + Sync>> =
+            if let Some(cb) = progress {
+                if !cb.is_callable() {
+                    return Err(PyValueError::new_err("progress must be callable"));
+                }
+                let cb = cb.clone().unbind();
+                Some(Box::new(move |p: TrackedFilesProgress| {
+                    Python::attach(|py| {
+                        let total: Option<usize> = p.manifests_total;
+                        match cb.call1(py, (p.manifests_processed, total)) {
+                            Ok(_) => (),
+                            Err(e) => {
+                                log::error!("Error in tracked_files progress callback: {}", e);
+                            }
+                        }
+                    });
+                }))
+            } else {
+                None
+            };
+
+        let options = TrackedFilesOptions {
+            min_version,
+            progress: progress_cb,
+        };
+        let stream = rt().block_on(None, self.ds.tracked_files_with_options(options))?;
+        let reader = Box::new(LanceReader::from_stream(DatasetRecordBatchStream::new(
+            stream,
+        )));
+        Ok(PyArrowType(reader))
+    }
+
+    fn all_files(&self) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
+        let stream = rt().block_on(None, self.ds.all_files())?;
+        let reader = Box::new(LanceReader::from_stream(DatasetRecordBatchStream::new(
+            stream,
+        )));
+        Ok(PyArrowType(reader))
+    }
+
     #[pyo3(signature = (keys))]
     fn delete_config_keys(&mut self, keys: Vec<String>) -> PyResult<()> {
         let mut new_self = self.ds.as_ref().clone();
@@ -3098,11 +3330,7 @@ impl Dataset {
 
             let vindex = self
                 .ds
-                .open_vector_index(
-                    column_name,
-                    &idx_meta.uuid.to_string(),
-                    &NoOpMetricsCollector,
-                )
+                .open_vector_index(column_name, &idx_meta.uuid, &NoOpMetricsCollector)
                 .await
                 .infer_error()?;
 
@@ -3407,6 +3635,229 @@ impl Dataset {
             self.ds.clone(),
         ))
     }
+
+    /// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index.
+    ///
+    /// This function loads a specific partition from every segment of an IVF_FLAT
+    /// index on a hash column, computes pairwise hamming distances between all
+    /// hashes in the combined partition, filters by threshold, and clusters the
+    /// results using union-find.
+    ///
+    /// Parameters
+    /// ----------
+    /// index_name : str
+    ///     Name of the IVF_FLAT index on the hash column
+    /// partition_id : int
+    ///     The partition ID within the IVF_FLAT index
+    /// hamming_threshold : int
+    ///     Maximum hamming distance to consider as similar
+    /// index_segments : list of str, optional
+    ///     If specified, only these physical index segment UUIDs of the named
+    ///     logical index contribute rows. Defaults to all segments.
+    ///
+    /// Returns
+    /// -------
+    /// pyarrow.RecordBatchReader
+    ///     A reader yielding batches with columns:
+    ///     - 'representative': uint64 - The representative row ID for each cluster
+    ///     - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    #[pyo3(signature = (index_name, partition_id, hamming_threshold, index_segments=None))]
+    fn hamming_clustering_for_ivf_partition(
+        &self,
+        py: Python<'_>,
+        index_name: &str,
+        partition_id: usize,
+        hamming_threshold: u32,
+        index_segments: Option<Vec<String>>,
+    ) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
+        use lance::index::vector::hamming::{
+            hamming_clustering_for_ivf_partition, hamming_clustering_for_ivf_partition_segments,
+        };
+
+        let segment_ids = parse_index_segment_ids(index_segments)?;
+        let ds = self.ds.as_ref();
+        let reader = rt()
+            .block_on(Some(py), async {
+                match segment_ids.as_deref() {
+                    Some(segment_ids) => {
+                        hamming_clustering_for_ivf_partition_segments(
+                            ds,
+                            index_name,
+                            segment_ids,
+                            partition_id,
+                            hamming_threshold,
+                        )
+                        .await
+                    }
+                    None => {
+                        hamming_clustering_for_ivf_partition(
+                            ds,
+                            index_name,
+                            partition_id,
+                            hamming_threshold,
+                        )
+                        .await
+                    }
+                }
+            })?
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        Ok(PyArrowType(reader))
+    }
+
+    /// Get partition information for an IVF_FLAT index.
+    ///
+    /// Partition sizes are aggregated across all segments of the logical index
+    /// unless a subset is selected via ``index_segments``.
+    ///
+    /// Parameters
+    /// ----------
+    /// index_name : str
+    ///     Name of the IVF_FLAT index
+    /// index_segments : list of str, optional
+    ///     If specified, only these physical index segment UUIDs of the named
+    ///     logical index contribute to the sizes. Defaults to all segments.
+    ///
+    /// Returns
+    /// -------
+    /// List[dict]
+    ///     List of partition info dicts with 'partition_id' and 'size'
+    #[pyo3(signature = (index_name, index_segments=None))]
+    fn get_ivf_partition_info(
+        &self,
+        py: Python<'_>,
+        index_name: &str,
+        index_segments: Option<Vec<String>>,
+    ) -> PyResult<Vec<Py<PyDict>>> {
+        use lance::index::vector::hamming::{
+            get_ivf_partition_info, get_ivf_partition_info_segments,
+        };
+
+        let segment_ids = parse_index_segment_ids(index_segments)?;
+        let ds = self.ds.as_ref();
+        let result = rt()
+            .block_on(Some(py), async {
+                match segment_ids.as_deref() {
+                    Some(segment_ids) => {
+                        get_ivf_partition_info_segments(ds, index_name, segment_ids).await
+                    }
+                    None => get_ivf_partition_info(ds, index_name).await,
+                }
+            })?
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        let partitions: PyResult<Vec<_>> = result
+            .iter()
+            .map(|p| {
+                let dict = PyDict::new(py);
+                dict.set_item("partition_id", p.partition_id)?;
+                dict.set_item("size", p.size)?;
+                Ok(dict.into())
+            })
+            .collect();
+
+        partitions
+    }
+
+    /// Perform pairwise hamming distance clustering on sampled rows from a dataset.
+    ///
+    /// This function samples N rows randomly from the dataset, extracts hashes,
+    /// computes pairwise hamming distances, and clusters the results.
+    /// It's useful for benchmarking and testing without requiring an IVF index.
+    ///
+    /// Parameters
+    /// ----------
+    /// column : str
+    ///     Name of the hash column (must be FixedSizeList<UInt8, N> where N is
+    ///     a positive multiple of 8 bytes)
+    /// sample_size : int, optional
+    ///     Number of rows to sample (if None or >= total rows, uses all rows)
+    /// hamming_threshold : int
+    ///     Maximum hamming distance to consider as similar
+    ///
+    /// Returns
+    /// -------
+    /// pyarrow.RecordBatchReader
+    ///     A reader yielding batches with columns:
+    ///     - 'representative': uint64 - The representative row ID for each cluster
+    ///     - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    #[pyo3(signature = (column, sample_size, hamming_threshold))]
+    fn hamming_clustering_for_sample(
+        &self,
+        py: Python<'_>,
+        column: &str,
+        sample_size: Option<usize>,
+        hamming_threshold: u32,
+    ) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
+        use lance::index::vector::hamming::hamming_clustering_for_sample;
+
+        let ds = self.ds.as_ref();
+        let reader = rt()
+            .block_on(
+                Some(py),
+                hamming_clustering_for_sample(ds, column, sample_size, hamming_threshold),
+            )?
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        Ok(PyArrowType(reader))
+    }
+
+    /// Perform pairwise hamming distance clustering on a contiguous range of rows from a fragment.
+    ///
+    /// This function reads a contiguous range of rows from a specific fragment,
+    /// extracts hashes, computes pairwise hamming distances, and clusters the results.
+    /// Unlike sampling, this reads sequential rows which is useful for distributed
+    /// processing where each worker handles a specific range of a fragment.
+    ///
+    /// Parameters
+    /// ----------
+    /// column : str
+    ///     Name of the hash column (must be FixedSizeList<UInt8, N> where N is
+    ///     a positive multiple of 8 bytes)
+    /// fragment_id : int
+    ///     The fragment ID to read from
+    /// start_row : int
+    ///     The starting row offset within the fragment
+    /// num_rows : int
+    ///     Number of rows to read from the start position
+    /// hamming_threshold : int
+    ///     Maximum hamming distance to consider as similar
+    ///
+    /// Returns
+    /// -------
+    /// pyarrow.RecordBatchReader
+    ///     A reader yielding batches with columns:
+    ///     - 'representative': uint64 - The representative row ID for each cluster
+    ///     - 'duplicates': list<uint64> - List of duplicate row IDs in each cluster
+    #[pyo3(signature = (column, fragment_id, start_row, num_rows, hamming_threshold))]
+    fn hamming_clustering_for_range(
+        &self,
+        py: Python<'_>,
+        column: &str,
+        fragment_id: usize,
+        start_row: usize,
+        num_rows: usize,
+        hamming_threshold: u32,
+    ) -> PyResult<PyArrowType<Box<dyn RecordBatchReader + Send>>> {
+        use lance::index::vector::hamming::hamming_clustering_for_range;
+
+        let ds = self.ds.as_ref();
+        let reader = rt()
+            .block_on(
+                Some(py),
+                hamming_clustering_for_range(
+                    ds,
+                    column,
+                    fragment_id,
+                    start_row,
+                    num_rows,
+                    hamming_threshold,
+                ),
+            )?
+            .map_err(|err| PyValueError::new_err(err.to_string()))?;
+
+        Ok(PyArrowType(reader))
+    }
 }
 
 #[pyclass(name = "SqlQuery", module = "_lib", subclass, skip_from_py_object)]
@@ -3606,9 +4057,18 @@ impl PyWriteDest {
             Self::Uri(uri) => WriteDestination::Uri(uri),
         }
     }
+
+    /// The table root uri of this destination (a branch dataset resolves to
+    /// its main location). Used to root the namespace manifest store.
+    pub fn table_root_uri(&self) -> PyResult<String> {
+        match self {
+            Self::Dataset(ds) => Ok(ds.ds.branch_location().find_main().infer_error()?.uri),
+            Self::Uri(uri) => Ok(uri.to_string()),
+        }
+    }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 enum IndexProgressEventType {
     Start,
     Progress,
@@ -3764,7 +4224,20 @@ impl IndexProgressDispatcher {
 
     fn drain(&mut self) -> PyResult<()> {
         while let Ok(event) = self.receiver.try_recv() {
-            self.dispatch(event)?;
+            let is_complete = event.event == IndexProgressEventType::Complete;
+            if let Err(err) = self.dispatch(event) {
+                if is_complete {
+                    // Complete events are purely informational — the stage's work
+                    // is already done. Propagating a callback error here would
+                    // abort the operation after the real work has succeeded.
+                    log::warn!(
+                        "Ignoring progress callback error on stage-complete event: {}",
+                        err
+                    );
+                } else {
+                    return Err(err);
+                }
+            }
         }
         Ok(())
     }
@@ -3948,7 +4421,7 @@ pub fn write_dataset(
     dest: PyWriteDest,
     options: &Bound<'_, PyDict>,
 ) -> PyResult<Dataset> {
-    let params = get_write_params(options)?;
+    let params = get_write_params(options, &dest.table_root_uri()?)?;
     let py = options.py();
     let ds = if reader.is_instance_of::<Scanner>() {
         let scanner: Scanner = reader.extract()?;
@@ -4017,8 +4490,13 @@ fn get_dict_opt<'py, D: FromPyObjectOwned<'py>>(
         .transpose()
 }
 
+/// `table_uri` is the destination table's root uri; it roots the namespace
+/// manifest store when `namespace_client_managed_versioning` is requested.
 #[allow(deprecated)]
-pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WriteParams>> {
+pub fn get_write_params(
+    options: &Bound<'_, PyDict>,
+    table_uri: &str,
+) -> PyResult<Option<WriteParams>> {
     let params = if options.is_none() {
         None
     } else {
@@ -4136,6 +4614,11 @@ pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WritePar
             p = p.with_target_base_names_or_paths(target_bases_list);
         }
 
+        // Handle target_all_bases parameter (bool: include primary storage)
+        if let Some(target_all_bases) = get_dict_opt::<bool>(options, "target_all_bases")? {
+            p = p.with_target_all_bases(target_all_bases);
+        }
+
         // Handle base_store_params: per-base storage options keyed by base path URI
         if let Some(base_store_params) =
             get_dict_opt::<HashMap<String, HashMap<String, String>>>(options, "base_store_params")?
@@ -4188,9 +4671,15 @@ pub fn get_write_params(options: &Bound<'_, PyDict>) -> PyResult<Option<WritePar
             && let (Some(ns_client), Some(table_id)) =
                 (namespace_client_opt.as_ref(), table_id_opt.as_ref())
         {
+            // The store derives the branch a request targets from the base path
+            // it is handed, resolved against the table root.
             let ns_client = extract_namespace_arc(options.py(), ns_client)?;
-            let external_store =
-                LanceNamespaceExternalManifestStore::new(ns_client, table_id.clone());
+            let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
+                ns_client,
+                table_id.clone(),
+                table_uri,
+            )
+            .infer_error()?;
             let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
                 external_manifest_store: Arc::new(external_store),
             });
@@ -4361,6 +4850,13 @@ fn prepare_vector_index_params(
             pq_params.codebook = Some(codebook.values().clone())
         };
 
+        if let Some(r) = kwargs.get_item("rabitq_model")? {
+            let json: String = r.extract()?;
+            let meta: RabitQuantizationMetadata = serde_json::from_str(&json)
+                .map_err(|e| PyValueError::new_err(format!("Invalid rabitq_model JSON: {e}")))?;
+            rq_params.rotation = Some(meta);
+        };
+
         if let Some(version) = kwargs.get_item("index_file_version")? {
             let version: String = version.extract()?;
             index_file_version = IndexFileVersion::try_from(&version)
@@ -4681,6 +5177,7 @@ type VectorQueryParams = (
     bool,
     Option<usize>,
     i32,
+    ApproxMode,
 );
 
 fn extract_query_parallelism(value: &Bound<'_, PyAny>) -> PyResult<i32> {
@@ -4702,6 +5199,23 @@ fn vector_query_query_parallelism_from_dict(dict: &Bound<'_, PyDict>) -> PyResul
     }
 }
 
+fn vector_query_approx_mode_from_dict(dict: &Bound<'_, PyDict>) -> PyResult<ApproxMode> {
+    if let Some(approx_mode) = dict.get_item("approx_mode")?
+        && !approx_mode.is_none()
+    {
+        match approx_mode.to_string().to_lowercase().as_str() {
+            "fast" => Ok(ApproxMode::Fast),
+            "normal" => Ok(ApproxMode::Normal),
+            "accurate" => Ok(ApproxMode::Accurate),
+            value => Err(PyValueError::new_err(format!(
+                "approx_mode must be one of 'fast', 'normal', or 'accurate', got '{value}'"
+            ))),
+        }
+    } else {
+        Ok(ApproxMode::Normal)
+    }
+}
+
 fn vector_query_params_from_dict(
     dict: &Bound<'_, PyDict>,
     default_k: usize,
@@ -4808,6 +5322,7 @@ fn vector_query_params_from_dict(
     };
 
     let query_parallelism = vector_query_query_parallelism_from_dict(dict)?;
+    let approx_mode = vector_query_approx_mode_from_dict(dict)?;
 
     Ok((
         column,
@@ -4820,6 +5335,7 @@ fn vector_query_params_from_dict(
         use_index,
         ef,
         query_parallelism,
+        approx_mode,
     ))
 }
 
@@ -4856,6 +5372,7 @@ impl PySearchFilter {
             use_index,
             ef,
             query_parallelism,
+            approx_mode,
         ) = vector_query_params_from_dict(query, default_k)?;
 
         let metric_type = Some(metric_type_opt.unwrap_or(MetricType::L2));
@@ -4874,6 +5391,7 @@ impl PySearchFilter {
             use_index,
             query_parallelism,
             dist_q_c: 0.0,
+            approx_mode,
         };
 
         Ok(Self {
diff --git a/python/src/dataset/cleanup.rs b/python/src/dataset/cleanup.rs
index 4f1655e6df6..fb675124c4e 100644
--- a/python/src/dataset/cleanup.rs
+++ b/python/src/dataset/cleanup.rs
@@ -14,8 +14,8 @@
 
 use pyo3::{pyclass, pymethods};
 
-#[pyclass(get_all)]
-#[derive(Debug)]
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone, Debug)]
 pub struct CleanupStats {
     pub bytes_removed: u64,
     pub old_versions: u64,
@@ -31,3 +31,53 @@ impl CleanupStats {
         format!("{self:?}")
     }
 }
+
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone, Debug)]
+pub struct CleanupCandidateFile {
+    pub path: String,
+    pub kind: String,
+    pub unverified: bool,
+    pub size_bytes: u64,
+}
+
+#[pymethods]
+impl CleanupCandidateFile {
+    fn __repr__(&self) -> String {
+        format!("{self:?}")
+    }
+}
+
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone, Debug)]
+pub struct CleanupReferencedBranch {
+    pub name: String,
+    pub referenced_version: u64,
+    pub cleanup_candidate: bool,
+}
+
+#[pymethods]
+impl CleanupReferencedBranch {
+    fn __repr__(&self) -> String {
+        format!("{self:?}")
+    }
+}
+
+#[pyclass(get_all, skip_from_py_object)]
+#[derive(Clone, Debug)]
+pub struct CleanupExplanation {
+    pub read_version: u64,
+    pub stats: CleanupStats,
+    pub candidate_files: Vec<CleanupCandidateFile>,
+    pub candidate_files_truncated: bool,
+    pub candidate_file_limit: usize,
+    pub referenced_branches: Vec<CleanupReferencedBranch>,
+    pub warnings: Vec<String>,
+}
+
+#[pymethods]
+impl CleanupExplanation {
+    fn __repr__(&self) -> String {
+        format!("{self:?}")
+    }
+}
diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs
index 321d7157b86..4bb29246f45 100644
--- a/python/src/dataset/optimize.rs
+++ b/python/src/dataset/optimize.rs
@@ -58,6 +58,9 @@ fn parse_compaction_options(
             "batch_size" => {
                 opts.batch_size = value.extract()?;
             }
+            "io_buffer_size" => {
+                opts.io_buffer_size = value.extract()?;
+            }
             "compaction_mode" => {
                 let mode_str: Option<String> = value.extract()?;
                 if let Some(mode_str) = mode_str {
@@ -551,26 +554,34 @@ impl PyCompaction {
     ///     new version once committed.
     /// rewrites : List[RewriteResult]
     ///     The results of the compaction tasks to include in the commit.
+    /// options : dict, optional
+    ///     Compaction options to apply at commit time.
+    ///     When absent or ``None``, defaults to ``CompactionOptions::default()``.
     ///
     /// Returns
     /// -------
     /// CompactionMetrics
     #[staticmethod]
+    #[pyo3(signature = (dataset, rewrites, options = None))]
     pub fn commit(
         dataset: Bound<PyAny>,
         rewrites: Vec<PyRewriteResult>,
+        options: Option<Bound<PyDict>>,
     ) -> PyResult<PyCompactionMetrics> {
         let dataset_ref = unwrap_dataset(dataset)?;
         let dataset = dataset_ref.borrow().clone();
+        let config = dataset.ds.manifest.config.clone();
+        let opts = match options {
+            Some(ref dict) => parse_compaction_options(dict, &config)?,
+            None => CompactionOptions::default(),
+        };
         let rewrites: Vec<RewriteResult> = rewrites.into_iter().map(|r| r.0).collect();
         let mut new_ds = dataset.ds.as_ref().clone();
-        // TODO: pass compaction option from plan and execute time
-        let options: CompactionOptions = CompactionOptions::default();
         let fut = commit_compaction(
             &mut new_ds,
             rewrites,
             Arc::new(DatasetIndexRemapperOptions::default()),
-            &options,
+            &opts,
         );
         let metrics = rt()
             .block_on(None, fut)?
diff --git a/python/src/file.rs b/python/src/file.rs
index ab5bda77fb0..2a3dd09e17f 100644
--- a/python/src/file.rs
+++ b/python/src/file.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::blob::{PyDedicatedBlobWriter, PyPackedBlobWriter};
 use crate::namespace::extract_namespace_arc;
 use crate::{error::PythonErrorExt, rt};
 use arrow::pyarrow::PyArrowType;
@@ -347,8 +348,10 @@ impl LanceFileWriter {
     }
 
     pub fn finish(&self) -> PyResult<u64> {
-        rt().block_on(None, async { self.inner.lock().await.finish().await })?
-            .infer_error()
+        rt().block_on(None, async {
+            self.inner.lock().await.finish().await.map(|s| s.num_rows)
+        })?
+        .infer_error()
     }
 
     pub fn add_global_buffer(&self, bytes: Vec<u8>) -> PyResult<u32> {
@@ -522,6 +525,30 @@ impl LanceFileSession {
         )?
     }
 
+    pub fn open_packed_blob_writer(
+        &self,
+        path: String,
+        blob_id: u32,
+    ) -> PyResult<PyPackedBlobWriter> {
+        let path = self.base_path.child_path(&Path::from(path));
+        rt().block_on(
+            None,
+            PyPackedBlobWriter::try_new(self.object_store.clone(), path, blob_id),
+        )?
+    }
+
+    pub fn open_dedicated_blob_writer(
+        &self,
+        path: String,
+        blob_id: u32,
+    ) -> PyResult<PyDedicatedBlobWriter> {
+        let path = self.base_path.child_path(&Path::from(path));
+        rt().block_on(
+            None,
+            PyDedicatedBlobWriter::try_new(self.object_store.clone(), path, blob_id),
+        )?
+    }
+
     pub fn contains(&self, path: String) -> PyResult<bool> {
         let full_path = self.base_path.child_path(&Path::from(path));
         rt().block_on(None, async {
@@ -572,6 +599,57 @@ impl LanceFileSession {
         })?
     }
 
+    /// Non-recursive, delimited list of a single directory level.
+    ///
+    /// Returns a tuple `(common_prefixes, objects)` of paths relative to the
+    /// session's `base_path`, where `common_prefixes` are the immediate child
+    /// "directories" and `objects` are the immediate child files. Unlike
+    /// `list`, this does not recurse into the subtree.
+    #[pyo3(signature=(path=None))]
+    pub fn list_with_delimiter(
+        &self,
+        path: Option<String>,
+    ) -> PyResult<(Vec<String>, Vec<String>)> {
+        rt().block_on(None, async {
+            let list_path = if let Some(prefix) = path {
+                self.base_path.child_path(&Path::from(prefix))
+            } else {
+                self.base_path.clone()
+            };
+
+            let result = self
+                .object_store
+                .list_with_delimiter(Some(&list_path))
+                .await
+                .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(format!("{}", e)))?;
+
+            // Strip the base_path prefix to make each path relative to the session.
+            let relativize = |location: &Path| -> PyResult<String> {
+                let relative_parts = location.prefix_match(&self.base_path).ok_or_else(|| {
+                    PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(format!(
+                        "Path '{}' does not start with base path '{}'",
+                        location.as_ref(),
+                        self.base_path.as_ref()
+                    ))
+                })?;
+                Ok(Path::from_iter(relative_parts).as_ref().to_string())
+            };
+
+            let common_prefixes = result
+                .common_prefixes
+                .iter()
+                .map(relativize)
+                .collect::<PyResult<Vec<String>>>()?;
+            let objects = result
+                .objects
+                .iter()
+                .map(|meta| relativize(&meta.location))
+                .collect::<PyResult<Vec<String>>>()?;
+
+            Ok((common_prefixes, objects))
+        })?
+    }
+
     /// Upload a file from local filesystem to the object store
     ///
     /// Parameters
@@ -604,6 +682,61 @@ impl LanceFileSession {
         })?
     }
 
+    /// Delete a file from the object store.
+    ///
+    /// The path is interpreted relative to the session's `base_path`, matching
+    /// `contains`/`upload_file`/`download_file`. Deleting a missing object
+    /// raises `OSError`, consistent with `download_file`.
+    ///
+    /// Parameters
+    /// ----------
+    /// path : str
+    ///     Path relative to `base_path` to delete.
+    pub fn delete_file(&self, path: String) -> PyResult<()> {
+        rt().block_on(None, async {
+            let full_path = self.base_path.child_path(&Path::from(path));
+            self.object_store
+                .inner
+                .delete(&full_path)
+                .await
+                .map_err(|e| PyIOError::new_err(format!("Failed to delete remote file: {}", e)))?;
+            Ok(())
+        })?
+    }
+
+    /// Read a byte range from a file in the object store.
+    ///
+    /// The path is interpreted relative to the session's `base_path`, matching
+    /// the other session methods. This issues a single ranged GET. Reading a
+    /// missing object raises `OSError`, consistent with `download_file`.
+    ///
+    /// Parameters
+    /// ----------
+    /// path : str
+    ///     Path relative to `base_path` to read from.
+    /// offset : int
+    ///     Byte offset at which to start reading.
+    /// length : int
+    ///     Number of bytes to read.
+    ///
+    /// Returns
+    /// -------
+    /// bytes
+    ///     The requested byte range.
+    pub fn read_range(&self, path: String, offset: usize, length: usize) -> PyResult<Vec<u8>> {
+        rt().block_on(None, async {
+            let full_path = self.base_path.child_path(&Path::from(path));
+            let bytes = self
+                .object_store
+                .read_one_range(&full_path, offset..offset + length)
+                .await
+                .map_err(|e| {
+                    PyIOError::new_err(format!("Failed to read range from remote file: {}", e))
+                })?;
+            Ok(bytes.to_vec())
+        })?
+    }
+
     /// Download a file from object store to local filesystem
     ///
     /// Parameters
diff --git a/python/src/fragment.rs b/python/src/fragment.rs
index 1da99492fac..6b832870280 100644
--- a/python/src/fragment.rs
+++ b/python/src/fragment.rs
@@ -23,9 +23,10 @@ use lance::Error;
 use lance::dataset::fragment::FileFragment as LanceFragment;
 use lance::dataset::scanner::ColumnOrdering;
 use lance::dataset::transaction::{Operation, Transaction};
-use lance::dataset::{InsertBuilder, NewColumnTransform};
+use lance::dataset::{InsertBuilder, NewColumnTransform, WriteParams};
 use lance_core::datatypes::BlobHandling;
 use lance_io::utils::CachedFileSize;
+use lance_table::format::overlay::DataOverlayFile;
 use lance_table::format::{
     DataFile, DeletionFile, DeletionFileType, Fragment, RowDatasetVersionMeta, RowIdMeta,
 };
@@ -119,7 +120,7 @@ impl FileFragment {
         kwargs: Option<&Bound<'_, PyDict>>,
     ) -> PyResult<PyLance<Fragment>> {
         let params = if let Some(kw_params) = kwargs {
-            get_write_params(kw_params)?
+            get_write_params(kw_params, dataset_uri)?
         } else {
             None
         };
@@ -385,6 +386,38 @@ impl FileFragment {
         }
     }
 
+    /// Delete rows by their local (within-fragment) physical row offsets.
+    ///
+    /// Adds the given 0-based offsets to this fragment's deletion vector and
+    /// writes a new deletion file. Returns the updated fragment, or None if every
+    /// row is now deleted. Unlike `delete(predicate)`, this deletes exactly the
+    /// supplied rows without re-evaluating a filter -- useful when the caller
+    /// already knows which rows to delete (e.g. offsets collected from a prior
+    /// scan), avoiding a redundant predicate evaluation.
+    fn delete_rows(&self, offsets: Vec<u32>) -> PyResult<Option<Self>> {
+        let old_fragment = self.fragment.clone();
+        // The core deletion path only errors once the deletion count reaches
+        // physical_rows, so out-of-range offsets must be rejected here.
+        let updated_fragment = rt()
+            .block_on(None, async {
+                let physical_rows = old_fragment.physical_rows().await?;
+                if let Some(&offset) = offsets.iter().find(|&&o| o as usize >= physical_rows) {
+                    return Err(Error::invalid_input(format!(
+                        "delete_rows offset {offset} is out of range for fragment {} \
+                         with {physical_rows} rows",
+                        old_fragment.id()
+                    )));
+                }
+                old_fragment.extend_deletions(offsets).await
+            })?
+            .infer_error()?;
+
+        match updated_fragment {
+            Some(frag) => Ok(Some(Self::new(frag))),
+            None => Ok(None),
+        }
+    }
+
     fn schema<'py>(self_: PyRef<'py, Self>) -> PyResult<Bound<'py, PyAny>> {
         let schema = self_.fragment.dataset().schema();
         let logical_schema = logical_schema_from_lance(schema);
@@ -435,10 +468,10 @@ fn do_write_fragments(
 ) -> PyResult<Transaction> {
     let batches = convert_reader(reader)?;
 
-    let params = kwargs
-        .and_then(|params| get_write_params(params).transpose())
-        .transpose()?
-        .unwrap_or_default();
+    let params = match kwargs {
+        Some(params) => get_write_params(params, &dest.table_root_uri()?)?.unwrap_or_default(),
+        None => WriteParams::default(),
+    };
 
     rt().block_on(
         Some(reader.py()),
@@ -793,6 +826,10 @@ impl FromPyObject<'_, '_> for PyLance<Fragment> {
             row_id_meta,
             last_updated_at_version_meta,
             created_at_version_meta,
+            // Round-tripped so overlays survive operations that pass existing
+            // fragments back (a manual Delete/Update/Merge commit). Sorting
+            // newest-last is deferred to the manifest reload after commit.
+            overlays: extract_vec::<DataOverlayFile>(&ob.getattr("overlays")?)?,
         }))
     }
 }
@@ -825,6 +862,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> {
             .created_at_version_meta
             .as_ref()
             .map(|r| PyRowDatasetVersionMeta(r.clone()));
+        let overlays = export_vec(py, &self.0.overlays)?;
 
         cls.call1((
             self.0.id,
@@ -834,6 +872,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> {
             row_id_meta,
             created_at_version_meta,
             last_updated_at_version_meta,
+            overlays,
         ))
     }
 }
diff --git a/python/src/indices.rs b/python/src/indices.rs
index fe988206117..1d5c476d795 100644
--- a/python/src/indices.rs
+++ b/python/src/indices.rs
@@ -145,11 +145,7 @@ async fn do_get_ivf_model(dataset: &Dataset, index_name: &str) -> PyResult<IvfMo
     // Open the vector index
     let vindex = dataset
         .ds
-        .open_vector_index(
-            column_name,
-            &idx_meta.uuid.to_string(),
-            &NoOpMetricsCollector,
-        )
+        .open_vector_index(column_name, &idx_meta.uuid, &NoOpMetricsCollector)
         .await
         .infer_error()?;
 
@@ -236,6 +232,7 @@ async fn do_train_pq_model(
     distance_type: &str,
     sample_rate: u32,
     max_iters: u32,
+    num_bits: u32,
     ivf_model: IvfModel,
     fragment_ids: Option<Vec<u32>>,
 ) -> PyResult<ArrayData> {
@@ -243,7 +240,7 @@ async fn do_train_pq_model(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let params = PQBuildParams {
         num_sub_vectors: num_subvectors as usize,
-        num_bits: 8,
+        num_bits: num_bits as usize,
         max_iters: max_iters as usize,
         sample_rate: sample_rate as usize,
         ..Default::default()
@@ -264,7 +261,7 @@ async fn do_train_pq_model(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
-#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None))]
+#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None, num_bits=8))]
 fn train_pq_model<'py>(
     py: Python<'py>,
     dataset: &Dataset,
@@ -276,6 +273,7 @@ fn train_pq_model<'py>(
     max_iters: u32,
     ivf_centroids: PyArrowType<ArrayData>,
     fragment_ids: Option<Vec<u32>>,
+    num_bits: u32,
 ) -> PyResult<Bound<'py, PyAny>> {
     let ivf_centroids = ivf_centroids.0;
     let ivf_centroids = FixedSizeListArray::from(ivf_centroids);
@@ -295,6 +293,7 @@ fn train_pq_model<'py>(
             distance_type,
             sample_rate,
             max_iters,
+            num_bits,
             ivf_model,
             fragment_ids,
         ),
@@ -302,6 +301,61 @@ fn train_pq_model<'py>(
     codebook.to_pyarrow(py)
 }
 
+/// Mint one RaBitQ rotation and return it as a JSON string.
+///
+/// Distributed IVF_RQ builds must pin a single rotation across all workers so that
+/// independently built per-fragment segments rotate vectors identically and their
+/// binary codes remain comparable when merged. A driver calls this once and broadcasts
+/// the resulting string to every `create_index_uncommitted(..., rabitq_model=...)` call.
+///
+/// The rotation is always the "fast" rotation since its sign vector is JSON-serializable,
+/// whereas the "matrix" rotation stores a dense matrix in a binary buffer that is dropped by
+/// the JSON wire format. `dtype` is accepted for API symmetry but does not affect the fast
+/// rotation.
+///
+/// # Example (Python)
+///
+/// ```python
+/// from lance.lance import indices
+///
+/// # Mint one model and broadcast `model` to every worker.
+/// model = indices.build_rq_model(dimension=128, num_bits=1)
+/// seg = ds.create_index_uncommitted(
+///     column="vector",
+///     index_type="IVF_RQ",
+///     num_partitions=256,
+///     ivf_centroids=centroids,
+///     rabitq_model=model,
+///     fragment_ids=my_fragments,
+/// )
+/// ```
+#[pyfunction]
+#[pyo3(signature = (dimension, num_bits=1, dtype="float32"))]
+pub fn build_rq_model(dimension: usize, num_bits: u8, dtype: &str) -> PyResult<String> {
+    use arrow::datatypes::{Float16Type, Float32Type, Float64Type};
+    use lance_index::vector::bq::RQRotationType;
+    use lance_index::vector::bq::builder::RabitQuantizer;
+    use lance_index::vector::quantizer::Quantization;
+
+    if !dimension.is_multiple_of(u8::BITS as usize) {
+        return Err(PyValueError::new_err(
+            "dimension must be divisible by 8 for IVF_RQ",
+        ));
+    }
+    let dim = dimension as i32;
+    let rotation = RQRotationType::Fast;
+    let quantizer = match dtype.to_lowercase().as_str() {
+        "float16" => RabitQuantizer::new_with_rotation::<Float16Type>(num_bits, dim, rotation),
+        "float32" => RabitQuantizer::new_with_rotation::<Float32Type>(num_bits, dim, rotation),
+        "float64" => RabitQuantizer::new_with_rotation::<Float64Type>(num_bits, dim, rotation),
+        other => {
+            return Err(PyValueError::new_err(format!("unsupported dtype: {other}")));
+        }
+    };
+    serde_json::to_string(&quantizer.metadata(None))
+        .map_err(|e| PyValueError::new_err(format!("failed to serialize RQ model: {e}")))
+}
+
 #[allow(clippy::too_many_arguments)]
 async fn do_transform_vectors(
     dataset: &Dataset,
@@ -347,7 +401,7 @@ async fn do_transform_vectors(
 
 #[pyfunction]
 #[allow(clippy::too_many_arguments)]
-#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None))]
+#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None, num_bits=8))]
 pub fn transform_vectors(
     py: Python<'_>,
     dataset: &Dataset,
@@ -360,6 +414,7 @@ pub fn transform_vectors(
     dst_uri: &str,
     fragments: Vec<FileFragment>,
     partitions_ds_uri: Option<&str>,
+    num_bits: u32,
 ) -> PyResult<()> {
     let ivf_centroids = ivf_centroids.0;
     let ivf_centroids = FixedSizeListArray::from(ivf_centroids);
@@ -368,7 +423,7 @@ pub fn transform_vectors(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let pq = ProductQuantizer::new(
         num_subvectors as usize,
-        /*num_bits=*/ 8,
+        num_bits,
         dimension,
         codebook,
         distance_type,
@@ -510,7 +565,7 @@ async fn do_load_shuffled_vectors(
 }
 
 #[pyfunction]
-#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None))]
+#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None, num_bits=8))]
 #[allow(clippy::too_many_arguments)]
 pub fn load_shuffled_vectors(
     filenames: Vec<String>,
@@ -523,6 +578,7 @@ pub fn load_shuffled_vectors(
     num_subvectors: u32,
     distance_type: &str,
     index_name: Option<&str>,
+    num_bits: u32,
 ) -> PyResult<()> {
     let mut default_idx_name = column.to_string();
     default_idx_name.push_str("_idx");
@@ -544,7 +600,7 @@ pub fn load_shuffled_vectors(
     let distance_type = DistanceType::try_from(distance_type).unwrap();
     let pq_model = ProductQuantizer::new(
         num_subvectors as usize,
-        /*num_bits=*/ 8,
+        num_bits,
         pq_dimension,
         codebook,
         distance_type,
@@ -579,6 +635,9 @@ pub struct PyIndexSegmentDescription {
     /// The total size in bytes of all files in this segment
     /// (None for backward compatibility with indices created before file tracking)
     pub size_bytes: Option<u64>,
+    /// The id of the dataset base path that stores this segment
+    /// (None when the segment is stored in the dataset's default base path)
+    pub base_id: Option<i64>,
 }
 
 impl PyIndexSegmentDescription {
@@ -597,18 +656,20 @@ impl PyIndexSegmentDescription {
             index_version: segment.index_version,
             created_at: segment.created_at,
             size_bytes,
+            base_id: segment.base_id.map(|id| id as i64),
         }
     }
 
     pub fn __repr__(&self) -> String {
         format!(
-            "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?}, size_bytes={:?})",
+            "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?}, size_bytes={:?}, base_id={:?})",
             self.uuid,
             self.dataset_version_at_last_update,
             self.fragment_ids,
             self.index_version,
             self.created_at,
-            self.size_bytes
+            self.size_bytes,
+            self.base_id
         )
     }
 }
@@ -623,7 +684,8 @@ pub struct PyIndexDescription {
     pub index_type: String,
     /// The ids of the fields that the index is built on
     pub fields: Vec<u32>,
-    /// The names of the fields that the index is built on
+    /// The full paths of the fields that the index is built on
+    /// (dotted, with backtick-quoted segments for non-identifier names)
     pub field_names: Vec<String>,
     /// The number of rows indexed by the index
     pub num_rows_indexed: u64,
@@ -644,9 +706,8 @@ impl PyIndexDescription {
             .map(|field| {
                 dataset
                     .schema()
-                    .field_by_id(*field as i32)
-                    .map(|f| f.name.clone())
-                    .unwrap_or("<unknown>".to_string())
+                    .field_path(*field as i32)
+                    .unwrap_or_else(|_| "<unknown>".to_string())
             })
             .collect();
 
@@ -696,6 +757,7 @@ pub fn register_indices(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     let indices = PyModule::new(py, "indices")?;
     indices.add_wrapped(wrap_pyfunction!(train_ivf_model))?;
     indices.add_wrapped(wrap_pyfunction!(train_pq_model))?;
+    indices.add_wrapped(wrap_pyfunction!(build_rq_model))?;
     indices.add_wrapped(wrap_pyfunction!(transform_vectors))?;
     indices.add_wrapped(wrap_pyfunction!(shuffle_transformed_vectors))?;
     indices.add_wrapped(wrap_pyfunction!(load_shuffled_vectors))?;
diff --git a/python/src/lib.rs b/python/src/lib.rs
index cf29b26c46a..c5631d26f10 100644
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -38,7 +38,9 @@ use datafusion_ffi::table_provider::FFI_TableProvider;
 #[cfg(feature = "datagen")]
 use datagen::register_datagen;
 use dataset::blob::LanceBlobFile;
-use dataset::cleanup::CleanupStats;
+use dataset::cleanup::{
+    CleanupCandidateFile, CleanupExplanation, CleanupReferencedBranch, CleanupStats,
+};
 use dataset::io_stats::IoStats;
 use dataset::optimize::{
     PyCompaction, PyCompactionMetrics, PyCompactionPlan, PyCompactionTask, PyRewriteResult,
@@ -60,6 +62,7 @@ use std::ffi::CString;
 use std::ptr::NonNull;
 
 pub(crate) mod arrow;
+pub(crate) mod blob;
 #[cfg(feature = "datagen")]
 pub(crate) mod datagen;
 pub(crate) mod dataset;
@@ -71,6 +74,7 @@ pub(crate) mod fragment;
 pub(crate) mod indices;
 pub(crate) mod mem_wal;
 pub(crate) mod namespace;
+pub(crate) mod otel;
 pub(crate) mod reader;
 pub(crate) mod scanner;
 pub(crate) mod schema;
@@ -94,6 +98,9 @@ pub use indices::register_indices;
 pub use reader::LanceReader;
 pub use scanner::Scanner;
 
+use crate::blob::{
+    PyBlobDescriptor, PyBlobDescriptorArrayBuilder, PyDedicatedBlobWriter, PyPackedBlobWriter,
+};
 use crate::executor::BackgroundExecutor;
 
 const CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION");
@@ -255,6 +262,10 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyRowDatasetVersionMeta>()?;
     m.add_class::<MergeInsertBuilder>()?;
     m.add_class::<LanceBlobFile>()?;
+    m.add_class::<PyBlobDescriptor>()?;
+    m.add_class::<PyBlobDescriptorArrayBuilder>()?;
+    m.add_class::<PyPackedBlobWriter>()?;
+    m.add_class::<PyDedicatedBlobWriter>()?;
     m.add_class::<LanceFileReader>()?;
     m.add_class::<LanceFileWriter>()?;
     m.add_class::<LanceFileSession>()?;
@@ -265,6 +276,9 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<LanceBufferDescriptor>()?;
     m.add_class::<BFloat16>()?;
     m.add_class::<CleanupStats>()?;
+    m.add_class::<CleanupCandidateFile>()?;
+    m.add_class::<CleanupReferencedBranch>()?;
+    m.add_class::<CleanupExplanation>()?;
     m.add_class::<IoStats>()?;
     m.add_class::<KMeans>()?;
     m.add_class::<Hnsw>()?;
@@ -293,6 +307,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<mem_wal::PyLsmPointLookupPlanner>()?;
     m.add_class::<mem_wal::PyLsmVectorSearchPlanner>()?;
     m.add_wrapped(wrap_pyfunction!(mem_wal::py_evaluate_sharding_spec))?;
+    m.add_wrapped(wrap_pyfunction!(mem_wal::py_write_pk_sidecar))?;
     m.add_wrapped(wrap_pyfunction!(bfloat16_array))?;
     m.add_wrapped(wrap_pyfunction!(write_dataset))?;
     m.add_wrapped(wrap_pyfunction!(write_fragments))?;
@@ -304,10 +319,17 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(trace_to_chrome))?;
     m.add_wrapped(wrap_pyfunction!(capture_trace_events))?;
     m.add_wrapped(wrap_pyfunction!(shutdown_tracing))?;
+    // OpenTelemetry metrics bridge
+    m.add_class::<otel::PyMetricPoint>()?;
+    m.add_class::<otel::PyMetricDescription>()?;
+    m.add_wrapped(wrap_pyfunction!(otel::register_lance_metrics_recorder))?;
+    m.add_wrapped(wrap_pyfunction!(otel::lance_metrics_catalog))?;
+    m.add_wrapped(wrap_pyfunction!(otel::snapshot_lance_metrics))?;
     m.add_wrapped(wrap_pyfunction!(manifest_needs_migration))?;
     m.add_wrapped(wrap_pyfunction!(language_model_home))?;
     m.add_wrapped(wrap_pyfunction!(bytes_read_counter))?;
     m.add_wrapped(wrap_pyfunction!(iops_counter))?;
+    m.add_wrapped(wrap_pyfunction!(simd_info))?;
     m.add_wrapped(wrap_pyfunction!(stable_version))?;
     // Debug functions
     m.add_wrapped(wrap_pyfunction!(debug::format_schema))?;
@@ -326,6 +348,37 @@ fn iops_counter() -> PyResult<u64> {
     Ok(::lance::io::iops_counter())
 }
 
+/// Returns a dict describing which SIMD tier the lance runtime dispatches to
+/// on this host, plus the raw CPU feature flags it detected.
+///
+/// Mirrors `pyarrow.runtime_info()`: a cheap, transparent way to verify that
+/// the host is hitting the expected SIMD tier (e.g., `"avx512_fp16"`,
+/// `"avx2"`) when debugging vector-search performance.
+///
+/// Returns:
+///   {
+///     "tier": str,                      # e.g. "avx2", "avx_fma", "neon", "none"
+///     "target_arch": str,               # e.g. "x86_64", "aarch64", "loongarch64"
+///     "host_features": list[str],       # raw CPU feature flags (x86_64 only)
+///   }
+///
+/// Examples:
+///   >>> import lance
+///   >>> info = lance.simd_info()
+///   >>> sorted(info)
+///   ['host_features', 'target_arch', 'tier']
+///   >>> isinstance(info["tier"], str)
+///   True
+#[pyfunction]
+pub fn simd_info(py: Python<'_>) -> PyResult<Py<PyAny>> {
+    let info = lance_core::utils::cpu::simd_info();
+    let dict = pyo3::types::PyDict::new(py);
+    dict.set_item("tier", info.tier.to_string())?;
+    dict.set_item("target_arch", info.target_arch)?;
+    dict.set_item("host_features", info.host_features)?;
+    Ok(dict.into())
+}
+
 #[pyfunction(name = "bytes_read_counter")]
 fn bytes_read_counter() -> PyResult<u64> {
     Ok(::lance::io::bytes_read_counter())
diff --git a/python/src/mem_wal.rs b/python/src/mem_wal.rs
index 25127c95ea4..e415acfc38d 100644
--- a/python/src/mem_wal.rs
+++ b/python/src/mem_wal.rs
@@ -19,6 +19,7 @@ use futures::TryStreamExt;
 use lance::dataset::Dataset as LanceDataset;
 use lance::dataset::mem_wal::scanner::{
     FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner,
+    parse_filter_expr as parse_lsm_filter_expr,
 };
 use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot};
 use lance::dataset::mem_wal::{LsmScanner, ShardSnapshot, ShardWriter, evaluate_sharding_spec};
@@ -51,6 +52,31 @@ pub fn py_evaluate_sharding_spec<'py>(
     result.to_pyarrow(py)
 }
 
+/// Write a primary-key dedup sidecar (`_pk_index/`) for a flushed-generation
+/// dataset already written at `gen_path`, mirroring what production flush emits.
+///
+/// Test-support only: lets Python tests stage a *faithful* flushed generation
+/// (dataset + sidecar). Production always writes the sidecar during flush, so a
+/// dataset-without-sidecar is not a state the system otherwise produces.
+#[pyfunction(name = "_write_pk_sidecar", signature = (gen_path, data, pk_columns))]
+pub fn py_write_pk_sidecar(
+    py: Python<'_>,
+    gen_path: String,
+    data: &Bound<'_, PyAny>,
+    pk_columns: Vec<String>,
+) -> PyResult<()> {
+    let reader = ArrowArrayStreamReader::from_pyarrow_bound(data)
+        .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?;
+    let batches: Vec<RecordBatch> = reader
+        .collect::<Result<_, _>>()
+        .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))?;
+    rt().block_on(Some(py), async move {
+        let pk_refs: Vec<&str> = pk_columns.iter().map(String::as_str).collect();
+        lance::dataset::mem_wal::scanner::write_pk_sidecar(&gen_path, &batches, &pk_refs).await
+    })?
+    .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))
+}
+
 fn sharding_spec_from_py(spec: &Bound<'_, PyAny>) -> PyResult<ShardingSpec> {
     let spec_id = get_py_value(spec, "spec_id")?.extract::<u32>()?;
     let fields_obj = get_py_value(spec, "fields")?;
@@ -212,6 +238,14 @@ struct ClosedShardWriterState {
     memtable_stats: MemTableStats,
 }
 
+fn collect_record_batches(data: &Bound<'_, PyAny>) -> PyResult<Vec<RecordBatch>> {
+    let reader = ArrowArrayStreamReader::from_pyarrow_bound(data)
+        .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?;
+    reader
+        .collect::<Result<_, _>>()
+        .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))
+}
+
 #[pymethods]
 impl PyShardWriter {
     /// Write data batches to the MemWAL.
@@ -219,11 +253,7 @@ impl PyShardWriter {
     /// Accepts any PyArrow-compatible data source (RecordBatch, Table,
     /// or an Arrow stream reader).
     pub fn put(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> {
-        let reader = ArrowArrayStreamReader::from_pyarrow_bound(data)
-            .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?;
-        let batches: Vec<RecordBatch> = reader
-            .collect::<Result<_, _>>()
-            .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))?;
+        let batches = collect_record_batches(data)?;
 
         if batches.is_empty() {
             return Ok(());
@@ -242,6 +272,31 @@ impl PyShardWriter {
         .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))
     }
 
+    /// Delete rows from the MemWAL by primary key.
+    ///
+    /// Accepts any PyArrow-compatible data source carrying the shard's primary
+    /// key column(s). Rust core validates that primary keys exist and builds the
+    /// tombstone rows.
+    pub fn delete(&self, py: Python<'_>, keys: &Bound<'_, PyAny>) -> PyResult<()> {
+        let batches = collect_record_batches(keys)?;
+
+        if batches.is_empty() {
+            return Ok(());
+        }
+
+        let inner = self.inner.clone();
+        rt().block_on(Some(py), async move {
+            let guard = inner.lock().await;
+            match guard.as_ref() {
+                Some(writer) => writer.delete(batches).await.map(|_| ()),
+                None => Err(lance_core::Error::invalid_input(
+                    "ShardWriter is already closed",
+                )),
+            }
+        })?
+        .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))
+    }
+
     /// Flush pending data and close the writer.
     ///
     /// After close(), calling put() will raise an error.
@@ -520,7 +575,11 @@ impl PyLsmScanner {
             .take()
             .ok_or_else(|| PyRuntimeError::new_err("Scanner has already been consumed"))?;
         let cols: Vec<&str> = columns.iter().map(|s| s.as_str()).collect();
-        slf.inner = Some(scanner.project(&cols));
+        slf.inner = Some(
+            scanner
+                .project(&cols)
+                .map_err(|e| PyValueError::new_err(e.to_string()))?,
+        );
         Ok(slf)
     }
 
@@ -538,18 +597,22 @@ impl PyLsmScanner {
         Ok(slf)
     }
 
-    /// Limit the number of rows returned.
-    #[pyo3(signature = (n, offset=None))]
+    /// Limit the number of rows returned, optionally with an offset.
+    #[pyo3(signature = (n=None, offset=None))]
     pub fn limit(
         mut slf: PyRefMut<'_, Self>,
-        n: usize,
+        n: Option<usize>,
         offset: Option<usize>,
     ) -> PyResult<PyRefMut<'_, Self>> {
         let scanner = slf
             .inner
             .take()
             .ok_or_else(|| PyRuntimeError::new_err("Scanner has already been consumed"))?;
-        slf.inner = Some(scanner.limit(n, offset));
+        slf.inner = Some(
+            scanner
+                .limit(n.map(|n| n as i64), offset.map(|o| o as i64))
+                .map_err(|e| PyValueError::new_err(e.to_string()))?,
+        );
         Ok(slf)
     }
 
@@ -704,13 +767,14 @@ pub struct PyLsmVectorSearchPlanner {
 #[pymethods]
 impl PyLsmVectorSearchPlanner {
     #[new]
-    #[pyo3(signature = (dataset, shard_snapshots, vector_column, pk_columns=None, distance_type=None))]
+    #[pyo3(signature = (dataset, shard_snapshots, vector_column, pk_columns=None, distance_type=None, filter=None))]
     pub fn new(
         dataset: &Bound<'_, PyDataset>,
         shard_snapshots: Vec<Bound<'_, PyShardSnapshot>>,
         vector_column: String,
         pk_columns: Option<Vec<String>>,
         distance_type: Option<String>,
+        filter: Option<String>,
     ) -> PyResult<Self> {
         let ds = dataset.borrow().ds.clone();
         let snapshots: Vec<ShardSnapshot> = shard_snapshots
@@ -726,9 +790,16 @@ impl PyLsmVectorSearchPlanner {
         let dist_type = parse_distance_type(distance_type.as_deref().unwrap_or("l2"))?;
 
         let vector_dim = get_vector_dim(&ds, &vector_column)?;
+        let filter = filter
+            .as_deref()
+            .map(|filter| {
+                parse_lsm_filter_expr(base_schema.as_ref(), filter)
+                    .map_err(|e| PyValueError::new_err(e.to_string()))
+            })
+            .transpose()?;
 
         let collector = LsmDataSourceCollector::new(ds.clone(), snapshots);
-        let planner = LsmVectorSearchPlanner::new(
+        let mut planner = LsmVectorSearchPlanner::new(
             collector,
             pk_cols,
             base_schema.clone(),
@@ -736,6 +807,9 @@ impl PyLsmVectorSearchPlanner {
             dist_type,
         )
         .with_dataset(ds);
+        if let Some(filter) = filter {
+            planner = planner.with_filter(Some(filter));
+        }
 
         Ok(Self {
             planner,
diff --git a/python/src/namespace.rs b/python/src/namespace.rs
index cf5f7c41b0f..e88ff40de2c 100644
--- a/python/src/namespace.rs
+++ b/python/src/namespace.rs
@@ -392,6 +392,44 @@ impl PyDirectoryNamespace {
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
+    // Table branch operations
+
+    fn create_table_branch<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_branch(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn list_table_branches<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_table_branches(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn delete_table_branch<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.delete_table_branch(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
     // Data manipulation operations
 
     fn count_table_rows(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<i64> {
@@ -1054,6 +1092,44 @@ impl PyRestNamespace {
         pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
     }
 
+    // Table branch operations
+
+    fn create_table_branch<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.create_table_branch(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn list_table_branches<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.list_table_branches(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
+    fn delete_table_branch<'py>(
+        &self,
+        py: Python<'py>,
+        request: &Bound<'_, PyAny>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let request = depythonize(request)?;
+        let response = crate::rt()
+            .block_on(Some(py), self.inner.delete_table_branch(request))?
+            .infer_error()?;
+        pythonize(py, &response).map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))
+    }
+
     // Data manipulation operations
 
     fn count_table_rows(&self, py: Python, request: &Bound<'_, PyAny>) -> PyResult<i64> {
@@ -1472,6 +1548,30 @@ fn get_dict_with_model_dump_class(py: Python<'_>) -> PyResult<Bound<'_, PyAny>>
     Ok(class)
 }
 
+/// Convert a Python namespace exception into a lance error, preserving the
+/// namespace error identity when the exception is a `lance_namespace`
+/// `LanceNamespaceError` carrying an error `code`, so callers can react to
+/// e.g. TableNotFound the same way they do for native clients. Foreign
+/// exceptions that happen to carry an integer `code` (e.g. SystemExit) must
+/// not be reinterpreted, so the extraction is gated on the exception type.
+fn namespace_error_from_py(method_name: &'static str, e: PyErr) -> lance_core::Error {
+    Python::attach(|py| {
+        let value = e.value(py);
+        let is_namespace_error = py
+            .import("lance_namespace.errors")
+            .and_then(|module| module.getattr("LanceNamespaceError"))
+            .and_then(|class| value.is_instance(&class))
+            .unwrap_or(false);
+        if is_namespace_error
+            && let Ok(code) = value.getattr("code").and_then(|code| code.extract::<u32>())
+        {
+            return lance_namespace::error::NamespaceError::from_code(code, value.to_string())
+                .into();
+        }
+        lance_core::Error::io(format!("Python error in {}: {}", method_name, e))
+    })
+}
+
 /// Helper to call a Python namespace method with JSON serialization.
 /// For methods that take a request and return a response.
 /// Uses DictWithModelDump to pass a dict that also has model_dump() method,
@@ -1519,7 +1619,7 @@ where
     })
     .await
     .map_err(|e| lance_core::Error::io(format!("Task join error for {}: {}", method_name, e)))?
-    .map_err(|e: PyErr| lance_core::Error::io(format!("Python error in {}: {}", method_name, e)))?;
+    .map_err(|e: PyErr| namespace_error_from_py(method_name, e))?;
 
     serde_json::from_str(&response_json).map_err(|e| {
         lance_core::Error::io(format!(
diff --git a/python/src/otel.rs b/python/src/otel.rs
new file mode 100644
index 00000000000..71d191d7db9
--- /dev/null
+++ b/python/src/otel.rs
@@ -0,0 +1,640 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Bridge from the [`metrics`] crate facade to Python OpenTelemetry.
+//!
+//! Lance core publishes metrics through the global [`metrics`] facade without
+//! choosing a backend. This module installs a process-global [`Recorder`] that
+//! aggregates those metrics into lock-free cumulative storage, and exposes that
+//! state to Python so the bindings can feed it into the user's OpenTelemetry
+//! `MeterProvider`.
+//!
+//! While it targets OpenTelemetry on the Python side, the recorder is agnostic
+//! to the metric *source*: it records any metric emitted through the facade,
+//! keyed by name and labels. Object store metrics are the first producer, but
+//! nothing here is specific to them. New metrics flow through automatically;
+//! they only need to be described (see [`describe_all`]) so the Python layer can
+//! discover their name, kind, and unit up front.
+//!
+//! ## Why pull, not push
+//!
+//! OpenTelemetry collects on its own schedule and invokes observable-instrument
+//! callbacks at collection time. Cumulative counters map directly onto OTel's
+//! `ObservableCounter` semantics. So the bridge aggregates in Rust and lets the
+//! Python collection thread pull a [`snapshot`](snapshot_lance_metrics). The
+//! snapshot is lock-free, but it still walks every registered series and
+//! allocates owned copies of their names and labels, so it runs with the GIL
+//! released to avoid stalling other Python threads during collection.
+//!
+//! ## Histograms
+//!
+//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot
+//! be pulled as-is. Instead each histogram is aggregated into fixed buckets
+//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count
+//! and sum, which the Python layer surfaces as observable counters.
+
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
+
+use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
+use metrics_util::registry::{Registry, Storage};
+use pyo3::prelude::*;
+
+/// Bucket boundaries used when a histogram has no registered bounds. Covers a
+/// broad latency range so unknown histograms still produce useful buckets.
+const DEFAULT_BOUNDS: &[f64] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
+];
+
+/// The kind of a metric, mirroring the three `metrics` instrument types.
+#[derive(Clone, Copy)]
+enum MetricKind {
+    Counter,
+    Gauge,
+    Histogram,
+}
+
+impl MetricKind {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Counter => "counter",
+            Self::Gauge => "gauge",
+            Self::Histogram => "histogram",
+        }
+    }
+}
+
+/// Description of a metric, populated by the recorder's `describe_*` methods.
+struct MetricDescription {
+    kind: MetricKind,
+    unit: Option<String>,
+    description: String,
+}
+
+/// Catalog of described metrics, keyed by metric name. The Python layer reads
+/// this to create one OpenTelemetry instrument per metric up front.
+static CATALOG: LazyLock<Mutex<HashMap<String, MetricDescription>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Per-metric histogram bucket boundaries, keyed by metric name. Producers
+/// register their recommended bounds before any metric is recorded.
+static HISTOGRAM_BOUNDS: LazyLock<RwLock<HashMap<String, Arc<[f64]>>>> =
+    LazyLock::new(|| RwLock::new(HashMap::new()));
+
+/// The installed recorder's registry, available once installation succeeds.
+static REGISTRY: OnceLock<Arc<Registry<Key, LanceStorage>>> = OnceLock::new();
+
+fn bounds_for(name: &str) -> Arc<[f64]> {
+    HISTOGRAM_BOUNDS
+        .read()
+        .unwrap()
+        .get(name)
+        .cloned()
+        .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS))
+}
+
+/// A histogram that buckets samples at record time into fixed boundaries,
+/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded
+/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets.
+struct BucketedHistogram {
+    /// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose
+    /// bound is `>= v`; samples above all bounds fall in the implicit `+Inf`
+    /// bucket stored as the final entry of `counts`.
+    bounds: Arc<[f64]>,
+    /// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`.
+    counts: Box<[AtomicU64]>,
+    count: AtomicU64,
+    /// Running sum of recorded values, stored as `f64` bits (there is no atomic
+    /// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]).
+    sum_bits: AtomicU64,
+}
+
+// All atomics here use `Ordering::Relaxed`: each metric counter is independent,
+// so no happens-before relationship is needed between them, and a snapshot
+// reader tolerates slightly stale values. This matches `metrics_util`'s
+// `AtomicStorage`.
+
+impl BucketedHistogram {
+    fn new(bounds: Arc<[f64]>) -> Self {
+        let counts = (0..bounds.len() + 1)
+            .map(|_| AtomicU64::new(0))
+            .collect::<Vec<_>>()
+            .into_boxed_slice();
+        Self {
+            bounds,
+            counts,
+            count: AtomicU64::new(0),
+            sum_bits: AtomicU64::new(0),
+        }
+    }
+
+    fn add_to_sum(&self, value: f64) {
+        // No atomic offers an f64 add, so read the current bit pattern, add in
+        // float space, and CAS it back, retrying if another thread won the race.
+        let mut current = self.sum_bits.load(Ordering::Relaxed);
+        loop {
+            let updated = (f64::from_bits(current) + value).to_bits();
+            match self.sum_bits.compare_exchange_weak(
+                current,
+                updated,
+                Ordering::Relaxed,
+                Ordering::Relaxed,
+            ) {
+                Ok(_) => break,
+                Err(actual) => current = actual,
+            }
+        }
+    }
+
+    /// Cumulative `le` buckets, total count, and sum at this instant.
+    fn snapshot(&self) -> MetricValue {
+        let mut cumulative = 0u64;
+        let mut buckets = Vec::with_capacity(self.bounds.len() + 1);
+        for (i, bound) in self.bounds.iter().enumerate() {
+            cumulative += self.counts[i].load(Ordering::Relaxed);
+            buckets.push((format!("{}", bound), cumulative));
+        }
+        cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed);
+        buckets.push(("+Inf".to_string(), cumulative));
+        MetricValue::Histogram {
+            buckets,
+            count: self.count.load(Ordering::Relaxed),
+            sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)),
+        }
+    }
+}
+
+impl metrics::HistogramFn for BucketedHistogram {
+    fn record(&self, value: f64) {
+        let idx = self.bounds.partition_point(|&bound| bound < value);
+        self.counts[idx].fetch_add(1, Ordering::Relaxed);
+        self.count.fetch_add(1, Ordering::Relaxed);
+        self.add_to_sum(value);
+    }
+}
+
+/// Storage backing the registry. Counters and gauges are plain atomics (as in
+/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`].
+struct LanceStorage;
+
+impl Storage<Key> for LanceStorage {
+    type Counter = Arc<AtomicU64>;
+    type Gauge = Arc<AtomicU64>;
+    type Histogram = Arc<BucketedHistogram>;
+
+    fn counter(&self, _key: &Key) -> Self::Counter {
+        Arc::new(AtomicU64::new(0))
+    }
+
+    fn gauge(&self, _key: &Key) -> Self::Gauge {
+        // The `metrics` facade writes the f64 bit pattern into this `u64` (the
+        // snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`.
+        // `0` decodes to `0.0`, the correct initial value.
+        Arc::new(AtomicU64::new(0))
+    }
+
+    fn histogram(&self, key: &Key) -> Self::Histogram {
+        Arc::new(BucketedHistogram::new(bounds_for(key.name())))
+    }
+}
+
+struct LanceRecorder {
+    registry: Arc<Registry<Key, LanceStorage>>,
+}
+
+impl LanceRecorder {
+    fn describe(
+        &self,
+        key: KeyName,
+        kind: MetricKind,
+        unit: Option<Unit>,
+        description: SharedString,
+    ) {
+        CATALOG.lock().unwrap().insert(
+            key.as_str().to_string(),
+            MetricDescription {
+                kind,
+                unit: unit.map(|u| u.as_canonical_label().to_string()),
+                description: description.into_owned(),
+            },
+        );
+    }
+}
+
+impl Recorder for LanceRecorder {
+    fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Counter, unit, description);
+    }
+
+    fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Gauge, unit, description);
+    }
+
+    fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
+        self.describe(key, MetricKind::Histogram, unit, description);
+    }
+
+    fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
+        self.registry
+            .get_or_create_counter(key, |c| Counter::from_arc(c.clone()))
+    }
+
+    fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge {
+        self.registry
+            .get_or_create_gauge(key, |g| Gauge::from_arc(g.clone()))
+    }
+
+    fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram {
+        self.registry
+            .get_or_create_histogram(key, |h| Histogram::from_arc(h.clone()))
+    }
+}
+
+/// Register the recommended histogram bounds for every metric-emitting
+/// subsystem. New subsystems add their `histogram_bounds()` here.
+fn register_bounds() {
+    let mut bounds = HISTOGRAM_BOUNDS.write().unwrap();
+    for (name, values) in lance_io::object_store::metrics::histogram_bounds() {
+        bounds.insert((*name).to_string(), Arc::from(*values));
+    }
+}
+
+/// Describe every metric-emitting subsystem so the catalog is populated. Must
+/// run after the recorder is installed. New subsystems add their
+/// `describe_metrics()` here.
+fn describe_all() {
+    lance_io::object_store::metrics::describe_metrics();
+}
+
+enum MetricValue {
+    Scalar(f64),
+    Histogram {
+        buckets: Vec<(String, u64)>,
+        count: u64,
+        sum: f64,
+    },
+}
+
+struct MetricPoint {
+    name: String,
+    kind: &'static str,
+    attributes: HashMap<String, String>,
+    value: MetricValue,
+}
+
+fn labels(key: &Key) -> HashMap<String, String> {
+    key.labels()
+        .map(|label| (label.key().to_string(), label.value().to_string()))
+        .collect()
+}
+
+fn collect_points(registry: &Registry<Key, LanceStorage>) -> Vec<MetricPoint> {
+    let mut points = Vec::new();
+    for (key, handle) in registry.get_counter_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "counter",
+            attributes: labels(&key),
+            // OpenTelemetry observations are float; counts stay well within the
+            // f64-exact integer range (2^53), so this cast is lossless in practice.
+            value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64),
+        });
+    }
+    for (key, handle) in registry.get_gauge_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "gauge",
+            attributes: labels(&key),
+            value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))),
+        });
+    }
+    for (key, handle) in registry.get_histogram_handles() {
+        points.push(MetricPoint {
+            name: key.name().to_string(),
+            kind: "histogram",
+            attributes: labels(&key),
+            value: handle.snapshot(),
+        });
+    }
+    points
+}
+
+/// One metric data point exposed to Python. For counters and gauges only
+/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`,
+/// and `sum` are set.
+#[pyclass(name = "MetricPoint", get_all)]
+pub struct PyMetricPoint {
+    name: String,
+    kind: String,
+    attributes: HashMap<String, String>,
+    value: Option<f64>,
+    buckets: Option<Vec<(String, u64)>>,
+    count: Option<u64>,
+    sum: Option<f64>,
+}
+
+impl From<MetricPoint> for PyMetricPoint {
+    fn from(point: MetricPoint) -> Self {
+        let (value, buckets, count, sum) = match point.value {
+            MetricValue::Scalar(v) => (Some(v), None, None, None),
+            MetricValue::Histogram {
+                buckets,
+                count,
+                sum,
+            } => (None, Some(buckets), Some(count), Some(sum)),
+        };
+        Self {
+            name: point.name,
+            kind: point.kind.to_string(),
+            attributes: point.attributes,
+            value,
+            buckets,
+            count,
+            sum,
+        }
+    }
+}
+
+/// A described metric, used by the Python layer to create instruments up front.
+#[pyclass(name = "MetricDescription", get_all)]
+pub struct PyMetricDescription {
+    name: String,
+    kind: String,
+    unit: Option<String>,
+    description: String,
+}
+
+/// Install the Lance metrics recorder as the process-global `metrics` recorder.
+///
+/// Returns `True` if the recorder is installed (now or previously). Returns
+/// `False` if a *different* recorder is already installed — `metrics` allows
+/// only one global recorder per process, so Lance cannot coexist with another.
+#[pyfunction]
+pub fn register_lance_metrics_recorder() -> bool {
+    if REGISTRY.get().is_some() {
+        return true;
+    }
+    let registry = Arc::new(Registry::new(LanceStorage));
+    let recorder = LanceRecorder {
+        registry: registry.clone(),
+    };
+    // Register histogram bounds *before* installing the recorder. Once the
+    // recorder is global, another thread can emit a metric and create the
+    // histogram handle concurrently; if the bounds aren't registered yet that
+    // handle would be built with the fallback bounds and keep them for the
+    // life of the process. `register_bounds()` doesn't need the recorder.
+    register_bounds();
+    match metrics::set_global_recorder(recorder) {
+        Ok(()) => {
+            let _ = REGISTRY.set(registry);
+            describe_all();
+            true
+        }
+        Err(_) => false,
+    }
+}
+
+/// The catalog of described Lance metrics. Empty until the recorder is installed.
+#[pyfunction]
+pub fn lance_metrics_catalog() -> Vec<PyMetricDescription> {
+    CATALOG
+        .lock()
+        .unwrap()
+        .iter()
+        .map(|(name, desc)| PyMetricDescription {
+            name: name.clone(),
+            kind: desc.kind.as_str().to_string(),
+            unit: desc.unit.clone(),
+            description: desc.description.clone(),
+        })
+        .collect()
+}
+
+/// A point-in-time snapshot of every recorded metric. Empty until the recorder
+/// is installed. The read is lock-free but walks every series and allocates, so
+/// it runs with the GIL released.
+#[pyfunction]
+pub fn snapshot_lance_metrics(py: Python<'_>) -> Vec<PyMetricPoint> {
+    let Some(registry) = REGISTRY.get() else {
+        return Vec::new();
+    };
+    let points = py.detach(|| collect_points(registry));
+    points.into_iter().map(PyMetricPoint::from).collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use metrics::HistogramFn;
+
+    fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 {
+        buckets
+            .iter()
+            .find(|(b, _)| b == le)
+            .map(|(_, c)| *c)
+            .unwrap_or_else(|| panic!("no bucket with le={le}"))
+    }
+
+    #[test]
+    fn bucketed_histogram_records_cumulative_buckets() {
+        let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
+        hist.record(0.05); // le=0.1
+        hist.record(0.5); // le=1
+        hist.record(0.5); // le=1
+        hist.record(50.0); // +Inf
+
+        let MetricValue::Histogram {
+            buckets,
+            count,
+            sum,
+        } = hist.snapshot()
+        else {
+            panic!("expected histogram");
+        };
+
+        // Buckets are cumulative (Prometheus `le` semantics).
+        assert_eq!(bucket_count(&buckets, "0.1"), 1);
+        assert_eq!(bucket_count(&buckets, "1"), 3);
+        assert_eq!(bucket_count(&buckets, "10"), 3);
+        assert_eq!(bucket_count(&buckets, "+Inf"), 4);
+        assert_eq!(count, 4);
+        assert!((sum - 51.05).abs() < 1e-9);
+    }
+
+    #[test]
+    fn bucketed_histogram_boundary_is_inclusive() {
+        let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice()));
+        hist.record(1.0); // exactly the bound -> le=1, not +Inf
+        let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
+            panic!("expected histogram");
+        };
+        assert_eq!(bucket_count(&buckets, "1"), 1);
+        assert_eq!(bucket_count(&buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn bucketed_histogram_boundary_is_inclusive_mid_range() {
+        // A value equal to a middle bound lands in that bucket, not the next.
+        let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
+        hist.record(1.0);
+        let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
+            panic!("expected histogram");
+        };
+        assert_eq!(bucket_count(&buckets, "0.1"), 0);
+        assert_eq!(bucket_count(&buckets, "1"), 1);
+        assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1
+        assert_eq!(bucket_count(&buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn recorder_aggregates_counters_with_labels() {
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
+                .increment(2);
+            metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
+                .increment(3);
+            // A distinct label set must produce a separate point, not merge.
+            metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs")
+                .increment(7);
+        });
+
+        let scalar = |attrs: &[(&str, &str)]| {
+            let points = collect_points(&registry);
+            let point = points
+                .into_iter()
+                .find(|p| {
+                    p.name == "test_requests_total"
+                        && attrs
+                            .iter()
+                            .all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v))
+                })
+                .expect("counter recorded for label set");
+            assert_eq!(point.kind, "counter");
+            match point.value {
+                MetricValue::Scalar(v) => v,
+                _ => panic!("expected scalar"),
+            }
+        };
+
+        // Same labels aggregate; distinct labels stay separate.
+        assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9);
+        assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9);
+    }
+
+    #[test]
+    fn recorder_records_gauges() {
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        // Gauges store the f64 bit pattern in a u64; the snapshot must decode it.
+        metrics::with_local_recorder(&recorder, || {
+            metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5);
+        });
+
+        let points = collect_points(&registry);
+        let point = points
+            .iter()
+            .find(|p| p.name == "test_gauge")
+            .expect("gauge recorded");
+        assert_eq!(point.kind, "gauge");
+        assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9));
+    }
+
+    #[test]
+    fn recorder_falls_back_to_default_bounds() {
+        // A histogram with no registered bounds uses DEFAULT_BOUNDS.
+        let name = "test_unregistered_histogram";
+        assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name));
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::histogram!(name).record(0.02);
+        });
+
+        let points = collect_points(&registry);
+        let point = points.iter().find(|p| p.name == name).expect("recorded");
+        let MetricValue::Histogram { buckets, count, .. } = &point.value else {
+            panic!("expected histogram");
+        };
+        assert_eq!(*count, 1);
+        // DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`.
+        assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1);
+        // 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry).
+        assert_eq!(bucket_count(buckets, "0.025"), 1);
+        assert_eq!(bucket_count(buckets, "0.01"), 0);
+        assert_eq!(bucket_count(buckets, "+Inf"), 1);
+    }
+
+    #[test]
+    fn recorder_uses_registered_histogram_bounds() {
+        let name = "test_recorder_bounds_seconds";
+        HISTOGRAM_BOUNDS
+            .write()
+            .unwrap()
+            .insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice()));
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder {
+            registry: registry.clone(),
+        };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::histogram!(name).record(0.05);
+            metrics::histogram!(name).record(5.0);
+        });
+
+        let points = collect_points(&registry);
+        let point = points.iter().find(|p| p.name == name).expect("recorded");
+        let MetricValue::Histogram { buckets, count, .. } = &point.value else {
+            panic!("expected histogram");
+        };
+        assert_eq!(*count, 2);
+        assert_eq!(bucket_count(buckets, "0.1"), 1);
+        assert_eq!(bucket_count(buckets, "+Inf"), 2);
+    }
+
+    #[test]
+    fn describe_populates_catalog() {
+        let name = "test_describe_catalog_total";
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder { registry };
+        metrics::with_local_recorder(&recorder, || {
+            metrics::describe_counter!(name, Unit::Count, "a test counter");
+        });
+
+        let catalog = CATALOG.lock().unwrap();
+        let desc = catalog.get(name).expect("described");
+        assert!(matches!(desc.kind, MetricKind::Counter));
+        assert_eq!(desc.description, "a test counter");
+    }
+
+    #[test]
+    fn describe_all_covers_object_store_metrics() {
+        use lance_io::object_store::metrics as os;
+
+        let registry = Arc::new(Registry::new(LanceStorage));
+        let recorder = LanceRecorder { registry };
+        metrics::with_local_recorder(&recorder, describe_all);
+
+        let catalog = CATALOG.lock().unwrap();
+        let kind = |name: &str| catalog.get(name).expect("described").kind;
+        // Every emitted object store metric must be described so the OTel
+        // bridge can create an instrument for it, including the gauge and the
+        // retryable counter that a plain request path might never emit.
+        assert!(matches!(kind(os::METRIC_REQUESTS), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_BYTES), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_ERRORS), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_THROTTLE), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_RETRYABLE), MetricKind::Counter));
+        assert!(matches!(kind(os::METRIC_IN_FLIGHT), MetricKind::Gauge));
+        assert!(matches!(kind(os::METRIC_DURATION), MetricKind::Histogram));
+    }
+}
diff --git a/python/src/scanner.rs b/python/src/scanner.rs
index 691f7f53294..bbf1b3f35a3 100644
--- a/python/src/scanner.rs
+++ b/python/src/scanner.rs
@@ -125,14 +125,17 @@ impl Scanner {
         Ok(res)
     }
 
-    #[pyo3(signature = (*))]
-    fn analyze_plan(self_: PyRef<'_, Self>) -> PyResult<String> {
+    #[pyo3(signature = (*, count_rows = false))]
+    fn analyze_plan(self_: PyRef<'_, Self>, count_rows: bool) -> PyResult<String> {
         let scanner = self_.scanner.clone();
         let res = rt()
-            .spawn(
-                Some(self_.py()),
-                async move { scanner.analyze_plan().await },
-            )?
+            .spawn(Some(self_.py()), async move {
+                if count_rows {
+                    scanner.analyze_count_plan().await
+                } else {
+                    scanner.analyze_plan().await
+                }
+            })?
             .map_err(|err| PyValueError::new_err(err.to_string()))?;
 
         Ok(res)
diff --git a/python/src/schema.rs b/python/src/schema.rs
index db4f7369710..8cdc2115cd1 100644
--- a/python/src/schema.rs
+++ b/python/src/schema.rs
@@ -179,7 +179,8 @@ impl LanceSchema {
             fields: Fields(fields),
             metadata,
         };
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)
+            .map_err(|err| PyValueError::new_err(format!("Failed to reconstruct schema: {err}")))?;
         Ok(Self(schema))
     }
 
diff --git a/python/src/session.rs b/python/src/session.rs
index c91329ec1ee..da3174a9bf9 100644
--- a/python/src/session.rs
+++ b/python/src/session.rs
@@ -3,7 +3,7 @@
 
 use std::sync::Arc;
 
-use pyo3::{pyclass, pymethods};
+use pyo3::{PyResult, pyclass, pymethods};
 
 use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE};
 use lance::session::Session as LanceSession;
@@ -63,6 +63,13 @@ impl Session {
         self.inner.size_bytes()
     }
 
+    /// Return the current size of the index cache in bytes.
+    pub fn index_cache_size_bytes(&self) -> PyResult<u64> {
+        rt().block_on(None, async move {
+            self.inner.index_cache_stats().await.size_bytes as u64
+        })
+    }
+
     /// Return whether the other session is the same as this one.
     pub fn is_same_as(&self, other: &Self) -> bool {
         Arc::ptr_eq(&self.inner, &other.inner)
diff --git a/python/src/transaction.rs b/python/src/transaction.rs
index 1b659395099..37085d60de7 100644
--- a/python/src/transaction.rs
+++ b/python/src/transaction.rs
@@ -7,10 +7,11 @@ use crate::utils::{PyLance, class_name, export_vec, extract_vec};
 use arrow::pyarrow::PyArrowType;
 use arrow_schema::Schema as ArrowSchema;
 use lance::dataset::transaction::{
-    DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap,
-    UpdateMapEntry, UpdateMode,
+    DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction,
+    UpdateMap, UpdateMapEntry, UpdateMode,
 };
 use lance::datatypes::Schema;
+use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
 use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata};
 use pyo3::exceptions::PyValueError;
 use pyo3::types::PySet;
@@ -206,6 +207,128 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataReplacementGroup> {
     }
 }
 
+// The Nth offset in an overlay list positionally maps to the Nth value row in
+// `data_file`, but `RoaringBitmap` stores offsets in ascending order and drops
+// duplicates. A caller-supplied list that isn't strictly ascending would be
+// silently reordered, breaking that mapping, so reject it here instead. This can
+// go away once we expose RoaringBitmap directly to Python (issue #7695).
+fn bitmap_from_sorted_offsets(offsets: Vec<u32>) -> PyResult<RoaringBitmap> {
+    if offsets.windows(2).any(|w| w[0] >= w[1]) {
+        return Err(PyValueError::new_err(
+            "DataOverlayFile.offsets must be strictly ascending with no duplicates; \
+             each offset positionally maps to a value row in data_file",
+        ));
+    }
+    Ok(RoaringBitmap::from_sorted_iter(offsets).expect("offsets verified strictly ascending"))
+}
+
+impl FromPyObject<'_, '_> for PyLance<DataOverlayFile> {
+    type Error = PyErr;
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
+        let data_file = ob.getattr("data_file")?.extract::<PyLance<DataFile>>()?.0;
+        let offsets = ob.getattr("offsets")?;
+
+        // A flat list of offsets is a dense overlay (one coverage shared by every
+        // field); a list of per-field lists is a sparse overlay. Differentiate by
+        // shape, trying the dense form first.
+        let coverage = if let Ok(shared) = offsets.extract::<Vec<u32>>() {
+            OverlayCoverage::dense(bitmap_from_sorted_offsets(shared)?)
+        } else if let Ok(per_field) = offsets.extract::<Vec<Vec<u32>>>() {
+            OverlayCoverage::sparse(
+                per_field
+                    .into_iter()
+                    .map(bitmap_from_sorted_offsets)
+                    .collect::<PyResult<Vec<_>>>()?,
+            )
+        } else {
+            return Err(PyValueError::new_err(
+                "DataOverlayFile.offsets must be a list of ints (dense coverage shared by \
+                 every field) or a list of per-field int lists (sparse coverage)",
+            ));
+        };
+
+        // Present (and preserved) when round-tripping an existing fragment's
+        // overlays; None/0 when creating an overlay to commit, since the
+        // DataOverlay commit stamps the effective version.
+        let committed_version = ob
+            .getattr("committed_version")?
+            .extract::<Option<u64>>()?
+            .unwrap_or(0);
+
+        Ok(Self(DataOverlayFile {
+            data_file,
+            coverage,
+            committed_version,
+        }))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayFile> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let namespace = py
+            .import(intern!(py, "lance"))
+            .and_then(|module| module.getattr(intern!(py, "LanceOperation")))
+            .expect("Failed to import LanceOperation namespace");
+
+        let data_file = PyLance(&self.0.data_file).into_pyobject(py)?;
+        let cls = namespace
+            .getattr("DataOverlayFile")
+            .expect("Failed to get DataOverlayFile class");
+
+        let committed_version = self.0.committed_version;
+
+        // Mirror the read side: a dense overlay becomes a flat list of offsets, a
+        // sparse overlay a list of per-field lists.
+        match &self.0.coverage {
+            OverlayCoverage::Shared(bitmap) => {
+                let offsets: Vec<u32> = bitmap.iter().collect();
+                cls.call1((data_file, offsets, committed_version))
+            }
+            OverlayCoverage::PerField(bitmaps) => {
+                let offsets: Vec<Vec<u32>> = bitmaps.iter().map(|b| b.iter().collect()).collect();
+                cls.call1((data_file, offsets, committed_version))
+            }
+        }
+    }
+}
+
+impl FromPyObject<'_, '_> for PyLance<DataOverlayGroup> {
+    type Error = PyErr;
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
+        let fragment_id = ob.getattr("fragment_id")?.extract::<u64>()?;
+        let overlays = extract_vec(&ob.getattr("overlays")?)?;
+        Ok(Self(DataOverlayGroup {
+            fragment_id,
+            overlays,
+        }))
+    }
+}
+
+impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayGroup> {
+    type Target = PyAny;
+    type Output = Bound<'py, Self::Target>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let namespace = py
+            .import(intern!(py, "lance"))
+            .and_then(|module| module.getattr(intern!(py, "LanceOperation")))
+            .expect("Failed to import LanceOperation namespace");
+
+        let fragment_id = self.0.fragment_id;
+        let overlays = export_vec(py, self.0.overlays.as_slice())?;
+
+        let cls = namespace
+            .getattr("DataOverlayGroup")
+            .expect("Failed to get DataOverlayGroup class");
+        cls.call1((fragment_id, overlays))
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct PyUpdateMode(pub UpdateMode);
 
@@ -350,6 +473,13 @@ impl FromPyObject<'_, '_> for PyLance<Operation> {
 
                 Ok(Self(op))
             }
+            "DataOverlay" => {
+                let groups = extract_vec(&ob.getattr("groups")?)?;
+
+                let op = Operation::DataOverlay { groups };
+
+                Ok(Self(op))
+            }
             "Project" => {
                 let schema = extract_schema(&ob.getattr("schema")?)?;
 
@@ -487,6 +617,13 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> {
                     .expect("Failed to get DataReplacement class");
                 cls.call1((replacements,))
             }
+            Operation::DataOverlay { groups } => {
+                let groups = export_vec(py, groups.as_slice())?;
+                let cls = namespace
+                    .getattr("DataOverlay")
+                    .expect("Failed to get DataOverlay class");
+                cls.call1((groups,))
+            }
             Operation::Delete {
                 updated_fragments,
                 deleted_fragment_ids,
diff --git a/python/src/utils.rs b/python/src/utils.rs
index 4f7d6d7dde2..edc376d35c8 100644
--- a/python/src/utils.rs
+++ b/python/src/utils.rs
@@ -26,7 +26,6 @@ use lance::Result;
 use lance::datatypes::Schema;
 use lance_arrow::FixedSizeListArrayExt;
 use lance_file::previous::writer::FileWriter as PreviousFileWriter;
-use lance_index::scalar::IndexWriter;
 use lance_index::vector::hnsw::{HNSW, builder::HnswBuildParams};
 use lance_index::vector::kmeans::{
     KMeans as LanceKMeans, KMeansAlgoFloat, KMeansParams, compute_partitions,
@@ -255,7 +254,7 @@ impl Hnsw {
         rt().block_on(Some(py), async {
             let batch = self.hnsw.to_batch()?;
             let metadata = batch.schema_ref().metadata().clone();
-            writer.write_record_batch(batch).await?;
+            writer.write(&[batch]).await?;
             writer.finish_with_metadata(&metadata).await?;
             Result::Ok(())
         })?
diff --git a/python/uv.lock b/python/uv.lock
index 69e061b8075..65e4c8b30ee 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -1,22 +1,12 @@
 version = 1
 revision = 3
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 resolution-markers = [
     "python_full_version >= '3.14'",
     "python_full_version == '3.13.*'",
     "python_full_version == '3.12.*'",
     "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-    "python_full_version < '3.10'",
-]
-
-[[package]]
-name = "absl-py"
-version = "2.3.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" },
+    "python_full_version < '3.11'",
 ]
 
 [[package]]
@@ -30,139 +20,139 @@ wheels = [
 
 [[package]]
 name = "aiohttp"
-version = "3.13.4"
+version = "3.14.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "aiohappyeyeballs", marker = "python_full_version >= '3.10'" },
-    { name = "aiosignal", marker = "python_full_version >= '3.10'" },
-    { name = "async-timeout", marker = "python_full_version == '3.10.*'" },
-    { name = "attrs", marker = "python_full_version >= '3.10'" },
-    { name = "frozenlist", marker = "python_full_version >= '3.10'" },
-    { name = "multidict", marker = "python_full_version >= '3.10'" },
-    { name = "propcache", marker = "python_full_version >= '3.10'" },
-    { name = "yarl", marker = "python_full_version >= '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/45/4a/064321452809dae953c1ed6e017504e72551a26b6f5708a5a80e4bf556ff/aiohttp-3.13.4.tar.gz", hash = "sha256:d97a6d09c66087890c2ab5d49069e1e570583f7ac0314ecf98294c1b6aaebd38", size = 7859748, upload-time = "2026-03-28T17:19:40.6Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2c/05/6817e0390eb47b0867cf8efdb535298191662192281bc3ca62a0cb7973eb/aiohttp-3.13.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6290fe12fe8cefa6ea3c1c5b969d32c010dfe191d4392ff9b599a3f473cbe722", size = 753094, upload-time = "2026-03-28T17:14:59.928Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/c1/e5b7f25f6dd1ab57da92aa9d226b2c8b56f223dd20475d3ddfddaba86ab8/aiohttp-3.13.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7520d92c0e8fbbe63f36f20a5762db349ff574ad38ad7bc7732558a650439845", size = 505213, upload-time = "2026-03-28T17:15:01.989Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/e5/8f42033c7ce98b54dfd3791f03e60231cfe4a2db4471b5fc188df2b8a6ad/aiohttp-3.13.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2710ae1e1b81d0f187883b6e9d66cecf8794b50e91aa1e73fc78bfb5503b5d9", size = 498580, upload-time = "2026-03-28T17:15:03.879Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/a4/bbc989f5362066b81930da1a66084a859a971d03faab799dc59a3ce3a220/aiohttp-3.13.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:717d17347567ded1e273aa09918650dfd6fd06f461549204570c7973537d4123", size = 1692718, upload-time = "2026-03-28T17:15:05.541Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/72/3775116969931f151be116689d2ae6ddafff2ec2887d8f9b4e7043f32e74/aiohttp-3.13.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:383880f7b8de5ac208fa829c7038d08e66377283b2de9e791b71e06e803153c2", size = 1660714, upload-time = "2026-03-28T17:15:08.23Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/e8/d2f1a2da2743e32fe348ebf8a4c59caad14a92f5f18af616fd33381275e1/aiohttp-3.13.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1867087e2c1963db1216aedf001efe3b129835ed2b05d97d058176a6d08b5726", size = 1744152, upload-time = "2026-03-28T17:15:10.828Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/a6/575886f417ac3c08e462f2ca237cc49f436bd992ca3f7ff95b7dd9c44205/aiohttp-3.13.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6234bf416a38d687c3ab7f79934d7fb2a42117a5b9813aca07de0a5398489023", size = 1836278, upload-time = "2026-03-28T17:15:12.537Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/4c/0051d4550fb9e8b5ca4e0fe1ccd58652340915180c5164999e6741bf2083/aiohttp-3.13.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3cdd3393130bf6588962441ffd5bde1d3ea2d63a64afa7119b3f3ba349cebbe7", size = 1687953, upload-time = "2026-03-28T17:15:14.248Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/54/841e87b8c51c2adc01a3ceb9919dc45c7899fe4c21deb70aada734ea5a38/aiohttp-3.13.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d0dbc6c76befa76865373d6aa303e480bb8c3486e7763530f7f6e527b471118", size = 1572484, upload-time = "2026-03-28T17:15:15.911Z" },
-    { url = "https://files.pythonhosted.org/packages/da/f1/21cbf5f7fa1e267af6301f886cab9b314f085e4d0097668d189d165cd7da/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10fb7b53262cf4144a083c9db0d2b4d22823d6708270a9970c4627b248c6064c", size = 1662851, upload-time = "2026-03-28T17:15:17.822Z" },
-    { url = "https://files.pythonhosted.org/packages/40/15/bcad6b68d7bef27ae7443288215767263c7753ede164267cf6cf63c94a87/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:eb10ce8c03850e77f4d9518961c227be569e12f71525a7e90d17bca04299921d", size = 1671984, upload-time = "2026-03-28T17:15:19.561Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/fa/ab316931afc7a73c7f493bb1b30fbd61e28ec2d3ea50353336e76293e8ec/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:7c65738ac5ae32b8feef699a4ed0dc91a0c8618b347781b7461458bbcaaac7eb", size = 1713880, upload-time = "2026-03-28T17:15:21.589Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/45/314e8e64c7f328174964b6db511dd5e9e60c9121ab5457bc2c908b7d03a4/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:6b335919ffbaf98df8ff3c74f7a6decb8775882632952fd1810a017e38f15aee", size = 1560315, upload-time = "2026-03-28T17:15:23.66Z" },
-    { url = "https://files.pythonhosted.org/packages/18/e7/93d5fa06fe00219a81466577dacae9e3732f3b4f767b12b2e2cc8c35c970/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ec75fc18cb9f4aca51c2cbace20cf6716e36850f44189644d2d69a875d5e0532", size = 1735115, upload-time = "2026-03-28T17:15:25.77Z" },
-    { url = "https://files.pythonhosted.org/packages/19/9f/f64b95392ddd4e204fd9ab7cd33dd18d14ac9e4b86866f1f6a69b7cda83d/aiohttp-3.13.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:463fa18a95c5a635d2b8c09babe240f9d7dbf2a2010a6c0b35d8c4dff2a0e819", size = 1673916, upload-time = "2026-03-28T17:15:27.526Z" },
-    { url = "https://files.pythonhosted.org/packages/52/c1/bb33be79fd285c69f32e5b074b299cae8847f748950149c3965c1b3b3adf/aiohttp-3.13.4-cp310-cp310-win32.whl", hash = "sha256:13168f5645d9045522c6cef818f54295376257ed8d02513a37c2ef3046fc7a97", size = 440277, upload-time = "2026-03-28T17:15:29.173Z" },
-    { url = "https://files.pythonhosted.org/packages/23/f9/7cf1688da4dd0885f914ee40bc8e1dce776df98fe6518766de975a570538/aiohttp-3.13.4-cp310-cp310-win_amd64.whl", hash = "sha256:a7058af1f53209fdf07745579ced525d38d481650a989b7aa4a3b484b901cdab", size = 463015, upload-time = "2026-03-28T17:15:30.802Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/7e/cb94129302d78c46662b47f9897d642fd0b33bdfef4b73b20c6ced35aa4c/aiohttp-3.13.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8ea0c64d1bcbf201b285c2246c51a0c035ba3bbd306640007bc5844a3b4658c1", size = 760027, upload-time = "2026-03-28T17:15:33.022Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/cd/2db3c9397c3bd24216b203dd739945b04f8b87bb036c640da7ddb63c75ef/aiohttp-3.13.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6f742e1fa45c0ed522b00ede565e18f97e4cf8d1883a712ac42d0339dfb0cce7", size = 508325, upload-time = "2026-03-28T17:15:34.714Z" },
-    { url = "https://files.pythonhosted.org/packages/36/a3/d28b2722ec13107f2e37a86b8a169897308bab6a3b9e071ecead9d67bd9b/aiohttp-3.13.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dcfb50ee25b3b7a1222a9123be1f9f89e56e67636b561441f0b304e25aaef8f", size = 502402, upload-time = "2026-03-28T17:15:36.409Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/d6/acd47b5f17c4430e555590990a4746efbcb2079909bb865516892bf85f37/aiohttp-3.13.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3262386c4ff370849863ea93b9ea60fd59c6cf56bf8f93beac625cf4d677c04d", size = 1771224, upload-time = "2026-03-28T17:15:38.223Z" },
-    { url = "https://files.pythonhosted.org/packages/98/af/af6e20113ba6a48fd1cd9e5832c4851e7613ef50c7619acdaee6ec5f1aff/aiohttp-3.13.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:473bb5aa4218dd254e9ae4834f20e31f5a0083064ac0136a01a62ddbae2eaa42", size = 1731530, upload-time = "2026-03-28T17:15:39.988Z" },
-    { url = "https://files.pythonhosted.org/packages/81/16/78a2f5d9c124ad05d5ce59a9af94214b6466c3491a25fb70760e98e9f762/aiohttp-3.13.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e56423766399b4c77b965f6aaab6c9546617b8994a956821cc507d00b91d978c", size = 1827925, upload-time = "2026-03-28T17:15:41.944Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/1f/79acf0974ced805e0e70027389fccbb7d728e6f30fcac725fb1071e63075/aiohttp-3.13.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8af249343fafd5ad90366a16d230fc265cf1149f26075dc9fe93cfd7c7173942", size = 1923579, upload-time = "2026-03-28T17:15:44.071Z" },
-    { url = "https://files.pythonhosted.org/packages/af/53/29f9e2054ea6900413f3b4c3eb9d8331f60678ec855f13ba8714c47fd48d/aiohttp-3.13.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bc0a5cf4f10ef5a2c94fdde488734b582a3a7a000b131263e27c9295bd682d9", size = 1767655, upload-time = "2026-03-28T17:15:45.911Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/57/462fe1d3da08109ba4aa8590e7aed57c059af2a7e80ec21f4bac5cfe1094/aiohttp-3.13.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5c7ff1028e3c9fc5123a865ce17df1cb6424d180c503b8517afbe89aa566e6be", size = 1630439, upload-time = "2026-03-28T17:15:48.11Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/4b/4813344aacdb8127263e3eec343d24e973421143826364fa9fc847f6283f/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ba5cf98b5dcb9bddd857da6713a503fa6d341043258ca823f0f5ab7ab4a94ee8", size = 1745557, upload-time = "2026-03-28T17:15:50.13Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/01/1ef1adae1454341ec50a789f03cfafe4c4ac9c003f6a64515ecd32fe4210/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d85965d3ba21ee4999e83e992fecb86c4614d6920e40705501c0a1f80a583c12", size = 1741796, upload-time = "2026-03-28T17:15:52.351Z" },
-    { url = "https://files.pythonhosted.org/packages/22/04/8cdd99af988d2aa6922714d957d21383c559835cbd43fbf5a47ddf2e0f05/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:49f0b18a9b05d79f6f37ddd567695943fcefb834ef480f17a4211987302b2dc7", size = 1805312, upload-time = "2026-03-28T17:15:54.407Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/7f/b48d5577338d4b25bbdbae35c75dbfd0493cb8886dc586fbfb2e90862239/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7f78cb080c86fbf765920e5f1ef35af3f24ec4314d6675d0a21eaf41f6f2679c", size = 1621751, upload-time = "2026-03-28T17:15:56.564Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/89/4eecad8c1858e6d0893c05929e22343e0ebe3aec29a8a399c65c3cc38311/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:67a3ec705534a614b68bbf1c70efa777a21c3da3895d1c44510a41f5a7ae0453", size = 1826073, upload-time = "2026-03-28T17:15:58.489Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/5c/9dc8293ed31b46c39c9c513ac7ca152b3c3d38e0ea111a530ad12001b827/aiohttp-3.13.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6630ec917e85c5356b2295744c8a97d40f007f96a1c76bf1928dc2e27465393", size = 1760083, upload-time = "2026-03-28T17:16:00.677Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/19/8bbf6a4994205d96831f97b7d21a0feed120136e6267b5b22d229c6dc4dc/aiohttp-3.13.4-cp311-cp311-win32.whl", hash = "sha256:54049021bc626f53a5394c29e8c444f726ee5a14b6e89e0ad118315b1f90f5e3", size = 439690, upload-time = "2026-03-28T17:16:02.902Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/f5/ac409ecd1007528d15c3e8c3a57d34f334c70d76cfb7128a28cffdebd4c1/aiohttp-3.13.4-cp311-cp311-win_amd64.whl", hash = "sha256:c033f2bc964156030772d31cbf7e5defea181238ce1f87b9455b786de7d30145", size = 463824, upload-time = "2026-03-28T17:16:05.058Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/bd/ede278648914cabbabfdf95e436679b5d4156e417896a9b9f4587169e376/aiohttp-3.13.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ee62d4471ce86b108b19c3364db4b91180d13fe3510144872d6bad5401957360", size = 752158, upload-time = "2026-03-28T17:16:06.901Z" },
-    { url = "https://files.pythonhosted.org/packages/90/de/581c053253c07b480b03785196ca5335e3c606a37dc73e95f6527f1591fe/aiohttp-3.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c0fd8f41b54b58636402eb493afd512c23580456f022c1ba2db0f810c959ed0d", size = 501037, upload-time = "2026-03-28T17:16:08.82Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/f9/a5ede193c08f13cc42c0a5b50d1e246ecee9115e4cf6e900d8dbd8fd6acb/aiohttp-3.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4baa48ce49efd82d6b1a0be12d6a36b35e5594d1dd42f8bfba96ea9f8678b88c", size = 501556, upload-time = "2026-03-28T17:16:10.63Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/10/88ff67cd48a6ec36335b63a640abe86135791544863e0cfe1f065d6cef7a/aiohttp-3.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d738ebab9f71ee652d9dbd0211057690022201b11197f9a7324fd4dba128aa97", size = 1757314, upload-time = "2026-03-28T17:16:12.498Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/15/fdb90a5cf5a1f52845c276e76298c75fbbcc0ac2b4a86551906d54529965/aiohttp-3.13.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0ce692c3468fa831af7dceed52edf51ac348cebfc8d3feb935927b63bd3e8576", size = 1731819, upload-time = "2026-03-28T17:16:14.558Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/df/28146785a007f7820416be05d4f28cc207493efd1e8c6c1068e9bdc29198/aiohttp-3.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e08abcfe752a454d2cb89ff0c08f2d1ecd057ae3e8cc6d84638de853530ebab", size = 1793279, upload-time = "2026-03-28T17:16:16.594Z" },
-    { url = "https://files.pythonhosted.org/packages/10/47/689c743abf62ea7a77774d5722f220e2c912a77d65d368b884d9779ef41b/aiohttp-3.13.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5977f701b3fff36367a11087f30ea73c212e686d41cd363c50c022d48b011d8d", size = 1891082, upload-time = "2026-03-28T17:16:18.71Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/b6/f7f4f318c7e58c23b761c9b13b9a3c9b394e0f9d5d76fbc6622fa98509f6/aiohttp-3.13.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54203e10405c06f8b6020bd1e076ae0fe6c194adcee12a5a78af3ffa3c57025e", size = 1773938, upload-time = "2026-03-28T17:16:21.125Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/06/f207cb3121852c989586a6fc16ff854c4fcc8651b86c5d3bd1fc83057650/aiohttp-3.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:358a6af0145bc4dda037f13167bef3cce54b132087acc4c295c739d05d16b1c3", size = 1579548, upload-time = "2026-03-28T17:16:23.588Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/58/e1289661a32161e24c1fe479711d783067210d266842523752869cc1d9c2/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:898ea1850656d7d61832ef06aa9846ab3ddb1621b74f46de78fbc5e1a586ba83", size = 1714669, upload-time = "2026-03-28T17:16:25.713Z" },
-    { url = "https://files.pythonhosted.org/packages/96/0a/3e86d039438a74a86e6a948a9119b22540bae037d6ba317a042ae3c22711/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7bc30cceb710cf6a44e9617e43eebb6e3e43ad855a34da7b4b6a73537d8a6763", size = 1754175, upload-time = "2026-03-28T17:16:28.18Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/30/e717fc5df83133ba467a560b6d8ef20197037b4bb5d7075b90037de1018e/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4a31c0c587a8a038f19a4c7e60654a6c899c9de9174593a13e7cc6e15ff271f9", size = 1762049, upload-time = "2026-03-28T17:16:30.941Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/28/8f7a2d4492e336e40005151bdd94baf344880a4707573378579f833a64c1/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2062f675f3fe6e06d6113eb74a157fb9df58953ffed0cdb4182554b116545758", size = 1570861, upload-time = "2026-03-28T17:16:32.953Z" },
-    { url = "https://files.pythonhosted.org/packages/78/45/12e1a3d0645968b1c38de4b23fdf270b8637735ea057d4f84482ff918ad9/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d1ba8afb847ff80626d5e408c1fdc99f942acc877d0702fe137015903a220a9", size = 1790003, upload-time = "2026-03-28T17:16:35.468Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/0f/60374e18d590de16dcb39d6ff62f39c096c1b958e6f37727b5870026ea30/aiohttp-3.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b08149419994cdd4d5eecf7fd4bc5986b5a9380285bcd01ab4c0d6bfca47b79d", size = 1737289, upload-time = "2026-03-28T17:16:38.187Z" },
-    { url = "https://files.pythonhosted.org/packages/02/bf/535e58d886cfbc40a8b0013c974afad24ef7632d645bca0b678b70033a60/aiohttp-3.13.4-cp312-cp312-win32.whl", hash = "sha256:fc432f6a2c4f720180959bc19aa37259651c1a4ed8af8afc84dd41c60f15f791", size = 434185, upload-time = "2026-03-28T17:16:40.735Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/1a/d92e3325134ebfff6f4069f270d3aac770d63320bd1fcd0eca023e74d9a8/aiohttp-3.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:6148c9ae97a3e8bff9a1fc9c757fa164116f86c100468339730e717590a3fb77", size = 461285, upload-time = "2026-03-28T17:16:42.713Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/ac/892f4162df9b115b4758d615f32ec63d00f3084c705ff5526630887b9b42/aiohttp-3.13.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:63dd5e5b1e43b8fb1e91b79b7ceba1feba588b317d1edff385084fcc7a0a4538", size = 745744, upload-time = "2026-03-28T17:16:44.67Z" },
-    { url = "https://files.pythonhosted.org/packages/97/a9/c5b87e4443a2f0ea88cb3000c93a8fdad1ee63bffc9ded8d8c8e0d66efc6/aiohttp-3.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:746ac3cc00b5baea424dacddea3ec2c2702f9590de27d837aa67004db1eebc6e", size = 498178, upload-time = "2026-03-28T17:16:46.766Z" },
-    { url = "https://files.pythonhosted.org/packages/94/42/07e1b543a61250783650df13da8ddcdc0d0a5538b2bd15cef6e042aefc61/aiohttp-3.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bda8f16ea99d6a6705e5946732e48487a448be874e54a4f73d514660ff7c05d3", size = 498331, upload-time = "2026-03-28T17:16:48.9Z" },
-    { url = "https://files.pythonhosted.org/packages/20/d6/492f46bf0328534124772d0cf58570acae5b286ea25006900650f69dae0e/aiohttp-3.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b061e7b5f840391e3f64d0ddf672973e45c4cfff7a0feea425ea24e51530fc2", size = 1744414, upload-time = "2026-03-28T17:16:50.968Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/4d/e02627b2683f68051246215d2d62b2d2f249ff7a285e7a858dc47d6b6a14/aiohttp-3.13.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b252e8d5cd66184b570d0d010de742736e8a4fab22c58299772b0c5a466d4b21", size = 1719226, upload-time = "2026-03-28T17:16:53.173Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/6c/5d0a3394dd2b9f9aeba6e1b6065d0439e4b75d41f1fb09a3ec010b43552b/aiohttp-3.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20af8aad61d1803ff11152a26146d8d81c266aa8c5aa9b4504432abb965c36a0", size = 1782110, upload-time = "2026-03-28T17:16:55.362Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/2d/c20791e3437700a7441a7edfb59731150322424f5aadf635602d1d326101/aiohttp-3.13.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:13a5cc924b59859ad2adb1478e31f410a7ed46e92a2a619d6d1dd1a63c1a855e", size = 1884809, upload-time = "2026-03-28T17:16:57.734Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/94/d99dbfbd1924a87ef643833932eb2a3d9e5eee87656efea7d78058539eff/aiohttp-3.13.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:534913dfb0a644d537aebb4123e7d466d94e3be5549205e6a31f72368980a81a", size = 1764938, upload-time = "2026-03-28T17:17:00.221Z" },
-    { url = "https://files.pythonhosted.org/packages/49/61/3ce326a1538781deb89f6cf5e094e2029cd308ed1e21b2ba2278b08426f6/aiohttp-3.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:320e40192a2dcc1cf4b5576936e9652981ab596bf81eb309535db7e2f5b5672f", size = 1570697, upload-time = "2026-03-28T17:17:02.985Z" },
-    { url = "https://files.pythonhosted.org/packages/b6/77/4ab5a546857bb3028fbaf34d6eea180267bdab022ee8b1168b1fcde4bfdd/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9e587fcfce2bcf06526a43cb705bdee21ac089096f2e271d75de9c339db3100c", size = 1702258, upload-time = "2026-03-28T17:17:05.28Z" },
-    { url = "https://files.pythonhosted.org/packages/79/63/d8f29021e39bc5af8e5d5e9da1b07976fb9846487a784e11e4f4eeda4666/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9eb9c2eea7278206b5c6c1441fdd9dc420c278ead3f3b2cc87f9b693698cc500", size = 1740287, upload-time = "2026-03-28T17:17:07.712Z" },
-    { url = "https://files.pythonhosted.org/packages/55/3a/cbc6b3b124859a11bc8055d3682c26999b393531ef926754a3445b99dfef/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:29be00c51972b04bf9d5c8f2d7f7314f48f96070ca40a873a53056e652e805f7", size = 1753011, upload-time = "2026-03-28T17:17:10.053Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/30/836278675205d58c1368b21520eab9572457cf19afd23759216c04483048/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90c06228a6c3a7c9f776fe4fc0b7ff647fffd3bed93779a6913c804ae00c1073", size = 1566359, upload-time = "2026-03-28T17:17:12.433Z" },
-    { url = "https://files.pythonhosted.org/packages/50/b4/8032cc9b82d17e4277704ba30509eaccb39329dc18d6a35f05e424439e32/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a533ec132f05fd9a1d959e7f34184cd7d5e8511584848dab85faefbaac573069", size = 1785537, upload-time = "2026-03-28T17:17:14.721Z" },
-    { url = "https://files.pythonhosted.org/packages/17/7d/5873e98230bde59f493bf1f7c3e327486a4b5653fa401144704df5d00211/aiohttp-3.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1c946f10f413836f82ea4cfb90200d2a59578c549f00857e03111cf45ad01ca5", size = 1740752, upload-time = "2026-03-28T17:17:17.387Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/f2/13e46e0df051494d7d3c68b7f72d071f48c384c12716fc294f75d5b1a064/aiohttp-3.13.4-cp313-cp313-win32.whl", hash = "sha256:48708e2706106da6967eff5908c78ca3943f005ed6bcb75da2a7e4da94ef8c70", size = 433187, upload-time = "2026-03-28T17:17:19.523Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/c0/649856ee655a843c8f8664592cfccb73ac80ede6a8c8db33a25d810c12db/aiohttp-3.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:74a2eb058da44fa3a877a49e2095b591d4913308bb424c418b77beb160c55ce3", size = 459778, upload-time = "2026-03-28T17:17:21.964Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/29/6657cc37ae04cacc2dbf53fb730a06b6091cc4cbe745028e047c53e6d840/aiohttp-3.13.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:e0a2c961fc92abeff61d6444f2ce6ad35bb982db9fc8ff8a47455beacf454a57", size = 749363, upload-time = "2026-03-28T17:17:24.044Z" },
-    { url = "https://files.pythonhosted.org/packages/90/7f/30ccdf67ca3d24b610067dc63d64dcb91e5d88e27667811640644aa4a85d/aiohttp-3.13.4-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:153274535985a0ff2bff1fb6c104ed547cec898a09213d21b0f791a44b14d933", size = 499317, upload-time = "2026-03-28T17:17:26.199Z" },
-    { url = "https://files.pythonhosted.org/packages/93/13/e372dd4e68ad04ee25dafb050c7f98b0d91ea643f7352757e87231102555/aiohttp-3.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:351f3171e2458da3d731ce83f9e6b9619e325c45cbd534c7759750cabf453ad7", size = 500477, upload-time = "2026-03-28T17:17:28.279Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/fe/ee6298e8e586096fb6f5eddd31393d8544f33ae0792c71ecbb4c2bef98ac/aiohttp-3.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f989ac8bc5595ff761a5ccd32bdb0768a117f36dd1504b1c2c074ed5d3f4df9c", size = 1737227, upload-time = "2026-03-28T17:17:30.587Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/b9/a7a0463a09e1a3fe35100f74324f23644bfc3383ac5fd5effe0722a5f0b7/aiohttp-3.13.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d36fc1709110ec1e87a229b201dd3ddc32aa01e98e7868083a794609b081c349", size = 1694036, upload-time = "2026-03-28T17:17:33.29Z" },
-    { url = "https://files.pythonhosted.org/packages/57/7c/8972ae3fb7be00a91aee6b644b2a6a909aedb2c425269a3bfd90115e6f8f/aiohttp-3.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:42adaeea83cbdf069ab94f5103ce0787c21fb1a0153270da76b59d5578302329", size = 1786814, upload-time = "2026-03-28T17:17:36.035Z" },
-    { url = "https://files.pythonhosted.org/packages/93/01/c81e97e85c774decbaf0d577de7d848934e8166a3a14ad9f8aa5be329d28/aiohttp-3.13.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:92deb95469928cc41fd4b42a95d8012fa6df93f6b1c0a83af0ffbc4a5e218cde", size = 1866676, upload-time = "2026-03-28T17:17:38.441Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/5f/5b46fe8694a639ddea2cd035bf5729e4677ea882cb251396637e2ef1590d/aiohttp-3.13.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0c7c07c4257ef3a1df355f840bc62d133bcdef5c1c5ba75add3c08553e2eed", size = 1740842, upload-time = "2026-03-28T17:17:40.783Z" },
-    { url = "https://files.pythonhosted.org/packages/20/a2/0d4b03d011cca6b6b0acba8433193c1e484efa8d705ea58295590fe24203/aiohttp-3.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f062c45de8a1098cb137a1898819796a2491aec4e637a06b03f149315dff4d8f", size = 1566508, upload-time = "2026-03-28T17:17:43.235Z" },
-    { url = "https://files.pythonhosted.org/packages/98/17/e689fd500da52488ec5f889effd6404dece6a59de301e380f3c64f167beb/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:76093107c531517001114f0ebdb4f46858ce818590363e3e99a4a2280334454a", size = 1700569, upload-time = "2026-03-28T17:17:46.165Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/0d/66402894dbcf470ef7db99449e436105ea862c24f7ea4c95c683e635af35/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:6f6ec32162d293b82f8b63a16edc80769662fbd5ae6fbd4936d3206a2c2cc63b", size = 1707407, upload-time = "2026-03-28T17:17:48.825Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/eb/af0ab1a3650092cbd8e14ef29e4ab0209e1460e1c299996c3f8288b3f1ff/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5903e2db3d202a00ad9f0ec35a122c005e85d90c9836ab4cda628f01edf425e2", size = 1752214, upload-time = "2026-03-28T17:17:51.206Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/bf/72326f8a98e4c666f292f03c385545963cc65e358835d2a7375037a97b57/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2d5bea57be7aca98dbbac8da046d99b5557c5cf4e28538c4c786313078aca09e", size = 1562162, upload-time = "2026-03-28T17:17:53.634Z" },
-    { url = "https://files.pythonhosted.org/packages/67/9f/13b72435f99151dd9a5469c96b3b5f86aa29b7e785ca7f35cf5e538f74c0/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:bcf0c9902085976edc0232b75006ef38f89686901249ce14226b6877f88464fb", size = 1768904, upload-time = "2026-03-28T17:17:55.991Z" },
-    { url = "https://files.pythonhosted.org/packages/18/bc/28d4970e7d5452ac7776cdb5431a1164a0d9cf8bd2fffd67b4fb463aa56d/aiohttp-3.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3295f98bfeed2e867cab588f2a146a9db37a85e3ae9062abf46ba062bd29165", size = 1723378, upload-time = "2026-03-28T17:17:58.348Z" },
-    { url = "https://files.pythonhosted.org/packages/53/74/b32458ca1a7f34d65bdee7aef2036adbe0438123d3d53e2b083c453c24dd/aiohttp-3.13.4-cp314-cp314-win32.whl", hash = "sha256:a598a5c5767e1369d8f5b08695cab1d8160040f796c4416af76fd773d229b3c9", size = 438711, upload-time = "2026-03-28T17:18:00.728Z" },
-    { url = "https://files.pythonhosted.org/packages/40/b2/54b487316c2df3e03a8f3435e9636f8a81a42a69d942164830d193beb56a/aiohttp-3.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:c555db4bc7a264bead5a7d63d92d41a1122fcd39cc62a4db815f45ad46f9c2c8", size = 464977, upload-time = "2026-03-28T17:18:03.367Z" },
-    { url = "https://files.pythonhosted.org/packages/47/fb/e41b63c6ce71b07a59243bb8f3b457ee0c3402a619acb9d2c0d21ef0e647/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:45abbbf09a129825d13c18c7d3182fecd46d9da3cfc383756145394013604ac1", size = 781549, upload-time = "2026-03-28T17:18:05.779Z" },
-    { url = "https://files.pythonhosted.org/packages/97/53/532b8d28df1e17e44c4d9a9368b78dcb6bf0b51037522136eced13afa9e8/aiohttp-3.13.4-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:74c80b2bc2c2adb7b3d1941b2b60701ee2af8296fc8aad8b8bc48bc25767266c", size = 514383, upload-time = "2026-03-28T17:18:08.096Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/1f/62e5d400603e8468cd635812d99cb81cfdc08127a3dc474c647615f31339/aiohttp-3.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c97989ae40a9746650fa196894f317dafc12227c808c774929dda0ff873a5954", size = 518304, upload-time = "2026-03-28T17:18:10.642Z" },
-    { url = "https://files.pythonhosted.org/packages/90/57/2326b37b10896447e3c6e0cbef4fe2486d30913639a5cfd1332b5d870f82/aiohttp-3.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dae86be9811493f9990ef44fff1685f5c1a3192e9061a71a109d527944eed551", size = 1893433, upload-time = "2026-03-28T17:18:13.121Z" },
-    { url = "https://files.pythonhosted.org/packages/d2/b4/a24d82112c304afdb650167ef2fe190957d81cbddac7460bedd245f765aa/aiohttp-3.13.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1db491abe852ca2fa6cc48a3341985b0174b3741838e1341b82ac82c8bd9e871", size = 1755901, upload-time = "2026-03-28T17:18:16.21Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/2d/0883ef9d878d7846287f036c162a951968f22aabeef3ac97b0bea6f76d5d/aiohttp-3.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0e5d701c0aad02a7dce72eef6b93226cf3734330f1a31d69ebbf69f33b86666e", size = 1876093, upload-time = "2026-03-28T17:18:18.703Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/52/9204bb59c014869b71971addad6778f005daa72a96eed652c496789d7468/aiohttp-3.13.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8ac32a189081ae0a10ba18993f10f338ec94341f0d5df8fff348043962f3c6f8", size = 1970815, upload-time = "2026-03-28T17:18:21.858Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/b5/e4eb20275a866dde0f570f411b36c6b48f7b53edfe4f4071aa1b0728098a/aiohttp-3.13.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e968cdaba43e45c73c3f306fca418c8009a957733bac85937c9f9cf3f4de27", size = 1816223, upload-time = "2026-03-28T17:18:24.729Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/23/e98075c5bb146aa61a1239ee1ac7714c85e814838d6cebbe37d3fe19214a/aiohttp-3.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca114790c9144c335d538852612d3e43ea0f075288f4849cf4b05d6cd2238ce7", size = 1649145, upload-time = "2026-03-28T17:18:27.269Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/c1/7bad8be33bb06c2bb224b6468874346026092762cbec388c3bdb65a368ee/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ea2e071661ba9cfe11eabbc81ac5376eaeb3061f6e72ec4cc86d7cdd1ffbdbbb", size = 1816562, upload-time = "2026-03-28T17:18:29.847Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/10/c00323348695e9a5e316825969c88463dcc24c7e9d443244b8a2c9cf2eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:34e89912b6c20e0fd80e07fa401fd218a410aa1ce9f1c2f1dad6db1bd0ce0927", size = 1800333, upload-time = "2026-03-28T17:18:32.269Z" },
-    { url = "https://files.pythonhosted.org/packages/84/43/9b2147a1df3559f49bd723e22905b46a46c068a53adb54abdca32c4de180/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0e217cf9f6a42908c52b46e42c568bd57adc39c9286ced31aaace614b6087965", size = 1820617, upload-time = "2026-03-28T17:18:35.238Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/7f/b3481a81e7a586d02e99387b18c6dafff41285f6efd3daa2124c01f87eae/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:0c296f1221e21ba979f5ac1964c3b78cfde15c5c5f855ffd2caab337e9cd9182", size = 1643417, upload-time = "2026-03-28T17:18:37.949Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/72/07181226bc99ce1124e0f89280f5221a82d3ae6a6d9d1973ce429d48e52b/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:d99a9d168ebaffb74f36d011750e490085ac418f4db926cce3989c8fe6cb6b1b", size = 1849286, upload-time = "2026-03-28T17:18:40.534Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/e6/1b3566e103eca6da5be4ae6713e112a053725c584e96574caf117568ffef/aiohttp-3.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cb19177205d93b881f3f89e6081593676043a6828f59c78c17a0fd6c1fbed2ba", size = 1782635, upload-time = "2026-03-28T17:18:43.073Z" },
-    { url = "https://files.pythonhosted.org/packages/37/58/1b11c71904b8d079eb0c39fe664180dd1e14bebe5608e235d8bfbadc8929/aiohttp-3.13.4-cp314-cp314t-win32.whl", hash = "sha256:c606aa5656dab6552e52ca368e43869c916338346bfaf6304e15c58fb113ea30", size = 472537, upload-time = "2026-03-28T17:18:46.286Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/8f/87c56a1a1977d7dddea5b31e12189665a140fdb48a71e9038ff90bb564ec/aiohttp-3.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:014dcc10ec8ab8db681f0d68e939d1e9286a5aa2b993cbbdb0db130853e02144", size = 506381, upload-time = "2026-03-28T17:18:48.74Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/f9/17e8a70abe874ec694395119338fde2f13ee1903bd14f3fd5b310b77a1ea/aiohttp-3.13.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b3f00bb9403728b08eb3951e982ca0a409c7a871d709684623daeab79465b181", size = 755716, upload-time = "2026-03-28T17:18:51.918Z" },
-    { url = "https://files.pythonhosted.org/packages/27/b3/fdb36e59b9fb37297b1651248d3d84e61faa49af2faabc1e243d3f75585f/aiohttp-3.13.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cb15595eb52870f84248d7cc97013a76f52ab02ff74d394be093b1d9b8b82bc0", size = 506500, upload-time = "2026-03-28T17:18:54.755Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/fb/dacf759c43cfb5fa32568bd369f054eeb23906ab23f4e3663e01e04c7988/aiohttp-3.13.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:907ad36b6a65cff7d88d7aca0f77c650546ba850a4f92c92ecb83590d4613249", size = 499881, upload-time = "2026-03-28T17:18:57.302Z" },
-    { url = "https://files.pythonhosted.org/packages/52/cd/7824ee57dde8ca7f62e7fbc247ebe1aa3b5495d3598f0c516f06de1ef7ab/aiohttp-3.13.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5539ec0d6a3a5c6799b661b7e79166ad1b7ae71ccb59a92fcb6b4ef89295bc94", size = 1681734, upload-time = "2026-03-28T17:19:00.057Z" },
-    { url = "https://files.pythonhosted.org/packages/7a/40/6f4ca61736a16deed2d2762a8dbeaaa48ad292974489be2a2f32f62a4e0b/aiohttp-3.13.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3b4e07d8803a70dd886b5f38588e5b49f894995ca8e132b06c31a2583ae2ef6e", size = 1653787, upload-time = "2026-03-28T17:19:03.026Z" },
-    { url = "https://files.pythonhosted.org/packages/89/80/3793f0a1148a42190f6824ce9a0af79910cd3df8dfc58fa784234a7d9e41/aiohttp-3.13.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ce7320a945aac4bf0bb8901600e4f9409eb602f25ce3ef4d275b48f6d704a862", size = 1737964, upload-time = "2026-03-28T17:19:05.77Z" },
-    { url = "https://files.pythonhosted.org/packages/15/fd/e41981d0f9e0dccfb8f2580d4e64e6c59d293b9b0815849950cc499fe53a/aiohttp-3.13.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:26ed03f7d3d6453634729e2c7600d7255d65e879559c5a48fe1bb78355cde74b", size = 1832226, upload-time = "2026-03-28T17:19:08.809Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/69/e6b566c638b37bfa14b98c2c429fcdba3b097a990acc9845fcc779ce39cc/aiohttp-3.13.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c3f733916e85506b8000dddc071c6b82f8c68f56c99adb328d6550017db062d", size = 1681476, upload-time = "2026-03-28T17:19:11.502Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/8c/f1b7f03e745fa6281dd949673297c7ac54d7cc54d2e58beb5135ac5c6204/aiohttp-3.13.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3d525648fe7c8b4977e460c18098f9f81d7991d72edfdc2f13cf96068f279bc", size = 1573061, upload-time = "2026-03-28T17:19:14.437Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/56/e7e972f1bed922297d72cc1d27bae6b2e28fdc2d6a895320e396a93c0f8a/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e2e68085730a03704beb2cff035fa8648f62c9f93758d7e6d70add7f7bb5b3b", size = 1653248, upload-time = "2026-03-28T17:19:17.432Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/98/3d63d2f2e06808911e103d6d47c400548cf26a23dd3275de594339ff8e96/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:797613182ffaaca0b9ad5f3b3d3ce5d21242c768f75e66c750b8292bd97c9de3", size = 1666599, upload-time = "2026-03-28T17:19:20.17Z" },
-    { url = "https://files.pythonhosted.org/packages/da/c8/31e487fb16d37c89cc6ee190a424b218471750ac48a227e042e200a17687/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2d15e7e4f1099d9e4d863eaf77a8eee5dcb002b7d7188061b0fbee37f845899e", size = 1709919, upload-time = "2026-03-28T17:19:22.872Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/86/3b742bd9204b7deb4f61e6723b1f42a8211ccc60dfddb3e52a6cd4329d46/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:19f60011ad60e40a01d242238bb335399e3a4d8df958c63cbb835add8d5c3b5a", size = 1560523, upload-time = "2026-03-28T17:19:25.879Z" },
-    { url = "https://files.pythonhosted.org/packages/72/63/6b80cef343a0527690588808d02aad7604cc4e23eaab207179e77dd607be/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c344c47e85678e410b064fc2ace14db86bb69db7ed5520c234bf13aed603ec30", size = 1731336, upload-time = "2026-03-28T17:19:29.02Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/3c/9b39bc9609cac87e19b3394b7ed4bbab3787b434b14e012b9e16be64e9d5/aiohttp-3.13.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d904084985ca66459e93797e5e05985c048a9c0633655331144c089943e53d12", size = 1667646, upload-time = "2026-03-28T17:19:31.797Z" },
-    { url = "https://files.pythonhosted.org/packages/21/72/3fb0ea857c891de89f6914f737f7423b7fa4dd1f46d8ce621eb07595ff4c/aiohttp-3.13.4-cp39-cp39-win32.whl", hash = "sha256:1746338dc2a33cf706cd7446575d13d451f28f9860bebc908c7632b22e71ae3f", size = 441019, upload-time = "2026-03-28T17:19:34.79Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/61/8a7191782a31ae3c7f7cee2cd2e37b3ee5849666767db116d449cfe20b88/aiohttp-3.13.4-cp39-cp39-win_amd64.whl", hash = "sha256:a5444dce2e6fba0a1dc2d58d026e674f25f21de178c6f844342629bcef019f2f", size = 464025, upload-time = "2026-03-28T17:19:37.362Z" },
+    { name = "aiohappyeyeballs" },
+    { name = "aiosignal" },
+    { name = "async-timeout", marker = "python_full_version < '3.11'" },
+    { name = "attrs" },
+    { name = "frozenlist" },
+    { name = "multidict" },
+    { name = "propcache" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "yarl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/f0/f81190ba488cd106c2fc6d92680e56bb223bbbbf1e6908c2617011290112/aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c", size = 760606, upload-time = "2026-06-01T19:36:39.054Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/54/444d37eebf0f15db661ca44ec7caf93962f3c5ca92eb4c9a5d888b70aaa2/aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2", size = 514677, upload-time = "2026-06-01T19:36:42.408Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/d1/da280e23321c132c0a3fa7c8cc2830621d79174edc64c829443346489a36/aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd", size = 510155, upload-time = "2026-06-01T19:36:44.072Z" },
+    { url = "https://files.pythonhosted.org/packages/09/b8/2e36d54d0991ec5bba451444004591ee0af58cb1662a3a81c562878b9c1f/aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869", size = 1699947, upload-time = "2026-06-01T19:36:45.762Z" },
+    { url = "https://files.pythonhosted.org/packages/57/95/a31d8ea1a0b9ecc084f5a7dd0b431ce64ef585918bb7bdc82afe11843877/aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4", size = 1664364, upload-time = "2026-06-01T19:36:47.653Z" },
+    { url = "https://files.pythonhosted.org/packages/01/f6/5de3ddffc87a9e8d09b3be38fbd6dd1a736b2ad477a7e787dcb85f57f338/aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb", size = 1761186, upload-time = "2026-06-01T19:36:49.355Z" },
+    { url = "https://files.pythonhosted.org/packages/33/8c/03c5438ec35d7e3a4f33fe895d6c3ec7540a7cec46065f21851211e1ee4d/aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca", size = 1849727, upload-time = "2026-06-01T19:36:51.478Z" },
+    { url = "https://files.pythonhosted.org/packages/22/32/5a05303b0874458920b73f48b8779cc3a93d503f121b38dcc0456dbd698c/aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f", size = 1708197, upload-time = "2026-06-01T19:36:53.241Z" },
+    { url = "https://files.pythonhosted.org/packages/7d/62/478f169488d61414c0a05e7fe423b59ae3d9dcc933d1f0e4acc2c5d5bc3e/aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6", size = 1578147, upload-time = "2026-06-01T19:36:55.154Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/af/b20af85765658972d3337834bd5eebba91b962794f2b4fc3e0ee8c85c0e1/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b", size = 1665836, upload-time = "2026-06-01T19:36:56.94Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/a3/771879cfd59948f4544b172189048905feff802f20f1c6c5411e998a3e06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15", size = 1680335, upload-time = "2026-06-01T19:36:58.642Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/16/582e36ad1d32133cd40659f3bc98e71c22179665a1cfbbb4713bce339c06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce", size = 1731180, upload-time = "2026-06-01T19:37:00.583Z" },
+    { url = "https://files.pythonhosted.org/packages/11/bc/80708fe3f64a07a2c306a42fc7b009118a952709761d215f6d1b4c57195b/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7", size = 1565805, upload-time = "2026-06-01T19:37:02.446Z" },
+    { url = "https://files.pythonhosted.org/packages/57/8f/8d25897f8273a32fe4ad40a8885eec4f397377ed46e8e383078169f60316/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b", size = 1742496, upload-time = "2026-06-01T19:37:04.222Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/7d/c341d32ab2dec56c8478740695743dc6c21b383cace9376a3eab16311a07/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25", size = 1691240, upload-time = "2026-06-01T19:37:06.277Z" },
+    { url = "https://files.pythonhosted.org/packages/37/0f/a81207dd7a2d4a4f645b3a3f8b5a1da1159dc63117ffb137b698fd6df50f/aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f", size = 454686, upload-time = "2026-06-01T19:37:07.96Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/ae/842357f2afb9c915715c6f5775239d987f5d0f845abf7675fa794e0a9d40/aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594", size = 478677, upload-time = "2026-06-01T19:37:09.652Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/d1/330fb22c9535ec177b52396905131c6e39447244b6ca876262939af668ef/aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a", size = 450364, upload-time = "2026-06-01T19:37:11.279Z" },
+    { url = "https://files.pythonhosted.org/packages/67/47/7727bfe8db93f8835a001bd4359d8480cc68d1259b8bce334668f8be97bd/aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a", size = 759147, upload-time = "2026-06-01T19:37:12.918Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/f2/cd3fedff6fade73d71df9ec908c210cec518ef90fd00289250684b90aecf/aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803", size = 513705, upload-time = "2026-06-01T19:37:14.633Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/fe/49746b6b610144a06323bebd8e1211a390310d8c69b98dd6d52df341bc3e/aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e", size = 509627, upload-time = "2026-06-01T19:37:16.385Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/3f/28f2f6cf3d5c0e7b01b27140d0e7873fd11fb341169ad3ce78ad04aba628/aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903", size = 1769293, upload-time = "2026-06-01T19:37:18.067Z" },
+    { url = "https://files.pythonhosted.org/packages/97/6f/2e5f1b525d5474b12b3c60abf733a755845f3bceff21542081ada515f837/aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb", size = 1732363, upload-time = "2026-06-01T19:37:20.138Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/ce/596120faa85ca7b19cd061e3f2f3be23aa8f11a0aedf9191db9e0da1bd76/aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2", size = 1840375, upload-time = "2026-06-01T19:37:22.104Z" },
+    { url = "https://files.pythonhosted.org/packages/72/3c/a7ffe05a757a4a7867643da69357ec41f506879fbd1b231d2ed90af246b2/aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81", size = 1921484, upload-time = "2026-06-01T19:37:24.068Z" },
+    { url = "https://files.pythonhosted.org/packages/93/fa/2c861170bbd4a491de93a69e081db1d971092569e0d593a98ef62c384dc1/aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee", size = 1774153, upload-time = "2026-06-01T19:37:26.256Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/da/1d2f5a165f47ec9b1f69d37b8b977fdc4d501aa72ffb7930db27bb9e49ea/aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d", size = 1632569, upload-time = "2026-06-01T19:37:28.192Z" },
+    { url = "https://files.pythonhosted.org/packages/46/1d/7a6e295c4257252f70f69e90864fdad74b6a1293054fb3f9e65a15de6d63/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00", size = 1740325, upload-time = "2026-06-01T19:37:30.08Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/7e/e1899b1ca3ec62f1eab2a5cbde14039b97493f7f53eb88d9b668562ffa8d/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026", size = 1748691, upload-time = "2026-06-01T19:37:32.211Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/54/4e6b61c1fe7d3433f82bcc6bd7e4d7c683a742a10c9b12a025fd3695c047/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86", size = 1814477, upload-time = "2026-06-01T19:37:34.173Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/38/86fd51be2e08d8e45c83d879d255f10391903cd9fe2a16512f7591a15873/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f", size = 1623393, upload-time = "2026-06-01T19:37:36.281Z" },
+    { url = "https://files.pythonhosted.org/packages/78/49/466e947a42a88ee23c486d036e7e5d1b097f1bafd8084ad9c9a0a92f0f43/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93", size = 1824097, upload-time = "2026-06-01T19:37:38.421Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/89/35f3410bc284682338a1be6b6ea0c5abfa05f063942cfaa9256608440434/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996", size = 1764790, upload-time = "2026-06-01T19:37:40.755Z" },
+    { url = "https://files.pythonhosted.org/packages/42/80/2d4291bd5724d3d17e5951aff5a3e02281483fb47295f0788276ee66cd73/aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae", size = 454176, upload-time = "2026-06-01T19:37:42.837Z" },
+    { url = "https://files.pythonhosted.org/packages/59/ed/41d0ad4f6ececffc32bdf1f7b494e5498f7ca5c849ea2e3cc9bbd1668251/aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5", size = 479334, upload-time = "2026-06-01T19:37:44.776Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/86/c0b5e305c770053f8c3d069bb52b8196917ba91949d1962d52eb307fb0d2/aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43", size = 450262, upload-time = "2026-06-01T19:37:46.461Z" },
+    { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" },
+    { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" },
+    { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" },
+    { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" },
+    { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" },
+    { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" },
+    { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" },
+    { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" },
+    { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" },
+    { url = "https://files.pythonhosted.org/packages/21/61/d11f7d9a3144bffe825247d6367cd93053666da50b94707c9129c78868d5/aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127", size = 502399, upload-time = "2026-06-01T19:38:25.955Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/9b/a7e317625d36356844f8bb022cabd305b541f968856cc3c2e0b58e53ee6e/aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981", size = 510068, upload-time = "2026-06-01T19:38:27.828Z" },
+    { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/6a/42e030a46743841414402a3b00cd3d78419055e86c66fb5822c14b5abfc6/aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61", size = 1729674, upload-time = "2026-06-01T19:38:44.79Z" },
+    { url = "https://files.pythonhosted.org/packages/34/26/3199beb415202e3108e7b83ecebe10914d806d33fb9860c3e4aa60a19be3/aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277", size = 1798808, upload-time = "2026-06-01T19:38:47.01Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/94/b9b6fcf0ee17c21d0d19fb8c22bf83ad18f82e702a9c3bd901a868f5e446/aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882", size = 1891921, upload-time = "2026-06-01T19:38:49.233Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" },
+    { url = "https://files.pythonhosted.org/packages/21/2a/45be91ad1b860508557448d4cc2e165a2ee68dd865657b73bf66cc5a00fb/aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988", size = 1579554, upload-time = "2026-06-01T19:38:54.508Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/e4/1f1c8acbb3acd5c8f795473b92c9c3d44eb60a5692c6104256c8a1c83a0c/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3", size = 1749803, upload-time = "2026-06-01T19:38:59.367Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/c8/c45ea6e7ed84cebba939b9c334498a045ba19d79c61b0110df5f21580de3/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac", size = 1765023, upload-time = "2026-06-01T19:39:01.651Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/a1/a932941784432962fe390e1066823aaef64b4e5ac9fa595df57b5fe472a9/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e", size = 1571671, upload-time = "2026-06-01T19:39:04.044Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/01/e1280feac522597a4d46eb67a0cdfa053cfae263033030b761ab146f29fb/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec", size = 1789904, upload-time = "2026-06-01T19:39:06.294Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" },
+    { url = "https://files.pythonhosted.org/packages/af/cc/c122eabd7a1b7e0c9bbdd6be60e4715905b858399145d9df872bb94f1427/aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456", size = 448656, upload-time = "2026-06-01T19:39:11.171Z" },
+    { url = "https://files.pythonhosted.org/packages/41/a5/bab07d79848a00eedd8ed979ccb302aaea3ac6eb9fa16bd0ed87135869b4/aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11", size = 475803, upload-time = "2026-06-01T19:39:13.439Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/a0/f03ade8566c153666a3871afccbedf6d99911da006325e1fc6cf72a2de99/aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b", size = 443889, upload-time = "2026-06-01T19:39:15.945Z" },
+    { url = "https://files.pythonhosted.org/packages/28/03/5f36ab196a88ba5e9648ae5643e6531e67a3a8c0e96f9c6510ff41540fec/aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f", size = 503330, upload-time = "2026-06-01T19:39:18.195Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/ce/8b49ec2f30f68e02f314f4832186cd45e583360a5a386058be36855d23b6/aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42", size = 509822, upload-time = "2026-06-01T19:39:20.396Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" },
+    { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" },
+    { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/28/24e1409e605a9aa5d84abe0e2acb365354b70ae56d40948101cabe3341ab/aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f", size = 1705862, upload-time = "2026-06-01T19:39:38.968Z" },
+    { url = "https://files.pythonhosted.org/packages/8c/d0/e5eb3ff1daeaf644c7e36a957517672494122628e067c38b263fa04eda77/aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3", size = 1798909, upload-time = "2026-06-01T19:39:41.334Z" },
+    { url = "https://files.pythonhosted.org/packages/d3/ba/8943f906f0570342886ababb9a722a44e360f786a028c5e0b0e29e3f735b/aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6", size = 1868892, upload-time = "2026-06-01T19:39:43.807Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/62/da182e5910ab912b2e88aa919b61a16046a37a95714a5795b02eb57b2d18/aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c", size = 1578775, upload-time = "2026-06-01T19:39:48.902Z" },
+    { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/55/0e2732ca598c7a4dfe8a775662376d0ca2977cb1030e48386d4da5d9a456/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4", size = 1715016, upload-time = "2026-06-01T19:39:53.807Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/96/f0b73730798c9ca525afc30b39f1f81bbe24e245d9654c54d3b39d63212d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c", size = 1763810, upload-time = "2026-06-01T19:39:56.31Z" },
+    { url = "https://files.pythonhosted.org/packages/71/cc/11acb6c4518f448323405a7312b6f255d0f974a34373ad1db7633c4aadc8/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3", size = 1573064, upload-time = "2026-06-01T19:39:58.718Z" },
+    { url = "https://files.pythonhosted.org/packages/de/2d/28c31dde0a7dc98c0ee7d0da2ddcec3f7688c4fc131e5989e278d0c03c0a/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb", size = 1775765, upload-time = "2026-06-01T19:40:01.195Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/44/6126116fd8a316b712bb615660b855c78466bb67ba1bb1742427eafcf7ac/aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d", size = 453684, upload-time = "2026-06-01T19:40:06.277Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/d7/eff4c58a88c5cac5e38b55f44fb8a6d3929c3cbd77356e383e094d3220bd/aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7", size = 481758, upload-time = "2026-06-01T19:40:08.653Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/ed/17b5bd9fbcb46e688f02e572f517754a9a75831e7b54702f027761dc4fa5/aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6", size = 450557, upload-time = "2026-06-01T19:40:11.03Z" },
+    { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" },
+    { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" },
+    { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" },
+    { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/85/ce79bab0310d2e3fd2d7bc7e44412abeff7c8338f8a21dd0f2f1714989e5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a", size = 1778813, upload-time = "2026-06-01T19:40:23.726Z" },
+    { url = "https://files.pythonhosted.org/packages/05/54/ba62ac2d1bc87e010aad23751e383b8794e45d931df67677313a2da78823/aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936", size = 1899969, upload-time = "2026-06-01T19:40:26.406Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/82/7cc7907725d83a19f31551334061e1ab8e108b1d7ac52632a2a844a4acb5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e", size = 1991771, upload-time = "2026-06-01T19:40:29.061Z" },
+    { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/ae/3839726cd49150a53ed340cc24ce5ba09d4c2117020ef9d45542bec5eb2f/aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a", size = 1665437, upload-time = "2026-06-01T19:40:35.01Z" },
+    { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" },
+    { url = "https://files.pythonhosted.org/packages/98/02/a5a7a2524f92d3911761b405a7c067c751891942144adc13e2ad79611e39/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce", size = 1816907, upload-time = "2026-06-01T19:40:40.46Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/76/a8b9f0d09234d516af9f2d7dd715557f33b5da3b0b56ead41d1170e86e3c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c", size = 1840382, upload-time = "2026-06-01T19:40:43.48Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/8e/140e715a0a4bbc211979ea30ec8396ad2ed5bf90ab87d8058fc4668b1923/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7", size = 1659497, upload-time = "2026-06-01T19:40:46.265Z" },
+    { url = "https://files.pythonhosted.org/packages/10/c7/7ba5de8af9650b9767b063c675427b8685f43fa7ce563673a7bc3af60f08/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928", size = 1870829, upload-time = "2026-06-01T19:40:49.583Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" },
+    { url = "https://files.pythonhosted.org/packages/39/98/31b9ad9fbc01f0075ee7221002df5fd2d10b647f451ca5f30edc802d9dd6/aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6", size = 490597, upload-time = "2026-06-01T19:40:54.937Z" },
+    { url = "https://files.pythonhosted.org/packages/59/1f/299b21441c8de42ff70fddc7cfe65e92f810abcf740739a09b56f7835364/aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2", size = 525789, upload-time = "2026-06-01T19:40:57.306Z" },
+    { url = "https://files.pythonhosted.org/packages/70/11/7f83fcba9ee05d4c54d61b3f8104da0d43a59adac44dd28effc0c9a10422/aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24", size = 467399, upload-time = "2026-06-01T19:40:59.993Z" },
 ]
 
 [[package]]
@@ -170,8 +160,8 @@ name = "aiosignal"
 version = "1.4.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "frozenlist", marker = "python_full_version >= '3.10'" },
-    { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" },
+    { name = "frozenlist" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" }
 wheels = [
@@ -235,19 +225,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" },
     { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" },
     { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" },
-    { url = "https://files.pythonhosted.org/packages/07/c2/407d6bc19813fb74cc2b087ad3e959e102b29ff81e35dcc0ad0dfb5b946c/arro3_core-0.6.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:dfac7fac3c6a302399d94644d48682a19488a5b67bd1ccbdf6c560a7ffabde6d", size = 2680237, upload-time = "2025-10-13T23:11:10.876Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/73/c67156794d7e9734f4cc03d2eca7e44a1cc014686e6b7663f5110f58581d/arro3_core-0.6.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9fc70042e558d1cd5fbe917b58e8ef52701441e38ff30b1912858050f796a62c", size = 2386228, upload-time = "2025-10-13T23:11:14.02Z" },
-    { url = "https://files.pythonhosted.org/packages/79/e8/817ee1abb0cfa7e266ef00749b144553d2bb9c4679ca932ecbca9dc7dea9/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1583b29b2ba83927a33e5435e5d9d134114c45a6360a8bb4db4beda13dab4fd8", size = 2886476, upload-time = "2025-10-13T23:11:17.579Z" },
-    { url = "https://files.pythonhosted.org/packages/8e/d6/1b9beceab797c4510abfc25ef6e657e4c940d06a9615927ce506463691dd/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a170fe53f18dda4a4647fd3b8b4a9373fc11ac42c41a4b65f55d79ad531a33e", size = 2911941, upload-time = "2025-10-13T23:11:21.131Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/ed/4fe1fb9a24698fe6189111836d22c9582cbc92fa159b24b8664e924738dc/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83047b4e6e18835c91c8d12c5494e6ababc7c185c5a772d3429e8f9b0c185894", size = 3150419, upload-time = "2025-10-13T23:11:24.503Z" },
-    { url = "https://files.pythonhosted.org/packages/a1/91/d6215b782fa91493f504ae13623db889beeaf0519037c28fc6744464439a/arro3_core-0.6.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3d4393d281d1ef18927915a11187da27287d279f99d5325bc9afb417f76084f", size = 2777891, upload-time = "2025-10-13T23:11:28.11Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/de/0aa3504e6cbf406086de49b59cb0dcb3ab11f64acbb38602143e479831dc/arro3_core-0.6.5-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:f0c88d8babcf51affdd69390882e2f0ecb1890a1b8a5abfc087d003e7181eb6e", size = 2519673, upload-time = "2025-10-13T23:11:31.426Z" },
-    { url = "https://files.pythonhosted.org/packages/05/69/47bf9c9ab66bafc7056a41f6db9d2149639eea6417299e3fe6c01ef99b6c/arro3_core-0.6.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36424e1d62212466a5cacdc27d414e99bf0fdab1544cc2b7e5b81e41437e5970", size = 3026254, upload-time = "2025-10-13T23:11:36.199Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/e8/638582437ab41ba52d3c7f2a1b0a98e4a05a51e3f660985e594b4f6c18d5/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4eb4d96f7db618f100758a8b7ec1b221c8737d543073701b7ffee74bc5019d46", size = 2704582, upload-time = "2025-10-13T23:11:39.408Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/0a/7bc46ee799459cce72a2e15b0eb184170f26cac37eace0b813e855fbc4d8/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2cfe9b4b1dd663d256754f1aa7aae783a1cddd3eb5698892b9caf381431f0af7", size = 3155815, upload-time = "2025-10-13T23:11:43.304Z" },
-    { url = "https://files.pythonhosted.org/packages/99/8a/f20eff8f4ff5bd7db9b37b70ea058b37375a930a10e03d584a7597b6b740/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:a3b2621505f97eb5ce80f1c6fa8c77d18d757ab48d1f11d33a805e9ccbcd6fb6", size = 3107791, upload-time = "2025-10-13T23:11:46.735Z" },
-    { url = "https://files.pythonhosted.org/packages/79/da/60c66f0cc4a6af7f54e57973190540f77b84da1218fad2a9917e17bd897b/arro3_core-0.6.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6c1becbb96ceba0b20f3d4318dd35f3417ee9a49065813d99f52b0fa285fc569", size = 2957730, upload-time = "2025-10-13T23:11:49.875Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/8d/6e3235894196e1fd2be34e01ac2d4280dd24e6c9019e3b12603858651e91/arro3_core-0.6.5-cp39-cp39-win_amd64.whl", hash = "sha256:5459e7bd39bb9dd8c57aa06856d2bebc5c1ca782cbccab0e186c6c89530e4ca9", size = 2839298, upload-time = "2025-10-13T23:11:53.566Z" },
     { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788, upload-time = "2025-10-13T23:11:56.965Z" },
     { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809, upload-time = "2025-10-13T23:12:00.175Z" },
     { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818, upload-time = "2025-10-13T23:12:03.721Z" },
@@ -262,19 +239,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405, upload-time = "2025-10-13T23:12:35.328Z" },
 ]
 
-[[package]]
-name = "astunparse"
-version = "1.6.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "six" },
-    { name = "wheel" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" },
-]
-
 [[package]]
 name = "async-timeout"
 version = "5.0.1"
@@ -314,8 +278,7 @@ source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "jmespath" },
     { name = "python-dateutil" },
-    { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "urllib3" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164, upload-time = "2025-10-01T19:38:16.06Z" }
 wheels = [
@@ -392,20 +355,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" },
     { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" },
     { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/ca/9a0983dd5c8e9733565cf3db4df2b0a2e9a82659fd8aa2a868ac6e4a991f/charset_normalizer-3.4.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:70bfc5f2c318afece2f5838ea5e4c3febada0be750fcf4775641052bbba14d05", size = 207520, upload-time = "2025-08-09T07:57:11.026Z" },
-    { url = "https://files.pythonhosted.org/packages/39/c6/99271dc37243a4f925b09090493fb96c9333d7992c6187f5cfe5312008d2/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23b6b24d74478dc833444cbd927c338349d6ae852ba53a0d02a2de1fce45b96e", size = 147307, upload-time = "2025-08-09T07:57:12.4Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/69/132eab043356bba06eb333cc2cc60c6340857d0a2e4ca6dc2b51312886b3/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:34a7f768e3f985abdb42841e20e17b330ad3aaf4bb7e7aeeb73db2e70f077b99", size = 160448, upload-time = "2025-08-09T07:57:13.712Z" },
-    { url = "https://files.pythonhosted.org/packages/04/9a/914d294daa4809c57667b77470533e65def9c0be1ef8b4c1183a99170e9d/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb731e5deb0c7ef82d698b0f4c5bb724633ee2a489401594c5c88b02e6cb15f7", size = 157758, upload-time = "2025-08-09T07:57:14.979Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/a8/6f5bcf1bcf63cb45625f7c5cadca026121ff8a6c8a3256d8d8cd59302663/charset_normalizer-3.4.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:257f26fed7d7ff59921b78244f3cd93ed2af1800ff048c33f624c87475819dd7", size = 152487, upload-time = "2025-08-09T07:57:16.332Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/72/d3d0e9592f4e504f9dea08b8db270821c909558c353dc3b457ed2509f2fb/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1ef99f0456d3d46a50945c98de1774da86f8e992ab5c77865ea8b8195341fc19", size = 150054, upload-time = "2025-08-09T07:57:17.576Z" },
-    { url = "https://files.pythonhosted.org/packages/20/30/5f64fe3981677fe63fa987b80e6c01042eb5ff653ff7cec1b7bd9268e54e/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2c322db9c8c89009a990ef07c3bcc9f011a3269bc06782f916cd3d9eed7c9312", size = 161703, upload-time = "2025-08-09T07:57:20.012Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/ef/dd08b2cac9284fd59e70f7d97382c33a3d0a926e45b15fc21b3308324ffd/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:511729f456829ef86ac41ca78c63a5cb55240ed23b4b737faca0eb1abb1c41bc", size = 159096, upload-time = "2025-08-09T07:57:21.329Z" },
-    { url = "https://files.pythonhosted.org/packages/45/8c/dcef87cfc2b3f002a6478f38906f9040302c68aebe21468090e39cde1445/charset_normalizer-3.4.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:88ab34806dea0671532d3f82d82b85e8fc23d7b2dd12fa837978dad9bb392a34", size = 153852, upload-time = "2025-08-09T07:57:22.608Z" },
-    { url = "https://files.pythonhosted.org/packages/63/86/9cbd533bd37883d467fcd1bd491b3547a3532d0fbb46de2b99feeebf185e/charset_normalizer-3.4.3-cp39-cp39-win32.whl", hash = "sha256:16a8770207946ac75703458e2c743631c79c59c5890c80011d536248f8eaa432", size = 99840, upload-time = "2025-08-09T07:57:23.883Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/d6/7e805c8e5c46ff9729c49950acc4ee0aeb55efb8b3a56687658ad10c3216/charset_normalizer-3.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:d22dbedd33326a4a5190dd4fe9e9e693ef12160c77382d9e87919bce54f3d4ca", size = 107438, upload-time = "2025-08-09T07:57:25.287Z" },
     { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" },
 ]
 
+[[package]]
+name = "cloudpickle"
+version = "3.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" },
+]
+
 [[package]]
 name = "colorama"
 version = "0.4.6"
@@ -416,65 +377,144 @@ wheels = [
 ]
 
 [[package]]
-name = "datafusion"
-version = "53.0.0"
+name = "cuda-bindings"
+version = "13.3.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.13'" },
+    { name = "cuda-pathfinder" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" },
-    { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/1f/5ef51f5fbaa5d4d3201bb3d7555af028ec1aa4416275ccbf73c9e34e3d2d/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9851b0caa8bfd3bc6fa054eaf57bea7c8e9c3a62db2d2621224677f49f3c53d0", size = 6675244, upload-time = "2026-05-29T23:11:38.664Z" },
+    { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" },
+    { url = "https://files.pythonhosted.org/packages/95/7a/c5e3c34a409b148f5c0f5a4ea374158f95d488862c1dffedf9aa5c639df9/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708", size = 6674166, upload-time = "2026-05-29T23:11:45.478Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" },
+    { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" },
+    { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/81/bff68ce829999c1e4209c761bbf903b1c06ec570416ddb25020864ad5907/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8", size = 6013639, upload-time = "2026-05-29T23:12:03.509Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/e0/c8a1f0c8f9ffdea4f5fe6dbab89b326cef4d85caf489dad39e209da89416/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80", size = 6534419, upload-time = "2026-05-29T23:12:05.633Z" },
+    { url = "https://files.pythonhosted.org/packages/52/b8/83b1f563925b290f2d11a01a77a84013ba56052fe3653a5bef3ccfbb43d6/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76", size = 5809771, upload-time = "2026-05-29T23:12:10.422Z" },
+    { url = "https://files.pythonhosted.org/packages/12/20/e79b4bfe98f075195afb6343d41c498f9dbd2d161d7021d4d28bceb83581/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9", size = 6358584, upload-time = "2026-05-29T23:12:12.767Z" },
 ]
 
 [[package]]
-name = "datasets"
-version = "0.0.9"
+name = "cuda-pathfinder"
+version = "1.5.5"
 source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" },
+]
+
+[[package]]
+name = "cuda-toolkit"
+version = "13.0.3.0"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" },
+]
+
+[package.optional-dependencies]
+cublas = [
+    { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+cudart = [
+    { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+cufft = [
+    { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+cufile = [
+    { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" },
+]
+cupti = [
+    { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+curand = [
+    { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+cusolver = [
+    { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+cusparse = [
+    { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+nvjitlink = [
+    { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+nvrtc = [
+    { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+nvtx = [
+    { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" },
+]
+
+[[package]]
+name = "datafusion"
+version = "54.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cloudpickle" },
+    { name = "pyarrow" },
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" },
+    { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" },
+    { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" },
+    { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" },
+    { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" },
+    { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/cd/fe/4d2874473a753d59c83335691bd9532704f2605418a0d288a1d70fa003fc/datasets-0.0.9.zip", hash = "sha256:86d54441bab87aebb2aa3bf0853aa7fb7abed8c708f9bb08a88e86a498972010", size = 4013, upload-time = "2015-08-18T00:07:40.556Z" }
 
 [[package]]
 name = "datasets"
 version = "4.1.1"
 source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
 dependencies = [
-    { name = "dill", marker = "python_full_version >= '3.10'" },
-    { name = "filelock", marker = "python_full_version >= '3.10'" },
-    { name = "fsspec", extra = ["http"], marker = "python_full_version >= '3.10'" },
-    { name = "huggingface-hub", marker = "python_full_version >= '3.10'" },
-    { name = "multiprocess", marker = "python_full_version >= '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "dill" },
+    { name = "filelock" },
+    { name = "fsspec", extra = ["http"] },
+    { name = "huggingface-hub" },
+    { name = "multiprocess" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "packaging", marker = "python_full_version >= '3.10'" },
-    { name = "pandas", marker = "python_full_version >= '3.10'" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "pyyaml", marker = "python_full_version >= '3.10'" },
-    { name = "requests", marker = "python_full_version >= '3.10'" },
-    { name = "tqdm", marker = "python_full_version >= '3.10'" },
-    { name = "xxhash", marker = "python_full_version >= '3.10'" },
+    { name = "packaging" },
+    { name = "pandas" },
+    { name = "pyarrow" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "tqdm" },
+    { name = "xxhash" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324, upload-time = "2025-09-18T13:14:27.108Z" }
 wheels = [
     { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623, upload-time = "2025-09-18T13:14:25.111Z" },
 ]
 
+[[package]]
+name = "deprecated"
+version = "1.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" },
+]
+
 [[package]]
 name = "dill"
 version = "0.4.0"
@@ -514,12 +554,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289, upload-time = "2025-09-16T10:22:21.564Z" },
     { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547, upload-time = "2025-09-16T10:22:23.759Z" },
     { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467, upload-time = "2025-09-16T10:22:25.923Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/42/0f355319b3e8ee1703d0e17378dd829db391434306621f85c110134f2763/duckdb-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1c97ee61c582002b654331f7fd967d6b1e83bf7fdb0772f409dfd4b6af3a70f4", size = 31292373, upload-time = "2025-09-16T10:22:28.118Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/52/091dbef5eb2ac4e60a9c6d38fcc7c7530a75fafa0f37658450e8731a265b/duckdb-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:74e3d6295355160df5d3588b880e8bcae23fdd6f573f538793a8a1abf4c2c29d", size = 17288145, upload-time = "2025-09-16T10:22:30.346Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/6c/879317d9c3ac7a2a1f0618ca536a48ebfa4b9fe202f9783e07070e168192/duckdb-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0c76425e4ffe98069dd4fc4752ab919a4125dc0d176bb676b3065fdea152c42", size = 14816258, upload-time = "2025-09-16T10:22:32.442Z" },
-    { url = "https://files.pythonhosted.org/packages/95/87/83ac8e67c0530b69fe39f91bbb7f3bd0a49b0c24216cffa9c5561fb2845c/duckdb-1.4.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c122bd7d80ab5057f53024ee3922d7612a5cdc99583fae730990964aebc3fd4", size = 18391043, upload-time = "2025-09-16T10:22:34.616Z" },
-    { url = "https://files.pythonhosted.org/packages/d6/01/1d70bd6c594ef915c004edc0f1119d1602173dc5ce91c1eed7368f6aab34/duckdb-1.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30689c1436bca723526be6102fe1f4f82ea6d4780fb9ca196bda7ed5ec227950", size = 20385348, upload-time = "2025-09-16T10:22:36.982Z" },
-    { url = "https://files.pythonhosted.org/packages/b6/04/0650128cdcdc5208c4f51341a0a3f8db436ecaba51032c6065e20ea0baae/duckdb-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:4c55a367c1296617cff89c5e1c7153f1dc3c3b556ef70711a45b0236515f80c2", size = 12283322, upload-time = "2025-09-16T10:22:39.388Z" },
 ]
 
 [[package]]
@@ -543,34 +577,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
 ]
 
-[[package]]
-name = "flatbuffers"
-version = "2.0.7"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/d1/90/0532e737a11e1dc50e9e352c3ccc97338cb75991f83279c2edbc9234e022/flatbuffers-2.0.7.tar.gz", hash = "sha256:0ae7d69c5b82bf41962ca5fde9cc43033bc9501311d975fd5a25e8a7d29c1245", size = 22686, upload-time = "2022-08-23T22:50:07.903Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d7/0d/b5bfb553a6ac66d6ec2b6d7f1e814a908fba7188356ac94bb36ae3d905c3/flatbuffers-2.0.7-py2.py3-none-any.whl", hash = "sha256:71e135d533be527192819aaab757c5e3d109cb10fbb01e687f6bdb7a61ad39d1", size = 26562, upload-time = "2022-08-23T22:50:56.342Z" },
-]
-
-[[package]]
-name = "flatbuffers"
-version = "25.9.23"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067, upload-time = "2025-09-24T05:25:30.106Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869, upload-time = "2025-09-24T05:25:28.912Z" },
-]
-
 [[package]]
 name = "frozenlist"
 version = "1.7.0"
@@ -662,23 +668,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" },
     { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" },
     { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/b1/ee59496f51cd244039330015d60f13ce5a54a0f2bd8d79e4a4a375ab7469/frozenlist-1.7.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cea3dbd15aea1341ea2de490574a4a37ca080b2ae24e4b4f4b51b9057b4c3630", size = 82434, upload-time = "2025-06-09T23:02:05.195Z" },
-    { url = "https://files.pythonhosted.org/packages/75/e1/d518391ce36a6279b3fa5bc14327dde80bcb646bb50d059c6ca0756b8d05/frozenlist-1.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7d536ee086b23fecc36c2073c371572374ff50ef4db515e4e503925361c24f71", size = 48232, upload-time = "2025-06-09T23:02:07.728Z" },
-    { url = "https://files.pythonhosted.org/packages/b7/8d/a0d04f28b6e821a9685c22e67b5fb798a5a7b68752f104bfbc2dccf080c4/frozenlist-1.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dfcebf56f703cb2e346315431699f00db126d158455e513bd14089d992101e44", size = 47186, upload-time = "2025-06-09T23:02:09.243Z" },
-    { url = "https://files.pythonhosted.org/packages/93/3a/a5334c0535c8b7c78eeabda1579179e44fe3d644e07118e59a2276dedaf1/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:974c5336e61d6e7eb1ea5b929cb645e882aadab0095c5a6974a111e6479f8878", size = 226617, upload-time = "2025-06-09T23:02:10.949Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/67/8258d971f519dc3f278c55069a775096cda6610a267b53f6248152b72b2f/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c70db4a0ab5ab20878432c40563573229a7ed9241506181bba12f6b7d0dc41cb", size = 224179, upload-time = "2025-06-09T23:02:12.603Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/89/8225905bf889b97c6d935dd3aeb45668461e59d415cb019619383a8a7c3b/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1137b78384eebaf70560a36b7b229f752fb64d463d38d1304939984d5cb887b6", size = 235783, upload-time = "2025-06-09T23:02:14.678Z" },
-    { url = "https://files.pythonhosted.org/packages/54/6e/ef52375aa93d4bc510d061df06205fa6dcfd94cd631dd22956b09128f0d4/frozenlist-1.7.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e793a9f01b3e8b5c0bc646fb59140ce0efcc580d22a3468d70766091beb81b35", size = 229210, upload-time = "2025-06-09T23:02:16.313Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/55/62c87d1a6547bfbcd645df10432c129100c5bd0fd92a384de6e3378b07c1/frozenlist-1.7.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74739ba8e4e38221d2c5c03d90a7e542cb8ad681915f4ca8f68d04f810ee0a87", size = 215994, upload-time = "2025-06-09T23:02:17.9Z" },
-    { url = "https://files.pythonhosted.org/packages/45/d2/263fea1f658b8ad648c7d94d18a87bca7e8c67bd6a1bbf5445b1bd5b158c/frozenlist-1.7.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e63344c4e929b1a01e29bc184bbb5fd82954869033765bfe8d65d09e336a677", size = 225122, upload-time = "2025-06-09T23:02:19.479Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/22/7145e35d12fb368d92124f679bea87309495e2e9ddf14c6533990cb69218/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2ea2a7369eb76de2217a842f22087913cdf75f63cf1307b9024ab82dfb525938", size = 224019, upload-time = "2025-06-09T23:02:20.969Z" },
-    { url = "https://files.pythonhosted.org/packages/44/1e/7dae8c54301beb87bcafc6144b9a103bfd2c8f38078c7902984c9a0c4e5b/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:836b42f472a0e006e02499cef9352ce8097f33df43baaba3e0a28a964c26c7d2", size = 239925, upload-time = "2025-06-09T23:02:22.466Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/1e/99c93e54aa382e949a98976a73b9b20c3aae6d9d893f31bbe4991f64e3a8/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e22b9a99741294b2571667c07d9f8cceec07cb92aae5ccda39ea1b6052ed4319", size = 220881, upload-time = "2025-06-09T23:02:24.521Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/9c/ca5105fa7fb5abdfa8837581be790447ae051da75d32f25c8f81082ffc45/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:9a19e85cc503d958abe5218953df722748d87172f71b73cf3c9257a91b999890", size = 234046, upload-time = "2025-06-09T23:02:26.206Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/4d/e99014756093b4ddbb67fb8f0df11fe7a415760d69ace98e2ac6d5d43402/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f22dac33bb3ee8fe3e013aa7b91dc12f60d61d05b7fe32191ffa84c3aafe77bd", size = 235756, upload-time = "2025-06-09T23:02:27.79Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/72/a19a40bcdaa28a51add2aaa3a1a294ec357f36f27bd836a012e070c5e8a5/frozenlist-1.7.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9ccec739a99e4ccf664ea0775149f2749b8a6418eb5b8384b4dc0a7d15d304cb", size = 222894, upload-time = "2025-06-09T23:02:29.848Z" },
-    { url = "https://files.pythonhosted.org/packages/08/49/0042469993e023a758af81db68c76907cd29e847d772334d4d201cbe9a42/frozenlist-1.7.0-cp39-cp39-win32.whl", hash = "sha256:b3950f11058310008a87757f3eee16a8e1ca97979833239439586857bc25482e", size = 39848, upload-time = "2025-06-09T23:02:31.413Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/45/827d86ee475c877f5f766fbc23fb6acb6fada9e52f1c9720e2ba3eae32da/frozenlist-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:43a82fce6769c70f2f5a06248b614a7d268080a9d20f7457ef10ecee5af82b63", size = 44102, upload-time = "2025-06-09T23:02:32.808Z" },
     { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" },
 ]
 
@@ -693,102 +682,66 @@ wheels = [
 
 [package.optional-dependencies]
 http = [
-    { name = "aiohttp", marker = "python_full_version >= '3.10'" },
-]
-
-[[package]]
-name = "gast"
-version = "0.4.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/83/4a/07c7e59cef23fb147454663c3271c21da68ba2ab141427c20548ae5a8a4d/gast-0.4.0.tar.gz", hash = "sha256:40feb7b8b8434785585ab224d1568b857edb18297e5a3047f1ba012bc83b42c1", size = 13804, upload-time = "2020-08-07T21:45:23.526Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b6/48/583c032b79ae5b3daa02225a675aeb673e58d2cb698e78510feceb11958c/gast-0.4.0-py3-none-any.whl", hash = "sha256:b7adcdd5adbebf1adf17378da5ba3f543684dbec47b1cda1f3997e573cd542c4", size = 9824, upload-time = "2020-08-07T21:45:21.32Z" },
-]
-
-[[package]]
-name = "gast"
-version = "0.6.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708, upload-time = "2024-06-27T20:31:49.527Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173, upload-time = "2024-07-09T13:15:15.615Z" },
+    { name = "aiohttp" },
 ]
 
 [[package]]
 name = "geoarrow-rust-core"
-version = "0.6.1"
+version = "0.6.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "arro3-core" },
-    { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/77/2d/3e994dd76223fac0eb597a6f55647cca51bd5a4f446d09b668697f901724/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:84d972cc3dd45a797fd99588d7ee68f257e4083ebdcecad9ec773260067f71a6", size = 3570129, upload-time = "2025-12-03T18:51:07.148Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/2a/e19df203b4ffb225f39627e1bd1b89ce7b2220e39f1d6972692174820c57/geoarrow_rust_core-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc0f382d4ed41e85d2d89fc2c7c8c3d046681c9a5e19350ce79e0e930cf69821", size = 3333881, upload-time = "2025-11-21T01:49:28.959Z" },
-    { url = "https://files.pythonhosted.org/packages/52/98/b749a2165dfc5d9c54a1c19eb3e6a75b6d005ecde42289b25c1c355346b7/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e719edcaf6698ed2b1aa9525bd97cf79e23a500a39b1e83566cd9a16a294d3", size = 3806366, upload-time = "2025-11-21T01:48:03.525Z" },
-    { url = "https://files.pythonhosted.org/packages/84/93/7c0e42ba7d46208fb0f851e06c05de071962170f3a3b2a2260d8a3f66e7a/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0f3546a15503329880063aca31266b301b0b781f618f832585bcd1c9efcc876", size = 3981800, upload-time = "2025-11-21T01:48:17.789Z" },
-    { url = "https://files.pythonhosted.org/packages/de/43/9c5736569dead60b33e46b7c485e24804d950693df70dee306e153547789/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6937f3cabebf673f8b726d60d8ca160b46401de8b08c8e257be22772c12c2001", size = 5068955, upload-time = "2025-11-21T01:48:32.569Z" },
-    { url = "https://files.pythonhosted.org/packages/71/5e/f26f9bea2af96b0d070e980dcc2196d369a678e06141ed260de5ca72bcc2/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f29ba92053e8ad4bd60d72188518f033ca4abc1f34eecebeb41ee7b790612e00", size = 4104946, upload-time = "2025-11-21T01:48:45.801Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/08/473796b3e0c03b35292220de88c8efa3e74d6174e807b26a371f2523a4b0/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a5d05a312fbb76821566b1d144c64d0923fcbd790b2c7376ee11f62472b2fe", size = 3917533, upload-time = "2025-11-21T01:49:14.631Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/7a/7b62b839c3a9878a7d91b8395e0b7b04483e4bec687e073df0fbd4056583/geoarrow_rust_core-0.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88fe8fd33b16a06e9b3b7638b51d24047f1d01af12cc2e3e2653140877bddef6", size = 4318837, upload-time = "2025-11-21T01:48:58.953Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/86/309c55a9c63f316e3a04949ade8847b8e5acbdd21645696911175f0e1814/geoarrow_rust_core-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:dbecc2487cc95526ac77797cd70c199e196811b0a9e877c1b61fcaca508575fa", size = 3320081, upload-time = "2025-11-21T01:49:58.861Z" },
-    { url = "https://files.pythonhosted.org/packages/1a/ed/514cff089185d71242a62e774e2c59dda147baab65929851b66d72198d5d/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:e26ca240d7a6a0fa1b4f56a9ebe07b2e14fc7c1c9507aa862bd31ef14e0521f0", size = 3572326, upload-time = "2025-12-03T18:51:08.477Z" },
-    { url = "https://files.pythonhosted.org/packages/77/21/22f8233235bd020db22b4f2bf888f9aeed08813eda7b8b421a6963bdc7e4/geoarrow_rust_core-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46876e3528685673e08b4cbc696dca7f22fb073a83318708b0eaf640107b923b", size = 3335166, upload-time = "2025-11-21T01:49:30.632Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/eb/0c2e40a6a1bd450347a8a9fc7648ca840710bc177ff6eed3fc5da6ef981a/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5502bd12ede712d9b4725753df4db231a0aa6d3e131079bc4b6452c436e37b7", size = 3800540, upload-time = "2025-11-21T01:48:05.583Z" },
-    { url = "https://files.pythonhosted.org/packages/4c/42/22d3b8441bb7041a6fcdb4cf0a1108e150513a52f8a407715188412bc71f/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f04dd7dd03449dba6d15f7d35c6c708637ac05f125638f56206e876756cd4c5", size = 3984840, upload-time = "2025-11-21T01:48:19.719Z" },
-    { url = "https://files.pythonhosted.org/packages/12/44/477b6b2389398dc983026a4ab7dbb7ec121284ad5fb864a1b7a4658c3881/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2afce33d0c3fa87d5d4d24d6617732e4297da3372b1746569b759f9b62aede1", size = 5067358, upload-time = "2025-11-21T01:48:34.373Z" },
-    { url = "https://files.pythonhosted.org/packages/62/50/6995e9d11462635972b2fc09c8e1e510928563ca4fb0fd2c9145cf6ef771/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e63cdb661652a9836dc86cb5995ad269817d88b80f4cce6ed236a7f80f0aba", size = 4105773, upload-time = "2025-11-21T01:48:47.461Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/21/b369208495f213db0a0e7d563358307a706cc6af0cb9c897dacf28ae06a1/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adbaf97cb770aef69df8a16437c9faa67adb2b04856faf45bcb61d5b986101dc", size = 3914659, upload-time = "2025-11-21T01:49:16.35Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/49/fccb14c6ee9bb715451e4d5bbe3d571eb59a8a1abe21b2abe0d9d48a7fac/geoarrow_rust_core-0.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:202f35b301caa5154d95fd74424a1ef6449306e4f6fbfb5140270e48e94188a5", size = 4315153, upload-time = "2025-11-21T01:49:01.075Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/1c/88b16510e24a4a3332284669085673701b9fe4d6a511b4466c90655a9daf/geoarrow_rust_core-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:491405dfcc821a2c599e381cc9923e04a758deb1cc84fdb5794b519446c2f8a8", size = 3320510, upload-time = "2025-11-21T01:50:15.545Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/5f/1dbdbc1dde2140937cff20188cb25034b6f39e1734c14ca6510cf464bf77/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a8145a562e94419402dd0882bb62429853804c53d47dbea944f2a24abc57abd2", size = 3568115, upload-time = "2025-12-03T18:51:09.743Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/e1/b62676f89ef3b866676967989ee8dbbd3d16c77f69aa4287825703268c42/geoarrow_rust_core-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:51040a5afcfa0cd3ab372d981375c7fe8eb652d155e3964d52ed51d14faa04e8", size = 3325336, upload-time = "2025-11-21T01:49:32.67Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/89/94e20f255712ff0eaccf9bfeac4bf51953ebcef0599cfc92f67037f8ab1a/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fbf8506848b0254b3c89b27c045be38bbef6372b21714cad45d76b0c8cb92ce", size = 3808535, upload-time = "2025-11-21T01:48:07.618Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/e4/37c7e2c9e251148be17292d0656d7d1ab35019678f6bd11090a41c270d18/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1a0d9c14bf2f36676016c753517d9470381969c2a67859716cceae33735f3ee", size = 3978997, upload-time = "2025-11-21T01:48:21.551Z" },
-    { url = "https://files.pythonhosted.org/packages/71/27/c4ba353d9b77889136bdfd1c0cd1a04d6eade9da6e0748b06719c458afb5/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6df97301782ecbaf5f2f0252011a9ff309471cde25435bdf1e17b29c263ebc16", size = 5066492, upload-time = "2025-11-21T01:48:36.142Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/81/34107fc9aacc489e41afed420202645675b41d85b46dc70d5ba222312791/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1948cfdd0e1c7d03a0c2067821dd536ab34d1e726515202e51fbd6b0d9f775f", size = 4106130, upload-time = "2025-11-21T01:48:49.144Z" },
-    { url = "https://files.pythonhosted.org/packages/92/5f/2e348b884738fb213fb3b4745955baeeaf047aecb37639e39a4dd8f12d99/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95b1611b66c386cc6c74e990df4f114bcf24956a35e18e51bf6331c079a36688", size = 3913166, upload-time = "2025-11-21T01:49:18.228Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/81/fdda8bb5f84df82bc9e000435a88be46d46dda41eb5149f624ed96b7031c/geoarrow_rust_core-0.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1751357a1aaa26aeb5feb6f66873b6a2d369655039f7278dedcb692b512111cc", size = 4313573, upload-time = "2025-11-21T01:49:03.184Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/14/ca0bc7d3b158094e769ba2bbc43d203330e7e457ed67b50af97d3eac45df/geoarrow_rust_core-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:16fe159043a444579948864808ebec8c49ec167ec0df3cb772dfb88de268bc91", size = 3318746, upload-time = "2025-11-21T01:50:17.319Z" },
-    { url = "https://files.pythonhosted.org/packages/85/b8/94e4f8fb32ef705cf65031a24c58cdc441042a68a794b74757a6561cbc60/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6c1b692f76b613757438bf23cfe3be4a8715f0268afd8ad3ca0063c257a3be4b", size = 3568328, upload-time = "2025-12-03T18:51:11.291Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/45/a96e64f9febc3436766c5055508c4e823cce56577529d7b76c4e4f584bc4/geoarrow_rust_core-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a2b4f9a8cfe852a0ba9a667258307db9e354b470b7e0a03edffd0b7daf9b6f5", size = 3325879, upload-time = "2025-11-21T01:49:34.941Z" },
-    { url = "https://files.pythonhosted.org/packages/58/c0/c719ce3fb4e982e28c71f65a80cf697d07d733336e6b74d7d1b8a7daf9d0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8248330f5c3e7ec5852d0a23c23b31a08395300ef9544109e2991317beddfee3", size = 3809144, upload-time = "2025-11-21T01:48:09.562Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/8e/2ab3563b2ffd13f2dd69c050a901de0a4bb325879531a66f56d30bc7337e/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:775e9fe45c06d02be59b1497c60aa4f7a7c1d460387bf5f63142faf39b8ad4ff", size = 3978886, upload-time = "2025-11-21T01:48:23.335Z" },
-    { url = "https://files.pythonhosted.org/packages/db/0a/31625caa0a32e8e9e7aaf2514a840dda0dadf8e2452710ebc10e5f469494/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94de8fb01da3f22332eab28b03570c43cc36492ce482c254fe87e851ae21285b", size = 5065429, upload-time = "2025-11-21T01:48:37.896Z" },
-    { url = "https://files.pythonhosted.org/packages/11/8d/ee247bd4ccf3b0791b8669357d440e3960d4dbd5cca940a2e226e8910c31/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c70a63d1d36687a53dc6c2933446b1435c187e4c616cd84844d89b6ba13bc4f6", size = 4105436, upload-time = "2025-11-21T01:48:50.874Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/fb/c1e92716ee5aa00d48b650f0cb43220a1bf4088c8d572dfc21d400b16723/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e505312f2761393fe5158242f3f2d77e9daa5cca63badd8d66e6d1d69fc17bf", size = 3913672, upload-time = "2025-11-21T01:49:19.873Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/6f/ef47f6070c5d5cf0d061d5f5ba95aed7e895e4720a784b84c911c0209fc0/geoarrow_rust_core-0.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a732e58549108df8267ab72fa6cc7c54e5a9e30b818d8d869e301a9de9d3029e", size = 4313496, upload-time = "2025-11-21T01:49:04.953Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/ac/2696b979623ea02129e342f8820c89d03fa5a253a913ad00b588d6dd2948/geoarrow_rust_core-0.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:9e1d6492b1388b9d5ae898728838ada78dbf2340d2e9dd25ad3df6ccdd058813", size = 3318780, upload-time = "2025-11-21T01:50:18.928Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/42/0cb3af24b01d3897a9eee6af5cc0676bf6b80364e0d4638e45a5fc873d35/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3748cc8e8cb2bcedaede27cefed6749d4eea93e358b49a2f0b061d8974dd1b91", size = 3560313, upload-time = "2025-12-03T18:51:12.897Z" },
-    { url = "https://files.pythonhosted.org/packages/51/bc/33f8c918e46188707ab358752b993bee9184fa62e580998c1ec4c37885c1/geoarrow_rust_core-0.6.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1b0e232fe4e239ca435d0bab638934eee87d758024c1727ee24a2b8bc4d8bc7b", size = 3321855, upload-time = "2025-12-03T18:51:00.056Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/d7/aeb2a3922670ad57f62cb591bd0309a8300ceeec6efc7f925a563c9da672/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:843444ada2c7f7670fd9df3bdebd93e5247b376d1dd20c4fb3828632847ab78e", size = 3799057, upload-time = "2025-12-03T18:50:28.982Z" },
-    { url = "https://files.pythonhosted.org/packages/76/08/606e55fc2a0e85b02e0fde7dec2014eb8f1463e8a823496d72a3095de73d/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:880641183a09ebfbca3a6357071f137d1a4b0f1ba606fb9127a01cf58faaef56", size = 3968892, upload-time = "2025-12-03T18:50:34.661Z" },
-    { url = "https://files.pythonhosted.org/packages/10/1f/e75fd5b59e9e582190c11ec73c91728d96e90608a22e0aed7365439d9534/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bb69024257d2fd20da691d1e15bcced874d278884218b64690256982fa30cb1", size = 5049247, upload-time = "2025-12-03T18:50:40.542Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/95/2257b9b148c8c6557387e67828a5096ebc519b997a158ffb67a0987589e5/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:85464a1bab81068789de5fb19684e43709d2ba6d64d5655aace7c50b35893d6d", size = 4099850, upload-time = "2025-12-03T18:50:45.341Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/07/8c8aaf8755ee7c137f0898823bd005ffb16edaa6accc0cc1a9a747d56ddc/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7eb773a101f1d9716d750bb326991885a7c4576e85d9a016a567a3b07380bf07", size = 3908308, upload-time = "2025-12-03T18:50:55.587Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/7e/b8f1933be03d9a3a6416edf29fc23d520e45f00fbde6bd8f0614ad6f8a69/geoarrow_rust_core-0.6.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:920e6fed857acd2145a8fca7c6fad17094873f586ac5efed7049ce43a7af4ff6", size = 4307178, upload-time = "2025-12-03T18:50:50.429Z" },
-    { url = "https://files.pythonhosted.org/packages/df/95/a8ba3d7e51ec02ec954d0247c6021b36de5935a9a3845c1cf6c1348cd6e3/geoarrow_rust_core-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:9887119cc31a763c34ed8676d06434b47971517e86f8e35c640b494d05e7d5ac", size = 3316511, upload-time = "2025-12-03T18:51:18.831Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/6d/4b2f51d0e4ac683217852d79c3acef719ca116f418d9ce8f4dcc6d717716/geoarrow_rust_core-0.6.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:420a720217b5a7ec6f7977cfe7e7a729c73381ed5e63112fdef33bd805b9cf8a", size = 3572216, upload-time = "2025-12-03T18:51:14.544Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/55/85a2948b10ad9ea347597f90355d8992745f00fedae54916205c8c9b80fb/geoarrow_rust_core-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fb9c8c6bba4e712edf475ce3c78bf13f7b10f750256f57deb29c3222eaef033", size = 3335928, upload-time = "2025-11-21T01:49:51.601Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/98/fdd6c34ff8acd878c31e9f5fe4792f49d437e0465e0b60c24d6cdc287ed7/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9547ead76eac906b7a583ee65fa137e6b8ed34c0f128c1745a290c451726f27", size = 3808249, upload-time = "2025-11-21T01:48:11.192Z" },
-    { url = "https://files.pythonhosted.org/packages/8a/a1/fd6741b5c1d7d48b5f6ab58a994a91c86e29d19ee7bca2636590b8ac9a54/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eaa8e8f40ca8fcd367735cb4226c5aa5171a713d75bc2caab9a03bd9f59d7bf2", size = 3984081, upload-time = "2025-11-21T01:48:25.595Z" },
-    { url = "https://files.pythonhosted.org/packages/91/1e/2b5a9b65bf19a79d212ea0fe60fa5632ec4c89bb64ee446272b47e5cd6ac/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08992719a2accbf993837a6aad615e3f2bf1954d2d9152e507dd79621c87e9d3", size = 5071749, upload-time = "2025-11-21T01:48:39.673Z" },
-    { url = "https://files.pythonhosted.org/packages/08/7a/6b37f5e52300b60854b74f4cdc9fbe613c692a15c3ae42f1952f3849bc86/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:071c0e72c4c2047326ebec8d76ce2debcdd59e187207433c3a29ac2da861ca92", size = 4107621, upload-time = "2025-11-21T01:48:52.632Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/3e/f849642ef4e1f54bcc651903f19a219c3d2be68d27f4ceb282a07ebba7cd/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c49d5a9e7b73c30dd1790a3e0faf30b7a4ee393c127c5a799d543653d1d80f0c", size = 3919352, upload-time = "2025-11-21T01:49:21.495Z" },
-    { url = "https://files.pythonhosted.org/packages/84/c8/57318cb04d061788d5ba523984915c98523e9eb9b7ba4937ff3438e045ef/geoarrow_rust_core-0.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:909152922ee42197b8ae846a8b6c5383c6f3ab39fe627ec8539765e3a634de68", size = 4320006, upload-time = "2025-11-21T01:49:06.588Z" },
-    { url = "https://files.pythonhosted.org/packages/13/9f/be16e191fdedbac4d9c01096327917a948625619423c666ec3db2191b4ab/geoarrow_rust_core-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:796c84184fe5e65e30df9f9f45aa8c1680f07689ea71ed1960faa7324fb67e52", size = 3321071, upload-time = "2025-11-21T01:50:20.844Z" },
+    { url = "https://files.pythonhosted.org/packages/70/a7/9de5cdcb86089ef4d9a24940838a72ef0655d5be11b46dc4ee807b0d7772/geoarrow_rust_core-0.6.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:e1dbbca927858c05ef4eaa5e13a3977a62183cfa3f17fe7b19dd2d88ecf24e91", size = 3855749, upload-time = "2026-06-11T19:24:32.965Z" },
+    { url = "https://files.pythonhosted.org/packages/54/48/da86c2bd1db71849f003f5a8eb78ce54f7a33341d5b33ddcdb480b5aafb4/geoarrow_rust_core-0.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce7e126d340f335bcc108327cbf7264539e856cb6a299f59757a6ee8329f6643", size = 3710538, upload-time = "2026-06-11T19:24:34.925Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/65/7f8ecc05447a85f14643170de8a29715e7c3e732fbb7132617772d39eac7/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88eb7982c1345fc4c4b18d9895602f0148c9495fe7ac00df03a92c20c8058149", size = 4198382, upload-time = "2026-06-11T19:24:37.02Z" },
+    { url = "https://files.pythonhosted.org/packages/41/57/b11fbb277fab166d8a8940bc1151bbd1aeef537e70c55f495ff85178f827/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c8368b91d4cab5cb5ad1b0f7369da4cec196d82bf73aa3823618a99c1bd4cf04", size = 4270350, upload-time = "2026-06-11T19:24:38.726Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/16/0c35e5aff4aca77d818b28d79f9ce20fe1c282ef26d6a2fcc764f3a55f26/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2955d82d0204197c8e96adbfb70f252fa5987821dd8f202e712a84bfb5b876d3", size = 5602389, upload-time = "2026-06-11T19:24:40.198Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/06/58e4d0c94f7d8897ca5e2469fe5db0dd937bfc3cd676dea43c6ce488effe/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cacbc2231b03c674975d5a25ff549c367dd8c07147c41edb5461c8ebda693739", size = 4414385, upload-time = "2026-06-11T19:24:41.779Z" },
+    { url = "https://files.pythonhosted.org/packages/09/65/902e986d01d4978e752c1d0d5b15873de712321ce3f61c285f491e4149b9/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f5726fd638563d11dfefd7d17dd769e679ac1efb868178791573de19d16b41f", size = 4251263, upload-time = "2026-06-11T19:24:43.556Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/f1/b1e0f93ea5288706f08ac7c01f332eb0feaa128251f3c2c9896e5f42cba5/geoarrow_rust_core-0.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df7a0319cdec5d0e4ffc3f17a171e16787e7719f85f82c8cf0035d873ec31e62", size = 4747229, upload-time = "2026-06-11T19:24:45.281Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/f3/77ebd20cb5cf5eb18c5bb0e32e07f76ec915a728ea123e075365f0b6c53c/geoarrow_rust_core-0.6.3-cp310-cp310-win_amd64.whl", hash = "sha256:19ce5fb18025480461253d0a03f20cbb635163214b5f193b0700bc1a407dfe4d", size = 3601298, upload-time = "2026-06-11T19:24:46.721Z" },
+    { url = "https://files.pythonhosted.org/packages/02/a8/d50e482a56d9543119be40000bc405b725242b6056809bbee3a75eff2411/geoarrow_rust_core-0.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d91b5249d5e1da53a79268759601c107beb69a8944dd3b5b225e9515ab63d519", size = 3856056, upload-time = "2026-06-11T19:24:48.331Z" },
+    { url = "https://files.pythonhosted.org/packages/04/e3/f4de7795959d95d88b32b85740d5d2d6b0a2e17233258f0331aee6cb7b13/geoarrow_rust_core-0.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:14412f02c1e60c92d2f88bc9f92835cf6d80f1da37fe8ba462eafdb7bd570f3c", size = 3710092, upload-time = "2026-06-11T19:24:49.802Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/48/04888477c2a12fbe6a6f8898bd026facdc3a929b4e747d7b569e6d20dd58/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d6db2341568b1e44678ccc0ade1ca1e7660a2c186ebf8bf847acdb160f2cf", size = 4197891, upload-time = "2026-06-11T19:24:51.245Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/2d/c16b6eb6f9f2ab213dcd0cd2ac0dec2eae1e2ce5922b3fbeb7bb1ac2a865/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:45f4193b9d6f6caae969d8448f3687a19f0998d757519a091df609c06ffa68a0", size = 4269771, upload-time = "2026-06-11T19:24:52.781Z" },
+    { url = "https://files.pythonhosted.org/packages/47/fd/2ee73341c37d554ce8d0b67a95525700ec32194fa785261c17262afadfc8/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf9ca054562fb4610c8e5ea140fa1bf746ccc16de505d3a5684abd2fa11f9538", size = 5601846, upload-time = "2026-06-11T19:24:54.63Z" },
+    { url = "https://files.pythonhosted.org/packages/67/05/229234ae7bf1d39306e41896f3055a2ae847707ce58f21bd0872b9a5764e/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ec9530fefb653f9a2e605cc26fc1c0d1ffa5c4923ec1037323ba9a16744f8ccc", size = 4413741, upload-time = "2026-06-11T19:24:56.015Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/5a/7875548a48231b02f909d3d8c7d74ba47867b2af3396e7aed59cd3b2b40d/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2813aceabb29567d96f29fd2d3099d6f8decd0f5f968ff81ed1a664751dc84a3", size = 4251434, upload-time = "2026-06-11T19:24:57.527Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/46/ed0370def1a950f185edda603a02276bb412a9c95ad5a052c9e919b2df78/geoarrow_rust_core-0.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:49686767d1379ff3b165f9d35a73e96fc25daba786ce27cf3359c5feac880fd0", size = 4746598, upload-time = "2026-06-11T19:24:58.979Z" },
+    { url = "https://files.pythonhosted.org/packages/44/bc/3a1720be855d7d0011416b7f0a7b7e33546b0fc7320faf59b05e401adff7/geoarrow_rust_core-0.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:fd9cc8c47af736dd087575306088e73b28a720f52e5c3342968851ddd2fb5778", size = 3601329, upload-time = "2026-06-11T19:25:00.459Z" },
+    { url = "https://files.pythonhosted.org/packages/24/b2/65db3af5fcc7d64ac7ac86d7debc6a90803bb076c8f7d4599c167be79fd6/geoarrow_rust_core-0.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:86aaa60e5b6d99be08f9adc9e58bd088135e1dcfebd290085228ed8a0e93e90f", size = 3848323, upload-time = "2026-06-11T19:25:02.079Z" },
+    { url = "https://files.pythonhosted.org/packages/27/9a/37bdd36d7feb9d591b9ccdc1952c6171b04dc777b999e2082b810eb1dd45/geoarrow_rust_core-0.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fec148cd63e616d9a7aa00c4ab08693eeec55aca7c9d700aa6451cd8001d0e08", size = 3707679, upload-time = "2026-06-11T19:25:03.594Z" },
+    { url = "https://files.pythonhosted.org/packages/45/b7/8d2998284de21d0feb2a0935c41636f8ebf2b65723d8139026e7f9f3d5e8/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b1944f3d548b6296e9fbd668602accae0ad68e49ee0f5b8df9e7ea4f474e4ae", size = 4190279, upload-time = "2026-06-11T19:25:05.21Z" },
+    { url = "https://files.pythonhosted.org/packages/25/f3/140209f53a70f261ef1459b08eea25c4edef3ad9f6ec0924033b5285ee7e/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f5c04195cbedf5d1684a50203e862d979cda0d6218aac32f607d6e3f7cd65c8", size = 4264876, upload-time = "2026-06-11T19:25:06.654Z" },
+    { url = "https://files.pythonhosted.org/packages/14/32/0097bfb92816ef91b38f7e757f65fe8456e56152ca51cd7a05b1be8a2e40/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:671c6be9cbc68295a68598fc8c6ddd875de063a795d64b2cfd10d36abd1ee324", size = 5586563, upload-time = "2026-06-11T19:25:08.376Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/86/508fe299aa44afe95399d9fa73cdbc7a451841803b8f1431e8c3d0b26ec1/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5f4726fbe09d545a507993f2f76c2be7812fef3c20c994ff33c32aaa96aaa212", size = 4402886, upload-time = "2026-06-11T19:25:10.302Z" },
+    { url = "https://files.pythonhosted.org/packages/46/81/fc34afcce2b0f17424610405481f69f3c6e4d670c5c94170d71ed6719794/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0fa37a90312e7ca06921be56cee183c12c442b345fadd982480cd1f8ed2eede", size = 4247331, upload-time = "2026-06-11T19:25:11.857Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/0d/af42431f80282a2f7e1f3e496c39483dd2362e11f8008c65033be9d2ba4c/geoarrow_rust_core-0.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f41a8c0a9f3558d73537dcad83c88b29c2a169bcc7766dc677e8245a98a5e95", size = 4741954, upload-time = "2026-06-11T19:25:13.964Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/e5/be80aa4384f16be6a20828fd4cc67da18bd2266366f80c9bfefa481559f8/geoarrow_rust_core-0.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:382f0914c75d84b87420aef7b6f11e8b5d4d58b5f5db7c8d199815e4dd282a42", size = 3599115, upload-time = "2026-06-11T19:25:15.357Z" },
+    { url = "https://files.pythonhosted.org/packages/19/52/93bbf15979ce656d09821f02f82420957fdc99ee4cd37e5e2d8c99a324da/geoarrow_rust_core-0.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:c11190008ed6a571b8ca4ef769198e95434dbe7c3caefa9acd5f0ceba1ed868f", size = 3848682, upload-time = "2026-06-11T19:25:16.914Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/1e/1665171a3756b1977b7240a8f518bbbdfa778dcc156e0f90d659723468fb/geoarrow_rust_core-0.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1308ad09912fb67a43ff7dd7dbc685ca8a8fbd8028d3876eb187b6b082a98a7b", size = 3707868, upload-time = "2026-06-11T19:25:22.483Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/38/e344ccb72473b8756c8f2dae3a8a9339e1821884a2a50befbad45150d178/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1247b961c61656596631ca3380d405f8d0a2f60f045f8b8a3a335b1a849dc55", size = 4189835, upload-time = "2026-06-11T19:25:24.116Z" },
+    { url = "https://files.pythonhosted.org/packages/22/10/bc92b9fcdc628fa1ff7e234219701cd575b0a78da5fdf3a6c8884e5ca445/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5c2cb90116255c3f74d5aee563405f3a440bd4eb75471adac13cd0c80a2564dc", size = 4265584, upload-time = "2026-06-11T19:25:25.628Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/ed/67edd70967851bef3ef9e35d8ccef242923ed69104ecb885ad3adf4de9a2/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a993d3a0964b8cf55a51bd404225dc3037b51f34b01c6bb1312611ce61f9b2d", size = 5586300, upload-time = "2026-06-11T19:25:27.32Z" },
+    { url = "https://files.pythonhosted.org/packages/76/a6/a20fba654caa314b4688ad9dceb5e99fa7956bbf92b3059baa36e06c59b3/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbd153a3348d166ecb57b2770b69b17c2df14cf303d41cd9168adba77532a31b", size = 4402375, upload-time = "2026-06-11T19:25:28.799Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/5d/c8949bb5916ff80186c854792b9ddadc9f3069db09d31311f24d82ba7096/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fb5aaf3a6f104145b4c5a3188b1be589849b2599626c0e40181a18fc2e79f68", size = 4246712, upload-time = "2026-06-11T19:25:31.015Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/36/c9b7afa2929b697a164ae18f35aba517bcab85efcf19cb48ffa5ac66642b/geoarrow_rust_core-0.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c3b33be8308a479f3a3a6d3a664861d6b5f8b1ad8822798f5a7e5d9af0b924eb", size = 4742863, upload-time = "2026-06-11T19:25:32.468Z" },
+    { url = "https://files.pythonhosted.org/packages/57/5c/55a8d753bff924959837c39c9aa37c7813c5929570a2629ae4ece811505f/geoarrow_rust_core-0.6.3-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090191ae224e8490a95e68038db7a14df8f0326706f10c2e958621bf6c06ef5", size = 1979216, upload-time = "2026-06-11T19:25:33.905Z" },
+    { url = "https://files.pythonhosted.org/packages/71/c7/a9f93af9306fd3743a96cc61bfdd7fc9194c38026f7904c067d4b4a99f0c/geoarrow_rust_core-0.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:2606d6f5afacdb49145b39d3e024efadf33f847b596c19c9b6d3030d6beb2721", size = 3599237, upload-time = "2026-06-11T19:25:35.452Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/7a/6993bd89e12d0b227b611a53c657b38e63f906dfca773accae3a1f3815a4/geoarrow_rust_core-0.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:370cd1ef46bf18fa598f3038fe6f417b016da211ffe060f2b60e47dd2f684a34", size = 3854961, upload-time = "2026-06-11T19:25:37.045Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/c4/92cbcabd2a6add1b69a76a22a349fa219bdfed8026dfab4b8ec230bf9943/geoarrow_rust_core-0.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4dbf733db0bc57859d1a34c4bc8c50805f19e60081496967588e43f1f606e885", size = 3708325, upload-time = "2026-06-11T19:25:38.638Z" },
+    { url = "https://files.pythonhosted.org/packages/07/b3/8fc34c5efa95cd597328876b6295fbe280d4b71df615655aaa2cd1618881/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45ac6715e790b1ca9be38ceb8ee39cdfe29395d29c83541f7a1190812290d81d", size = 4196828, upload-time = "2026-06-11T19:25:40.329Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/f2/bd2026862995ff96eb6b94d2fc56f7bf737d13f6bac9662481eaae23d079/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d14917d471dce8ee5a0976ec50b5da800bab0117bfd72bc56e23518a1dbbdb3a", size = 4265577, upload-time = "2026-06-11T19:25:41.91Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/01/73d69c5205a34e043026a73048d210f448a986ebb577deee7ceb1923fb5a/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43a371299305388663131321f0d623fc70ca4a3840f973598946b5183e5ba4e4", size = 5592303, upload-time = "2026-06-11T19:25:43.503Z" },
+    { url = "https://files.pythonhosted.org/packages/98/20/fe35466e526a5d363ebd9c9dd16985dbad7fd677b90e1f123a8180bceb44/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23eddb8dd65dfefb397762cc3c3f6bfaffb4271641bd9dc8043a9ab3aa4cd72a", size = 4409972, upload-time = "2026-06-11T19:25:45.114Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/c8/dc588827ad6e8dad75413bc1d35b5189c8a011a2be4827499a4ab9402253/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43ce7b3aaeb0e8c8ad7c37c84ceed49e10d0929a5a92042c3f6ec5ef33271de4", size = 4250885, upload-time = "2026-06-11T19:25:46.649Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/e2/a9923e4c5848ace6e3e6f09a40d3860955f7d836675affe35bc79bc27033/geoarrow_rust_core-0.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c67201bd662e4732a822f91651111bc024329b3e71eba9f4eed19e58c9cf789b", size = 4742518, upload-time = "2026-06-11T19:25:48.098Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/c7/3112def9e93e88341210dd22b4d04c598fb4d0726adef2114b68157354d5/geoarrow_rust_core-0.6.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:8461e6d07a7b39ab099c9885a68d5e7983d4e83a82a42dd5b331c543683c9d6e", size = 1959191, upload-time = "2026-06-11T19:25:49.668Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/0f/de74ce2171c408e4b4a7660f69f6dfaa294797a18a209fa85b1ea79be141/geoarrow_rust_core-0.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:5d2fd45d09bf700e0ca4d30b51ebcd59fb8d1a9eb4a4d7b4fc5f53a6cca59475", size = 3603948, upload-time = "2026-06-11T19:25:51.078Z" },
 ]
 
 [[package]]
@@ -797,8 +750,7 @@ version = "0.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "arro3-core" },
-    { name = "pyproj", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "pyproj", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "pyproj", version = "3.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 wheels = [
@@ -842,14 +794,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/41/14/1ec1ba4df851b477d802285e8b770f65e6774f0d6272e4e8548c8758892c/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a10e67d95a134dbb5f657fe3436ea645c6760a4ffef44df211f7d9b8fb687e6", size = 10499137, upload-time = "2025-12-03T19:02:24.514Z" },
     { url = "https://files.pythonhosted.org/packages/a5/66/7ad618415790671664e76596c000e812e0bd39e8f347f4eb7b8e3f519a55/geoarrow_rust_io-0.6.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:61ccbb528bbe4834849c501e5990a4a6f4b87976ca6a22df7859f16760c79590", size = 10394123, upload-time = "2025-12-03T19:02:01.248Z" },
     { url = "https://files.pythonhosted.org/packages/43/4b/4520af8c694ca0932f995c91d604837741522bd02b66414fdff4521abc98/geoarrow_rust_io-0.6.1-cp314-cp314-win_amd64.whl", hash = "sha256:aa46f6beda6c267f420ea390f071fadd0161094c1db8d71ad54002c006fe7f21", size = 8989484, upload-time = "2025-12-03T19:02:40.081Z" },
-    { url = "https://files.pythonhosted.org/packages/69/87/efadbf1bb9d359f55791f7198cf9aa87f0272be6a2d373f5844f5e59cd1e/geoarrow_rust_io-0.6.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:46e3e41b726b250b44a829ab41489e5008280acb8af8e68001230babf04bafd8", size = 9780411, upload-time = "2025-11-21T02:11:30.128Z" },
-    { url = "https://files.pythonhosted.org/packages/95/73/5e108b286b219d3a46042cfa0830e0f075f4addd01f83f7c851a933919ae/geoarrow_rust_io-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb95364b726c34c23fb93ebc9c08b8fa1d52062a4a9c1ac614ff8761a339ba7a", size = 9316307, upload-time = "2025-11-21T02:11:21.195Z" },
-    { url = "https://files.pythonhosted.org/packages/06/76/89c387d6d4d303feef328fc9c63df76cea52963e2046f2c092b434fb04a9/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:691a67ef3a5214fb704d1a19d33a9ddf173483c3943056fb965101c19b0edd28", size = 10309182, upload-time = "2025-11-21T02:10:34.063Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/08/34ed2d76ebfb34ed6bf3312defad16b2b5246e40d59e46443a6fe19e85dd/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91c82e9cbae6759798a8e4a87adb13ea617090a5498f384fc56c44775653d7f0", size = 11291230, upload-time = "2025-11-21T02:10:57.771Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/f5/9c25512c1f31101125555367e55ff28f72f449c8f56ff06c5be9e3feb9e5/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d9c2b609addc7a810eab5cd573243710d95afe8486f829edd05b311d51bbb5af", size = 13300664, upload-time = "2025-11-21T02:10:46.082Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/aa/14be165b439d3a3ffc6ced96f971b02df255e86b82c7e1f9f340d35689c3/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6abdc80e130f472f55598543a4bb9ba522d6502a5d80017a952027a9e9c1d1ce", size = 10486589, upload-time = "2025-11-21T02:11:09.681Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/df/1c36bae723561785ce47e463f6366a3c52994795a168d7c4ed5e457e9a37/geoarrow_rust_io-0.6.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c4638a89d61629110dde474b3d410ee2e71c89d2035ab2f2557857e7eee4ea30", size = 10395106, upload-time = "2025-11-21T02:10:20.832Z" },
-    { url = "https://files.pythonhosted.org/packages/47/d4/4e9cffad7647c07a5cd1cce68c97102dd011652168e3e09a2dedc1253a5e/geoarrow_rust_io-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:4811e96b1777fcf12ac2416872407b1e4717f9a59fe5b80ce02b1e9a087d1b5e", size = 8988735, upload-time = "2025-11-21T02:11:39.164Z" },
     { url = "https://files.pythonhosted.org/packages/e6/9f/32059400bb853eafe5d37d8c4ae9e48cd9c43820287e435cc1566f42208e/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef94f84ba4efb42d63588241733e1b62bbdb4edeac5513baeb7bfb07db4f204a", size = 10303111, upload-time = "2025-11-21T02:10:36.067Z" },
     { url = "https://files.pythonhosted.org/packages/6c/a2/7db0a685eafa41e9565a3c4e441f41d2630c084f616d2669c5fe8f5805ef/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:872dd92c52b2df342d34ac42d1b710c91c58e9dd93f5c88098816f9cd9dc8a84", size = 11299498, upload-time = "2025-11-21T02:11:00.19Z" },
     { url = "https://files.pythonhosted.org/packages/13/b4/1bfbfbe828ca51b4f314d9f70514c2ff19923714aa7d51ef1b0ec8600aed/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:235a7ea94faa95a4699f6577765a5e5a88bee079828c3d9015d9d5c6c240459c", size = 13299230, upload-time = "2025-11-21T02:10:48.12Z" },
@@ -857,94 +801,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6b/7e/6196a7b6c63c0875474a2c2319f2a2d92bb4acd4a8d260e1e10726ccff2b/geoarrow_rust_io-0.6.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:209ddc68c06a2f8577deaf4d744eac21696872f21d367a3ec0b15dc7cf824d5b", size = 10404698, upload-time = "2025-11-21T02:10:23.556Z" },
 ]
 
-[[package]]
-name = "google-pasta"
-version = "0.2.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "six" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" },
-]
-
-[[package]]
-name = "grpcio"
-version = "1.75.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534, upload-time = "2025-09-26T09:00:52.225Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953, upload-time = "2025-09-26T09:01:03.699Z" },
-    { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785, upload-time = "2025-09-26T09:01:07.504Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708, upload-time = "2025-09-26T09:01:11.028Z" },
-    { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912, upload-time = "2025-09-26T09:01:14.393Z" },
-    { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497, upload-time = "2025-09-26T09:01:17.759Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284, upload-time = "2025-09-26T09:01:20.896Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761, upload-time = "2025-09-26T09:01:28.528Z" },
-    { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084, upload-time = "2025-09-26T09:01:34.596Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403, upload-time = "2025-09-26T09:01:36.736Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166, upload-time = "2025-09-26T09:01:39.474Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828, upload-time = "2025-09-26T09:01:41.782Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421, upload-time = "2025-09-26T09:01:45.835Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290, upload-time = "2025-09-26T09:01:49.28Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" },
-    { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" },
-    { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" },
-    { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779, upload-time = "2025-09-26T09:02:19.11Z" },
-    { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838, upload-time = "2025-09-26T09:02:26.426Z" },
-    { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663, upload-time = "2025-09-26T09:02:28.724Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149, upload-time = "2025-09-26T09:02:30.971Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989, upload-time = "2025-09-26T09:02:33.233Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717, upload-time = "2025-09-26T09:02:36.011Z" },
-    { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822, upload-time = "2025-09-26T09:02:38.26Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319, upload-time = "2025-09-26T09:02:44.742Z" },
-    { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706, upload-time = "2025-09-26T09:02:50.4Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501, upload-time = "2025-09-26T09:02:52.711Z" },
-    { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492, upload-time = "2025-09-26T09:02:55.542Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061, upload-time = "2025-09-26T09:02:58.261Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849, upload-time = "2025-09-26T09:03:00.548Z" },
-    { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478, upload-time = "2025-09-26T09:03:03.096Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/e2/33efd823a879dc7b60c10192df1900ee5c200f8e782663a41a3b2aecd143/grpcio-1.75.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:c09fba33327c3ac11b5c33dbdd8218eef8990d78f83b1656d628831812a8c0fb", size = 5706679, upload-time = "2025-09-26T09:03:10.218Z" },
-    { url = "https://files.pythonhosted.org/packages/77/90/b80e75f8cce758425b2772742eed4e9db765a965d902ba4b7f239b2513de/grpcio-1.75.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c12121e509b9f8b0914d10054d24120237d19e870b1cd82acbb8a9b9ddd198a3", size = 6291926, upload-time = "2025-09-26T09:03:16.282Z" },
-    { url = "https://files.pythonhosted.org/packages/40/5f/e6033d8f99063350e20873a46225468b73045b9ef2c8cba73d66a87c3fd5/grpcio-1.75.1-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:73577a93e692b3474b1bfe84285d098de36705dbd838bb4d6a056d326e4dc880", size = 6950040, upload-time = "2025-09-26T09:03:18.874Z" },
-    { url = "https://files.pythonhosted.org/packages/01/12/34076c079b45af5aed40f037fffe388d7fbe90dd539ed01e4744c926d227/grpcio-1.75.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e19e7dfa0d7ca7dea22be464339e18ac608fd75d88c56770c646cdabe54bc724", size = 6465780, upload-time = "2025-09-26T09:03:21.219Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/c5/ee6fd69a9f6e7288d04da010ad7480a0566d2aac81097ff4dafbc5ffa9b6/grpcio-1.75.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4e1c28f51c1cf67eccdfc1065e8e866c9ed622f09773ca60947089c117f848a1", size = 7098308, upload-time = "2025-09-26T09:03:23.875Z" },
-    { url = "https://files.pythonhosted.org/packages/78/32/f2be13f13035361768923159fe20470a7d22db2c7c692b952e21284f56e5/grpcio-1.75.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:030a6164bc2ca726052778c0cf8e3249617a34e368354f9e6107c27ad4af8c28", size = 8042268, upload-time = "2025-09-26T09:03:26.268Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/2d/1bb0572f0a2eaab100b4635c6c2cd0d37e3cda5554037e3f90b1bc428d56/grpcio-1.75.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:67697efef5a98d46d5db7b1720fa4043536f8b8e5072a5d61cfca762f287e939", size = 7491470, upload-time = "2025-09-26T09:03:28.906Z" },
-]
-
-[[package]]
-name = "h5py"
-version = "3.14.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" },
-    { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" },
-    { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" },
-    { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" },
-    { url = "https://files.pythonhosted.org/packages/66/40/b423b57696514e05aa7bb06150ef96667d0e0006cc6de7ab52c71734ab51/h5py-3.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:573c33ad056ac7c1ab6d567b6db9df3ffc401045e3f605736218f96c1e0490c6", size = 4326368, upload-time = "2025-06-06T14:06:00.782Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/07/e088f89f04fdbe57ddf9de377f857158d3daa38cf5d0fb20ef9bd489e313/h5py-3.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccbe17dc187c0c64178f1a10aa274ed3a57d055117588942b8a08793cc448216", size = 4559686, upload-time = "2025-06-06T14:06:07.416Z" },
-]
-
 [[package]]
 name = "hf-xet"
 version = "1.1.10"
@@ -965,14 +821,14 @@ name = "huggingface-hub"
 version = "0.35.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "filelock", marker = "python_full_version >= '3.10'" },
-    { name = "fsspec", marker = "python_full_version >= '3.10'" },
-    { name = "hf-xet", marker = "(python_full_version >= '3.10' and platform_machine == 'aarch64') or (python_full_version >= '3.10' and platform_machine == 'amd64') or (python_full_version >= '3.10' and platform_machine == 'arm64') or (python_full_version >= '3.10' and platform_machine == 'x86_64')" },
-    { name = "packaging", marker = "python_full_version >= '3.10'" },
-    { name = "pyyaml", marker = "python_full_version >= '3.10'" },
-    { name = "requests", marker = "python_full_version >= '3.10'" },
-    { name = "tqdm", marker = "python_full_version >= '3.10'" },
-    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
+    { name = "filelock" },
+    { name = "fsspec" },
+    { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "packaging" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "tqdm" },
+    { name = "typing-extensions" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" }
 wheels = [
@@ -981,23 +837,23 @@ wheels = [
 
 [[package]]
 name = "idna"
-version = "3.10"
+version = "3.15"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
 ]
 
 [[package]]
 name = "importlib-metadata"
-version = "8.7.1"
+version = "8.5.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "zipp", marker = "python_full_version < '3.10'" },
+    { name = "zipp" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" },
 ]
 
 [[package]]
@@ -1030,119 +886,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
 ]
 
-[[package]]
-name = "keras"
-version = "2.7.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/6b/8b/065f94ba03282fa41b2d76942b87a180a9913312c4611ea7d6508fbbc114/keras-2.7.0-py2.py3-none-any.whl", hash = "sha256:0c33ae1f728064ca0d35dfba999e9c316f03623bf5688c82fb83cc74a80ea248", size = 1332171, upload-time = "2021-11-03T16:16:34.318Z" },
-]
-
-[[package]]
-name = "keras"
-version = "3.11.3"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
-dependencies = [
-    { name = "absl-py", marker = "python_full_version >= '3.10'" },
-    { name = "h5py", marker = "python_full_version >= '3.10'" },
-    { name = "ml-dtypes", marker = "python_full_version >= '3.10'" },
-    { name = "namex", marker = "python_full_version >= '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "optree", marker = "python_full_version >= '3.10'" },
-    { name = "packaging", marker = "python_full_version >= '3.10'" },
-    { name = "rich", marker = "python_full_version >= '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906, upload-time = "2025-08-21T22:08:57.643Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438, upload-time = "2025-08-21T22:08:55.858Z" },
-]
-
-[[package]]
-name = "keras-preprocessing"
-version = "1.1.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "six", marker = "python_full_version < '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/5e/f1/b44337faca48874333769a29398fe4666686733c8880aa160b9fd5dfe600/Keras_Preprocessing-1.1.2.tar.gz", hash = "sha256:add82567c50c8bc648c14195bf544a5ce7c1f76761536956c3d2978970179ef3", size = 163598, upload-time = "2020-05-14T03:53:48.526Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/79/4c/7c3275a01e12ef9368a892926ab932b33bb13d55794881e3573482b378a7/Keras_Preprocessing-1.1.2-py2.py3-none-any.whl", hash = "sha256:7b82029b130ff61cc99b55f3bd27427df4838576838c5b2f65940e4fcec99a7b", size = 42581, upload-time = "2020-05-14T03:53:47.192Z" },
-]
-
 [[package]]
 name = "lance-namespace"
-version = "0.8.0"
+version = "0.8.6"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "lance-namespace-urllib3-client" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/21/80/2b6eaa08c5e25915acaa6368a70211a25b5ba9d2d6006450e68a73936164/lance_namespace-0.8.0.tar.gz", hash = "sha256:c4a79ee221a3b2315c29863ad12d85fcf219a13158e26149d63e21dc4b4673a7", size = 10756, upload-time = "2026-06-01T08:47:10.183Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/4b/bd/7b40a08fb132fab39a6caebf832fdf6b9befc71be9413beb9be0a9d927d4/lance_namespace-0.8.0-py3-none-any.whl", hash = "sha256:782cf9e332f46bf06836722dd98b53ca8495ad98bb541501ff6876c89b67ec90", size = 12579, upload-time = "2026-06-01T08:47:10.91Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" },
 ]
 
 [[package]]
 name = "lance-namespace-urllib3-client"
-version = "0.8.0"
+version = "0.8.6"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic" },
     { name = "python-dateutil" },
     { name = "typing-extensions" },
-    { name = "urllib3", version = "1.26.20", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "urllib3" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8c/37/06fcd5a8969381e0ba953d51990af8d331bdccbc62458bf2eed30d064573/lance_namespace_urllib3_client-0.8.0.tar.gz", hash = "sha256:4f060f05ebf3c04aeaeb0d2022cbe77648a3df290f02cd2c305e5797d0fc1fdd", size = 203710, upload-time = "2026-06-01T08:47:13.404Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/43/e280727feee958f303bc58d5fa912b07734a0831f756d841654d500c2c34/lance_namespace_urllib3_client-0.8.0-py3-none-any.whl", hash = "sha256:6734e341b726e5cc96a0cd257cef27eb9d03013f2d151526ee426cef8e63e228", size = 336669, upload-time = "2026-06-01T08:47:11.88Z" },
-]
-
-[[package]]
-name = "libclang"
-version = "18.1.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" },
-]
-
-[[package]]
-name = "markdown"
-version = "3.9"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" },
-]
-
-[[package]]
-name = "markdown-it-py"
-version = "4.0.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "mdurl", marker = "python_full_version >= '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" },
 ]
 
 [[package]]
@@ -1228,17 +996,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" },
     { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" },
     { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
-    { url = "https://files.pythonhosted.org/packages/56/23/0d8c13a44bde9154821586520840643467aee574d8ce79a17da539ee7fed/markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26", size = 11623, upload-time = "2025-09-27T18:37:29.296Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/23/07a2cb9a8045d5f3f0890a8c3bc0859d7a47bfd9a560b563899bec7b72ed/markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc", size = 12049, upload-time = "2025-09-27T18:37:30.234Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/e4/6be85eb81503f8e11b61c0b6369b6e077dcf0a74adbd9ebf6b349937b4e9/markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c", size = 21923, upload-time = "2025-09-27T18:37:31.177Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/bc/4dc914ead3fe6ddaef035341fee0fc956949bbd27335b611829292b89ee2/markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42", size = 20543, upload-time = "2025-09-27T18:37:32.168Z" },
-    { url = "https://files.pythonhosted.org/packages/89/6e/5fe81fbcfba4aef4093d5f856e5c774ec2057946052d18d168219b7bd9f9/markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b", size = 20585, upload-time = "2025-09-27T18:37:33.166Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/f6/e0e5a3d3ae9c4020f696cd055f940ef86b64fe88de26f3a0308b9d3d048c/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758", size = 21387, upload-time = "2025-09-27T18:37:34.185Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/25/651753ef4dea08ea790f4fbb65146a9a44a014986996ca40102e237aa49a/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2", size = 20133, upload-time = "2025-09-27T18:37:35.138Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/0a/c3cf2b4fef5f0426e8a6d7fce3cb966a17817c568ce59d76b92a233fdbec/markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d", size = 20588, upload-time = "2025-09-27T18:37:36.096Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/1b/a7782984844bd519ad4ffdbebbba2671ec5d0ebbeac34736c15fb86399e8/markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7", size = 14566, upload-time = "2025-09-27T18:37:37.09Z" },
-    { url = "https://files.pythonhosted.org/packages/18/1f/8d9c20e1c9440e215a44be5ab64359e207fcb4f675543f1cf9a2a7f648d0/markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e", size = 15053, upload-time = "2025-09-27T18:37:38.054Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/d3/fe08482b5cd995033556d45041a4f4e76e7f0521112a9c9991d40d39825f/markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8", size = 13928, upload-time = "2025-09-27T18:37:39.037Z" },
 ]
 
 [[package]]
@@ -1265,22 +1022,12 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6c/93/e32e79333f0902ba292b996f504f5f06be59587f7d02ab8d5ed1e3066445/maturin-1.13.3-py3-none-win_arm64.whl", hash = "sha256:2389fe92d017cea9d94e521fa0175314a4c52f79a1057b901fbc9f8686ef7d0b", size = 9706562, upload-time = "2026-05-11T07:43:31.743Z" },
 ]
 
-[[package]]
-name = "mdurl"
-version = "0.1.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" },
-]
-
 [[package]]
 name = "ml-dtypes"
 version = "0.5.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316, upload-time = "2025-07-29T18:39:19.454Z" }
@@ -1315,10 +1062,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/53/21/783dfb51f40d2660afeb9bccf3612b99f6a803d980d2a09132b0f9d216ab/ml_dtypes-0.5.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3", size = 689324, upload-time = "2025-07-29T18:39:07.567Z" },
     { url = "https://files.pythonhosted.org/packages/09/f7/a82d249c711abf411ac027b7163f285487f5e615c3e0716c61033ce996ab/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93", size = 5275917, upload-time = "2025-07-29T18:39:09.339Z" },
     { url = "https://files.pythonhosted.org/packages/7f/3c/541c4b30815ab90ebfbb51df15d0b4254f2f9f1e2b4907ab229300d5e6f2/ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39", size = 5285284, upload-time = "2025-07-29T18:39:11.532Z" },
-    { url = "https://files.pythonhosted.org/packages/19/2d/c61af51173083bbf2a3b0f1a1a01d50ef1830436880027433d1b75271083/ml_dtypes-0.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5ee72568d46b9533ad54f78b1e1f3067c0534c5065120ea8ecc6f210d22748b3", size = 663552, upload-time = "2025-07-29T18:39:13.102Z" },
-    { url = "https://files.pythonhosted.org/packages/61/0e/a628f2aefd719745e8a13492375a55cedea77c0cfc917b1ce11bde435c68/ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01de48de4537dc3c46e684b969a40ec36594e7eeb7c69e9a093e7239f030a28a", size = 4952704, upload-time = "2025-07-29T18:39:14.829Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/2e/5ba92f1f99d1f5f62bffec614a5b8161e55c3961257c902fa26dbe909baa/ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b1a6e231b0770f2894910f1dce6d2f31d65884dbf7668f9b08d73623cdca909", size = 4923538, upload-time = "2025-07-29T18:39:16.581Z" },
-    { url = "https://files.pythonhosted.org/packages/70/3b/f801c69027866ea6e387224551185fedef62ad8e2e71181ec0d9dda905f7/ml_dtypes-0.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:a4f39b9bf6555fab9bfb536cf5fdd1c1c727e8d22312078702e9ff005354b37f", size = 206567, upload-time = "2025-07-29T18:39:18.047Z" },
 ]
 
 [[package]]
@@ -1335,7 +1078,7 @@ name = "multidict"
 version = "6.6.4"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version == '3.10.*'" },
+    { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" }
 wheels = [
@@ -1429,24 +1172,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" },
     { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" },
     { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" },
-    { url = "https://files.pythonhosted.org/packages/d4/d3/f04c5db316caee9b5b2cbba66270b358c922a959855995bedde87134287c/multidict-6.6.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:af7618b591bae552b40dbb6f93f5518328a949dac626ee75927bba1ecdeea9f4", size = 76977, upload-time = "2025-08-11T12:08:16.667Z" },
-    { url = "https://files.pythonhosted.org/packages/70/39/a6200417d883e510728ab3caec02d3b66ff09e1c85e0aab2ba311abfdf06/multidict-6.6.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b6819f83aef06f560cb15482d619d0e623ce9bf155115150a85ab11b8342a665", size = 44878, upload-time = "2025-08-11T12:08:18.157Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/7e/815be31ed35571b137d65232816f61513fcd97b2717d6a9d7800b5a0c6e0/multidict-6.6.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4d09384e75788861e046330308e7af54dd306aaf20eb760eb1d0de26b2bea2cb", size = 44546, upload-time = "2025-08-11T12:08:19.694Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/f1/21b5bff6a8c3e2aff56956c241941ace6b8820e1abe6b12d3c52868a773d/multidict-6.6.4-cp39-cp39-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a59c63061f1a07b861c004e53869eb1211ffd1a4acbca330e3322efa6dd02978", size = 223020, upload-time = "2025-08-11T12:08:21.554Z" },
-    { url = "https://files.pythonhosted.org/packages/15/59/37083f1dd3439979a0ffeb1906818d978d88b4cc7f4600a9f89b1cb6713c/multidict-6.6.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350f6b0fe1ced61e778037fdc7613f4051c8baf64b1ee19371b42a3acdb016a0", size = 240528, upload-time = "2025-08-11T12:08:23.45Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/f0/f054d123c87784307a27324c829eb55bcfd2e261eb785fcabbd832c8dc4a/multidict-6.6.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0c5cbac6b55ad69cb6aa17ee9343dfbba903118fd530348c330211dc7aa756d1", size = 219540, upload-time = "2025-08-11T12:08:24.965Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/26/8f78ce17b7118149c17f238f28fba2a850b660b860f9b024a34d0191030f/multidict-6.6.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:630f70c32b8066ddfd920350bc236225814ad94dfa493fe1910ee17fe4365cbb", size = 251182, upload-time = "2025-08-11T12:08:26.511Z" },
-    { url = "https://files.pythonhosted.org/packages/00/c3/a21466322d69f6594fe22d9379200f99194d21c12a5bbf8c2a39a46b83b6/multidict-6.6.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f8d4916a81697faec6cb724a273bd5457e4c6c43d82b29f9dc02c5542fd21fc9", size = 249371, upload-time = "2025-08-11T12:08:28.075Z" },
-    { url = "https://files.pythonhosted.org/packages/c2/8e/2e673124eb05cf8dc82e9265eccde01a36bcbd3193e27799b8377123c976/multidict-6.6.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e42332cf8276bb7645d310cdecca93a16920256a5b01bebf747365f86a1675b", size = 239235, upload-time = "2025-08-11T12:08:29.937Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/2d/bdd9f05e7c89e30a4b0e4faf0681a30748f8d1310f68cfdc0e3571e75bd5/multidict-6.6.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f3be27440f7644ab9a13a6fc86f09cdd90b347c3c5e30c6d6d860de822d7cb53", size = 237410, upload-time = "2025-08-11T12:08:31.872Z" },
-    { url = "https://files.pythonhosted.org/packages/46/4c/3237b83f8ca9a2673bb08fc340c15da005a80f5cc49748b587c8ae83823b/multidict-6.6.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:21f216669109e02ef3e2415ede07f4f8987f00de8cdfa0cc0b3440d42534f9f0", size = 232979, upload-time = "2025-08-11T12:08:33.399Z" },
-    { url = "https://files.pythonhosted.org/packages/55/a6/a765decff625ae9bc581aed303cd1837955177dafc558859a69f56f56ba8/multidict-6.6.4-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:d9890d68c45d1aeac5178ded1d1cccf3bc8d7accf1f976f79bf63099fb16e4bd", size = 240979, upload-time = "2025-08-11T12:08:35.02Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/2d/9c75975cb0c66ea33cae1443bb265b2b3cd689bffcbc68872565f401da23/multidict-6.6.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:edfdcae97cdc5d1a89477c436b61f472c4d40971774ac4729c613b4b133163cb", size = 246849, upload-time = "2025-08-11T12:08:37.038Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/71/d21ac0843c1d8751fb5dcf8a1f436625d39d4577bc27829799d09b419af7/multidict-6.6.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:0b2e886624be5773e69cf32bcb8534aecdeb38943520b240fed3d5596a430f2f", size = 241798, upload-time = "2025-08-11T12:08:38.669Z" },
-    { url = "https://files.pythonhosted.org/packages/94/3d/1d8911e53092837bd11b1c99d71de3e2a9a26f8911f864554677663242aa/multidict-6.6.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:be5bf4b3224948032a845d12ab0f69f208293742df96dc14c4ff9b09e508fc17", size = 235315, upload-time = "2025-08-11T12:08:40.266Z" },
-    { url = "https://files.pythonhosted.org/packages/86/c5/4b758df96376f73e936b1942c6c2dfc17e37ed9d5ff3b01a811496966ca0/multidict-6.6.4-cp39-cp39-win32.whl", hash = "sha256:10a68a9191f284fe9d501fef4efe93226e74df92ce7a24e301371293bd4918ae", size = 41434, upload-time = "2025-08-11T12:08:41.965Z" },
-    { url = "https://files.pythonhosted.org/packages/58/16/f1dfa2a0f25f2717a5e9e5fe8fd30613f7fe95e3530cec8d11f5de0b709c/multidict-6.6.4-cp39-cp39-win_amd64.whl", hash = "sha256:ee25f82f53262f9ac93bd7e58e47ea1bdcc3393cef815847e397cba17e284210", size = 46186, upload-time = "2025-08-11T12:08:43.367Z" },
-    { url = "https://files.pythonhosted.org/packages/88/7d/a0568bac65438c494cb6950b29f394d875a796a237536ac724879cf710c9/multidict-6.6.4-cp39-cp39-win_arm64.whl", hash = "sha256:f9867e55590e0855bcec60d4f9a092b69476db64573c9fe17e92b0c50614c16a", size = 43115, upload-time = "2025-08-11T12:08:45.126Z" },
     { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" },
 ]
 
@@ -1455,14 +1180,12 @@ name = "multiprocess"
 version = "0.70.16"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "dill", marker = "python_full_version >= '3.10'" },
+    { name = "dill" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" }
 wheels = [
     { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980, upload-time = "2024-01-28T18:52:15.731Z" },
     { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982, upload-time = "2024-01-28T18:52:17.783Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/94/8638a89f93c80df329116e6781a060506c7e91e1f4370dc831e9d17a041d/multiprocess-0.70.16-pp39-pypy39_pp73-macosx_10_13_x86_64.whl", hash = "sha256:0dfd078c306e08d46d7a8d06fb120313d87aa43af60d66da43ffff40b44d2f41", size = 133497, upload-time = "2024-01-28T18:52:22.644Z" },
-    { url = "https://files.pythonhosted.org/packages/89/21/222066f6bb8d8af287923ae3bd26cf4699a9ce020228ac273caca1de8250/multiprocess-0.70.16-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e7b9d0f307cd9bd50851afaac0dba2cb6c44449efff697df7c7645f7d3f2be3a", size = 133498, upload-time = "2024-01-28T18:52:24.576Z" },
     { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" },
     { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" },
     { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" },
@@ -1470,33 +1193,12 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" },
 ]
 
-[[package]]
-name = "namex"
-version = "0.1.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" },
-]
-
-[[package]]
-name = "networkx"
-version = "3.2.1"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/c4/80/a84676339aaae2f1cfdf9f418701dd634aef9cc76f708ef55c36ff39c3ca/networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6", size = 2073928, upload-time = "2023-10-28T08:41:39.364Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d5/f0/8fbc882ca80cf077f1b246c0e3c3465f7f415439bdea6b899f6b19f61f70/networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2", size = 1647772, upload-time = "2023-10-28T08:41:36.945Z" },
-]
-
 [[package]]
 name = "networkx"
 version = "3.4.2"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version == '3.10.*'",
+    "python_full_version < '3.11'",
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" }
 wheels = [
@@ -1527,67 +1229,12 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" },
 ]
 
-[[package]]
-name = "numpy"
-version = "2.0.2"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245, upload-time = "2024-08-26T20:04:14.625Z" },
-    { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540, upload-time = "2024-08-26T20:04:36.784Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623, upload-time = "2024-08-26T20:04:46.491Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774, upload-time = "2024-08-26T20:04:58.173Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081, upload-time = "2024-08-26T20:05:19.098Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451, upload-time = "2024-08-26T20:05:47.479Z" },
-    { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572, upload-time = "2024-08-26T20:06:17.137Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722, upload-time = "2024-08-26T20:06:39.16Z" },
-    { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170, upload-time = "2024-08-26T20:06:50.361Z" },
-    { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558, upload-time = "2024-08-26T20:07:13.881Z" },
-    { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137, upload-time = "2024-08-26T20:07:45.345Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552, upload-time = "2024-08-26T20:08:06.666Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957, upload-time = "2024-08-26T20:08:15.83Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573, upload-time = "2024-08-26T20:08:27.185Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330, upload-time = "2024-08-26T20:08:48.058Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895, upload-time = "2024-08-26T20:09:16.536Z" },
-    { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253, upload-time = "2024-08-26T20:09:46.263Z" },
-    { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074, upload-time = "2024-08-26T20:10:08.483Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640, upload-time = "2024-08-26T20:10:19.732Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230, upload-time = "2024-08-26T20:10:43.413Z" },
-    { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803, upload-time = "2024-08-26T20:11:13.916Z" },
-    { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835, upload-time = "2024-08-26T20:11:34.779Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499, upload-time = "2024-08-26T20:11:43.902Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497, upload-time = "2024-08-26T20:11:55.09Z" },
-    { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158, upload-time = "2024-08-26T20:12:14.95Z" },
-    { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173, upload-time = "2024-08-26T20:12:44.049Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174, upload-time = "2024-08-26T20:13:13.634Z" },
-    { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701, upload-time = "2024-08-26T20:13:34.851Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313, upload-time = "2024-08-26T20:13:45.653Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179, upload-time = "2024-08-26T20:14:08.786Z" },
-    { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942, upload-time = "2024-08-26T20:14:40.108Z" },
-    { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512, upload-time = "2024-08-26T20:15:00.985Z" },
-    { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976, upload-time = "2024-08-26T20:15:10.876Z" },
-    { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494, upload-time = "2024-08-26T20:15:22.055Z" },
-    { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596, upload-time = "2024-08-26T20:15:42.452Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099, upload-time = "2024-08-26T20:16:11.048Z" },
-    { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823, upload-time = "2024-08-26T20:16:40.171Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424, upload-time = "2024-08-26T20:17:02.604Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809, upload-time = "2024-08-26T20:17:13.553Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314, upload-time = "2024-08-26T20:17:36.72Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288, upload-time = "2024-08-26T20:18:07.732Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793, upload-time = "2024-08-26T20:18:19.125Z" },
-    { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885, upload-time = "2024-08-26T20:18:47.237Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784, upload-time = "2024-08-26T20:19:11.19Z" },
-]
-
 [[package]]
 name = "numpy"
 version = "2.2.6"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version == '3.10.*'",
+    "python_full_version < '3.11'",
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" }
 wheels = [
@@ -1735,195 +1382,195 @@ wheels = [
 ]
 
 [[package]]
-name = "nvidia-cublas-cu12"
-version = "12.8.4.1"
+name = "nvidia-cublas"
+version = "13.1.1.3"
 source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "nvidia-cuda-nvrtc" },
+]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" },
 ]
 
 [[package]]
-name = "nvidia-cuda-cupti-cu12"
-version = "12.8.90"
+name = "nvidia-cuda-cupti"
+version = "13.0.85"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" },
+    { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" },
 ]
 
 [[package]]
-name = "nvidia-cuda-nvrtc-cu12"
-version = "12.8.93"
+name = "nvidia-cuda-nvrtc"
+version = "13.0.88"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" },
 ]
 
 [[package]]
-name = "nvidia-cuda-runtime-cu12"
-version = "12.8.90"
+name = "nvidia-cuda-runtime"
+version = "13.0.96"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" },
+    { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" },
 ]
 
 [[package]]
-name = "nvidia-cudnn-cu12"
-version = "9.10.2.21"
+name = "nvidia-cudnn-cu13"
+version = "9.20.0.48"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12" },
+    { name = "nvidia-cublas" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" },
+    { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" },
 ]
 
 [[package]]
-name = "nvidia-cufft-cu12"
-version = "11.3.3.83"
+name = "nvidia-cufft"
+version = "12.0.0.61"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-nvjitlink" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" },
 ]
 
 [[package]]
-name = "nvidia-cufile-cu12"
-version = "1.13.1.3"
+name = "nvidia-cufile"
+version = "1.15.1.6"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" },
 ]
 
 [[package]]
-name = "nvidia-curand-cu12"
-version = "10.3.9.90"
+name = "nvidia-curand"
+version = "10.4.0.35"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" },
 ]
 
 [[package]]
-name = "nvidia-cusolver-cu12"
-version = "11.7.3.90"
+name = "nvidia-cusolver"
+version = "12.0.4.66"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-cublas-cu12" },
-    { name = "nvidia-cusparse-cu12" },
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-cublas" },
+    { name = "nvidia-cusparse" },
+    { name = "nvidia-nvjitlink" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" },
+    { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" },
 ]
 
 [[package]]
-name = "nvidia-cusparse-cu12"
-version = "12.5.8.93"
+name = "nvidia-cusparse"
+version = "12.6.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "nvidia-nvjitlink-cu12" },
+    { name = "nvidia-nvjitlink" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
+    { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" },
 ]
+
+[[package]]
+name = "nvidia-cusparselt-cu13"
+version = "0.8.1"
+source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" },
+    { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" },
+    { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" },
 ]
 
 [[package]]
-name = "nvidia-cusparselt-cu12"
-version = "0.7.1"
+name = "nvidia-nccl-cu13"
+version = "2.29.7"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" },
+    { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" },
+    { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" },
 ]
 
 [[package]]
-name = "nvidia-nccl-cu12"
-version = "2.27.3"
+name = "nvidia-nvjitlink"
+version = "13.0.88"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/5b/4e4fff7bad39adf89f735f2bc87248c81db71205b62bcc0d5ca5b606b3c3/nvidia_nccl_cu12-2.27.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adf27ccf4238253e0b826bce3ff5fa532d65fc42322c8bfdfaf28024c0fbe039", size = 322364134, upload-time = "2025-06-03T21:58:04.013Z" },
+    { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" },
 ]
 
 [[package]]
-name = "nvidia-nvjitlink-cu12"
-version = "12.8.93"
+name = "nvidia-nvshmem-cu13"
+version = "3.4.5"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" },
+    { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" },
 ]
 
 [[package]]
-name = "nvidia-nvtx-cu12"
-version = "12.8.90"
+name = "nvidia-nvtx"
+version = "13.0.85"
 source = { registry = "https://pypi.org/simple" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" },
 ]
 
 [[package]]
-name = "opt-einsum"
-version = "3.4.0"
+name = "opentelemetry-api"
+version = "1.30.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" }
+dependencies = [
+    { name = "deprecated" },
+    { name = "importlib-metadata" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2b/6d/bbbf879826b7f3c89a45252010b5796fb1f1a0d45d9dc4709db0ef9a06c8/opentelemetry_api-1.30.0.tar.gz", hash = "sha256:375893400c1435bf623f7dfb3bcd44825fe6b56c34d0667c542ea8257b1a1240", size = 63703, upload-time = "2025-02-04T18:17:13.789Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" },
+    { url = "https://files.pythonhosted.org/packages/36/0a/eea862fae6413d8181b23acf8e13489c90a45f17986ee9cf4eab8a0b9ad9/opentelemetry_api-1.30.0-py3-none-any.whl", hash = "sha256:d5f5284890d73fdf47f843dda3210edf37a38d66f44f2b5aedc1e89ed455dc09", size = 64955, upload-time = "2025-02-04T18:16:46.167Z" },
 ]
 
 [[package]]
-name = "optree"
-version = "0.17.0"
+name = "opentelemetry-sdk"
+version = "1.30.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111, upload-time = "2025-07-25T11:26:11.586Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398, upload-time = "2025-07-25T11:24:26.672Z" },
-    { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834, upload-time = "2025-07-25T11:24:28.988Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116, upload-time = "2025-07-25T11:24:30.396Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491, upload-time = "2025-07-25T11:24:31.725Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957, upload-time = "2025-07-25T11:24:32.759Z" },
-    { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797, upload-time = "2025-07-25T11:24:39.509Z" },
-    { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450, upload-time = "2025-07-25T11:24:40.555Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557, upload-time = "2025-07-25T11:24:42.396Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174, upload-time = "2025-07-25T11:24:43.51Z" },
-    { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000, upload-time = "2025-07-25T11:24:44.819Z" },
-    { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869, upload-time = "2025-07-25T11:24:51.807Z" },
-    { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465, upload-time = "2025-07-25T11:24:53.124Z" },
-    { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686, upload-time = "2025-07-25T11:24:54.212Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225, upload-time = "2025-07-25T11:24:55.213Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819, upload-time = "2025-07-25T11:24:56.315Z" },
-    { url = "https://files.pythonhosted.org/packages/21/04/9706d11b880186e9e9d66d7c21ce249b2ce0212645137cc13fdd18247c26/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5995a3efce4b00a14049268a81ab0379656a41ddf3c3761e3b88937fca44d48", size = 348177, upload-time = "2025-07-25T11:25:00.999Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/4b/0415c18816818ac871c9f3d5c7c5f4ceb83baff03ed511c9c94591ace4bc/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d06e8143d16fe6c0708f3cc2807b5b65f815d60ee2b52f3d79e4022c95563482", size = 354389, upload-time = "2025-07-25T11:25:02.337Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513, upload-time = "2025-07-25T11:25:05.548Z" },
-    { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378, upload-time = "2025-07-25T11:25:06.918Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294, upload-time = "2025-07-25T11:25:08.043Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633, upload-time = "2025-07-25T11:25:09.141Z" },
-    { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866, upload-time = "2025-07-25T11:25:10.214Z" },
-    { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945, upload-time = "2025-07-25T11:25:17.728Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177, upload-time = "2025-07-25T11:25:18.749Z" },
-    { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198, upload-time = "2025-07-25T11:25:19.865Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391, upload-time = "2025-07-25T11:25:20.942Z" },
-    { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555, upload-time = "2025-07-25T11:25:21.968Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929, upload-time = "2025-07-25T11:25:28.897Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381, upload-time = "2025-07-25T11:25:29.984Z" },
-    { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461, upload-time = "2025-07-25T11:25:31.116Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234, upload-time = "2025-07-25T11:25:32.289Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863, upload-time = "2025-07-25T11:25:33.379Z" },
-    { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975, upload-time = "2025-07-25T11:25:40.376Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173, upload-time = "2025-07-25T11:25:41.474Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080, upload-time = "2025-07-25T11:25:42.605Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422, upload-time = "2025-07-25T11:25:43.672Z" },
-    { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579, upload-time = "2025-07-25T11:25:44.765Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/29/3bb53de2de3b36a51e46b6d9ada7ee1a3a312ac461cd54292a023adc807c/optree-0.17.0-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:537498cf7bf7a4fe71f7ffd815e72b8672aea0fac82e1513f6b6e35e8569f5aa", size = 350302, upload-time = "2025-07-25T11:25:52.016Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/3b/d17a31447ed7ef6f10bd0caf40742b016fcdeaa3abb7568307b04a0f50cf/optree-0.17.0-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:3b3bb2326b550ddb048e3454fad40183b7fed74dda4351b016d20362809180af", size = 405358, upload-time = "2025-07-25T11:25:53.085Z" },
-    { url = "https://files.pythonhosted.org/packages/db/f3/b9f0a8c98fd0c7f53fa9d9a46d75bb1182aeecd7ecde6f353d3e69ec9618/optree-0.17.0-cp39-cp39-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c0d3d702044e5acbec2cf8349789f6b096057bd00dc8e1e1c97b990347279fda", size = 402694, upload-time = "2025-07-25T11:25:54.537Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/dd/0d9d7426fd6b5d90ad40e4d93717a955d4257d06574dfe7a1da0d24cb06c/optree-0.17.0-cp39-cp39-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9155e82717be1dda1f3c1244e9cb5b3733d5dd3ba47702730c7816be083a5cb", size = 398857, upload-time = "2025-07-25T11:25:55.921Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/57/dacec3f8c70f4685bb07fce19cf3361037fde2b596f6f7228e1a4b39677b/optree-0.17.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8e825501f55360e8381718623b094579dedc485e57010e01593d72a43b43e68", size = 387849, upload-time = "2025-07-25T11:25:57.046Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952, upload-time = "2025-07-25T11:26:02.692Z" },
-    { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568, upload-time = "2025-07-25T11:26:04.094Z" },
-    { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728, upload-time = "2025-07-25T11:26:07.959Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649, upload-time = "2025-07-25T11:26:09.463Z" },
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-semantic-conventions" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/93/ee/d710062e8a862433d1be0b85920d0c653abe318878fef2d14dfe2c62ff7b/opentelemetry_sdk-1.30.0.tar.gz", hash = "sha256:c9287a9e4a7614b9946e933a67168450b9ab35f08797eb9bc77d998fa480fa18", size = 158633, upload-time = "2025-02-04T18:17:28.908Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/97/28/64d781d6adc6bda2260067ce2902bd030cf45aec657e02e28c5b4480b976/opentelemetry_sdk-1.30.0-py3-none-any.whl", hash = "sha256:14fe7afc090caad881addb6926cec967129bd9260c4d33ae6a217359f6b61091", size = 118717, upload-time = "2025-02-04T18:17:09.353Z" },
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.51b0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "deprecated" },
+    { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1e/c0/0f9ef4605fea7f2b83d55dd0b0d7aebe8feead247cd6facd232b30907b4f/opentelemetry_semantic_conventions-0.51b0.tar.gz", hash = "sha256:3fabf47f35d1fd9aebcdca7e6802d86bd5ebc3bc3408b7e3248dde6e87a18c47", size = 107191, upload-time = "2025-02-04T18:17:29.903Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/75/d7bdbb6fd8630b4cafb883482b75c4fc276b6426619539d266e32ac53266/opentelemetry_semantic_conventions-0.51b0-py3-none-any.whl", hash = "sha256:fdc777359418e8d06c86012c3dc92c88a6453ba662e941593adb062e48c2eeae", size = 177416, upload-time = "2025-02-04T18:17:11.305Z" },
 ]
 
 [[package]]
@@ -1940,8 +1587,7 @@ name = "pandas"
 version = "2.3.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
     { name = "python-dateutil" },
     { name = "pytz" },
@@ -1996,13 +1642,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a4/1e/1bac1a839d12e6a82ec6cb40cda2edde64a2013a66963293696bbf31fbbb/pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5", size = 12121582, upload-time = "2025-09-29T23:30:43.391Z" },
     { url = "https://files.pythonhosted.org/packages/44/91/483de934193e12a3b1d6ae7c8645d083ff88dec75f46e827562f1e4b4da6/pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788", size = 12699963, upload-time = "2025-09-29T23:31:10.009Z" },
     { url = "https://files.pythonhosted.org/packages/70/44/5191d2e4026f86a2a109053e194d3ba7a31a2d10a9c2348368c63ed4e85a/pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87", size = 13202175, upload-time = "2025-09-29T23:31:59.173Z" },
-    { url = "https://files.pythonhosted.org/packages/56/b4/52eeb530a99e2a4c55ffcd352772b599ed4473a0f892d127f4147cf0f88e/pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2", size = 11567720, upload-time = "2025-09-29T23:33:06.209Z" },
-    { url = "https://files.pythonhosted.org/packages/48/4a/2d8b67632a021bced649ba940455ed441ca854e57d6e7658a6024587b083/pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8", size = 10810302, upload-time = "2025-09-29T23:33:35.846Z" },
-    { url = "https://files.pythonhosted.org/packages/13/e6/d2465010ee0569a245c975dc6967b801887068bc893e908239b1f4b6c1ac/pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff", size = 12154874, upload-time = "2025-09-29T23:33:49.939Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/18/aae8c0aa69a386a3255940e9317f793808ea79d0a525a97a903366bb2569/pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29", size = 12790141, upload-time = "2025-09-29T23:34:05.655Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/26/617f98de789de00c2a444fbe6301bb19e66556ac78cff933d2c98f62f2b4/pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73", size = 13208697, upload-time = "2025-09-29T23:34:21.835Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/fb/25709afa4552042bd0e15717c75e9b4a2294c3dc4f7e6ea50f03c5136600/pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9", size = 13879233, upload-time = "2025-09-29T23:34:35.079Z" },
-    { url = "https://files.pythonhosted.org/packages/98/af/7be05277859a7bc399da8ba68b88c96b27b48740b6cf49688899c6eb4176/pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa", size = 11359119, upload-time = "2025-09-29T23:34:46.339Z" },
 ]
 
 [[package]]
@@ -2091,17 +1730,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f0/77/bc6f92a3e8e6e46c0ca78abfffec0037845800ea38c73483760362804c41/pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12", size = 6377370, upload-time = "2025-07-01T09:15:46.673Z" },
     { url = "https://files.pythonhosted.org/packages/4a/82/3a721f7d69dca802befb8af08b7c79ebcab461007ce1c18bd91a5d5896f9/pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db", size = 7121500, upload-time = "2025-07-01T09:15:48.512Z" },
     { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835, upload-time = "2025-07-01T09:15:50.399Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/8e/9c089f01677d1264ab8648352dcb7773f37da6ad002542760c80107da816/pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f", size = 5316478, upload-time = "2025-07-01T09:15:52.209Z" },
-    { url = "https://files.pythonhosted.org/packages/b5/a9/5749930caf674695867eb56a581e78eb5f524b7583ff10b01b6e5048acb3/pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081", size = 4686522, upload-time = "2025-07-01T09:15:54.162Z" },
-    { url = "https://files.pythonhosted.org/packages/43/46/0b85b763eb292b691030795f9f6bb6fcaf8948c39413c81696a01c3577f7/pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4", size = 5853376, upload-time = "2025-07-03T13:11:01.066Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/c6/1a230ec0067243cbd60bc2dad5dc3ab46a8a41e21c15f5c9b52b26873069/pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc", size = 7626020, upload-time = "2025-07-03T13:11:06.479Z" },
-    { url = "https://files.pythonhosted.org/packages/63/dd/f296c27ffba447bfad76c6a0c44c1ea97a90cb9472b9304c94a732e8dbfb/pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06", size = 5956732, upload-time = "2025-07-01T09:15:56.111Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/a0/98a3630f0b57f77bae67716562513d3032ae70414fcaf02750279c389a9e/pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a", size = 6624404, upload-time = "2025-07-01T09:15:58.245Z" },
-    { url = "https://files.pythonhosted.org/packages/de/e6/83dfba5646a290edd9a21964da07674409e410579c341fc5b8f7abd81620/pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978", size = 6067760, upload-time = "2025-07-01T09:16:00.003Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/41/15ab268fe6ee9a2bc7391e2bbb20a98d3974304ab1a406a992dcb297a370/pillow-11.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:97afb3a00b65cc0804d1c7abddbf090a81eaac02768af58cbdcaaa0a931e0b6d", size = 6700534, upload-time = "2025-07-01T09:16:02.29Z" },
-    { url = "https://files.pythonhosted.org/packages/64/79/6d4f638b288300bed727ff29f2a3cb63db054b33518a95f27724915e3fbc/pillow-11.3.0-cp39-cp39-win32.whl", hash = "sha256:ea944117a7974ae78059fcc1800e5d3295172bb97035c0c1d9345fca1419da71", size = 6277091, upload-time = "2025-07-01T09:16:04.4Z" },
-    { url = "https://files.pythonhosted.org/packages/46/05/4106422f45a05716fd34ed21763f8ec182e8ea00af6e9cb05b93a247361a/pillow-11.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:e5c5858ad8ec655450a7c7df532e9842cf8df7cc349df7225c60d5d348c8aada", size = 6986091, upload-time = "2025-07-01T09:16:06.342Z" },
-    { url = "https://files.pythonhosted.org/packages/63/c6/287fd55c2c12761d0591549d48885187579b7c257bef0c6660755b0b59ae/pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb", size = 2422632, upload-time = "2025-07-01T09:16:08.142Z" },
     { url = "https://files.pythonhosted.org/packages/6f/8b/209bd6b62ce8367f47e68a218bffac88888fdf2c9fcf1ecadc6c3ec1ebc7/pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967", size = 5270556, upload-time = "2025-07-01T09:16:09.961Z" },
     { url = "https://files.pythonhosted.org/packages/2e/e6/231a0b76070c2cfd9e260a7a5b504fb72da0a95279410fa7afd99d9751d6/pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe", size = 4654625, upload-time = "2025-07-01T09:16:11.913Z" },
     { url = "https://files.pythonhosted.org/packages/13/f4/10cf94fda33cb12765f2397fc285fa6d8eb9c29de7f3185165b702fc7386/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c", size = 4874207, upload-time = "2025-07-03T13:11:10.201Z" },
@@ -2142,12 +1770,10 @@ wheels = [
 [package.optional-dependencies]
 pandas = [
     { name = "pandas" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "pyarrow" },
 ]
 pyarrow = [
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "pyarrow" },
 ]
 
 [[package]]
@@ -2250,59 +1876,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" },
     { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" },
     { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" },
-    { url = "https://files.pythonhosted.org/packages/6c/39/8ea9bcfaaff16fd0b0fc901ee522e24c9ec44b4ca0229cfffb8066a06959/propcache-0.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a7fad897f14d92086d6b03fdd2eb844777b0c4d7ec5e3bac0fbae2ab0602bbe5", size = 74678, upload-time = "2025-06-09T22:55:41.227Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/85/cab84c86966e1d354cf90cdc4ba52f32f99a5bca92a1529d666d957d7686/propcache-0.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1f43837d4ca000243fd7fd6301947d7cb93360d03cd08369969450cc6b2ce3b4", size = 43829, upload-time = "2025-06-09T22:55:42.417Z" },
-    { url = "https://files.pythonhosted.org/packages/23/f7/9cb719749152d8b26d63801b3220ce2d3931312b2744d2b3a088b0ee9947/propcache-0.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:261df2e9474a5949c46e962065d88eb9b96ce0f2bd30e9d3136bcde84befd8f2", size = 43729, upload-time = "2025-06-09T22:55:43.651Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/a2/0b2b5a210ff311260002a315f6f9531b65a36064dfb804655432b2f7d3e3/propcache-0.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e514326b79e51f0a177daab1052bc164d9d9e54133797a3a58d24c9c87a3fe6d", size = 204483, upload-time = "2025-06-09T22:55:45.327Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/e0/7aff5de0c535f783b0c8be5bdb750c305c1961d69fbb136939926e155d98/propcache-0.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4a996adb6904f85894570301939afeee65f072b4fd265ed7e569e8d9058e4ec", size = 217425, upload-time = "2025-06-09T22:55:46.729Z" },
-    { url = "https://files.pythonhosted.org/packages/92/1d/65fa889eb3b2a7d6e4ed3c2b568a9cb8817547a1450b572de7bf24872800/propcache-0.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:76cace5d6b2a54e55b137669b30f31aa15977eeed390c7cbfb1dafa8dfe9a701", size = 214723, upload-time = "2025-06-09T22:55:48.342Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/e2/eecf6989870988dfd731de408a6fa366e853d361a06c2133b5878ce821ad/propcache-0.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31248e44b81d59d6addbb182c4720f90b44e1efdc19f58112a3c3a1615fb47ef", size = 200166, upload-time = "2025-06-09T22:55:49.775Z" },
-    { url = "https://files.pythonhosted.org/packages/12/06/c32be4950967f18f77489268488c7cdc78cbfc65a8ba8101b15e526b83dc/propcache-0.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abb7fa19dbf88d3857363e0493b999b8011eea856b846305d8c0512dfdf8fbb1", size = 194004, upload-time = "2025-06-09T22:55:51.335Z" },
-    { url = "https://files.pythonhosted.org/packages/46/6c/17b521a6b3b7cbe277a4064ff0aa9129dd8c89f425a5a9b6b4dd51cc3ff4/propcache-0.3.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d81ac3ae39d38588ad0549e321e6f773a4e7cc68e7751524a22885d5bbadf886", size = 203075, upload-time = "2025-06-09T22:55:52.681Z" },
-    { url = "https://files.pythonhosted.org/packages/62/cb/3bdba2b736b3e45bc0e40f4370f745b3e711d439ffbffe3ae416393eece9/propcache-0.3.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:cc2782eb0f7a16462285b6f8394bbbd0e1ee5f928034e941ffc444012224171b", size = 195407, upload-time = "2025-06-09T22:55:54.048Z" },
-    { url = "https://files.pythonhosted.org/packages/29/bd/760c5c6a60a4a2c55a421bc34a25ba3919d49dee411ddb9d1493bb51d46e/propcache-0.3.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:db429c19a6c7e8a1c320e6a13c99799450f411b02251fb1b75e6217cf4a14fcb", size = 196045, upload-time = "2025-06-09T22:55:55.485Z" },
-    { url = "https://files.pythonhosted.org/packages/76/58/ced2757a46f55b8c84358d6ab8de4faf57cba831c51e823654da7144b13a/propcache-0.3.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:21d8759141a9e00a681d35a1f160892a36fb6caa715ba0b832f7747da48fb6ea", size = 208432, upload-time = "2025-06-09T22:55:56.884Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/ec/d98ea8d5a4d8fe0e372033f5254eddf3254344c0c5dc6c49ab84349e4733/propcache-0.3.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2ca6d378f09adb13837614ad2754fa8afaee330254f404299611bce41a8438cb", size = 210100, upload-time = "2025-06-09T22:55:58.498Z" },
-    { url = "https://files.pythonhosted.org/packages/56/84/b6d8a7ecf3f62d7dd09d9d10bbf89fad6837970ef868b35b5ffa0d24d9de/propcache-0.3.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:34a624af06c048946709f4278b4176470073deda88d91342665d95f7c6270fbe", size = 200712, upload-time = "2025-06-09T22:55:59.906Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/32/889f4903ddfe4a9dc61da71ee58b763758cf2d608fe1decede06e6467f8d/propcache-0.3.2-cp39-cp39-win32.whl", hash = "sha256:4ba3fef1c30f306b1c274ce0b8baaa2c3cdd91f645c48f06394068f37d3837a1", size = 38187, upload-time = "2025-06-09T22:56:01.212Z" },
-    { url = "https://files.pythonhosted.org/packages/67/74/d666795fb9ba1dc139d30de64f3b6fd1ff9c9d3d96ccfdb992cd715ce5d2/propcache-0.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:7a2368eed65fc69a7a7a40b27f22e85e7627b74216f0846b04ba5c116e191ec9", size = 42025, upload-time = "2025-06-09T22:56:02.875Z" },
     { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" },
 ]
 
-[[package]]
-name = "protobuf"
-version = "3.19.6"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/51/d1/79bfd1f481469b661a2eddab551255536401892722189433282bfb13cfb1/protobuf-3.19.6.tar.gz", hash = "sha256:5f5540d57a43042389e87661c6eaa50f47c19c6176e8cf1c4f287aeefeccb5c4", size = 218071, upload-time = "2022-09-29T22:07:23.03Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/4b/3b/90f805b9e5ecacf8a216f2e5acabc2d3ad965b62803510be41804e6bfbfe/protobuf-3.19.6-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:010be24d5a44be7b0613750ab40bc8b8cedc796db468eae6c779b395f50d1fa1", size = 913631, upload-time = "2022-09-29T21:17:39.095Z" },
-    { url = "https://files.pythonhosted.org/packages/26/ef/bd6ba3b4ff9a35944bdd325e2c9ee56f71e855757f7d43938232499f0278/protobuf-3.19.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11478547958c2dfea921920617eb457bc26867b0d1aa065ab05f35080c5d9eb6", size = 1055327, upload-time = "2022-09-29T21:17:41.054Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/db/8b33c9558f1f27dd74e7f9ad730c6b32efab431419af556b1659e125b041/protobuf-3.19.6-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:30a15015d86b9c3b8d6bf78d5b8c7749f2512c29f168ca259c9d7727604d0e39", size = 913657, upload-time = "2022-09-29T21:18:18.359Z" },
-    { url = "https://files.pythonhosted.org/packages/51/61/e80b7a04f4e1b4eecc86582335205fd876abca0abafee4a6c001f70a375e/protobuf-3.19.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:878b4cd080a21ddda6ac6d1e163403ec6eea2e206cf225982ae04567d39be7b0", size = 1055457, upload-time = "2022-09-29T21:18:20.212Z" },
-    { url = "https://files.pythonhosted.org/packages/32/27/1141a8232723dcb10a595cc0ce4321dcbbd5215300bf4acfc142343205bf/protobuf-3.19.6-py2.py3-none-any.whl", hash = "sha256:14082457dc02be946f60b15aad35e9f5c69e738f80ebbc0900a19bc83734a5a4", size = 162648, upload-time = "2022-09-29T22:07:20.303Z" },
-]
-
-[[package]]
-name = "protobuf"
-version = "6.32.1"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" },
-    { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" },
-]
-
 [[package]]
 name = "psutil"
 version = "7.1.0"
@@ -2330,119 +1906,59 @@ wheels = [
 
 [[package]]
 name = "pyarrow"
-version = "21.0.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version == '3.10.*'",
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ef/c2/ea068b8f00905c06329a3dfcd40d0fcc2b7d0f2e355bdb25b65e0a0e4cd4/pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc", size = 1133487, upload-time = "2025-07-18T00:57:31.761Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/17/d9/110de31880016e2afc52d8580b397dbe47615defbf09ca8cf55f56c62165/pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26", size = 31196837, upload-time = "2025-07-18T00:54:34.755Z" },
-    { url = "https://files.pythonhosted.org/packages/df/5f/c1c1997613abf24fceb087e79432d24c19bc6f7259cab57c2c8e5e545fab/pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79", size = 32659470, upload-time = "2025-07-18T00:54:38.329Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/ed/b1589a777816ee33ba123ba1e4f8f02243a844fed0deec97bde9fb21a5cf/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7be45519b830f7c24b21d630a31d48bcebfd5d4d7f9d3bdb49da9cdf6d764edb", size = 41055619, upload-time = "2025-07-18T00:54:42.172Z" },
-    { url = "https://files.pythonhosted.org/packages/44/28/b6672962639e85dc0ac36f71ab3a8f5f38e01b51343d7aa372a6b56fa3f3/pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51", size = 42733488, upload-time = "2025-07-18T00:54:47.132Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/cc/de02c3614874b9089c94eac093f90ca5dfa6d5afe45de3ba847fd950fdf1/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bd04ec08f7f8bd113c55868bd3fc442a9db67c27af098c5f814a3091e71cc61a", size = 43329159, upload-time = "2025-07-18T00:54:51.686Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/3e/99473332ac40278f196e105ce30b79ab8affab12f6194802f2593d6b0be2/pyarrow-21.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9b0b14b49ac10654332a805aedfc0147fb3469cbf8ea951b3d040dab12372594", size = 45050567, upload-time = "2025-07-18T00:54:56.679Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/f5/c372ef60593d713e8bfbb7e0c743501605f0ad00719146dc075faf11172b/pyarrow-21.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:9d9f8bcb4c3be7738add259738abdeddc363de1b80e3310e04067aa1ca596634", size = 26217959, upload-time = "2025-07-18T00:55:00.482Z" },
-    { url = "https://files.pythonhosted.org/packages/94/dc/80564a3071a57c20b7c32575e4a0120e8a330ef487c319b122942d665960/pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b", size = 31243234, upload-time = "2025-07-18T00:55:03.812Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/cc/3b51cb2db26fe535d14f74cab4c79b191ed9a8cd4cbba45e2379b5ca2746/pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10", size = 32714370, upload-time = "2025-07-18T00:55:07.495Z" },
-    { url = "https://files.pythonhosted.org/packages/24/11/a4431f36d5ad7d83b87146f515c063e4d07ef0b7240876ddb885e6b44f2e/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e", size = 41135424, upload-time = "2025-07-18T00:55:11.461Z" },
-    { url = "https://files.pythonhosted.org/packages/74/dc/035d54638fc5d2971cbf1e987ccd45f1091c83bcf747281cf6cc25e72c88/pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569", size = 42823810, upload-time = "2025-07-18T00:55:16.301Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/3b/89fced102448a9e3e0d4dded1f37fa3ce4700f02cdb8665457fcc8015f5b/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e", size = 43391538, upload-time = "2025-07-18T00:55:23.82Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/bb/ea7f1bd08978d39debd3b23611c293f64a642557e8141c80635d501e6d53/pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c", size = 45120056, upload-time = "2025-07-18T00:55:28.231Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/0b/77ea0600009842b30ceebc3337639a7380cd946061b620ac1a2f3cb541e2/pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6", size = 26220568, upload-time = "2025-07-18T00:55:32.122Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/d4/d4f817b21aacc30195cf6a46ba041dd1be827efa4a623cc8bf39a1c2a0c0/pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd", size = 31160305, upload-time = "2025-07-18T00:55:35.373Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/9c/dcd38ce6e4b4d9a19e1d36914cb8e2b1da4e6003dd075474c4cfcdfe0601/pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876", size = 32684264, upload-time = "2025-07-18T00:55:39.303Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/74/2a2d9f8d7a59b639523454bec12dba35ae3d0a07d8ab529dc0809f74b23c/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d", size = 41108099, upload-time = "2025-07-18T00:55:42.889Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/90/2660332eeb31303c13b653ea566a9918484b6e4d6b9d2d46879a33ab0622/pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e", size = 42829529, upload-time = "2025-07-18T00:55:47.069Z" },
-    { url = "https://files.pythonhosted.org/packages/33/27/1a93a25c92717f6aa0fca06eb4700860577d016cd3ae51aad0e0488ac899/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82", size = 43367883, upload-time = "2025-07-18T00:55:53.069Z" },
-    { url = "https://files.pythonhosted.org/packages/05/d9/4d09d919f35d599bc05c6950095e358c3e15148ead26292dfca1fb659b0c/pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623", size = 45133802, upload-time = "2025-07-18T00:55:57.714Z" },
-    { url = "https://files.pythonhosted.org/packages/71/30/f3795b6e192c3ab881325ffe172e526499eb3780e306a15103a2764916a2/pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18", size = 26203175, upload-time = "2025-07-18T00:56:01.364Z" },
-    { url = "https://files.pythonhosted.org/packages/16/ca/c7eaa8e62db8fb37ce942b1ea0c6d7abfe3786ca193957afa25e71b81b66/pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a", size = 31154306, upload-time = "2025-07-18T00:56:04.42Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/e8/e87d9e3b2489302b3a1aea709aaca4b781c5252fcb812a17ab6275a9a484/pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe", size = 32680622, upload-time = "2025-07-18T00:56:07.505Z" },
-    { url = "https://files.pythonhosted.org/packages/84/52/79095d73a742aa0aba370c7942b1b655f598069489ab387fe47261a849e1/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd", size = 41104094, upload-time = "2025-07-18T00:56:10.994Z" },
-    { url = "https://files.pythonhosted.org/packages/89/4b/7782438b551dbb0468892a276b8c789b8bbdb25ea5c5eb27faadd753e037/pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61", size = 42825576, upload-time = "2025-07-18T00:56:15.569Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/62/0f29de6e0a1e33518dec92c65be0351d32d7ca351e51ec5f4f837a9aab91/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d", size = 43368342, upload-time = "2025-07-18T00:56:19.531Z" },
-    { url = "https://files.pythonhosted.org/packages/90/c7/0fa1f3f29cf75f339768cc698c8ad4ddd2481c1742e9741459911c9ac477/pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99", size = 45131218, upload-time = "2025-07-18T00:56:23.347Z" },
-    { url = "https://files.pythonhosted.org/packages/01/63/581f2076465e67b23bc5a37d4a2abff8362d389d29d8105832e82c9c811c/pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636", size = 26087551, upload-time = "2025-07-18T00:56:26.758Z" },
-    { url = "https://files.pythonhosted.org/packages/c9/ab/357d0d9648bb8241ee7348e564f2479d206ebe6e1c47ac5027c2e31ecd39/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da", size = 31290064, upload-time = "2025-07-18T00:56:30.214Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/8a/5685d62a990e4cac2043fc76b4661bf38d06efed55cf45a334b455bd2759/pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7", size = 32727837, upload-time = "2025-07-18T00:56:33.935Z" },
-    { url = "https://files.pythonhosted.org/packages/fc/de/c0828ee09525c2bafefd3e736a248ebe764d07d0fd762d4f0929dbc516c9/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6", size = 41014158, upload-time = "2025-07-18T00:56:37.528Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/26/a2865c420c50b7a3748320b614f3484bfcde8347b2639b2b903b21ce6a72/pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8", size = 42667885, upload-time = "2025-07-18T00:56:41.483Z" },
-    { url = "https://files.pythonhosted.org/packages/0a/f9/4ee798dc902533159250fb4321267730bc0a107d8c6889e07c3add4fe3a5/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503", size = 43276625, upload-time = "2025-07-18T00:56:48.002Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/da/e02544d6997037a4b0d22d8e5f66bc9315c3671371a8b18c79ade1cefe14/pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79", size = 44951890, upload-time = "2025-07-18T00:56:52.568Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006, upload-time = "2025-07-18T00:56:56.379Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/cc/ce4939f4b316457a083dc5718b3982801e8c33f921b3c98e7a93b7c7491f/pyarrow-21.0.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a7f6524e3747e35f80744537c78e7302cd41deee8baa668d56d55f77d9c464b3", size = 31211248, upload-time = "2025-07-18T00:56:59.7Z" },
-    { url = "https://files.pythonhosted.org/packages/1f/c2/7a860931420d73985e2f340f06516b21740c15b28d24a0e99a900bb27d2b/pyarrow-21.0.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:203003786c9fd253ebcafa44b03c06983c9c8d06c3145e37f1b76a1f317aeae1", size = 32676896, upload-time = "2025-07-18T00:57:03.884Z" },
-    { url = "https://files.pythonhosted.org/packages/68/a8/197f989b9a75e59b4ca0db6a13c56f19a0ad8a298c68da9cc28145e0bb97/pyarrow-21.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b4d97e297741796fead24867a8dabf86c87e4584ccc03167e4a811f50fdf74d", size = 41067862, upload-time = "2025-07-18T00:57:07.587Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/82/6ecfa89487b35aa21accb014b64e0a6b814cc860d5e3170287bf5135c7d8/pyarrow-21.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:898afce396b80fdda05e3086b4256f8677c671f7b1d27a6976fa011d3fd0a86e", size = 42747508, upload-time = "2025-07-18T00:57:13.917Z" },
-    { url = "https://files.pythonhosted.org/packages/3b/b7/ba252f399bbf3addc731e8643c05532cf32e74cebb5e32f8f7409bc243cf/pyarrow-21.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:067c66ca29aaedae08218569a114e413b26e742171f526e828e1064fcdec13f4", size = 43345293, upload-time = "2025-07-18T00:57:19.828Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/0a/a20819795bd702b9486f536a8eeb70a6aa64046fce32071c19ec8230dbaa/pyarrow-21.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0c4e75d13eb76295a49e0ea056eb18dbd87d81450bfeb8afa19a7e5a75ae2ad7", size = 45060670, upload-time = "2025-07-18T00:57:24.477Z" },
-    { url = "https://files.pythonhosted.org/packages/10/15/6b30e77872012bbfe8265d42a01d5b3c17ef0ac0f2fae531ad91b6a6c02e/pyarrow-21.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdc4c17afda4dab2a9c0b79148a43a7f4e1094916b3e18d8975bfd6d6d52241f", size = 26227521, upload-time = "2025-07-18T00:57:29.119Z" },
-]
-
-[[package]]
-name = "pyarrow"
-version = "23.0.1"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/bc/a8/24e5dc6855f50a62936ceb004e6e9645e4219a8065f304145d7fb8a79d5d/pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56", size = 34307390, upload-time = "2026-02-16T10:08:08.654Z" },
-    { url = "https://files.pythonhosted.org/packages/bc/8e/4be5617b4aaae0287f621ad31c6036e5f63118cfca0dc57d42121ff49b51/pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c", size = 35853761, upload-time = "2026-02-16T10:08:17.811Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/08/3e56a18819462210432ae37d10f5c8eed3828be1d6c751b6e6a2e93c286a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258", size = 44493116, upload-time = "2026-02-16T10:08:25.792Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/82/c40b68001dbec8a3faa4c08cd8c200798ac732d2854537c5449dc859f55a/pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2", size = 47564532, upload-time = "2026-02-16T10:08:34.27Z" },
-    { url = "https://files.pythonhosted.org/packages/20/bc/73f611989116b6f53347581b02177f9f620efdf3cd3f405d0e83cdf53a83/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5", size = 48183685, upload-time = "2026-02-16T10:08:42.889Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/cc/6c6b3ecdae2a8c3aced99956187e8302fc954cc2cca2a37cf2111dad16ce/pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222", size = 50605582, upload-time = "2026-02-16T10:08:51.641Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/94/d359e708672878d7638a04a0448edf7c707f9e5606cee11e15aaa5c7535a/pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d", size = 27521148, upload-time = "2026-02-16T10:08:58.077Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" },
-    { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" },
-    { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" },
-    { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" },
-    { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" },
-    { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" },
-    { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" },
-    { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" },
-    { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" },
-    { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" },
-    { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" },
-    { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" },
-    { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" },
-    { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" },
-    { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" },
-    { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" },
-    { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" },
-    { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" },
-    { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" },
-    { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" },
-    { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" },
-    { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" },
-    { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" },
+version = "24.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" },
+    { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" },
+    { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" },
+    { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" },
+    { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" },
+    { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" },
+    { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" },
+    { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" },
+    { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" },
+    { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" },
+    { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" },
+    { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" },
+    { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" },
+    { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" },
+    { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" },
+    { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" },
+    { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" },
+    { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" },
+    { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" },
+    { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" },
+    { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" },
+    { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" },
+    { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" },
+    { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" },
 ]
 
 [[package]]
@@ -2552,19 +2068,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" },
     { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" },
     { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" },
-    { url = "https://files.pythonhosted.org/packages/54/db/160dffb57ed9a3705c4cbcbff0ac03bdae45f1ca7d58ab74645550df3fbd/pydantic_core-2.41.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8bfeaf8735be79f225f3fefab7f941c712aaca36f1128c9d7e2352ee1aa87bdf", size = 2107999, upload-time = "2025-11-04T13:42:03.885Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/7d/88e7de946f60d9263cc84819f32513520b85c0f8322f9b8f6e4afc938383/pydantic_core-2.41.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:346285d28e4c8017da95144c7f3acd42740d637ff41946af5ce6e5e420502dd5", size = 1929745, upload-time = "2025-11-04T13:42:06.075Z" },
-    { url = "https://files.pythonhosted.org/packages/d5/c2/aef51e5b283780e85e99ff19db0f05842d2d4a8a8cd15e63b0280029b08f/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a75dafbf87d6276ddc5b2bf6fae5254e3d0876b626eb24969a574fff9149ee5d", size = 1920220, upload-time = "2025-11-04T13:42:08.457Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/97/492ab10f9ac8695cd76b2fdb24e9e61f394051df71594e9bcc891c9f586e/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7b93a4d08587e2b7e7882de461e82b6ed76d9026ce91ca7915e740ecc7855f60", size = 2067296, upload-time = "2025-11-04T13:42:10.817Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/23/984149650e5269c59a2a4c41d234a9570adc68ab29981825cfaf4cfad8f4/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8465ab91a4bd96d36dde3263f06caa6a8a6019e4113f24dc753d79a8b3a3f82", size = 2231548, upload-time = "2025-11-04T13:42:13.843Z" },
-    { url = "https://files.pythonhosted.org/packages/71/0c/85bcbb885b9732c28bec67a222dbed5ed2d77baee1f8bba2002e8cd00c5c/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:299e0a22e7ae2b85c1a57f104538b2656e8ab1873511fd718a1c1c6f149b77b5", size = 2362571, upload-time = "2025-11-04T13:42:16.208Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/4a/412d2048be12c334003e9b823a3fa3d038e46cc2d64dd8aab50b31b65499/pydantic_core-2.41.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:707625ef0983fcfb461acfaf14de2067c5942c6bb0f3b4c99158bed6fedd3cf3", size = 2068175, upload-time = "2025-11-04T13:42:18.911Z" },
-    { url = "https://files.pythonhosted.org/packages/73/f4/c58b6a776b502d0a5540ad02e232514285513572060f0d78f7832ca3c98b/pydantic_core-2.41.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f41eb9797986d6ebac5e8edff36d5cef9de40def462311b3eb3eeded1431e425", size = 2177203, upload-time = "2025-11-04T13:42:22.578Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/ae/f06ea4c7e7a9eead3d165e7623cd2ea0cb788e277e4f935af63fc98fa4e6/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0384e2e1021894b1ff5a786dbf94771e2986ebe2869533874d7e43bc79c6f504", size = 2148191, upload-time = "2025-11-04T13:42:24.89Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/57/25a11dcdc656bf5f8b05902c3c2934ac3ea296257cc4a3f79a6319e61856/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:f0cd744688278965817fd0839c4a4116add48d23890d468bc436f78beb28abf5", size = 2343907, upload-time = "2025-11-04T13:42:27.683Z" },
-    { url = "https://files.pythonhosted.org/packages/96/82/e33d5f4933d7a03327c0c43c65d575e5919d4974ffc026bc917a5f7b9f61/pydantic_core-2.41.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:753e230374206729bf0a807954bcc6c150d3743928a73faffee51ac6557a03c3", size = 2322174, upload-time = "2025-11-04T13:42:30.776Z" },
-    { url = "https://files.pythonhosted.org/packages/81/45/4091be67ce9f469e81656f880f3506f6a5624121ec5eb3eab37d7581897d/pydantic_core-2.41.5-cp39-cp39-win32.whl", hash = "sha256:873e0d5b4fb9b89ef7c2d2a963ea7d02879d9da0da8d9d4933dee8ee86a8b460", size = 1990353, upload-time = "2025-11-04T13:42:33.111Z" },
-    { url = "https://files.pythonhosted.org/packages/44/8a/a98aede18db6e9cd5d66bcacd8a409fcf8134204cdede2e7de35c5a2c5ef/pydantic_core-2.41.5-cp39-cp39-win_amd64.whl", hash = "sha256:e4f4a984405e91527a0d62649ee21138f8e3d0ef103be488c1dc11a80d7f184b", size = 2015698, upload-time = "2025-11-04T13:42:35.484Z" },
     { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" },
     { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" },
     { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" },
@@ -2605,11 +2108,9 @@ name = "pylance"
 source = { editable = "." }
 dependencies = [
     { name = "lance-namespace" },
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
-    { name = "pyarrow", version = "23.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
+    { name = "pyarrow" },
 ]
 
 [package.optional-dependencies]
@@ -2624,11 +2125,14 @@ geo = [
     { name = "geoarrow-rust-core" },
     { name = "geoarrow-rust-io" },
 ]
+otel = [
+    { name = "opentelemetry-api" },
+    { name = "opentelemetry-sdk" },
+]
 tests = [
     { name = "boto3" },
-    { name = "datafusion", marker = "python_full_version >= '3.10'" },
-    { name = "datasets", version = "0.0.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "datasets", version = "4.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "datafusion" },
+    { name = "datasets" },
     { name = "duckdb" },
     { name = "ml-dtypes" },
     { name = "pandas" },
@@ -2636,8 +2140,6 @@ tests = [
     { name = "polars", extra = ["pandas", "pyarrow"] },
     { name = "psutil" },
     { name = "pytest" },
-    { name = "tensorflow", version = "2.7.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10' and sys_platform == 'linux'" },
-    { name = "tensorflow", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and sys_platform == 'linux'" },
     { name = "tqdm" },
 ]
 torch = [
@@ -2655,30 +2157,32 @@ dev = [
 ]
 tests = [
     { name = "boto3" },
-    { name = "datafusion", marker = "python_full_version >= '3.10'" },
-    { name = "datasets", version = "4.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "datafusion" },
+    { name = "datasets" },
     { name = "duckdb" },
     { name = "ml-dtypes" },
+    { name = "opentelemetry-sdk" },
     { name = "pandas" },
     { name = "pillow" },
     { name = "polars", extra = ["pandas", "pyarrow"] },
     { name = "psutil" },
     { name = "pytest" },
-    { name = "tensorflow", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and sys_platform == 'linux'" },
     { name = "tqdm" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "boto3", marker = "extra == 'tests'" },
-    { name = "datafusion", marker = "python_full_version >= '3.10' and extra == 'tests'", specifier = ">=53,<54" },
+    { name = "datafusion", marker = "extra == 'tests'", specifier = ">=54,<55" },
     { name = "datasets", marker = "extra == 'tests'" },
     { name = "duckdb", marker = "extra == 'tests'" },
     { name = "geoarrow-rust-core", marker = "extra == 'geo'" },
     { name = "geoarrow-rust-io", marker = "extra == 'geo'" },
-    { name = "lance-namespace", specifier = ">=0.8.0,<0.9" },
+    { name = "lance-namespace", specifier = ">=0.8.5,<0.9" },
     { name = "ml-dtypes", marker = "extra == 'tests'" },
     { name = "numpy", specifier = ">=1.22" },
+    { name = "opentelemetry-api", marker = "extra == 'otel'" },
+    { name = "opentelemetry-sdk", marker = "extra == 'otel'" },
     { name = "pandas", marker = "extra == 'tests'" },
     { name = "pillow", marker = "extra == 'tests'" },
     { name = "polars", extras = ["pyarrow", "pandas"], marker = "extra == 'tests'" },
@@ -2688,11 +2192,10 @@ requires-dist = [
     { name = "pytest", marker = "extra == 'tests'" },
     { name = "pytest-benchmark", marker = "extra == 'benchmarks'" },
     { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.2" },
-    { name = "tensorflow", marker = "sys_platform == 'linux' and extra == 'tests'" },
     { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0" },
     { name = "tqdm", marker = "extra == 'tests'" },
 ]
-provides-extras = ["benchmarks", "dev", "geo", "tests", "torch"]
+provides-extras = ["benchmarks", "dev", "geo", "otel", "tests", "torch"]
 
 [package.metadata.requires-dev]
 benchmarks = [{ name = "pytest-benchmark", specifier = "==5.1.0" }]
@@ -2703,68 +2206,28 @@ dev = [
 ]
 tests = [
     { name = "boto3", specifier = "==1.40.43" },
-    { name = "datafusion", marker = "python_full_version >= '3.10'", specifier = "==53.0.0" },
-    { name = "datasets", marker = "python_full_version >= '3.10'", specifier = "==4.1.1" },
+    { name = "datafusion", specifier = "==54.0.0" },
+    { name = "datasets", specifier = "==4.1.1" },
     { name = "duckdb", specifier = "==1.4.0" },
     { name = "ml-dtypes", specifier = "==0.5.3" },
+    { name = "opentelemetry-sdk", specifier = "==1.30.0" },
     { name = "pandas", specifier = "==2.3.3" },
     { name = "pillow", specifier = "==11.3.0" },
     { name = "polars", extras = ["pyarrow", "pandas"], specifier = "==1.34.0" },
     { name = "psutil", specifier = "==7.1.0" },
     { name = "pytest", specifier = "==8.4.2" },
-    { name = "tensorflow", marker = "python_full_version >= '3.10' and sys_platform == 'linux'", specifier = "==2.20.0" },
     { name = "tqdm", specifier = "==4.67.1" },
 ]
 
-[[package]]
-name = "pyproj"
-version = "3.6.1"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-dependencies = [
-    { name = "certifi", marker = "python_full_version < '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/7d/84/2b39bbf888c753ea48b40d47511548c77aa03445465c35cc4c4e9649b643/pyproj-3.6.1.tar.gz", hash = "sha256:44aa7c704c2b7d8fb3d483bbf75af6cb2350d30a63b144279a09b75fead501bf", size = 225131, upload-time = "2023-09-21T02:07:51.593Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c5/32/63cf474f4a8d4804b3bdf7c16b8589f38142e8e2f8319dcea27e0bc21a87/pyproj-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ab7aa4d9ff3c3acf60d4b285ccec134167a948df02347585fdd934ebad8811b4", size = 6142763, upload-time = "2023-09-21T02:07:12.844Z" },
-    { url = "https://files.pythonhosted.org/packages/18/86/2e7cb9de40492f1bafbf11f4c9072edc394509a40b5e4c52f8139546f039/pyproj-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4bc0472302919e59114aa140fd7213c2370d848a7249d09704f10f5b062031fe", size = 4877123, upload-time = "2023-09-21T02:10:37.905Z" },
-    { url = "https://files.pythonhosted.org/packages/5e/c5/928d5a26995dbefbebd7507d982141cd9153bc7e4392b334fff722c4af12/pyproj-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5279586013b8d6582e22b6f9e30c49796966770389a9d5b85e25a4223286cd3f", size = 6190576, upload-time = "2023-09-21T02:17:08.637Z" },
-    { url = "https://files.pythonhosted.org/packages/f6/2b/b60cf73b0720abca313bfffef34e34f7f7dae23852b2853cf0368d49426b/pyproj-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fafd1f3eb421694857f254a9bdbacd1eb22fc6c24ca74b136679f376f97d35", size = 8328075, upload-time = "2023-09-21T02:07:15.353Z" },
-    { url = "https://files.pythonhosted.org/packages/d9/a8/7193f46032636be917bc775506ae987aad72c931b1f691b775ca812a2917/pyproj-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c41e80ddee130450dcb8829af7118f1ab69eaf8169c4bf0ee8d52b72f098dc2f", size = 5635713, upload-time = "2023-09-21T02:07:17.548Z" },
-    { url = "https://files.pythonhosted.org/packages/89/8f/27350c8fba71a37cd0d316f100fbd96bf139cc2b5ff1ab0dcbc7ac64010a/pyproj-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:db3aedd458e7f7f21d8176f0a1d924f1ae06d725228302b872885a1c34f3119e", size = 6087932, upload-time = "2023-09-21T02:07:19.793Z" },
-    { url = "https://files.pythonhosted.org/packages/84/a6/a300c1b14b2112e966e9f90b18f9c13b586bdcf417207cee913ae9005da3/pyproj-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebfbdbd0936e178091309f6cd4fcb4decd9eab12aa513cdd9add89efa3ec2882", size = 6147442, upload-time = "2023-09-21T02:07:21.879Z" },
-    { url = "https://files.pythonhosted.org/packages/30/bd/b9bd3761f08754e8dbb34c5a647db2099b348ab5da338e90980caf280e37/pyproj-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:447db19c7efad70ff161e5e46a54ab9cc2399acebb656b6ccf63e4bc4a04b97a", size = 4880331, upload-time = "2023-09-21T02:10:40.828Z" },
-    { url = "https://files.pythonhosted.org/packages/f4/0a/d82aeeb605b5d6870bc72307c3b5e044e632eb7720df8885e144f51a8eac/pyproj-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7e13c40183884ec7f94eb8e0f622f08f1d5716150b8d7a134de48c6110fee85", size = 6192425, upload-time = "2023-09-21T02:17:09.049Z" },
-    { url = "https://files.pythonhosted.org/packages/64/90/dfe5c00de1ca4dbb82606e79790659d4ed7f0ed8d372bccb3baca2a5abe0/pyproj-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65ad699e0c830e2b8565afe42bd58cc972b47d829b2e0e48ad9638386d994915", size = 8571478, upload-time = "2023-09-21T02:07:23.771Z" },
-    { url = "https://files.pythonhosted.org/packages/14/6d/ae373629a1723f0db80d7b8c93598b00d9ecb930ed9ebf4f35826a33e97c/pyproj-3.6.1-cp311-cp311-win32.whl", hash = "sha256:8b8acc31fb8702c54625f4d5a2a6543557bec3c28a0ef638778b7ab1d1772132", size = 5634575, upload-time = "2023-09-21T02:07:26.535Z" },
-    { url = "https://files.pythonhosted.org/packages/79/95/eb68113c5b5737c342bde1bab92705dabe69c16299c5a122616e50f1fbd6/pyproj-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:38a3361941eb72b82bd9a18f60c78b0df8408416f9340521df442cebfc4306e2", size = 6088494, upload-time = "2023-09-21T02:07:28.75Z" },
-    { url = "https://files.pythonhosted.org/packages/0b/64/93232511a7906a492b1b7dfdfc17f4e95982d76a24ef4f86d18cfe7ae2c9/pyproj-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1e9fbaf920f0f9b4ee62aab832be3ae3968f33f24e2e3f7fbb8c6728ef1d9746", size = 6135280, upload-time = "2023-09-21T02:07:30.911Z" },
-    { url = "https://files.pythonhosted.org/packages/10/f2/b550b1f65cc7e51c9116b220b50aade60c439103432a3fd5b12efbc77e15/pyproj-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d227a865356f225591b6732430b1d1781e946893789a609bb34f59d09b8b0f8", size = 4880030, upload-time = "2023-09-21T02:10:43.067Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/4b/2f8f6f94643b9fe2083338eff294feda84d916409b5840b7a402d2be93f8/pyproj-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83039e5ae04e5afc974f7d25ee0870a80a6bd6b7957c3aca5613ccbe0d3e72bf", size = 6184439, upload-time = "2023-09-21T02:17:43.499Z" },
-    { url = "https://files.pythonhosted.org/packages/19/9b/c57569132174786aa3f72275ac306956859a639dad0ce8d95c8411ce8209/pyproj-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb059ba3bced6f6725961ba758649261d85ed6ce670d3e3b0a26e81cf1aa8d", size = 8660747, upload-time = "2023-09-21T02:07:32.586Z" },
-    { url = "https://files.pythonhosted.org/packages/0e/ab/1c2159ec757677c5a6b8803f6be45c2b550dc42c84ec4a228dc219849bbb/pyproj-3.6.1-cp312-cp312-win32.whl", hash = "sha256:2d6ff73cc6dbbce3766b6c0bce70ce070193105d8de17aa2470009463682a8eb", size = 5626805, upload-time = "2023-09-21T02:07:35.28Z" },
-    { url = "https://files.pythonhosted.org/packages/c7/f3/2f32fe143cd7ba1d4d68f1b6dce9ca402d909cbd5a5830e3a8fa3d1acbbf/pyproj-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:7a27151ddad8e1439ba70c9b4b2b617b290c39395fa9ddb7411ebb0eb86d6fb0", size = 6079779, upload-time = "2023-09-21T02:07:37.486Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/50/d369bbe62d7a0d1e2cb40bc211da86a3f6e0f3c99f872957a72c3d5492d6/pyproj-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ba1f9b03d04d8cab24d6375609070580a26ce76eaed54631f03bab00a9c737b", size = 6144755, upload-time = "2023-09-21T02:07:39.611Z" },
-    { url = "https://files.pythonhosted.org/packages/2c/c2/8d4f61065dfed965e53badd41201ad86a05af0c1bbc75dffb12ef0f5a7dd/pyproj-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18faa54a3ca475bfe6255156f2f2874e9a1c8917b0004eee9f664b86ccc513d3", size = 4879187, upload-time = "2023-09-21T02:10:45.519Z" },
-    { url = "https://files.pythonhosted.org/packages/31/38/2cf8777cb2d5622a78195e690281b7029098795fde4751aec8128238b8bb/pyproj-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd43bd9a9b9239805f406fd82ba6b106bf4838d9ef37c167d3ed70383943ade1", size = 6192339, upload-time = "2023-09-21T02:17:09.942Z" },
-    { url = "https://files.pythonhosted.org/packages/97/0a/b1525be9680369cc06dd288e12c59d24d5798b4afcdcf1b0915836e1caa6/pyproj-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50100b2726a3ca946906cbaa789dd0749f213abf0cbb877e6de72ca7aa50e1ae", size = 8332638, upload-time = "2023-09-21T02:07:41.777Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/e8/e826e0a962f36bd925a933829cf6ef218efe2055db5ea292be40974a929d/pyproj-3.6.1-cp39-cp39-win32.whl", hash = "sha256:9274880263256f6292ff644ca92c46d96aa7e57a75c6df3f11d636ce845a1877", size = 5638159, upload-time = "2023-09-21T02:07:43.49Z" },
-    { url = "https://files.pythonhosted.org/packages/43/d0/cbe29a4dcf38ee7e72bf695d0d3f2bee21b4f22ee6cf579ad974de9edfc8/pyproj-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:36b64c2cb6ea1cc091f329c5bd34f9c01bb5da8c8e4492c709bda6a09f96808f", size = 6090565, upload-time = "2023-09-21T02:07:45.735Z" },
-    { url = "https://files.pythonhosted.org/packages/43/28/e8d2ca71dd56c27cbe668e4226963d61956cded222a2e839e6fec1ab6d82/pyproj-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd93c1a0c6c4aedc77c0fe275a9f2aba4d59b8acf88cebfc19fe3c430cfabf4f", size = 6034252, upload-time = "2023-09-21T02:07:47.906Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/39/1ce27cb86f51a1f5aed3a1617802a6131b59ea78492141d1fbe36722595e/pyproj-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6420ea8e7d2a88cb148b124429fba8cd2e0fae700a2d96eab7083c0928a85110", size = 6386263, upload-time = "2023-09-21T02:07:49.586Z" },
-]
-
 [[package]]
 name = "pyproj"
 version = "3.7.1"
 source = { registry = "https://pypi.org/simple" }
 resolution-markers = [
-    "python_full_version == '3.10.*'",
+    "python_full_version < '3.11'",
 ]
 dependencies = [
-    { name = "certifi", marker = "python_full_version == '3.10.*'" },
+    { name = "certifi", marker = "python_full_version < '3.11'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339, upload-time = "2025-02-16T04:28:46.621Z" }
 wheels = [
@@ -3000,15 +2463,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" },
     { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" },
     { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" },
-    { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" },
-    { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" },
 ]
 
 [[package]]
@@ -3016,29 +2470,16 @@ name = "requests"
 version = "2.33.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "certifi", marker = "python_full_version >= '3.10'" },
-    { name = "charset-normalizer", marker = "python_full_version >= '3.10'" },
-    { name = "idna", marker = "python_full_version >= '3.10'" },
-    { name = "urllib3", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+    { name = "certifi" },
+    { name = "charset-normalizer" },
+    { name = "idna" },
+    { name = "urllib3" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" }
 wheels = [
     { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" },
 ]
 
-[[package]]
-name = "rich"
-version = "14.1.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "markdown-it-py", marker = "python_full_version >= '3.10'" },
-    { name = "pygments", marker = "python_full_version >= '3.10'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" },
-]
-
 [[package]]
 name = "ruff"
 version = "0.11.2"
@@ -3106,152 +2547,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" },
 ]
 
-[[package]]
-name = "tensorboard"
-version = "2.20.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "absl-py" },
-    { name = "grpcio" },
-    { name = "markdown" },
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "packaging" },
-    { name = "pillow" },
-    { name = "protobuf", version = "3.19.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "protobuf", version = "6.32.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "setuptools" },
-    { name = "tensorboard-data-server" },
-    { name = "werkzeug" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" },
-]
-
-[[package]]
-name = "tensorboard-data-server"
-version = "0.7.2"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" },
-    { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" },
-]
-
-[[package]]
-name = "tensorflow"
-version = "2.7.4"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-dependencies = [
-    { name = "absl-py", marker = "python_full_version < '3.10'" },
-    { name = "astunparse", marker = "python_full_version < '3.10'" },
-    { name = "flatbuffers", version = "2.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "gast", version = "0.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "google-pasta", marker = "python_full_version < '3.10'" },
-    { name = "grpcio", marker = "python_full_version < '3.10'" },
-    { name = "h5py", marker = "python_full_version < '3.10'" },
-    { name = "keras", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "keras-preprocessing", marker = "python_full_version < '3.10'" },
-    { name = "libclang", marker = "python_full_version < '3.10'" },
-    { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "opt-einsum", marker = "python_full_version < '3.10'" },
-    { name = "protobuf", version = "3.19.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "six", marker = "python_full_version < '3.10'" },
-    { name = "tensorboard", marker = "python_full_version < '3.10'" },
-    { name = "tensorflow-estimator", marker = "python_full_version < '3.10'" },
-    { name = "tensorflow-io-gcs-filesystem", marker = "python_full_version < '3.10'" },
-    { name = "termcolor", marker = "python_full_version < '3.10'" },
-    { name = "typing-extensions", marker = "python_full_version < '3.10'" },
-    { name = "wheel", marker = "python_full_version < '3.10'" },
-    { name = "wrapt", marker = "python_full_version < '3.10'" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5e/31/d49a3dff9c4ca6e6c09c2c5fea95f58cf59cc3cd4f0d557069c7dccd6f57/tensorflow-2.7.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c4597635dd71fc6809b7fffcb462524d73e2ade09da61844059e6a2fead71140", size = 496066688, upload-time = "2022-09-02T19:11:01.631Z" },
-]
-
-[[package]]
-name = "tensorflow"
-version = "2.20.0"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
-dependencies = [
-    { name = "absl-py", marker = "python_full_version >= '3.10'" },
-    { name = "astunparse", marker = "python_full_version >= '3.10'" },
-    { name = "flatbuffers", version = "25.9.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "gast", version = "0.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "google-pasta", marker = "python_full_version >= '3.10'" },
-    { name = "grpcio", marker = "python_full_version >= '3.10'" },
-    { name = "h5py", marker = "python_full_version >= '3.10'" },
-    { name = "keras", version = "3.11.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "libclang", marker = "python_full_version >= '3.10'" },
-    { name = "ml-dtypes", marker = "python_full_version >= '3.10'" },
-    { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
-    { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "opt-einsum", marker = "python_full_version >= '3.10'" },
-    { name = "packaging", marker = "python_full_version >= '3.10'" },
-    { name = "protobuf", version = "6.32.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
-    { name = "requests", marker = "python_full_version >= '3.10'" },
-    { name = "setuptools", marker = "python_full_version >= '3.10'" },
-    { name = "six", marker = "python_full_version >= '3.10'" },
-    { name = "tensorboard", marker = "python_full_version >= '3.10'" },
-    { name = "termcolor", marker = "python_full_version >= '3.10'" },
-    { name = "typing-extensions", marker = "python_full_version >= '3.10'" },
-    { name = "wrapt", marker = "python_full_version >= '3.10'" },
-]
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122, upload-time = "2025-08-13T16:50:47.909Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510, upload-time = "2025-08-13T16:51:02.608Z" },
-    { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882, upload-time = "2025-08-13T16:51:39.546Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129, upload-time = "2025-08-13T16:51:55.104Z" },
-    { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109, upload-time = "2025-08-13T16:52:31.513Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547, upload-time = "2025-08-13T16:52:46.396Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" },
-    { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" },
-    { url = "https://files.pythonhosted.org/packages/83/ff/a26d49895586207b2704403366ef976dcaa6ed07514699dae9a4fc3fa1a9/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bc33759249c98eabcee9debd24e74506bbe29ac139e050cf0c74aa9888ebdf", size = 259307564, upload-time = "2025-08-13T16:54:17.691Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/fe/f3d738dc7c93ed5f67f9ace8dd3ed66971dab7c5a47f2d1c504ef0d0cf1d/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0deb5c583dfc53b54fd158a194ce0087b406bb6518af400ca3809735e4548ec3", size = 620427169, upload-time = "2025-08-13T16:54:33.431Z" },
-]
-
-[[package]]
-name = "tensorflow-estimator"
-version = "2.7.0"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/db/de/3a71ad41b87f9dd424e3aec3b0794a60f169fa7e9a9a1e3dd44290b86dd6/tensorflow_estimator-2.7.0-py2.py3-none-any.whl", hash = "sha256:325b5a224864379242b7b76c6987ca544239be82579d33e68ec7c2bda57abc9d", size = 463110, upload-time = "2021-10-29T23:02:47.14Z" },
-]
-
-[[package]]
-name = "tensorflow-io-gcs-filesystem"
-version = "0.37.1"
-source = { registry = "https://pypi.org/simple" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e2/19/9095c69e22c879cb3896321e676c69273a549a3148c4f62aa4bc5ebdb20f/tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8febbfcc67c61e542a5ac1a98c7c20a91a5e1afc2e14b1ef0cb7c28bc3b6aa70", size = 4842078, upload-time = "2024-07-01T23:44:18.977Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/48/47b7d25572961a48b1de3729b7a11e835b888e41e0203cca82df95d23b91/tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9679b36e3a80921876f31685ab6f7270f3411a4cc51bc2847e80d0e4b5291e27", size = 5085736, upload-time = "2024-07-01T23:44:21.034Z" },
-    { url = "https://files.pythonhosted.org/packages/de/bf/ba597d3884c77d05a78050f3c178933d69e3f80200a261df6eaa920656cd/tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e1f2796b57e799a8ca1b75bf47c2aaa437c968408cc1a402a9862929e104cda", size = 4842079, upload-time = "2024-07-01T23:44:26.825Z" },
-    { url = "https://files.pythonhosted.org/packages/66/7f/e36ae148c2f03d61ca1bff24bc13a0fef6d6825c966abef73fc6f880a23b/tensorflow_io_gcs_filesystem-0.37.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee7c8ee5fe2fd8cb6392669ef16e71841133041fee8a330eff519ad9b36e4556", size = 5085736, upload-time = "2024-07-01T23:44:28.618Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/46/962f47af08bd39fc9feb280d3192825431a91a078c856d17a78ae4884eb1/tensorflow_io_gcs_filesystem-0.37.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fbb33f1745f218464a59cecd9a18e32ca927b0f4d77abd8f8671b645cc1a182f", size = 4842077, upload-time = "2024-07-01T23:44:33.86Z" },
-    { url = "https://files.pythonhosted.org/packages/f0/9b/790d290c232bce9b691391cf16e95a96e469669c56abfb1d9d0f35fa437c/tensorflow_io_gcs_filesystem-0.37.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:286389a203a5aee1a4fa2e53718c661091aa5fea797ff4fa6715ab8436b02e6c", size = 5085733, upload-time = "2024-07-01T23:44:36.663Z" },
-    { url = "https://files.pythonhosted.org/packages/66/5f/334a011caa1eb97689274d1141df8e6b7a25e389f0390bdcd90235de9783/tensorflow_io_gcs_filesystem-0.37.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:426de1173cb81fbd62becec2012fc00322a295326d90eb6c737fab636f182aed", size = 4842075, upload-time = "2024-07-01T23:44:42.094Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/cb/7dcee55fc5a7d7d8a862e12519322851cd5fe5b086f946fd71e4ae1ef281/tensorflow_io_gcs_filesystem-0.37.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df00891669390078a003cedbdd3b8e645c718b111917535fa1d7725e95cdb95", size = 5087496, upload-time = "2024-07-01T23:44:43.797Z" },
-]
-
-[[package]]
-name = "termcolor"
-version = "3.1.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" },
-]
-
 [[package]]
 name = "tomli"
 version = "2.2.1"
@@ -3293,59 +2588,50 @@ wheels = [
 
 [[package]]
 name = "torch"
-version = "2.8.0"
+version = "2.13.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
+    { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" },
+    { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" },
     { name = "filelock" },
     { name = "fsspec" },
     { name = "jinja2" },
-    { name = "networkx", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
-    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
+    { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
     { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
-    { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
-    { name = "setuptools", marker = "python_full_version >= '3.12'" },
+    { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" },
+    { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" },
+    { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" },
+    { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" },
+    { name = "setuptools" },
     { name = "sympy" },
-    { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
+    { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" },
     { name = "typing-extensions" },
 ]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/63/28/110f7274254f1b8476c561dada127173f994afa2b1ffc044efb773c15650/torch-2.8.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0be92c08b44009d4131d1ff7a8060d10bafdb7ddcb7359ef8d8c5169007ea905", size = 102052793, upload-time = "2025-08-06T14:53:15.852Z" },
-    { url = "https://files.pythonhosted.org/packages/70/1c/58da560016f81c339ae14ab16c98153d51c941544ae568da3cb5b1ceb572/torch-2.8.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:89aa9ee820bb39d4d72b794345cccef106b574508dd17dbec457949678c76011", size = 888025420, upload-time = "2025-08-06T14:54:18.014Z" },
-    { url = "https://files.pythonhosted.org/packages/70/87/f69752d0dd4ba8218c390f0438130c166fa264a33b7025adb5014b92192c/torch-2.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e8e5bf982e87e2b59d932769938b698858c64cc53753894be25629bdf5cf2f46", size = 241363614, upload-time = "2025-08-06T14:53:31.496Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/d6/e6d4c57e61c2b2175d3aafbfb779926a2cfd7c32eeda7c543925dceec923/torch-2.8.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:a3f16a58a9a800f589b26d47ee15aca3acf065546137fc2af039876135f4c760", size = 73611154, upload-time = "2025-08-06T14:53:10.919Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/c4/3e7a3887eba14e815e614db70b3b529112d1513d9dae6f4d43e373360b7f/torch-2.8.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:220a06fd7af8b653c35d359dfe1aaf32f65aa85befa342629f716acb134b9710", size = 102073391, upload-time = "2025-08-06T14:53:20.937Z" },
-    { url = "https://files.pythonhosted.org/packages/5a/63/4fdc45a0304536e75a5e1b1bbfb1b56dd0e2743c48ee83ca729f7ce44162/torch-2.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c12fa219f51a933d5f80eeb3a7a5d0cbe9168c0a14bbb4055f1979431660879b", size = 888063640, upload-time = "2025-08-06T14:55:05.325Z" },
-    { url = "https://files.pythonhosted.org/packages/84/57/2f64161769610cf6b1c5ed782bd8a780e18a3c9d48931319f2887fa9d0b1/torch-2.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:8c7ef765e27551b2fbfc0f41bcf270e1292d9bf79f8e0724848b1682be6e80aa", size = 241366752, upload-time = "2025-08-06T14:53:38.692Z" },
-    { url = "https://files.pythonhosted.org/packages/a4/5e/05a5c46085d9b97e928f3f037081d3d2b87fb4b4195030fc099aaec5effc/torch-2.8.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:5ae0524688fb6707c57a530c2325e13bb0090b745ba7b4a2cd6a3ce262572916", size = 73621174, upload-time = "2025-08-06T14:53:25.44Z" },
-    { url = "https://files.pythonhosted.org/packages/49/0c/2fd4df0d83a495bb5e54dca4474c4ec5f9c62db185421563deeb5dabf609/torch-2.8.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e2fab4153768d433f8ed9279c8133a114a034a61e77a3a104dcdf54388838705", size = 101906089, upload-time = "2025-08-06T14:53:52.631Z" },
-    { url = "https://files.pythonhosted.org/packages/99/a8/6acf48d48838fb8fe480597d98a0668c2beb02ee4755cc136de92a0a956f/torch-2.8.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b2aca0939fb7e4d842561febbd4ffda67a8e958ff725c1c27e244e85e982173c", size = 887913624, upload-time = "2025-08-06T14:56:44.33Z" },
-    { url = "https://files.pythonhosted.org/packages/af/8a/5c87f08e3abd825c7dfecef5a0f1d9aa5df5dd0e3fd1fa2f490a8e512402/torch-2.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f4ac52f0130275d7517b03a33d2493bab3693c83dcfadf4f81688ea82147d2e", size = 241326087, upload-time = "2025-08-06T14:53:46.503Z" },
-    { url = "https://files.pythonhosted.org/packages/be/66/5c9a321b325aaecb92d4d1855421e3a055abd77903b7dab6575ca07796db/torch-2.8.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:619c2869db3ada2c0105487ba21b5008defcc472d23f8b80ed91ac4a380283b0", size = 73630478, upload-time = "2025-08-06T14:53:57.144Z" },
-    { url = "https://files.pythonhosted.org/packages/10/4e/469ced5a0603245d6a19a556e9053300033f9c5baccf43a3d25ba73e189e/torch-2.8.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2b2f96814e0345f5a5aed9bf9734efa913678ed19caf6dc2cddb7930672d6128", size = 101936856, upload-time = "2025-08-06T14:54:01.526Z" },
-    { url = "https://files.pythonhosted.org/packages/16/82/3948e54c01b2109238357c6f86242e6ecbf0c63a1af46906772902f82057/torch-2.8.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:65616ca8ec6f43245e1f5f296603e33923f4c30f93d65e103d9e50c25b35150b", size = 887922844, upload-time = "2025-08-06T14:55:50.78Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/54/941ea0a860f2717d86a811adf0c2cd01b3983bdd460d0803053c4e0b8649/torch-2.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:659df54119ae03e83a800addc125856effda88b016dfc54d9f65215c3975be16", size = 241330968, upload-time = "2025-08-06T14:54:45.293Z" },
-    { url = "https://files.pythonhosted.org/packages/de/69/8b7b13bba430f5e21d77708b616f767683629fc4f8037564a177d20f90ed/torch-2.8.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:1a62a1ec4b0498930e2543535cf70b1bef8c777713de7ceb84cd79115f553767", size = 73915128, upload-time = "2025-08-06T14:54:34.769Z" },
-    { url = "https://files.pythonhosted.org/packages/15/0e/8a800e093b7f7430dbaefa80075aee9158ec22e4c4fc3c1a66e4fb96cb4f/torch-2.8.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:83c13411a26fac3d101fe8035a6b0476ae606deb8688e904e796a3534c197def", size = 102020139, upload-time = "2025-08-06T14:54:39.047Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/15/5e488ca0bc6162c86a33b58642bc577c84ded17c7b72d97e49b5833e2d73/torch-2.8.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:8f0a9d617a66509ded240add3754e462430a6c1fc5589f86c17b433dd808f97a", size = 887990692, upload-time = "2025-08-06T14:56:18.286Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/a8/6a04e4b54472fc5dba7ca2341ab219e529f3c07b6941059fbf18dccac31f/torch-2.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a7242b86f42be98ac674b88a4988643b9bc6145437ec8f048fea23f72feb5eca", size = 241603453, upload-time = "2025-08-06T14:55:22.945Z" },
-    { url = "https://files.pythonhosted.org/packages/04/6e/650bb7f28f771af0cb791b02348db8b7f5f64f40f6829ee82aa6ce99aabe/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7b677e17f5a3e69fdef7eb3b9da72622f8d322692930297e4ccb52fefc6c8211", size = 73632395, upload-time = "2025-08-06T14:55:28.645Z" },
-    { url = "https://files.pythonhosted.org/packages/5b/b0/a321f27270049baa12f5c3fb0d6ceea005634787e3af9a8d75dce8306b0a/torch-2.8.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:da6afa31c13b669d4ba49d8a2169f0db2c3ec6bec4af898aa714f401d4c38904", size = 102059214, upload-time = "2025-08-06T14:55:33.433Z" },
-    { url = "https://files.pythonhosted.org/packages/fd/dd/1630cb51b10d3d2e97db95e5a84c32def81fc26b005bce6fc880b0e6db81/torch-2.8.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:06fcee8000e5c62a9f3e52a688b9c5abb7c6228d0e56e3452983416025c41381", size = 888024302, upload-time = "2025-08-06T14:57:28.23Z" },
-    { url = "https://files.pythonhosted.org/packages/b9/dc/1f1f621afe15e3c496e1e8f94f8903f75f87e7d642d5a985e92210cc208d/torch-2.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5128fe752a355d9308e56af1ad28b15266fe2da5948660fad44de9e3a9e36e8c", size = 241249338, upload-time = "2025-08-06T14:57:05.669Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/95/ae26263aceb3d57b821179f827d0e321373ed49423e603dd5906ab14a730/torch-2.8.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e9f071f5b52a9f6970dc8a919694b27a91ae9dc08898b2b988abbef5eddfd1ae", size = 73610795, upload-time = "2025-08-06T14:57:11.513Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/e7/19894fdb51c7dbaf94f5a79bb0871da0992e8e4241e579cb006da46d2e58/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", size = 111178962, upload-time = "2026-07-08T16:05:49.855Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/5c/b1d5de470c54e339b30a92d96683a71bcebd78f5f2a7fc714cd6dc6bbd68/torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045", size = 427198333, upload-time = "2026-07-08T16:05:36.847Z" },
+    { url = "https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4", size = 526555154, upload-time = "2026-07-08T16:05:06.507Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/c9/0bb9d097b03cbaf96bb75b15e867347b8e41bfcdfe0539452d17d9e63993/torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb", size = 122015602, upload-time = "2026-07-08T16:05:45.25Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" },
+    { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" },
+    { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/fd/0f2ce40f58aefbdb3392f9acce3c8171940943ae2d661f70558bfa73befb/torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330", size = 122015870, upload-time = "2026-07-08T16:05:27.59Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" },
+    { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" },
+    { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" },
+    { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" },
+    { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" },
+    { url = "https://files.pythonhosted.org/packages/36/76/6dcc7f0c07052102dd36f83cbc5800842a909c8c3fbf1a7f8a5844954de9/torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c", size = 111227066, upload-time = "2026-07-08T16:03:33.6Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" },
+    { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/0c/7d1deb6bce5bc3e6042caf39100ac768eba3b9a098e1dddd16f75bd6489b/torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd", size = 122051871, upload-time = "2026-07-08T16:03:23.521Z" },
+    { url = "https://files.pythonhosted.org/packages/f4/ce/aa8b7f9949d32e0f2f624f342bc3b48112c1b8a130288465938bc83bcbf9/torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1", size = 111537025, upload-time = "2026-07-08T16:02:44.28Z" },
+    { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" },
+    { url = "https://files.pythonhosted.org/packages/56/94/655c91992a882bd5071aa0b5d22a07dbb130d801e872be97c0b627a7c693/torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8", size = 122306773, upload-time = "2026-07-08T16:02:39.832Z" },
 ]
 
 [[package]]
@@ -3362,19 +2648,21 @@ wheels = [
 
 [[package]]
 name = "triton"
-version = "3.4.0"
+version = "3.7.1"
 source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
-    { name = "setuptools" },
-]
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/62/ee/0ee5f64a87eeda19bbad9bc54ae5ca5b98186ed00055281fd40fb4beb10e/triton-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7ff2785de9bc02f500e085420273bb5cc9c9bb767584a4aa28d6e360cec70128", size = 155430069, upload-time = "2025-07-30T19:58:21.715Z" },
-    { url = "https://files.pythonhosted.org/packages/7d/39/43325b3b651d50187e591eefa22e236b2981afcebaefd4f2fc0ea99df191/triton-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b70f5e6a41e52e48cfc087436c8a28c17ff98db369447bcaff3b887a3ab4467", size = 155531138, upload-time = "2025-07-30T19:58:29.908Z" },
-    { url = "https://files.pythonhosted.org/packages/d0/66/b1eb52839f563623d185f0927eb3530ee4d5ffe9d377cdaf5346b306689e/triton-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c1d84a5c0ec2c0f8e8a072d7fd150cab84a9c239eaddc6706c081bfae4eb04", size = 155560068, upload-time = "2025-07-30T19:58:37.081Z" },
-    { url = "https://files.pythonhosted.org/packages/30/7b/0a685684ed5322d2af0bddefed7906674f67974aa88b0fae6e82e3b766f6/triton-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00be2964616f4c619193cb0d1b29a99bd4b001d7dc333816073f92cf2a8ccdeb", size = 155569223, upload-time = "2025-07-30T19:58:44.017Z" },
-    { url = "https://files.pythonhosted.org/packages/20/63/8cb444ad5cdb25d999b7d647abac25af0ee37d292afc009940c05b82dda0/triton-3.4.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7936b18a3499ed62059414d7df563e6c163c5e16c3773678a3ee3d417865035d", size = 155659780, upload-time = "2025-07-30T19:58:51.171Z" },
-    { url = "https://files.pythonhosted.org/packages/12/34/1251beb5a3cb93f3950ebe68732752014646003ef6eb11eb5f1a37ca78cd/triton-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98e5c1442eaeabae2e2452ae765801bd53cd4ce873cab0d1bdd59a32ab2d9397", size = 155430799, upload-time = "2025-07-30T19:58:57.664Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/ea/629cc37436ca5df93ce98956d09cd2ca1498bfee8ef4972d2fe48b9f958c/triton-3.7.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3daf64305d6cea88d3334c65ebc9bcd0c64c9564a977084366aa768d57cbcf64", size = 184551013, upload-time = "2026-06-17T20:03:37.551Z" },
+    { url = "https://files.pythonhosted.org/packages/15/76/c79c34311625227a288df3e483fc5cdf3d596624cbd4b4758c4cbdc14af3/triton-3.7.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee89fbf782ec2ad50391dd1cf26cbea4f4467154c37f4773026da8fc31c0f58e", size = 197596267, upload-time = "2026-06-17T19:53:06.898Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/f9/19d842d06a08559534fa1eaab6ca551b1bcf40f06620bddec1babaa2772d/triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6", size = 184664887, upload-time = "2026-06-17T20:03:42.913Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/5e/fce69606f7f240297f163e25539906732b199530d486ce67ae319877e821/triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5", size = 197701306, upload-time = "2026-06-17T19:53:13.673Z" },
+    { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" },
+    { url = "https://files.pythonhosted.org/packages/00/42/c5089d4d9327fcd1e862c599cc2927f39418f84dd11a84cb2ccff9d4787a/triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a", size = 184694629, upload-time = "2026-06-17T20:03:53.444Z" },
+    { url = "https://files.pythonhosted.org/packages/07/42/2c3ac59253ae8892b6f307875263dd23dc875cdf732d3aea40d6d41fb7cb/triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb", size = 197729241, upload-time = "2026-06-17T19:53:27.801Z" },
+    { url = "https://files.pythonhosted.org/packages/40/71/e01aa7ad573883ed9456f130226babdec70b005e098c4d6226a6238e761b/triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa", size = 184705764, upload-time = "2026-06-17T20:03:59.064Z" },
+    { url = "https://files.pythonhosted.org/packages/a4/09/5683146fda6a2b569deb78ccfd8fbfea8bfe55f726b081c0a6bb18dd6f28/triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2", size = 197729537, upload-time = "2026-06-17T19:53:35.516Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/f8/448220c3092019f9fdfab39ec47985968181d67da34b44f6a7f6280a5cbb/triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7", size = 184814760, upload-time = "2026-06-17T20:04:04.984Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/ac/229b7d4589d2e5937310e72c6d46e89599d16a4a12b479ffa1499fee8eb8/triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68", size = 197824404, upload-time = "2026-06-17T19:53:42.772Z" },
 ]
 
 [[package]]
@@ -3407,89 +2695,81 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
 ]
 
-[[package]]
-name = "urllib3"
-version = "1.26.20"
-source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version < '3.10'",
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380, upload-time = "2024-08-29T15:43:11.37Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225, upload-time = "2024-08-29T15:43:08.921Z" },
-]
-
 [[package]]
 name = "urllib3"
 version = "2.5.0"
 source = { registry = "https://pypi.org/simple" }
-resolution-markers = [
-    "python_full_version >= '3.14'",
-    "python_full_version == '3.13.*'",
-    "python_full_version == '3.12.*'",
-    "python_full_version == '3.11.*'",
-    "python_full_version == '3.10.*'",
-]
 sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" }
 wheels = [
     { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" },
 ]
 
-[[package]]
-name = "werkzeug"
-version = "3.1.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "markupsafe" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
-]
-
-[[package]]
-name = "wheel"
-version = "0.45.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" },
-]
-
 [[package]]
 name = "wrapt"
 version = "1.17.3"
 source = { registry = "https://pypi.org/simple" }
 sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" }
 wheels = [
+    { url = "https://files.pythonhosted.org/packages/3f/23/bb82321b86411eb51e5a5db3fb8f8032fd30bd7c2d74bfe936136b2fa1d6/wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04", size = 53482, upload-time = "2025-08-12T05:51:44.467Z" },
+    { url = "https://files.pythonhosted.org/packages/45/69/f3c47642b79485a30a59c63f6d739ed779fb4cc8323205d047d741d55220/wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2", size = 38676, upload-time = "2025-08-12T05:51:32.636Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/71/e7e7f5670c1eafd9e990438e69d8fb46fa91a50785332e06b560c869454f/wrapt-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c", size = 38957, upload-time = "2025-08-12T05:51:54.655Z" },
     { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" },
     { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" },
     { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" },
     { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/83/ed6baf89ba3a56694700139698cf703aac9f0f9eb03dab92f57551bd5385/wrapt-1.17.3-cp310-cp310-win32.whl", hash = "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390", size = 36464, upload-time = "2025-08-12T05:53:01.204Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/90/ee61d36862340ad7e9d15a02529df6b948676b9a5829fd5e16640156627d/wrapt-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6", size = 38748, upload-time = "2025-08-12T05:53:00.209Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/c3/cefe0bd330d389c9983ced15d326f45373f4073c9f4a8c2f99b50bfea329/wrapt-1.17.3-cp310-cp310-win_arm64.whl", hash = "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18", size = 36810, upload-time = "2025-08-12T05:52:51.906Z" },
+    { url = "https://files.pythonhosted.org/packages/52/db/00e2a219213856074a213503fdac0511203dceefff26e1daa15250cc01a0/wrapt-1.17.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7", size = 53482, upload-time = "2025-08-12T05:51:45.79Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/30/ca3c4a5eba478408572096fe9ce36e6e915994dd26a4e9e98b4f729c06d9/wrapt-1.17.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85", size = 38674, upload-time = "2025-08-12T05:51:34.629Z" },
+    { url = "https://files.pythonhosted.org/packages/31/25/3e8cc2c46b5329c5957cec959cb76a10718e1a513309c31399a4dad07eb3/wrapt-1.17.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f", size = 38959, upload-time = "2025-08-12T05:51:56.074Z" },
     { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" },
     { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" },
     { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" },
     { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/59/e0adfc831674a65694f18ea6dc821f9fcb9ec82c2ce7e3d73a88ba2e8718/wrapt-1.17.3-cp311-cp311-win32.whl", hash = "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89", size = 36457, upload-time = "2025-08-12T05:53:03.936Z" },
+    { url = "https://files.pythonhosted.org/packages/83/88/16b7231ba49861b6f75fc309b11012ede4d6b0a9c90969d9e0db8d991aeb/wrapt-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77", size = 38745, upload-time = "2025-08-12T05:53:02.885Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/1e/c4d4f3398ec073012c51d1c8d87f715f56765444e1a4b11e5180577b7e6e/wrapt-1.17.3-cp311-cp311-win_arm64.whl", hash = "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a", size = 36806, upload-time = "2025-08-12T05:52:53.368Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/41/cad1aba93e752f1f9268c77270da3c469883d56e2798e7df6240dcb2287b/wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0", size = 53998, upload-time = "2025-08-12T05:51:47.138Z" },
+    { url = "https://files.pythonhosted.org/packages/60/f8/096a7cc13097a1869fe44efe68dace40d2a16ecb853141394047f0780b96/wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba", size = 39020, upload-time = "2025-08-12T05:51:35.906Z" },
+    { url = "https://files.pythonhosted.org/packages/33/df/bdf864b8997aab4febb96a9ae5c124f700a5abd9b5e13d2a3214ec4be705/wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd", size = 39098, upload-time = "2025-08-12T05:51:57.474Z" },
     { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" },
     { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" },
     { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" },
     { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/b1/43b286ca1392a006d5336412d41663eeef1ad57485f3e52c767376ba7e5a/wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe", size = 36705, upload-time = "2025-08-12T05:53:07.123Z" },
+    { url = "https://files.pythonhosted.org/packages/28/de/49493f962bd3c586ab4b88066e967aa2e0703d6ef2c43aa28cb83bf7b507/wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c", size = 38877, upload-time = "2025-08-12T05:53:05.436Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/48/0f7102fe9cb1e8a5a77f80d4f0956d62d97034bbe88d33e94699f99d181d/wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6", size = 36885, upload-time = "2025-08-12T05:52:54.367Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/f6/759ece88472157acb55fc195e5b116e06730f1b651b5b314c66291729193/wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0", size = 54003, upload-time = "2025-08-12T05:51:48.627Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/a9/49940b9dc6d47027dc850c116d79b4155f15c08547d04db0f07121499347/wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77", size = 39025, upload-time = "2025-08-12T05:51:37.156Z" },
+    { url = "https://files.pythonhosted.org/packages/45/35/6a08de0f2c96dcdd7fe464d7420ddb9a7655a6561150e5fc4da9356aeaab/wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7", size = 39108, upload-time = "2025-08-12T05:51:58.425Z" },
     { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" },
     { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" },
     { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" },
     { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" },
+    { url = "https://files.pythonhosted.org/packages/e8/cf/7d848740203c7b4b27eb55dbfede11aca974a51c3d894f6cc4b865f42f58/wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8", size = 36711, upload-time = "2025-08-12T05:53:10.074Z" },
+    { url = "https://files.pythonhosted.org/packages/57/54/35a84d0a4d23ea675994104e667ceff49227ce473ba6a59ba2c84f250b74/wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb", size = 38885, upload-time = "2025-08-12T05:53:08.695Z" },
+    { url = "https://files.pythonhosted.org/packages/01/77/66e54407c59d7b02a3c4e0af3783168fff8e5d61def52cda8728439d86bc/wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16", size = 36896, upload-time = "2025-08-12T05:52:55.34Z" },
+    { url = "https://files.pythonhosted.org/packages/02/a2/cd864b2a14f20d14f4c496fab97802001560f9f41554eef6df201cd7f76c/wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39", size = 54132, upload-time = "2025-08-12T05:51:49.864Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/46/d011725b0c89e853dc44cceb738a307cde5d240d023d6d40a82d1b4e1182/wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235", size = 39091, upload-time = "2025-08-12T05:51:38.935Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/9e/3ad852d77c35aae7ddebdbc3b6d35ec8013af7d7dddad0ad911f3d891dae/wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c", size = 39172, upload-time = "2025-08-12T05:51:59.365Z" },
     { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" },
     { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" },
     { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" },
     { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/b8/da8560695e9284810b8d3df8a19396a6e40e7518059584a1a394a2b35e0a/wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10", size = 37178, upload-time = "2025-08-12T05:53:12.605Z" },
+    { url = "https://files.pythonhosted.org/packages/db/c8/b71eeb192c440d67a5a0449aaee2310a1a1e8eca41676046f99ed2487e9f/wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6", size = 39310, upload-time = "2025-08-12T05:53:11.106Z" },
+    { url = "https://files.pythonhosted.org/packages/45/20/2cda20fd4865fa40f86f6c46ed37a2a8356a7a2fde0773269311f2af56c7/wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58", size = 37266, upload-time = "2025-08-12T05:52:56.531Z" },
+    { url = "https://files.pythonhosted.org/packages/77/ed/dd5cf21aec36c80443c6f900449260b80e2a65cf963668eaef3b9accce36/wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a", size = 56544, upload-time = "2025-08-12T05:51:51.109Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/96/450c651cc753877ad100c7949ab4d2e2ecc4d97157e00fa8f45df682456a/wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067", size = 40283, upload-time = "2025-08-12T05:51:39.912Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/86/2fcad95994d9b572db57632acb6f900695a648c3e063f2cd344b3f5c5a37/wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454", size = 40366, upload-time = "2025-08-12T05:52:00.693Z" },
     { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" },
     { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" },
     { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" },
     { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" },
-    { url = "https://files.pythonhosted.org/packages/43/46/dd0791943613885f62619f18ee6107e6133237a6b6ed8a9ecfac339d0b4f/wrapt-1.17.3-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e18f01b0c3e4a07fe6dfdb00e29049ba17eadbc5e7609a2a3a4af83ab7d710a", size = 81745, upload-time = "2025-08-12T05:52:49.62Z" },
-    { url = "https://files.pythonhosted.org/packages/dd/ec/bb2d19bd1a614cc4f438abac13ae26c57186197920432d2a915183b15a8b/wrapt-1.17.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f5f51a6466667a5a356e6381d362d259125b57f059103dd9fdc8c0cf1d14139", size = 82833, upload-time = "2025-08-12T05:52:27.738Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/eb/66579aea6ad36f07617fedca8e282e49c7c9bab64c63b446cfe4f7f47a49/wrapt-1.17.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:59923aa12d0157f6b82d686c3fd8e1166fa8cdfb3e17b42ce3b6147ff81528df", size = 81889, upload-time = "2025-08-12T05:52:29.023Z" },
-    { url = "https://files.pythonhosted.org/packages/04/9c/a56b5ac0e2473bdc3fb11b22dd69ff423154d63861cf77911cdde5e38fd2/wrapt-1.17.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:46acc57b331e0b3bcb3e1ca3b421d65637915cfcd65eb783cb2f78a511193f9b", size = 81344, upload-time = "2025-08-12T05:52:50.869Z" },
+    { url = "https://files.pythonhosted.org/packages/be/44/a1bd64b723d13bb151d6cc91b986146a1952385e0392a78567e12149c7b4/wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977", size = 38717, upload-time = "2025-08-12T05:53:15.214Z" },
+    { url = "https://files.pythonhosted.org/packages/79/d9/7cfd5a312760ac4dd8bf0184a6ee9e43c33e47f3dadc303032ce012b8fa3/wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116", size = 41334, upload-time = "2025-08-12T05:53:14.178Z" },
+    { url = "https://files.pythonhosted.org/packages/46/78/10ad9781128ed2f99dbc474f43283b13fea8ba58723e98844367531c18e9/wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6", size = 38471, upload-time = "2025-08-12T05:52:57.784Z" },
     { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" },
 ]
 
@@ -3604,21 +2884,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" },
     { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" },
     { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" },
-    { url = "https://files.pythonhosted.org/packages/03/ff/1b4bb3f397552116c1df6266c1b83a21aeeb26061ab1f462984b499a3870/xxhash-3.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc604dc06027dbeb8281aeac5899c35fcfe7c77b25212833709f0bff4ce74d2a", size = 32844, upload-time = "2025-10-02T14:36:39.157Z" },
-    { url = "https://files.pythonhosted.org/packages/c1/db/27146d0bee4346a9a31f7b498a81fc02747f6f1e6c52a2e7989504278051/xxhash-3.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:277175a73900ad43a8caeb8b99b9604f21fe8d7c842f2f9061a364a7e220ddb7", size = 30806, upload-time = "2025-10-02T14:36:40.621Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/2b/4896188df564908817a75de19bf7f2384b99a75af2d528f9c49326f76458/xxhash-3.6.0-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cfbc5b91397c8c2972fdac13fb3e4ed2f7f8ccac85cd2c644887557780a9b6e2", size = 193448, upload-time = "2025-10-02T14:36:41.797Z" },
-    { url = "https://files.pythonhosted.org/packages/51/c5/be8953f62e772340319a826ce1e07489935600089756cf83b628cd36ebe3/xxhash-3.6.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2762bfff264c4e73c0e507274b40634ff465e025f0eaf050897e88ec8367575d", size = 212547, upload-time = "2025-10-02T14:36:43.581Z" },
-    { url = "https://files.pythonhosted.org/packages/51/1a/1e9f0b911d1cf00dd537c074ae3fae15b535a7f0d9e7edd42a9d2c4f78ce/xxhash-3.6.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f171a900d59d51511209f7476933c34a0c2c711078d3c80e74e0fe4f38680ec", size = 211309, upload-time = "2025-10-02T14:36:45.307Z" },
-    { url = "https://files.pythonhosted.org/packages/63/88/b284c6a128d88dc47f201957f926e707db79fb7415a87072e15c0e490de0/xxhash-3.6.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:780b90c313348f030b811efc37b0fa1431163cb8db8064cf88a7936b6ce5f222", size = 444480, upload-time = "2025-10-02T14:36:47.226Z" },
-    { url = "https://files.pythonhosted.org/packages/87/e4/798293a2bf9e4fac5f6d53ce59cba4739930778dfc6c7c73f40044ab0e6e/xxhash-3.6.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b242455eccdfcd1fa4134c431a30737d2b4f045770f8fe84356b3469d4b919", size = 192957, upload-time = "2025-10-02T14:36:48.968Z" },
-    { url = "https://files.pythonhosted.org/packages/78/55/bfd0d7db447a927897469048b953caececa3532e743b940dd1f5c1032d24/xxhash-3.6.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a75ffc1bd5def584129774c158e108e5d768e10b75813f2b32650bb041066ed6", size = 209850, upload-time = "2025-10-02T14:36:50.258Z" },
-    { url = "https://files.pythonhosted.org/packages/31/06/d08ef9a792bfebfd2fb2bcbf04a541ad283bef74749ead6f089a0809d288/xxhash-3.6.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1fc1ed882d1e8df932a66e2999429ba6cc4d5172914c904ab193381fba825360", size = 197342, upload-time = "2025-10-02T14:36:51.651Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/1a/aebf90797c94e9ca407c28e23f54d71f7149d91a93406a08a09e44d06994/xxhash-3.6.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:44e342e8cc11b4e79dae5c57f2fb6360c3c20cc57d32049af8f567f5b4bcb5f4", size = 209757, upload-time = "2025-10-02T14:36:53.009Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/80/799eec3d0a144dc3edf8c19b4f139c27fb923c50b34352796089ca206429/xxhash-3.6.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c2f9ccd5c4be370939a2e17602fbc49995299203da72a3429db013d44d590e86", size = 412773, upload-time = "2025-10-02T14:36:54.691Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/f9/09df7545699de09219a205123b8463ce9ea83f48acc7aeeba0269507f9d3/xxhash-3.6.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:02ea4cb627c76f48cd9fb37cf7ab22bd51e57e1b519807234b473faebe526796", size = 190357, upload-time = "2025-10-02T14:36:56.363Z" },
-    { url = "https://files.pythonhosted.org/packages/07/40/2f8327f94e64a3f34d6ce3347c55207c322abbc80ae486ea45df4c62e7b3/xxhash-3.6.0-cp39-cp39-win32.whl", hash = "sha256:6551880383f0e6971dc23e512c9ccc986147ce7bfa1cd2e4b520b876c53e9f3d", size = 30585, upload-time = "2025-10-02T14:36:57.664Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/c8/2ecbc6799be9c02e8bf7b5a66cd94832b6ac13d59808746f0d402481c6ad/xxhash-3.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:7c35c4cdc65f2a29f34425c446f2f5cdcd0e3c34158931e1cc927ece925ab802", size = 31512, upload-time = "2025-10-02T14:36:58.837Z" },
-    { url = "https://files.pythonhosted.org/packages/19/94/1d5459a9c587c94d7b8bcc710bd08bbfa145cbd814ebde41b48494362a21/xxhash-3.6.0-cp39-cp39-win_arm64.whl", hash = "sha256:ffc578717a347baf25be8397cb10d2528802d24f94cfc005c0e44fef44b5cdd6", size = 27878, upload-time = "2025-10-02T14:37:00.201Z" },
     { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" },
     { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" },
     { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" },
@@ -3631,9 +2896,9 @@ name = "yarl"
 version = "1.20.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "idna", marker = "python_full_version >= '3.10'" },
-    { name = "multidict", marker = "python_full_version >= '3.10'" },
-    { name = "propcache", marker = "python_full_version >= '3.10'" },
+    { name = "idna" },
+    { name = "multidict" },
+    { name = "propcache" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" }
 wheels = [
@@ -3722,31 +2987,14 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" },
     { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" },
     { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" },
-    { url = "https://files.pythonhosted.org/packages/01/75/0d37402d208d025afa6b5b8eb80e466d267d3fd1927db8e317d29a94a4cb/yarl-1.20.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e42ba79e2efb6845ebab49c7bf20306c4edf74a0b20fc6b2ccdd1a219d12fad3", size = 134259, upload-time = "2025-06-10T00:45:29.882Z" },
-    { url = "https://files.pythonhosted.org/packages/73/84/1fb6c85ae0cf9901046f07d0ac9eb162f7ce6d95db541130aa542ed377e6/yarl-1.20.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:41493b9b7c312ac448b7f0a42a089dffe1d6e6e981a2d76205801a023ed26a2b", size = 91269, upload-time = "2025-06-10T00:45:32.917Z" },
-    { url = "https://files.pythonhosted.org/packages/f3/9c/eae746b24c4ea29a5accba9a06c197a70fa38a49c7df244e0d3951108861/yarl-1.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f5a5928ff5eb13408c62a968ac90d43f8322fd56d87008b8f9dabf3c0f6ee983", size = 89995, upload-time = "2025-06-10T00:45:35.066Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/30/693e71003ec4bc1daf2e4cf7c478c417d0985e0a8e8f00b2230d517876fc/yarl-1.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30c41ad5d717b3961b2dd785593b67d386b73feca30522048d37298fee981805", size = 325253, upload-time = "2025-06-10T00:45:37.052Z" },
-    { url = "https://files.pythonhosted.org/packages/0f/a2/5264dbebf90763139aeb0b0b3154763239398400f754ae19a0518b654117/yarl-1.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:59febc3969b0781682b469d4aca1a5cab7505a4f7b85acf6db01fa500fa3f6ba", size = 320897, upload-time = "2025-06-10T00:45:39.962Z" },
-    { url = "https://files.pythonhosted.org/packages/e7/17/77c7a89b3c05856489777e922f41db79ab4faf58621886df40d812c7facd/yarl-1.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2b6fb3622b7e5bf7a6e5b679a69326b4279e805ed1699d749739a61d242449e", size = 340696, upload-time = "2025-06-10T00:45:41.915Z" },
-    { url = "https://files.pythonhosted.org/packages/6d/55/28409330b8ef5f2f681f5b478150496ec9cf3309b149dab7ec8ab5cfa3f0/yarl-1.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:749d73611db8d26a6281086f859ea7ec08f9c4c56cec864e52028c8b328db723", size = 335064, upload-time = "2025-06-10T00:45:43.893Z" },
-    { url = "https://files.pythonhosted.org/packages/85/58/cb0257cbd4002828ff735f44d3c5b6966c4fd1fc8cc1cd3cd8a143fbc513/yarl-1.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9427925776096e664c39e131447aa20ec738bdd77c049c48ea5200db2237e000", size = 327256, upload-time = "2025-06-10T00:45:46.393Z" },
-    { url = "https://files.pythonhosted.org/packages/53/f6/c77960370cfa46f6fb3d6a5a79a49d3abfdb9ef92556badc2dcd2748bc2a/yarl-1.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff70f32aa316393eaf8222d518ce9118148eddb8a53073c2403863b41033eed5", size = 316389, upload-time = "2025-06-10T00:45:48.358Z" },
-    { url = "https://files.pythonhosted.org/packages/64/ab/be0b10b8e029553c10905b6b00c64ecad3ebc8ace44b02293a62579343f6/yarl-1.20.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c7ddf7a09f38667aea38801da8b8d6bfe81df767d9dfc8c88eb45827b195cd1c", size = 340481, upload-time = "2025-06-10T00:45:50.663Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/c3/3f327bd3905a4916029bf5feb7f86dcf864c7704f099715f62155fb386b2/yarl-1.20.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:57edc88517d7fc62b174fcfb2e939fbc486a68315d648d7e74d07fac42cec240", size = 336941, upload-time = "2025-06-10T00:45:52.554Z" },
-    { url = "https://files.pythonhosted.org/packages/d1/42/040bdd5d3b3bb02b4a6ace4ed4075e02f85df964d6e6cb321795d2a6496a/yarl-1.20.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:dab096ce479d5894d62c26ff4f699ec9072269d514b4edd630a393223f45a0ee", size = 339936, upload-time = "2025-06-10T00:45:54.919Z" },
-    { url = "https://files.pythonhosted.org/packages/0d/1c/911867b8e8c7463b84dfdc275e0d99b04b66ad5132b503f184fe76be8ea4/yarl-1.20.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:14a85f3bd2d7bb255be7183e5d7d6e70add151a98edf56a770d6140f5d5f4010", size = 360163, upload-time = "2025-06-10T00:45:56.87Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/31/8c389f6c6ca0379b57b2da87f1f126c834777b4931c5ee8427dd65d0ff6b/yarl-1.20.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c89b5c792685dd9cd3fa9761c1b9f46fc240c2a3265483acc1565769996a3f8", size = 359108, upload-time = "2025-06-10T00:45:58.869Z" },
-    { url = "https://files.pythonhosted.org/packages/7f/09/ae4a649fb3964324c70a3e2b61f45e566d9ffc0affd2b974cbf628957673/yarl-1.20.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:69e9b141de5511021942a6866990aea6d111c9042235de90e08f94cf972ca03d", size = 351875, upload-time = "2025-06-10T00:46:01.45Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/43/bbb4ed4c34d5bb62b48bf957f68cd43f736f79059d4f85225ab1ef80f4b9/yarl-1.20.1-cp39-cp39-win32.whl", hash = "sha256:b5f307337819cdfdbb40193cad84978a029f847b0a357fbe49f712063cfc4f06", size = 82293, upload-time = "2025-06-10T00:46:03.763Z" },
-    { url = "https://files.pythonhosted.org/packages/d7/cd/ce185848a7dba68ea69e932674b5c1a42a1852123584bccc5443120f857c/yarl-1.20.1-cp39-cp39-win_amd64.whl", hash = "sha256:eae7bfe2069f9c1c5b05fc7fe5d612e5bbc089a39309904ee8b829e322dcad00", size = 87385, upload-time = "2025-06-10T00:46:05.655Z" },
     { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" },
 ]
 
 [[package]]
 name = "zipp"
-version = "3.23.0"
+version = "4.1.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" },
 ]
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 5699bd4d536..c4058187023 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 # We keep this pinned to keep clippy and rustfmt in sync between local and CI.
 # Feel free to upgrade to bring in new lints.
 [toolchain]
-channel = "1.94.0"
+channel = "1.97.0"
 components = ["rustfmt", "clippy", "rust-analyzer"]
diff --git a/rust/AGENTS.md b/rust/AGENTS.md
index 6b2729c6692..70a803c6c76 100644
--- a/rust/AGENTS.md
+++ b/rust/AGENTS.md
@@ -17,6 +17,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards.
 - Delete obsolete internal (`pub(crate)` / private) methods in the same PR that introduces their replacements. For public API methods, follow the deprecation path in root AGENTS.md instead.
 - Choose log levels by audience: `debug!` for routine/high-frequency ops, `info!` for infrequent operator-visible state changes, `warn!` for unexpected conditions.
 
+## Concurrency
+
+- The closure passed to `spawn_cpu()` must only consume CPU and return — it must **never** wait on anything: **no channels** (blocking send/recv), **no I/O**, **no locks**, and no `block_on`/`.blocking_*`. The CPU pool can collapse to a single worker in resource-constrained environments (`<= 3` CPUs), so a parked closure can deadlock the whole pool with a silent 0% hang. Keep the waiting in surrounding async code and hand only the pure-CPU work to `spawn_cpu()`. Only dispatch substantial work (rule of thumb: ~100µs+ of CPU); below that the pool overhead outweighs the benefit and the work is better left inline. See the doc comment on `spawn_cpu` for the rationale.
+
 ## API Design
 
 - Use `with_`-prefixed builder methods for optional config (e.g., `MyStruct::new(required).with_option(v)`) — don't create separate constructor variants.
diff --git a/rust/compression/bitpacking/Cargo.toml b/rust/compression/bitpacking/Cargo.toml
index f086ede8baa..56db321ac6d 100644
--- a/rust/compression/bitpacking/Cargo.toml
+++ b/rust/compression/bitpacking/Cargo.toml
@@ -13,8 +13,12 @@ rust-version.workspace = true
 
 [dependencies]
 arrayref = "0.3"
+crunchy = "0.2"
 paste = "1"
 seq-macro = "0.3"
 
+[dev-dependencies]
+bitpacking = { workspace = true, features = ["bitpacker4x"] }
+
 [lints]
 workspace = true
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker4x.rs b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker4x.rs
new file mode 100644
index 00000000000..ca3c68908ef
--- /dev/null
+++ b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker4x.rs
@@ -0,0 +1,820 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use super::BitPacker;
+
+use crate::bitpacker_internal::{Available, UnsafeBitPacker};
+
+const BLOCK_LEN: usize = 32 * 4;
+
+#[cfg(target_arch = "x86_64")]
+mod sse3 {
+
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+
+    use std::arch::x86_64::__m128i as DataType;
+    use std::arch::x86_64::_mm_and_si128 as op_and;
+    use std::arch::x86_64::_mm_lddqu_si128 as load_unaligned;
+    use std::arch::x86_64::_mm_or_si128 as op_or;
+    use std::arch::x86_64::_mm_set1_epi32 as set1;
+    use std::arch::x86_64::_mm_slli_epi32 as left_shift_32;
+    use std::arch::x86_64::_mm_srli_epi32 as right_shift_32;
+    use std::arch::x86_64::_mm_storeu_si128 as store_unaligned;
+    use std::arch::x86_64::{
+        _mm_add_epi32, _mm_cvtsi128_si32, _mm_shuffle_epi32, _mm_slli_si128, _mm_srli_si128,
+        _mm_sub_epi32,
+    };
+
+    #[allow(non_snake_case)]
+    #[inline]
+    unsafe fn or_collapse_to_u32(accumulator: DataType) -> u32 {
+        let a__b__c__d_ = accumulator;
+        let ______a__b_ = _mm_srli_si128(a__b__c__d_, 8);
+        let a__b__ca_db = op_or(a__b__c__d_, ______a__b_);
+        let ___a__b__ca = _mm_srli_si128(a__b__ca_db, 4);
+        let _______cadb = op_or(a__b__ca_db, ___a__b__ca);
+        _mm_cvtsi128_si32(_______cadb) as u32
+    }
+
+    #[target_feature(enable = "sse3")]
+    unsafe fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        _mm_sub_epi32(
+            curr,
+            op_or(_mm_slli_si128(curr, 4), _mm_srli_si128(prev, 12)),
+        )
+    }
+
+    #[target_feature(enable = "sse3")]
+    #[allow(non_snake_case)]
+    #[inline]
+    unsafe fn integrate_delta(prev: DataType, delta: DataType) -> DataType {
+        let offset = _mm_shuffle_epi32(prev, 0xff);
+        let a__b__c__d_ = delta;
+        let ______a__b_ = _mm_slli_si128(delta, 8);
+        let a__b__ca_db = _mm_add_epi32(______a__b_, a__b__c__d_);
+        let ___a__b__ca = _mm_slli_si128(a__b__ca_db, 4);
+        let a_ab_abc_abcd: DataType = _mm_add_epi32(___a__b__ca, a__b__ca_db);
+        _mm_add_epi32(offset, a_ab_abc_abcd)
+    }
+
+    #[target_feature(enable = "sse3")]
+    #[inline]
+    unsafe fn add(left: DataType, right: DataType) -> DataType {
+        _mm_add_epi32(left, right)
+    }
+
+    unsafe fn sub(left: DataType, right: DataType) -> DataType {
+        _mm_sub_epi32(left, right)
+    }
+
+    declare_bitpacker!(target_feature(enable = "sse3"));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            is_x86_feature_detected!("sse3")
+        }
+    }
+}
+
+#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+mod neon {
+
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+    use std::arch::aarch64::{
+        uint32x4_t, vaddq_u32, vandq_u32, vdupq_n_u32, vextq_u32, vgetq_lane_u32, vld1q_u32,
+        vorrq_u32, vshlq_n_u32, vshrq_n_u32, vst1q_u32, vsubq_u32,
+    };
+
+    pub(crate) type DataType = uint32x4_t;
+
+    #[inline]
+    /// Creates a vector with all elements set to `el`.
+    unsafe fn set1(el: i32) -> DataType {
+        vdupq_n_u32(el as u32)
+    }
+
+    #[inline]
+    unsafe fn right_shift_32<const N: i32>(el: DataType) -> DataType {
+        const {
+            assert!(N >= 0);
+            assert!(N <= 32);
+        }
+
+        // We unroll here because vshrq_n_u32 only accepts constants from 1 to 32.
+        match N {
+            0 => el,
+            1 => vshrq_n_u32::<1>(el),
+            2 => vshrq_n_u32::<2>(el),
+            3 => vshrq_n_u32::<3>(el),
+            4 => vshrq_n_u32::<4>(el),
+            5 => vshrq_n_u32::<5>(el),
+            6 => vshrq_n_u32::<6>(el),
+            7 => vshrq_n_u32::<7>(el),
+            8 => vshrq_n_u32::<8>(el),
+            9 => vshrq_n_u32::<9>(el),
+            10 => vshrq_n_u32::<10>(el),
+            11 => vshrq_n_u32::<11>(el),
+            12 => vshrq_n_u32::<12>(el),
+            13 => vshrq_n_u32::<13>(el),
+            14 => vshrq_n_u32::<14>(el),
+            15 => vshrq_n_u32::<15>(el),
+            16 => vshrq_n_u32::<16>(el),
+            17 => vshrq_n_u32::<17>(el),
+            18 => vshrq_n_u32::<18>(el),
+            19 => vshrq_n_u32::<19>(el),
+            20 => vshrq_n_u32::<20>(el),
+            21 => vshrq_n_u32::<21>(el),
+            22 => vshrq_n_u32::<22>(el),
+            23 => vshrq_n_u32::<23>(el),
+            24 => vshrq_n_u32::<24>(el),
+            25 => vshrq_n_u32::<25>(el),
+            26 => vshrq_n_u32::<26>(el),
+            27 => vshrq_n_u32::<27>(el),
+            28 => vshrq_n_u32::<28>(el),
+            29 => vshrq_n_u32::<29>(el),
+            30 => vshrq_n_u32::<30>(el),
+            31 => vshrq_n_u32::<31>(el),
+            32 => vdupq_n_u32(0),
+            _ => core::hint::unreachable_unchecked(),
+        }
+    }
+
+    #[inline]
+    unsafe fn left_shift_32<const N: i32>(el: DataType) -> DataType {
+        const {
+            assert!(N >= 0);
+            assert!(N <= 32);
+        }
+
+        // We unroll here because vshlq_n_u32 only accepts constants from 0 to 31.
+        match N {
+            0 => el,
+            1 => vshlq_n_u32::<1>(el),
+            2 => vshlq_n_u32::<2>(el),
+            3 => vshlq_n_u32::<3>(el),
+            4 => vshlq_n_u32::<4>(el),
+            5 => vshlq_n_u32::<5>(el),
+            6 => vshlq_n_u32::<6>(el),
+            7 => vshlq_n_u32::<7>(el),
+            8 => vshlq_n_u32::<8>(el),
+            9 => vshlq_n_u32::<9>(el),
+            10 => vshlq_n_u32::<10>(el),
+            11 => vshlq_n_u32::<11>(el),
+            12 => vshlq_n_u32::<12>(el),
+            13 => vshlq_n_u32::<13>(el),
+            14 => vshlq_n_u32::<14>(el),
+            15 => vshlq_n_u32::<15>(el),
+            16 => vshlq_n_u32::<16>(el),
+            17 => vshlq_n_u32::<17>(el),
+            18 => vshlq_n_u32::<18>(el),
+            19 => vshlq_n_u32::<19>(el),
+            20 => vshlq_n_u32::<20>(el),
+            21 => vshlq_n_u32::<21>(el),
+            22 => vshlq_n_u32::<22>(el),
+            23 => vshlq_n_u32::<23>(el),
+            24 => vshlq_n_u32::<24>(el),
+            25 => vshlq_n_u32::<25>(el),
+            26 => vshlq_n_u32::<26>(el),
+            27 => vshlq_n_u32::<27>(el),
+            28 => vshlq_n_u32::<28>(el),
+            29 => vshlq_n_u32::<29>(el),
+            30 => vshlq_n_u32::<30>(el),
+            31 => vshlq_n_u32::<31>(el),
+            32 => vdupq_n_u32(0),
+            _ => core::hint::unreachable_unchecked(),
+        }
+    }
+
+    use vorrq_u32 as op_or;
+
+    #[inline]
+    unsafe fn op_and(left: DataType, right: DataType) -> DataType {
+        vandq_u32(left, right)
+    }
+
+    #[inline]
+    unsafe fn load_unaligned(addr: *const DataType) -> DataType {
+        vld1q_u32(addr.cast::<u32>())
+    }
+
+    #[inline]
+    unsafe fn store_unaligned(addr: *mut DataType, data: DataType) {
+        vst1q_u32(addr.cast::<u32>(), data);
+    }
+
+    #[inline]
+    /// Collapses the vector by performing a bitwise OR across all lanes
+    unsafe fn or_collapse_to_u32(acc: DataType) -> u32 {
+        vgetq_lane_u32(acc, 0)
+            | vgetq_lane_u32(acc, 1)
+            | vgetq_lane_u32(acc, 2)
+            | vgetq_lane_u32(acc, 3)
+    }
+
+    unsafe fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        // Build a vector with [prev[3], curr[0], curr[1], curr[2]]
+        let prev_shifted = vextq_u32(prev, curr, 3);
+        vsubq_u32(curr, prev_shifted)
+    }
+
+    #[allow(non_snake_case)]
+    #[inline]
+    unsafe fn integrate_delta(prev: DataType, delta: DataType) -> DataType {
+        let base = vdupq_n_u32(vgetq_lane_u32(prev, 3));
+        let zero = vdupq_n_u32(0);
+        let a__b__c__d_ = delta;
+        let ______a__b_ = vextq_u32(zero, a__b__c__d_, 2);
+        let a__b__ca_db = vaddq_u32(______a__b_, a__b__c__d_);
+        let ___a__b__ca = vextq_u32(zero, a__b__ca_db, 3);
+        let a_ab_abc_abcd = vaddq_u32(___a__b__ca, a__b__ca_db);
+        vaddq_u32(base, a_ab_abc_abcd)
+    }
+
+    #[inline]
+    unsafe fn add(left: DataType, right: DataType) -> DataType {
+        vaddq_u32(left, right)
+    }
+
+    #[inline]
+    unsafe fn sub(left: DataType, right: DataType) -> DataType {
+        vsubq_u32(left, right)
+    }
+
+    declare_bitpacker!(target_feature(enable = "neon"));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            std::arch::is_aarch64_feature_detected!("neon")
+        }
+    }
+}
+
+mod scalar {
+
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+    use std::ptr;
+
+    pub(crate) type DataType = [u32; 4];
+
+    pub(crate) fn set1(el: i32) -> DataType {
+        [el as u32; 4]
+    }
+
+    pub(crate) fn right_shift_32<const N: i32>(el: DataType) -> DataType {
+        [el[0] >> N, el[1] >> N, el[2] >> N, el[3] >> N]
+    }
+
+    pub(crate) fn left_shift_32<const N: i32>(el: DataType) -> DataType {
+        [el[0] << N, el[1] << N, el[2] << N, el[3] << N]
+    }
+
+    pub(crate) fn op_or(left: DataType, right: DataType) -> DataType {
+        [
+            left[0] | right[0],
+            left[1] | right[1],
+            left[2] | right[2],
+            left[3] | right[3],
+        ]
+    }
+
+    pub(crate) fn op_and(left: DataType, right: DataType) -> DataType {
+        [
+            left[0] & right[0],
+            left[1] & right[1],
+            left[2] & right[2],
+            left[3] & right[3],
+        ]
+    }
+
+    pub(crate) unsafe fn load_unaligned(addr: *const DataType) -> DataType {
+        ptr::read_unaligned(addr)
+    }
+
+    pub(crate) unsafe fn store_unaligned(addr: *mut DataType, data: DataType) {
+        ptr::write_unaligned(addr, data);
+    }
+
+    pub(crate) fn or_collapse_to_u32(accumulator: DataType) -> u32 {
+        (accumulator[0] | accumulator[1]) | (accumulator[2] | accumulator[3])
+    }
+
+    fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        [
+            curr[0].wrapping_sub(prev[3]),
+            curr[1].wrapping_sub(curr[0]),
+            curr[2].wrapping_sub(curr[1]),
+            curr[3].wrapping_sub(curr[2]),
+        ]
+    }
+
+    fn integrate_delta(offset: DataType, delta: DataType) -> DataType {
+        let el0 = offset[3].wrapping_add(delta[0]);
+        let el1 = el0.wrapping_add(delta[1]);
+        let el2 = el1.wrapping_add(delta[2]);
+        let el3 = el2.wrapping_add(delta[3]);
+        [el0, el1, el2, el3]
+    }
+
+    pub(crate) fn add(left: DataType, right: DataType) -> DataType {
+        [
+            left[0].wrapping_add(right[0]),
+            left[1].wrapping_add(right[1]),
+            left[2].wrapping_add(right[2]),
+            left[3].wrapping_add(right[3]),
+        ]
+    }
+
+    pub(crate) fn sub(left: DataType, right: DataType) -> DataType {
+        [
+            left[0].wrapping_sub(right[0]),
+            left[1].wrapping_sub(right[1]),
+            left[2].wrapping_sub(right[2]),
+            left[3].wrapping_sub(right[3]),
+        ]
+    }
+
+    // The `allow(unused)` is here to put an attribute that has no effect.
+    //
+    // For other bitpacker, we enable specific CPU instruction set, but for the
+    // scalar bitpacker none is required.
+    declare_bitpacker!(allow(unused));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            true
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum InstructionSet {
+    #[cfg(target_arch = "x86_64")]
+    SSE3,
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    NEON,
+    Scalar,
+}
+
+/// `BitPacker4x` packs integers in groups of 4. This gives an opportunity
+/// to leverage `SSE3` instructions to encode and decode the stream.
+///
+/// One block must contain `128 integers`.
+#[derive(Clone, Copy)]
+pub struct BitPacker4x(InstructionSet);
+
+impl BitPacker4x {
+    #[cfg(target_arch = "x86_64")]
+    pub(crate) fn new_sse() -> Option<Self> {
+        sse3::UnsafeBitPackerImpl::available().then_some(BitPacker4x(InstructionSet::SSE3))
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    pub(crate) fn new_sse() -> Option<Self> {
+        None
+    }
+
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    pub(crate) fn new_neon() -> Option<Self> {
+        neon::UnsafeBitPackerImpl::available().then_some(BitPacker4x(InstructionSet::NEON))
+    }
+
+    #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))]
+    pub(crate) fn new_neon() -> Option<Self> {
+        None
+    }
+
+    pub(crate) fn new_scalar() -> Self {
+        BitPacker4x(InstructionSet::Scalar)
+    }
+}
+
+impl BitPacker for BitPacker4x {
+    const BLOCK_LEN: usize = BLOCK_LEN;
+
+    fn new() -> Self {
+        Self::new_sse()
+            .or_else(Self::new_neon)
+            .unwrap_or_else(Self::new_scalar)
+    }
+
+    fn compress(&self, decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => {
+                    sse3::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+            }
+        }
+    }
+
+    fn compress_sorted(
+        &self,
+        initial: u32,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => sse3::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn compress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => sse3::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn decompress(&self, compressed: &[u8], decompressed: &mut [u32], num_bits: u8) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => {
+                    sse3::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+            }
+        }
+    }
+
+    fn decompress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => sse3::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn decompress_sorted(
+        &self,
+        initial: u32,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => sse3::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn num_bits(&self, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => sse3::UnsafeBitPackerImpl::num_bits(decompressed),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::num_bits(decompressed),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::num_bits(decompressed),
+            }
+        }
+    }
+
+    fn num_bits_sorted(&self, initial: u32, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => {
+                    sse3::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+            }
+        }
+    }
+
+    fn num_bits_strictly_sorted(&self, initial: Option<u32>, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::SSE3 => {
+                    sse3::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+            }
+        }
+    }
+}
+
+#[cfg(any(
+    target_arch = "x86_64",
+    all(target_arch = "aarch64", target_endian = "little")
+))]
+#[cfg(any())]
+mod tests {
+    use super::BLOCK_LEN;
+    use super::scalar;
+    use crate::bitpacker_internal::Available;
+    use crate::tests::test_util_compatible;
+    use crate::{BitPacker, BitPacker4x};
+
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_compatible_sse3() {
+        use super::sse3;
+        if sse3::UnsafeBitPackerImpl::available() {
+            test_util_compatible::<scalar::UnsafeBitPackerImpl, sse3::UnsafeBitPackerImpl>(
+                BLOCK_LEN,
+            );
+        }
+    }
+
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    #[test]
+    fn test_compatible_neon() {
+        use super::neon;
+        if neon::UnsafeBitPackerImpl::available() {
+            test_util_compatible::<scalar::UnsafeBitPackerImpl, neon::UnsafeBitPackerImpl>(
+                BLOCK_LEN,
+            );
+        }
+    }
+
+    #[test]
+    fn test_delta_bit_width_32() {
+        let values = vec![i32::max_value() as u32 + 1; BitPacker4x::BLOCK_LEN];
+        let bit_packer = BitPacker4x::new();
+        let bit_width = bit_packer.num_bits_sorted(0, &values);
+        assert_eq!(bit_width, 32);
+
+        let mut block = vec![0u8; BitPacker4x::compressed_block_size(bit_width)];
+        bit_packer.compress_sorted(0, &values, &mut block, bit_width);
+
+        let mut decoded_values = vec![0x10101010; BitPacker4x::BLOCK_LEN];
+        bit_packer.decompress_sorted(0, &block, &mut decoded_values, bit_width);
+
+        assert_eq!(values, decoded_values);
+    }
+
+    #[test]
+    fn test_bit_width_32() {
+        let mut values = vec![i32::max_value() as u32 + 1; BitPacker4x::BLOCK_LEN];
+        values[0] = 0;
+        let bit_packer = BitPacker4x::new();
+        let bit_width = bit_packer.num_bits(&values);
+        assert_eq!(bit_width, 32);
+
+        let mut block = vec![0u8; BitPacker4x::compressed_block_size(bit_width)];
+        bit_packer.compress(&values, &mut block, bit_width);
+
+        let mut decoded_values = vec![0x10101010; BitPacker4x::BLOCK_LEN];
+        bit_packer.decompress(&block, &mut decoded_values, bit_width);
+
+        assert_eq!(values, decoded_values);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{BLOCK_LEN, BitPacker4x};
+    use crate::bitpacker_internal::BitPacker;
+    use bitpacking::{BitPacker as ExternalBitPacker, BitPacker4x as ExternalBitPacker4x};
+
+    fn mask_for_width(width: u8) -> u32 {
+        match width {
+            0 => 0,
+            32 => u32::MAX,
+            _ => (1u32 << width) - 1,
+        }
+    }
+
+    fn raw_values(width: u8) -> Vec<u32> {
+        let mask = mask_for_width(width);
+        (0..BLOCK_LEN)
+            .map(|idx| ((idx * 17 + 3) as u32) & mask)
+            .collect()
+    }
+
+    fn sorted_values(width: u8) -> (u32, Vec<u32>) {
+        if width == 0 {
+            return (11, vec![11; BLOCK_LEN]);
+        }
+        if width == 32 {
+            return (0, vec![u32::MAX; BLOCK_LEN]);
+        }
+
+        let mask = mask_for_width(width).min(127);
+        let initial = 11u32;
+        let mut current = initial;
+        let values = (0..BLOCK_LEN)
+            .map(|idx| {
+                current += (idx as u32 * 7 + 1) & mask;
+                current
+            })
+            .collect();
+        (initial, values)
+    }
+
+    fn strictly_sorted_values(width: u8) -> (Option<u32>, Vec<u32>) {
+        let mask = mask_for_width(width).min(127);
+        let mut current = 0u32;
+        let values = (0..BLOCK_LEN)
+            .map(|idx| {
+                if idx == 0 {
+                    current = 0;
+                } else {
+                    current += 1 + ((idx as u32 * 5) & mask);
+                }
+                current
+            })
+            .collect();
+        (None, values)
+    }
+
+    #[test]
+    fn scalar_backend_matches_external_bitpacker4x() {
+        let scalar = BitPacker4x::new_scalar();
+        let external = ExternalBitPacker4x::new();
+
+        for width in 0..=32 {
+            let values = raw_values(width);
+            assert_eq!(scalar.num_bits(&values), external.num_bits(&values));
+
+            let mut actual = vec![0u8; BitPacker4x::compressed_block_size(width)];
+            let actual_len = scalar.compress(&values, &mut actual, width);
+
+            let mut expected = vec![0u8; ExternalBitPacker4x::compressed_block_size(width)];
+            let expected_len = external.compress(&values, &mut expected, width);
+
+            assert_eq!(actual_len, expected_len);
+            assert_eq!(actual, expected, "raw width {width}");
+
+            let mut decoded = vec![0u32; BLOCK_LEN];
+            assert_eq!(scalar.decompress(&actual, &mut decoded, width), actual_len);
+            assert_eq!(decoded, values);
+
+            let (initial, values) = sorted_values(width);
+            assert_eq!(
+                scalar.num_bits_sorted(initial, &values),
+                external.num_bits_sorted(initial, &values)
+            );
+
+            let mut actual = vec![0u8; BitPacker4x::compressed_block_size(width)];
+            let actual_len = scalar.compress_sorted(initial, &values, &mut actual, width);
+
+            let mut expected = vec![0u8; ExternalBitPacker4x::compressed_block_size(width)];
+            let expected_len = external.compress_sorted(initial, &values, &mut expected, width);
+
+            assert_eq!(actual_len, expected_len);
+            assert_eq!(actual, expected, "sorted width {width}");
+
+            let mut decoded = vec![0u32; BLOCK_LEN];
+            assert_eq!(
+                scalar.decompress_sorted(initial, &actual, &mut decoded, width),
+                actual_len
+            );
+            assert_eq!(decoded, values);
+        }
+    }
+
+    #[test]
+    fn scalar_backend_matches_external_strictly_sorted_bitpacker4x() {
+        let scalar = BitPacker4x::new_scalar();
+        let external = ExternalBitPacker4x::new();
+
+        for width in 0..=16 {
+            let (initial, values) = strictly_sorted_values(width);
+            let num_bits = external.num_bits_strictly_sorted(initial, &values);
+            assert_eq!(scalar.num_bits_strictly_sorted(initial, &values), num_bits);
+
+            let mut actual = vec![0u8; BitPacker4x::compressed_block_size(num_bits)];
+            let actual_len =
+                scalar.compress_strictly_sorted(initial, &values, &mut actual, num_bits);
+
+            let mut expected = vec![0u8; ExternalBitPacker4x::compressed_block_size(num_bits)];
+            let expected_len =
+                external.compress_strictly_sorted(initial, &values, &mut expected, num_bits);
+
+            assert_eq!(actual_len, expected_len);
+            assert_eq!(actual, expected, "strict width {width}");
+
+            let mut decoded = vec![0u32; BLOCK_LEN];
+            assert_eq!(
+                scalar.decompress_strictly_sorted(initial, &actual, &mut decoded, num_bits),
+                actual_len
+            );
+            assert_eq!(decoded, values);
+        }
+    }
+}
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs
new file mode 100644
index 00000000000..b17edacabb2
--- /dev/null
+++ b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs
@@ -0,0 +1,872 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use super::BitPacker;
+
+use crate::bitpacker_internal::{Available, UnsafeBitPacker};
+
+const BLOCK_LEN: usize = 32 * 8;
+
+#[cfg(target_arch = "x86_64")]
+mod avx2 {
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+
+    use std::arch::x86_64::__m256i as DataType;
+    use std::arch::x86_64::_mm256_and_si256 as op_and;
+    use std::arch::x86_64::_mm256_lddqu_si256 as load_unaligned;
+    use std::arch::x86_64::_mm256_or_si256 as op_or;
+    use std::arch::x86_64::_mm256_set1_epi32 as set1;
+    use std::arch::x86_64::_mm256_slli_epi32 as left_shift_32;
+    use std::arch::x86_64::_mm256_srli_epi32 as right_shift_32;
+    use std::arch::x86_64::_mm256_storeu_si256 as store_unaligned;
+
+    use std::arch::x86_64::{
+        _mm256_add_epi32, _mm256_extract_epi32, _mm256_permute2f128_si256, _mm256_shuffle_epi32,
+        _mm256_slli_si256, _mm256_srli_si256, _mm256_sub_epi32,
+    };
+
+    #[allow(non_snake_case)]
+    unsafe fn or_collapse_to_u32(accumulator: DataType) -> u32 {
+        let a__b__c__d__e__f__g__h_ = accumulator;
+        let ______a__b________e__f = _mm256_srli_si256(a__b__c__d__e__f__g__h_, 8);
+        let a__b__ca_db_e__f__ge_hf = op_or(a__b__c__d__e__f__g__h_, ______a__b________e__f);
+        let ___a__b__ca____e__f__ge = _mm256_srli_si256(a__b__ca_db_e__f__ge_hf, 4);
+        let _________cadb______gehf = op_or(a__b__ca_db_e__f__ge_hf, ___a__b__ca____e__f__ge);
+        let cadb = _mm256_extract_epi32(_________cadb______gehf, 0);
+        let gehf = _mm256_extract_epi32(_________cadb______gehf, 4);
+        (cadb | gehf) as u32
+    }
+
+    unsafe fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        let left_shift = _mm256_slli_si256(curr, 4);
+        let curr_shift = _mm256_srli_si256(curr, 12);
+        let curr_right_only = _mm256_permute2f128_si256(curr_shift, curr_shift, 8);
+        let prev_shift = _mm256_srli_si256(prev, 12);
+        let sub_left = _mm256_permute2f128_si256(prev_shift, prev_shift, 3 | (8 << 4));
+        let diff = op_or(left_shift, op_or(curr_right_only, sub_left));
+        _mm256_sub_epi32(curr, diff)
+    }
+
+    #[allow(non_snake_case)]
+    unsafe fn integrate_delta(prev: DataType, delta: DataType) -> DataType {
+        let offset_repeat = _mm256_shuffle_epi32(prev, 0xff);
+        let offset = _mm256_permute2f128_si256(offset_repeat, offset_repeat, 3 | (8 << 4));
+        let a__b__c__d__e__f__g__h__ = delta;
+        let ______a__b________e__f__ = _mm256_slli_si256(delta, 8);
+        let a__b__ca_db_e__f__ge_fh_ =
+            _mm256_add_epi32(a__b__c__d__e__f__g__h__, ______a__b________e__f__);
+        let ___a__b__ca____e__f__ge_ = _mm256_slli_si256(a__b__ca_db_e__f__ge_fh_, 4);
+        let halved_prefix_sum =
+            _mm256_add_epi32(___a__b__ca____e__f__ge_, a__b__ca_db_e__f__ge_fh_);
+        let offsetted_halved_prefix_sum = _mm256_add_epi32(halved_prefix_sum, offset);
+        let select_last_low = _mm256_shuffle_epi32(offsetted_halved_prefix_sum, 0xff);
+        let high_offset = _mm256_permute2f128_si256(select_last_low, select_last_low, 8);
+        _mm256_add_epi32(high_offset, offsetted_halved_prefix_sum)
+    }
+
+    unsafe fn add(left: DataType, right: DataType) -> DataType {
+        _mm256_add_epi32(left, right)
+    }
+
+    unsafe fn sub(left: DataType, right: DataType) -> DataType {
+        _mm256_sub_epi32(left, right)
+    }
+
+    declare_bitpacker!(target_feature(enable = "avx2"));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            is_x86_feature_detected!("avx2")
+        }
+    }
+}
+
+#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+mod neon {
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+    use std::arch::aarch64::{
+        uint32x4_t, vaddq_u32, vandq_u32, vdupq_n_u32, vextq_u32, vgetq_lane_u32, vld1q_u32,
+        vorrq_u32, vshlq_n_u32, vshrq_n_u32, vst1q_u32, vsubq_u32,
+    };
+
+    pub(crate) type DataType = [uint32x4_t; 2];
+
+    #[inline]
+    unsafe fn set1(el: i32) -> DataType {
+        let lanes = vdupq_n_u32(el as u32);
+        [lanes, lanes]
+    }
+
+    #[inline]
+    unsafe fn right_shift_32<const N: i32>(el: DataType) -> DataType {
+        const {
+            assert!(N >= 0);
+            assert!(N <= 32);
+        }
+
+        match N {
+            0 => el,
+            1 => [vshrq_n_u32::<1>(el[0]), vshrq_n_u32::<1>(el[1])],
+            2 => [vshrq_n_u32::<2>(el[0]), vshrq_n_u32::<2>(el[1])],
+            3 => [vshrq_n_u32::<3>(el[0]), vshrq_n_u32::<3>(el[1])],
+            4 => [vshrq_n_u32::<4>(el[0]), vshrq_n_u32::<4>(el[1])],
+            5 => [vshrq_n_u32::<5>(el[0]), vshrq_n_u32::<5>(el[1])],
+            6 => [vshrq_n_u32::<6>(el[0]), vshrq_n_u32::<6>(el[1])],
+            7 => [vshrq_n_u32::<7>(el[0]), vshrq_n_u32::<7>(el[1])],
+            8 => [vshrq_n_u32::<8>(el[0]), vshrq_n_u32::<8>(el[1])],
+            9 => [vshrq_n_u32::<9>(el[0]), vshrq_n_u32::<9>(el[1])],
+            10 => [vshrq_n_u32::<10>(el[0]), vshrq_n_u32::<10>(el[1])],
+            11 => [vshrq_n_u32::<11>(el[0]), vshrq_n_u32::<11>(el[1])],
+            12 => [vshrq_n_u32::<12>(el[0]), vshrq_n_u32::<12>(el[1])],
+            13 => [vshrq_n_u32::<13>(el[0]), vshrq_n_u32::<13>(el[1])],
+            14 => [vshrq_n_u32::<14>(el[0]), vshrq_n_u32::<14>(el[1])],
+            15 => [vshrq_n_u32::<15>(el[0]), vshrq_n_u32::<15>(el[1])],
+            16 => [vshrq_n_u32::<16>(el[0]), vshrq_n_u32::<16>(el[1])],
+            17 => [vshrq_n_u32::<17>(el[0]), vshrq_n_u32::<17>(el[1])],
+            18 => [vshrq_n_u32::<18>(el[0]), vshrq_n_u32::<18>(el[1])],
+            19 => [vshrq_n_u32::<19>(el[0]), vshrq_n_u32::<19>(el[1])],
+            20 => [vshrq_n_u32::<20>(el[0]), vshrq_n_u32::<20>(el[1])],
+            21 => [vshrq_n_u32::<21>(el[0]), vshrq_n_u32::<21>(el[1])],
+            22 => [vshrq_n_u32::<22>(el[0]), vshrq_n_u32::<22>(el[1])],
+            23 => [vshrq_n_u32::<23>(el[0]), vshrq_n_u32::<23>(el[1])],
+            24 => [vshrq_n_u32::<24>(el[0]), vshrq_n_u32::<24>(el[1])],
+            25 => [vshrq_n_u32::<25>(el[0]), vshrq_n_u32::<25>(el[1])],
+            26 => [vshrq_n_u32::<26>(el[0]), vshrq_n_u32::<26>(el[1])],
+            27 => [vshrq_n_u32::<27>(el[0]), vshrq_n_u32::<27>(el[1])],
+            28 => [vshrq_n_u32::<28>(el[0]), vshrq_n_u32::<28>(el[1])],
+            29 => [vshrq_n_u32::<29>(el[0]), vshrq_n_u32::<29>(el[1])],
+            30 => [vshrq_n_u32::<30>(el[0]), vshrq_n_u32::<30>(el[1])],
+            31 => [vshrq_n_u32::<31>(el[0]), vshrq_n_u32::<31>(el[1])],
+            32 => set1(0),
+            _ => core::hint::unreachable_unchecked(),
+        }
+    }
+
+    #[inline]
+    unsafe fn left_shift_32<const N: i32>(el: DataType) -> DataType {
+        const {
+            assert!(N >= 0);
+            assert!(N <= 32);
+        }
+
+        match N {
+            0 => el,
+            1 => [vshlq_n_u32::<1>(el[0]), vshlq_n_u32::<1>(el[1])],
+            2 => [vshlq_n_u32::<2>(el[0]), vshlq_n_u32::<2>(el[1])],
+            3 => [vshlq_n_u32::<3>(el[0]), vshlq_n_u32::<3>(el[1])],
+            4 => [vshlq_n_u32::<4>(el[0]), vshlq_n_u32::<4>(el[1])],
+            5 => [vshlq_n_u32::<5>(el[0]), vshlq_n_u32::<5>(el[1])],
+            6 => [vshlq_n_u32::<6>(el[0]), vshlq_n_u32::<6>(el[1])],
+            7 => [vshlq_n_u32::<7>(el[0]), vshlq_n_u32::<7>(el[1])],
+            8 => [vshlq_n_u32::<8>(el[0]), vshlq_n_u32::<8>(el[1])],
+            9 => [vshlq_n_u32::<9>(el[0]), vshlq_n_u32::<9>(el[1])],
+            10 => [vshlq_n_u32::<10>(el[0]), vshlq_n_u32::<10>(el[1])],
+            11 => [vshlq_n_u32::<11>(el[0]), vshlq_n_u32::<11>(el[1])],
+            12 => [vshlq_n_u32::<12>(el[0]), vshlq_n_u32::<12>(el[1])],
+            13 => [vshlq_n_u32::<13>(el[0]), vshlq_n_u32::<13>(el[1])],
+            14 => [vshlq_n_u32::<14>(el[0]), vshlq_n_u32::<14>(el[1])],
+            15 => [vshlq_n_u32::<15>(el[0]), vshlq_n_u32::<15>(el[1])],
+            16 => [vshlq_n_u32::<16>(el[0]), vshlq_n_u32::<16>(el[1])],
+            17 => [vshlq_n_u32::<17>(el[0]), vshlq_n_u32::<17>(el[1])],
+            18 => [vshlq_n_u32::<18>(el[0]), vshlq_n_u32::<18>(el[1])],
+            19 => [vshlq_n_u32::<19>(el[0]), vshlq_n_u32::<19>(el[1])],
+            20 => [vshlq_n_u32::<20>(el[0]), vshlq_n_u32::<20>(el[1])],
+            21 => [vshlq_n_u32::<21>(el[0]), vshlq_n_u32::<21>(el[1])],
+            22 => [vshlq_n_u32::<22>(el[0]), vshlq_n_u32::<22>(el[1])],
+            23 => [vshlq_n_u32::<23>(el[0]), vshlq_n_u32::<23>(el[1])],
+            24 => [vshlq_n_u32::<24>(el[0]), vshlq_n_u32::<24>(el[1])],
+            25 => [vshlq_n_u32::<25>(el[0]), vshlq_n_u32::<25>(el[1])],
+            26 => [vshlq_n_u32::<26>(el[0]), vshlq_n_u32::<26>(el[1])],
+            27 => [vshlq_n_u32::<27>(el[0]), vshlq_n_u32::<27>(el[1])],
+            28 => [vshlq_n_u32::<28>(el[0]), vshlq_n_u32::<28>(el[1])],
+            29 => [vshlq_n_u32::<29>(el[0]), vshlq_n_u32::<29>(el[1])],
+            30 => [vshlq_n_u32::<30>(el[0]), vshlq_n_u32::<30>(el[1])],
+            31 => [vshlq_n_u32::<31>(el[0]), vshlq_n_u32::<31>(el[1])],
+            32 => set1(0),
+            _ => core::hint::unreachable_unchecked(),
+        }
+    }
+
+    #[inline]
+    unsafe fn op_or(left: DataType, right: DataType) -> DataType {
+        [vorrq_u32(left[0], right[0]), vorrq_u32(left[1], right[1])]
+    }
+
+    #[inline]
+    unsafe fn op_and(left: DataType, right: DataType) -> DataType {
+        [vandq_u32(left[0], right[0]), vandq_u32(left[1], right[1])]
+    }
+
+    #[inline]
+    unsafe fn load_unaligned(addr: *const DataType) -> DataType {
+        let ptr = addr.cast::<u32>();
+        [vld1q_u32(ptr), vld1q_u32(ptr.add(4))]
+    }
+
+    #[inline]
+    unsafe fn store_unaligned(addr: *mut DataType, data: DataType) {
+        let ptr = addr.cast::<u32>();
+        vst1q_u32(ptr, data[0]);
+        vst1q_u32(ptr.add(4), data[1]);
+    }
+
+    #[inline]
+    unsafe fn or_collapse_to_u32(accumulator: DataType) -> u32 {
+        vgetq_lane_u32(accumulator[0], 0)
+            | vgetq_lane_u32(accumulator[0], 1)
+            | vgetq_lane_u32(accumulator[0], 2)
+            | vgetq_lane_u32(accumulator[0], 3)
+            | vgetq_lane_u32(accumulator[1], 0)
+            | vgetq_lane_u32(accumulator[1], 1)
+            | vgetq_lane_u32(accumulator[1], 2)
+            | vgetq_lane_u32(accumulator[1], 3)
+    }
+
+    #[inline]
+    unsafe fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        [
+            vsubq_u32(curr[0], vextq_u32(prev[1], curr[0], 3)),
+            vsubq_u32(curr[1], vextq_u32(curr[0], curr[1], 3)),
+        ]
+    }
+
+    #[allow(non_snake_case)]
+    #[inline]
+    unsafe fn integrate_half(base: u32, delta: uint32x4_t) -> uint32x4_t {
+        let base = vdupq_n_u32(base);
+        let zero = vdupq_n_u32(0);
+        let a__b__c__d_ = delta;
+        let ______a__b_ = vextq_u32(zero, a__b__c__d_, 2);
+        let a__b__ca_db = vaddq_u32(______a__b_, a__b__c__d_);
+        let ___a__b__ca = vextq_u32(zero, a__b__ca_db, 3);
+        let a_ab_abc_abcd = vaddq_u32(___a__b__ca, a__b__ca_db);
+        vaddq_u32(base, a_ab_abc_abcd)
+    }
+
+    #[inline]
+    unsafe fn integrate_delta(prev: DataType, delta: DataType) -> DataType {
+        let low = integrate_half(vgetq_lane_u32(prev[1], 3), delta[0]);
+        let high = integrate_half(vgetq_lane_u32(low, 3), delta[1]);
+        [low, high]
+    }
+
+    #[inline]
+    unsafe fn add(left: DataType, right: DataType) -> DataType {
+        [vaddq_u32(left[0], right[0]), vaddq_u32(left[1], right[1])]
+    }
+
+    #[inline]
+    unsafe fn sub(left: DataType, right: DataType) -> DataType {
+        [vsubq_u32(left[0], right[0]), vsubq_u32(left[1], right[1])]
+    }
+
+    declare_bitpacker!(target_feature(enable = "neon"));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            std::arch::is_aarch64_feature_detected!("neon")
+        }
+    }
+}
+
+mod scalar {
+    use super::BLOCK_LEN;
+    use crate::bitpacker_internal::Available;
+    use std::ptr;
+
+    pub(crate) type DataType = [u32; 8];
+
+    pub(crate) fn set1(el: i32) -> DataType {
+        [el as u32; 8]
+    }
+
+    pub(crate) fn right_shift_32<const N: i32>(el: DataType) -> DataType {
+        [
+            el[0] >> N,
+            el[1] >> N,
+            el[2] >> N,
+            el[3] >> N,
+            el[4] >> N,
+            el[5] >> N,
+            el[6] >> N,
+            el[7] >> N,
+        ]
+    }
+
+    pub(crate) fn left_shift_32<const N: i32>(el: DataType) -> DataType {
+        [
+            el[0] << N,
+            el[1] << N,
+            el[2] << N,
+            el[3] << N,
+            el[4] << N,
+            el[5] << N,
+            el[6] << N,
+            el[7] << N,
+        ]
+    }
+
+    pub(crate) fn op_or(left: DataType, right: DataType) -> DataType {
+        [
+            left[0] | right[0],
+            left[1] | right[1],
+            left[2] | right[2],
+            left[3] | right[3],
+            left[4] | right[4],
+            left[5] | right[5],
+            left[6] | right[6],
+            left[7] | right[7],
+        ]
+    }
+
+    pub(crate) fn op_and(left: DataType, right: DataType) -> DataType {
+        [
+            left[0] & right[0],
+            left[1] & right[1],
+            left[2] & right[2],
+            left[3] & right[3],
+            left[4] & right[4],
+            left[5] & right[5],
+            left[6] & right[6],
+            left[7] & right[7],
+        ]
+    }
+
+    pub(crate) unsafe fn load_unaligned(addr: *const DataType) -> DataType {
+        ptr::read_unaligned(addr)
+    }
+
+    pub(crate) unsafe fn store_unaligned(addr: *mut DataType, data: DataType) {
+        ptr::write_unaligned(addr, data);
+    }
+
+    pub(crate) fn or_collapse_to_u32(accumulator: DataType) -> u32 {
+        ((accumulator[0] | accumulator[1]) | (accumulator[2] | accumulator[3]))
+            | ((accumulator[4] | accumulator[5]) | (accumulator[6] | accumulator[7]))
+    }
+
+    fn compute_delta(curr: DataType, prev: DataType) -> DataType {
+        [
+            curr[0].wrapping_sub(prev[7]),
+            curr[1].wrapping_sub(curr[0]),
+            curr[2].wrapping_sub(curr[1]),
+            curr[3].wrapping_sub(curr[2]),
+            curr[4].wrapping_sub(curr[3]),
+            curr[5].wrapping_sub(curr[4]),
+            curr[6].wrapping_sub(curr[5]),
+            curr[7].wrapping_sub(curr[6]),
+        ]
+    }
+
+    fn integrate_delta(offset: DataType, delta: DataType) -> DataType {
+        let el0 = offset[7].wrapping_add(delta[0]);
+        let el1 = el0.wrapping_add(delta[1]);
+        let el2 = el1.wrapping_add(delta[2]);
+        let el3 = el2.wrapping_add(delta[3]);
+        let el4 = el3.wrapping_add(delta[4]);
+        let el5 = el4.wrapping_add(delta[5]);
+        let el6 = el5.wrapping_add(delta[6]);
+        let el7 = el6.wrapping_add(delta[7]);
+        [el0, el1, el2, el3, el4, el5, el6, el7]
+    }
+
+    pub(crate) fn add(left: DataType, right: DataType) -> DataType {
+        [
+            left[0].wrapping_add(right[0]),
+            left[1].wrapping_add(right[1]),
+            left[2].wrapping_add(right[2]),
+            left[3].wrapping_add(right[3]),
+            left[4].wrapping_add(right[4]),
+            left[5].wrapping_add(right[5]),
+            left[6].wrapping_add(right[6]),
+            left[7].wrapping_add(right[7]),
+        ]
+    }
+
+    pub(crate) fn sub(left: DataType, right: DataType) -> DataType {
+        [
+            left[0].wrapping_sub(right[0]),
+            left[1].wrapping_sub(right[1]),
+            left[2].wrapping_sub(right[2]),
+            left[3].wrapping_sub(right[3]),
+            left[4].wrapping_sub(right[4]),
+            left[5].wrapping_sub(right[5]),
+            left[6].wrapping_sub(right[6]),
+            left[7].wrapping_sub(right[7]),
+        ]
+    }
+
+    // The `allow(unused)` is here to put an attribute that has no effect.
+    //
+    // For other bitpackers, we enable a specific CPU instruction set, but for
+    // the scalar bitpacker none is required.
+    declare_bitpacker!(allow(unused));
+
+    impl Available for UnsafeBitPackerImpl {
+        fn available() -> bool {
+            true
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum InstructionSet {
+    #[cfg(target_arch = "x86_64")]
+    AVX2,
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    NEON,
+    Scalar,
+}
+
+/// 8-wide bitpacker implementation.
+///
+/// One block contains 256 integers.
+#[derive(Clone, Copy)]
+pub struct BitPacker8x(InstructionSet);
+
+impl BitPacker8x {
+    #[cfg(target_arch = "x86_64")]
+    pub(crate) fn new_avx2() -> Option<Self> {
+        avx2::UnsafeBitPackerImpl::available().then_some(BitPacker8x(InstructionSet::AVX2))
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    pub(crate) fn new_avx2() -> Option<Self> {
+        None
+    }
+
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    pub(crate) fn new_neon() -> Option<Self> {
+        neon::UnsafeBitPackerImpl::available().then_some(BitPacker8x(InstructionSet::NEON))
+    }
+
+    #[cfg(not(all(target_arch = "aarch64", target_endian = "little")))]
+    pub(crate) fn new_neon() -> Option<Self> {
+        None
+    }
+
+    pub(crate) fn new_scalar() -> Self {
+        BitPacker8x(InstructionSet::Scalar)
+    }
+}
+
+impl BitPacker for BitPacker8x {
+    const BLOCK_LEN: usize = BLOCK_LEN;
+
+    fn new() -> Self {
+        Self::new_avx2()
+            .or_else(Self::new_neon)
+            .unwrap_or_else(Self::new_scalar)
+    }
+
+    fn compress(&self, decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => {
+                    avx2::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::compress(decompressed, compressed, num_bits)
+                }
+            }
+        }
+    }
+
+    fn compress_sorted(
+        &self,
+        initial: u32,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => avx2::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::compress_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn compress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => avx2::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::compress_strictly_sorted(
+                    initial,
+                    decompressed,
+                    compressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn decompress(&self, compressed: &[u8], decompressed: &mut [u32], num_bits: u8) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => {
+                    avx2::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::decompress(compressed, decompressed, num_bits)
+                }
+            }
+        }
+    }
+
+    fn decompress_sorted(
+        &self,
+        initial: u32,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => avx2::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::decompress_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn decompress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => avx2::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::decompress_strictly_sorted(
+                    initial,
+                    compressed,
+                    decompressed,
+                    num_bits,
+                ),
+            }
+        }
+    }
+
+    fn num_bits(&self, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => avx2::UnsafeBitPackerImpl::num_bits(decompressed),
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => neon::UnsafeBitPackerImpl::num_bits(decompressed),
+                InstructionSet::Scalar => scalar::UnsafeBitPackerImpl::num_bits(decompressed),
+            }
+        }
+    }
+
+    fn num_bits_sorted(&self, initial: u32, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => {
+                    avx2::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::num_bits_sorted(initial, decompressed)
+                }
+            }
+        }
+    }
+
+    fn num_bits_strictly_sorted(&self, initial: Option<u32>, decompressed: &[u32]) -> u8 {
+        unsafe {
+            match self.0 {
+                #[cfg(target_arch = "x86_64")]
+                InstructionSet::AVX2 => {
+                    avx2::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+                #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+                InstructionSet::NEON => {
+                    neon::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+                InstructionSet::Scalar => {
+                    scalar::UnsafeBitPackerImpl::num_bits_strictly_sorted(initial, decompressed)
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::BitPacker8x;
+    use crate::bitpacker_internal::BitPacker;
+    use bitpacking::{BitPacker as ExternalBitPacker, BitPacker8x as ExternalBitPacker8x};
+
+    fn mask_for_width(width: u8) -> u32 {
+        match width {
+            0 => 0,
+            32 => u32::MAX,
+            _ => (1u32 << width) - 1,
+        }
+    }
+
+    fn raw_values(width: u8, seed: u64) -> Vec<u32> {
+        let mask = mask_for_width(width);
+        let mut state = seed;
+        (0..BitPacker8x::BLOCK_LEN)
+            .map(|idx| {
+                state ^= state << 13;
+                state ^= state >> 7;
+                state ^= state << 17;
+                match seed % 4 {
+                    0 => 0,
+                    1 => mask,
+                    2 => idx as u32 & mask,
+                    _ => state as u32 & mask,
+                }
+            })
+            .collect()
+    }
+
+    fn sorted_values(width: u8, seed: u64) -> (u32, Vec<u32>) {
+        if width == 0 {
+            return (17, vec![17; BitPacker8x::BLOCK_LEN]);
+        }
+        if width == 32 {
+            return (0, vec![u32::MAX; BitPacker8x::BLOCK_LEN]);
+        }
+
+        let mask = mask_for_width(width).min(127);
+        let mut state = seed;
+        let mut current = 17u32;
+        let values = (0..BitPacker8x::BLOCK_LEN)
+            .map(|_| {
+                state ^= state << 13;
+                state ^= state >> 7;
+                state ^= state << 17;
+                current += state as u32 & mask;
+                current
+            })
+            .collect();
+        (17, values)
+    }
+
+    fn strictly_sorted_values(width: u8, seed: u64) -> (Option<u32>, Vec<u32>) {
+        let mask = mask_for_width(width).min(127);
+        let mut state = seed;
+        let mut current = 0u32;
+        let values = (0..BitPacker8x::BLOCK_LEN)
+            .map(|idx| {
+                if idx == 0 {
+                    current = 0;
+                } else {
+                    state ^= state << 13;
+                    state ^= state >> 7;
+                    state ^= state << 17;
+                    current += 1 + (state as u32 & mask);
+                }
+                current
+            })
+            .collect();
+        (None, values)
+    }
+
+    fn assert_raw_compatible(ours: BitPacker8x, external: ExternalBitPacker8x) {
+        for width in 0..=32 {
+            for seed in [0, 1, 2, 123456789] {
+                let values = raw_values(width, seed);
+                assert_eq!(ours.num_bits(&values), external.num_bits(&values));
+
+                let mut actual = vec![0u8; BitPacker8x::compressed_block_size(width)];
+                let actual_len = ours.compress(&values, &mut actual, width);
+
+                let mut expected = vec![0u8; ExternalBitPacker8x::compressed_block_size(width)];
+                let expected_len = external.compress(&values, &mut expected, width);
+
+                assert_eq!(actual_len, expected_len);
+                assert_eq!(actual, expected, "raw width {width} seed {seed}");
+
+                let mut decoded = vec![0u32; BitPacker8x::BLOCK_LEN];
+                assert_eq!(ours.decompress(&actual, &mut decoded, width), actual_len);
+                assert_eq!(decoded, values);
+            }
+        }
+    }
+
+    fn assert_sorted_compatible(ours: BitPacker8x, external: ExternalBitPacker8x) {
+        for width in 0..=32 {
+            for seed in [0, 1, 2, 123456789] {
+                let (initial, values) = sorted_values(width, seed);
+                assert_eq!(
+                    ours.num_bits_sorted(initial, &values),
+                    external.num_bits_sorted(initial, &values)
+                );
+
+                let mut actual = vec![0u8; BitPacker8x::compressed_block_size(width)];
+                let actual_len = ours.compress_sorted(initial, &values, &mut actual, width);
+
+                let mut expected = vec![0u8; ExternalBitPacker8x::compressed_block_size(width)];
+                let expected_len = external.compress_sorted(initial, &values, &mut expected, width);
+
+                assert_eq!(actual_len, expected_len);
+                assert_eq!(actual, expected, "sorted width {width} seed {seed}");
+
+                let mut decoded = vec![0u32; BitPacker8x::BLOCK_LEN];
+                assert_eq!(
+                    ours.decompress_sorted(initial, &actual, &mut decoded, width),
+                    actual_len
+                );
+                assert_eq!(decoded, values);
+            }
+        }
+    }
+
+    fn assert_strictly_sorted_compatible(ours: BitPacker8x, external: ExternalBitPacker8x) {
+        for width in 0..=16 {
+            for seed in [0, 1, 2, 123456789] {
+                let (initial, values) = strictly_sorted_values(width, seed);
+                let num_bits = external.num_bits_strictly_sorted(initial, &values);
+                assert_eq!(ours.num_bits_strictly_sorted(initial, &values), num_bits);
+
+                let mut actual = vec![0u8; BitPacker8x::compressed_block_size(num_bits)];
+                let actual_len =
+                    ours.compress_strictly_sorted(initial, &values, &mut actual, num_bits);
+
+                let mut expected = vec![0u8; ExternalBitPacker8x::compressed_block_size(num_bits)];
+                let expected_len =
+                    external.compress_strictly_sorted(initial, &values, &mut expected, num_bits);
+
+                assert_eq!(actual_len, expected_len);
+                assert_eq!(actual, expected, "strict width {width} seed {seed}");
+
+                let mut decoded = vec![0u32; BitPacker8x::BLOCK_LEN];
+                assert_eq!(
+                    ours.decompress_strictly_sorted(initial, &actual, &mut decoded, num_bits),
+                    actual_len
+                );
+                assert_eq!(decoded, values);
+            }
+        }
+    }
+
+    #[test]
+    fn bitpacker8x_raw_compatible_with_external_bitpacking() {
+        assert_raw_compatible(BitPacker8x::new(), ExternalBitPacker8x::new());
+    }
+
+    #[test]
+    fn bitpacker8x_sorted_compatible_with_external_bitpacking() {
+        assert_sorted_compatible(BitPacker8x::new(), ExternalBitPacker8x::new());
+    }
+
+    #[test]
+    fn scalar_backend_matches_external_bitpacker8x() {
+        let scalar = BitPacker8x::new_scalar();
+        let external = ExternalBitPacker8x::new();
+
+        assert_raw_compatible(scalar, external);
+        assert_sorted_compatible(scalar, external);
+    }
+
+    #[test]
+    fn scalar_backend_matches_external_strictly_sorted_bitpacker8x() {
+        let scalar = BitPacker8x::new_scalar();
+        let external = ExternalBitPacker8x::new();
+
+        assert_strictly_sorted_compatible(scalar, external);
+    }
+
+    #[cfg(all(target_arch = "aarch64", target_endian = "little"))]
+    #[test]
+    fn neon_backend_matches_external_bitpacker8x() {
+        if let Some(neon) = BitPacker8x::new_neon() {
+            let external = ExternalBitPacker8x::new();
+
+            assert_raw_compatible(neon, external);
+            assert_sorted_compatible(neon, external);
+            assert_strictly_sorted_compatible(neon, external);
+        }
+    }
+}
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/macros.rs b/rust/compression/bitpacking/src/bitpacker_internal/macros.rs
new file mode 100644
index 00000000000..e79686fe9af
--- /dev/null
+++ b/rust/compression/bitpacking/src/bitpacker_internal/macros.rs
@@ -0,0 +1,669 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+macro_rules! pack_unpack_with_bits {
+
+    ($name:ident, $n:expr, $cpufeature:meta) => {
+
+
+        mod $name {
+
+            use crunchy::unroll;
+            use super::BLOCK_LEN;
+            use super::{Sink, Transformer};
+            use super::{DataType,
+                set1,
+                right_shift_32,
+                left_shift_32,
+                op_or,
+                op_and,
+                load_unaligned,
+                store_unaligned};
+
+            const NUM_BITS: usize = $n;
+            const NUM_BYTES_PER_BLOCK: usize = NUM_BITS * BLOCK_LEN / 8;
+
+            #[$cpufeature]
+            pub(crate) unsafe fn pack<TDeltaComputer: Transformer>(input_arr: &[u32], output_arr: &mut [u8], mut delta_computer: TDeltaComputer) -> usize {
+                assert_eq!(input_arr.len(), BLOCK_LEN, "Input block too small {}, (expected {})", input_arr.len(), BLOCK_LEN);
+                assert!(output_arr.len() >= NUM_BYTES_PER_BLOCK, "Output array too small (numbits {}). {} <= {}", NUM_BITS, output_arr.len(), NUM_BYTES_PER_BLOCK);
+
+                let input_ptr = input_arr.as_ptr().cast::<DataType>();
+                let mut output_ptr = output_arr.as_mut_ptr().cast::<DataType>();
+                let mut out_register: DataType = delta_computer.transform(load_unaligned(input_ptr));
+
+                unroll! {
+                    for iter in 0..30 {
+                        const i: usize = 1 + iter;
+
+                        const bits_filled: usize = i * NUM_BITS;
+                        const inner_cursor: usize = bits_filled % 32;
+                        const remaining: usize = 32 - inner_cursor;
+
+                        let offset_ptr = input_ptr.add(i);
+                        let in_register: DataType = delta_computer.transform(load_unaligned(offset_ptr));
+
+                        out_register =
+                            if inner_cursor > 0 {
+                                op_or(out_register, left_shift_32::<{inner_cursor as i32}>(in_register))
+                            } else {
+                                in_register
+                            };
+
+                        if remaining <= NUM_BITS {
+                            store_unaligned(output_ptr, out_register);
+                            output_ptr = output_ptr.add(1);
+                            if 0 < remaining && remaining < NUM_BITS {
+                                out_register = right_shift_32::<{remaining as i32}>(in_register);
+                            }
+                        }
+                    }
+                }
+                let in_register: DataType = delta_computer.transform(load_unaligned(input_ptr.add(31)));
+                out_register = if 32 - NUM_BITS > 0 {
+                    op_or(out_register, left_shift_32::<{32 - NUM_BITS as i32}>(in_register))
+                } else {
+                    op_or(out_register, in_register)
+                };
+                store_unaligned(output_ptr, out_register);
+
+                NUM_BYTES_PER_BLOCK
+            }
+
+            #[$cpufeature]
+            pub(crate) unsafe fn unpack<Output: Sink>(compressed: &[u8], mut output: Output) -> usize {
+
+                assert!(compressed.len() >= NUM_BYTES_PER_BLOCK, "Compressed array seems too small. ({} < {}) ", compressed.len(), NUM_BYTES_PER_BLOCK);
+
+                let mut input_ptr = compressed.as_ptr().cast::<DataType>();
+
+                let mask_scalar: u32 = ((1u64 << NUM_BITS) - 1u64) as u32;
+                let mask = set1(mask_scalar as i32);
+
+                let mut in_register: DataType = load_unaligned(input_ptr);
+
+                let out_register = op_and(in_register, mask);
+                output.process(out_register);
+
+                unroll! {
+                    for iter in 0..31 {
+                        const i: usize = iter + 1;
+
+                        const inner_cursor: usize = (i * NUM_BITS) % 32;
+                        const inner_capacity: usize = 32 - inner_cursor;
+
+                        let shifted_in_register = if inner_cursor != 0 {
+                            right_shift_32::<{inner_cursor as i32}>(in_register)
+                        } else {
+                            in_register
+                        };
+                        let mut out_register: DataType = op_and(shifted_in_register, mask);
+
+                        // We consumed our current quadruplets entirely.
+                        // We therefore read another one.
+                        if inner_capacity <= NUM_BITS && i != 31 {
+                            input_ptr = input_ptr.add(1);
+                            in_register = load_unaligned(input_ptr);
+
+                            // This quadruplets is actually cutting one of
+                            // our `DataType`. We need to read the next one.
+                            if inner_capacity < NUM_BITS {
+                                let shifted = if inner_capacity != 0 {
+                                    left_shift_32::<{inner_capacity as i32}>(in_register)
+                                } else {
+                                    in_register
+                                };
+                                let masked = op_and(shifted, mask);
+                                out_register = op_or(out_register, masked);
+                            }
+                        }
+
+                        output.process(out_register);
+                    }
+                }
+
+
+                NUM_BYTES_PER_BLOCK
+            }
+        }
+    }
+}
+
+macro_rules! pack_unpack_with_bits_32 {
+    ($cpufeature:meta) => {
+        mod pack_unpack_with_bits_32 {
+            use super::BLOCK_LEN;
+            use super::{DataType, load_unaligned, store_unaligned};
+            use super::{Sink, Transformer};
+            use crunchy::unroll;
+
+            const NUM_BITS: usize = 32;
+            const NUM_BYTES_PER_BLOCK: usize = NUM_BITS * BLOCK_LEN / 8;
+
+            #[$cpufeature]
+            pub(crate) unsafe fn pack<TDeltaComputer: Transformer>(
+                input_arr: &[u32],
+                output_arr: &mut [u8],
+                mut delta_computer: TDeltaComputer,
+            ) -> usize {
+                assert_eq!(
+                    input_arr.len(),
+                    BLOCK_LEN,
+                    "Input block too small {}, (expected {})",
+                    input_arr.len(),
+                    BLOCK_LEN
+                );
+                assert!(
+                    output_arr.len() >= NUM_BYTES_PER_BLOCK,
+                    "Output array too small (numbits {}). {} <= {}",
+                    NUM_BITS,
+                    output_arr.len(),
+                    NUM_BYTES_PER_BLOCK
+                );
+
+                let input_ptr: *const DataType = input_arr.as_ptr().cast::<DataType>();
+                let output_ptr = output_arr.as_mut_ptr().cast::<DataType>();
+                unroll! {
+                    for i in 0..32 {
+                        let input_offset_ptr = input_ptr.add(i);
+                        let output_offset_ptr = output_ptr.add(i);
+                        let input_register = load_unaligned(input_offset_ptr);
+                        let output_register = delta_computer.transform(input_register);
+                        store_unaligned(output_offset_ptr, output_register);
+                    }
+                }
+                NUM_BYTES_PER_BLOCK
+            }
+
+            #[$cpufeature]
+            pub(crate) unsafe fn unpack<Output: Sink>(
+                compressed: &[u8],
+                mut output: Output,
+            ) -> usize {
+                assert!(
+                    compressed.len() >= NUM_BYTES_PER_BLOCK,
+                    "Compressed array seems too small. ({} < {}) ",
+                    compressed.len(),
+                    NUM_BYTES_PER_BLOCK
+                );
+                let input_ptr = compressed.as_ptr().cast::<DataType>();
+                for i in 0..32 {
+                    let input_offset_ptr = input_ptr.add(i);
+                    let in_register: DataType = load_unaligned(input_offset_ptr);
+                    output.process(in_register);
+                }
+                NUM_BYTES_PER_BLOCK
+            }
+        }
+    };
+}
+
+macro_rules! declare_bitpacker {
+    ($cpufeature:meta) => {
+        use super::super::UnsafeBitPacker;
+        use super::super::most_significant_bit;
+        use crunchy::unroll;
+
+        pack_unpack_with_bits!(pack_unpack_with_bits_1, 1, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_2, 2, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_3, 3, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_4, 4, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_5, 5, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_6, 6, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_7, 7, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_8, 8, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_9, 9, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_10, 10, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_11, 11, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_12, 12, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_13, 13, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_14, 14, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_15, 15, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_16, 16, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_17, 17, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_18, 18, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_19, 19, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_20, 20, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_21, 21, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_22, 22, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_23, 23, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_24, 24, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_25, 25, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_26, 26, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_27, 27, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_28, 28, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_29, 29, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_30, 30, $cpufeature);
+        pack_unpack_with_bits!(pack_unpack_with_bits_31, 31, $cpufeature);
+        pack_unpack_with_bits_32!($cpufeature);
+
+        unsafe fn compress_generic<DeltaComputer: Transformer>(
+            decompressed: &[u32],
+            compressed: &mut [u8],
+            num_bits: u8,
+            delta_computer: DeltaComputer,
+        ) -> usize {
+            match num_bits {
+                0 => 0,
+                1 => pack_unpack_with_bits_1::pack(decompressed, compressed, delta_computer),
+                2 => pack_unpack_with_bits_2::pack(decompressed, compressed, delta_computer),
+                3 => pack_unpack_with_bits_3::pack(decompressed, compressed, delta_computer),
+                4 => pack_unpack_with_bits_4::pack(decompressed, compressed, delta_computer),
+                5 => pack_unpack_with_bits_5::pack(decompressed, compressed, delta_computer),
+                6 => pack_unpack_with_bits_6::pack(decompressed, compressed, delta_computer),
+                7 => pack_unpack_with_bits_7::pack(decompressed, compressed, delta_computer),
+                8 => pack_unpack_with_bits_8::pack(decompressed, compressed, delta_computer),
+                9 => pack_unpack_with_bits_9::pack(decompressed, compressed, delta_computer),
+                10 => pack_unpack_with_bits_10::pack(decompressed, compressed, delta_computer),
+                11 => pack_unpack_with_bits_11::pack(decompressed, compressed, delta_computer),
+                12 => pack_unpack_with_bits_12::pack(decompressed, compressed, delta_computer),
+                13 => pack_unpack_with_bits_13::pack(decompressed, compressed, delta_computer),
+                14 => pack_unpack_with_bits_14::pack(decompressed, compressed, delta_computer),
+                15 => pack_unpack_with_bits_15::pack(decompressed, compressed, delta_computer),
+                16 => pack_unpack_with_bits_16::pack(decompressed, compressed, delta_computer),
+                17 => pack_unpack_with_bits_17::pack(decompressed, compressed, delta_computer),
+                18 => pack_unpack_with_bits_18::pack(decompressed, compressed, delta_computer),
+                19 => pack_unpack_with_bits_19::pack(decompressed, compressed, delta_computer),
+                20 => pack_unpack_with_bits_20::pack(decompressed, compressed, delta_computer),
+                21 => pack_unpack_with_bits_21::pack(decompressed, compressed, delta_computer),
+                22 => pack_unpack_with_bits_22::pack(decompressed, compressed, delta_computer),
+                23 => pack_unpack_with_bits_23::pack(decompressed, compressed, delta_computer),
+                24 => pack_unpack_with_bits_24::pack(decompressed, compressed, delta_computer),
+                25 => pack_unpack_with_bits_25::pack(decompressed, compressed, delta_computer),
+                26 => pack_unpack_with_bits_26::pack(decompressed, compressed, delta_computer),
+                27 => pack_unpack_with_bits_27::pack(decompressed, compressed, delta_computer),
+                28 => pack_unpack_with_bits_28::pack(decompressed, compressed, delta_computer),
+                29 => pack_unpack_with_bits_29::pack(decompressed, compressed, delta_computer),
+                30 => pack_unpack_with_bits_30::pack(decompressed, compressed, delta_computer),
+                31 => pack_unpack_with_bits_31::pack(decompressed, compressed, delta_computer),
+                32 => pack_unpack_with_bits_32::pack(decompressed, compressed, delta_computer),
+                _ => {
+                    panic!("Num bits must be <= 32. Was {}.", num_bits);
+                }
+            }
+        }
+
+        pub trait Transformer {
+            unsafe fn transform(&mut self, data: DataType) -> DataType;
+        }
+
+        struct NoDelta;
+
+        impl Transformer for NoDelta {
+            #[inline]
+            unsafe fn transform(&mut self, current: DataType) -> DataType {
+                current
+            }
+        }
+
+        struct DeltaComputer {
+            pub previous: DataType,
+        }
+
+        impl Transformer for DeltaComputer {
+            #[inline]
+            unsafe fn transform(&mut self, current: DataType) -> DataType {
+                let result = compute_delta(current, self.previous);
+                self.previous = current;
+                result
+            }
+        }
+
+        struct StrictDeltaComputer {
+            pub previous: DataType,
+        }
+
+        impl Transformer for StrictDeltaComputer {
+            #[inline]
+            unsafe fn transform(&mut self, current: DataType) -> DataType {
+                let result = compute_delta(current, self.previous);
+                self.previous = current;
+                sub(result, set1(1))
+            }
+        }
+
+        pub trait Sink {
+            unsafe fn process(&mut self, data_type: DataType);
+        }
+
+        struct Store {
+            output_ptr: *mut DataType,
+        }
+
+        impl Store {
+            fn new(output_ptr: *mut DataType) -> Store {
+                Store { output_ptr }
+            }
+        }
+
+        struct DeltaIntegrate {
+            current: DataType,
+            output_ptr: *mut DataType,
+        }
+
+        impl DeltaIntegrate {
+            unsafe fn new(initial: u32, output_ptr: *mut DataType) -> DeltaIntegrate {
+                DeltaIntegrate {
+                    current: set1(initial as i32),
+                    output_ptr,
+                }
+            }
+        }
+
+        impl Sink for DeltaIntegrate {
+            #[inline]
+            unsafe fn process(&mut self, delta: DataType) {
+                self.current = integrate_delta(self.current, delta);
+                store_unaligned(self.output_ptr, self.current);
+                self.output_ptr = self.output_ptr.add(1);
+            }
+        }
+
+        struct StrictDeltaIntegrate {
+            current: DataType,
+            output_ptr: *mut DataType,
+        }
+
+        impl StrictDeltaIntegrate {
+            unsafe fn new(initial: u32, output_ptr: *mut DataType) -> StrictDeltaIntegrate {
+                StrictDeltaIntegrate {
+                    current: set1(initial as i32),
+                    output_ptr,
+                }
+            }
+        }
+
+        impl Sink for StrictDeltaIntegrate {
+            #[inline]
+            unsafe fn process(&mut self, delta: DataType) {
+                self.current = integrate_delta(self.current, add(delta, set1(1)));
+                store_unaligned(self.output_ptr, self.current);
+                self.output_ptr = self.output_ptr.add(1);
+            }
+        }
+
+        impl Sink for Store {
+            #[inline]
+            unsafe fn process(&mut self, out_register: DataType) {
+                store_unaligned(self.output_ptr, out_register);
+                self.output_ptr = self.output_ptr.add(1);
+            }
+        }
+
+        #[inline]
+        unsafe fn decompress_to<Output: Sink>(
+            compressed: &[u8],
+            mut sink: Output,
+            num_bits: u8,
+        ) -> usize {
+            match num_bits {
+                0 => {
+                    let zero = set1(0i32);
+                    for _ in 0..32 {
+                        sink.process(zero);
+                    }
+                    0
+                }
+                1 => pack_unpack_with_bits_1::unpack(compressed, sink),
+                2 => pack_unpack_with_bits_2::unpack(compressed, sink),
+                3 => pack_unpack_with_bits_3::unpack(compressed, sink),
+                4 => pack_unpack_with_bits_4::unpack(compressed, sink),
+                5 => pack_unpack_with_bits_5::unpack(compressed, sink),
+                6 => pack_unpack_with_bits_6::unpack(compressed, sink),
+                7 => pack_unpack_with_bits_7::unpack(compressed, sink),
+                8 => pack_unpack_with_bits_8::unpack(compressed, sink),
+                9 => pack_unpack_with_bits_9::unpack(compressed, sink),
+                10 => pack_unpack_with_bits_10::unpack(compressed, sink),
+                11 => pack_unpack_with_bits_11::unpack(compressed, sink),
+                12 => pack_unpack_with_bits_12::unpack(compressed, sink),
+                13 => pack_unpack_with_bits_13::unpack(compressed, sink),
+                14 => pack_unpack_with_bits_14::unpack(compressed, sink),
+                15 => pack_unpack_with_bits_15::unpack(compressed, sink),
+                16 => pack_unpack_with_bits_16::unpack(compressed, sink),
+                17 => pack_unpack_with_bits_17::unpack(compressed, sink),
+                18 => pack_unpack_with_bits_18::unpack(compressed, sink),
+                19 => pack_unpack_with_bits_19::unpack(compressed, sink),
+                20 => pack_unpack_with_bits_20::unpack(compressed, sink),
+                21 => pack_unpack_with_bits_21::unpack(compressed, sink),
+                22 => pack_unpack_with_bits_22::unpack(compressed, sink),
+                23 => pack_unpack_with_bits_23::unpack(compressed, sink),
+                24 => pack_unpack_with_bits_24::unpack(compressed, sink),
+                25 => pack_unpack_with_bits_25::unpack(compressed, sink),
+                26 => pack_unpack_with_bits_26::unpack(compressed, sink),
+                27 => pack_unpack_with_bits_27::unpack(compressed, sink),
+                28 => pack_unpack_with_bits_28::unpack(compressed, sink),
+                29 => pack_unpack_with_bits_29::unpack(compressed, sink),
+                30 => pack_unpack_with_bits_30::unpack(compressed, sink),
+                31 => pack_unpack_with_bits_31::unpack(compressed, sink),
+                32 => pack_unpack_with_bits_32::unpack(compressed, sink),
+                _ => {
+                    panic!("Num bits must be <= 32. Was {}.", num_bits);
+                }
+            }
+        }
+
+        pub struct UnsafeBitPackerImpl;
+
+        impl UnsafeBitPacker for UnsafeBitPackerImpl {
+            const BLOCK_LEN: usize = BLOCK_LEN;
+
+            #[$cpufeature]
+            unsafe fn compress(decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize {
+                compress_generic(decompressed, compressed, num_bits, NoDelta)
+            }
+
+            #[$cpufeature]
+            unsafe fn compress_sorted(
+                initial: u32,
+                decompressed: &[u32],
+                compressed: &mut [u8],
+                num_bits: u8,
+            ) -> usize {
+                let delta_computer = DeltaComputer {
+                    previous: set1(initial as i32),
+                };
+                compress_generic(decompressed, compressed, num_bits, delta_computer)
+            }
+
+            #[$cpufeature]
+            unsafe fn compress_strictly_sorted(
+                initial: Option<u32>,
+                decompressed: &[u32],
+                compressed: &mut [u8],
+                num_bits: u8,
+            ) -> usize {
+                // to allow encoding [0, 1, 2, ..], we need to permit an initial value "lower" than
+                // zero. To get a clean api, that value is None, but in practice, as we work on
+                // wrapping integers, u32::MAX/-1 does the job just fine.
+                let initial = initial.unwrap_or(u32::MAX);
+                let delta_computer = StrictDeltaComputer {
+                    previous: set1(initial as i32),
+                };
+                compress_generic(decompressed, compressed, num_bits, delta_computer)
+            }
+
+            #[$cpufeature]
+            unsafe fn decompress(
+                compressed: &[u8],
+                decompressed: &mut [u32],
+                num_bits: u8,
+            ) -> usize {
+                assert!(
+                    decompressed.len() >= BLOCK_LEN,
+                    "The output array is not large enough : ({} >= {})",
+                    decompressed.len(),
+                    BLOCK_LEN
+                );
+                let output_ptr = decompressed.as_mut_ptr().cast::<DataType>();
+                let output = Store::new(output_ptr);
+                decompress_to(compressed, output, num_bits)
+            }
+
+            #[$cpufeature]
+            unsafe fn decompress_sorted(
+                initial: u32,
+                compressed: &[u8],
+                decompressed: &mut [u32],
+                num_bits: u8,
+            ) -> usize {
+                assert!(
+                    decompressed.len() >= BLOCK_LEN,
+                    "The output array is not large enough : ({} >= {})",
+                    decompressed.len(),
+                    BLOCK_LEN
+                );
+                let output_ptr = decompressed.as_mut_ptr().cast::<DataType>();
+                let output = DeltaIntegrate::new(initial, output_ptr);
+                decompress_to(compressed, output, num_bits)
+            }
+
+            #[$cpufeature]
+            unsafe fn decompress_strictly_sorted(
+                initial: Option<u32>,
+                compressed: &[u8],
+                decompressed: &mut [u32],
+                num_bits: u8,
+            ) -> usize {
+                assert!(
+                    decompressed.len() >= BLOCK_LEN,
+                    "The output array is not large enough : ({} >= {})",
+                    decompressed.len(),
+                    BLOCK_LEN
+                );
+                let initial = initial.unwrap_or(u32::MAX);
+                let output_ptr = decompressed.as_mut_ptr().cast::<DataType>();
+                let output = StrictDeltaIntegrate::new(initial, output_ptr);
+                decompress_to(compressed, output, num_bits)
+            }
+
+            #[$cpufeature]
+            unsafe fn num_bits(decompressed: &[u32]) -> u8 {
+                assert_eq!(
+                    decompressed.len(),
+                    BLOCK_LEN,
+                    "`decompressed`'s len is not `BLOCK_LEN={}`",
+                    BLOCK_LEN
+                );
+                let data: *const DataType = decompressed.as_ptr().cast::<DataType>();
+                let mut accumulator = load_unaligned(data);
+                unroll! {
+                    for iter in 0..31 {
+                        let i = iter + 1;
+                        let newvec = load_unaligned(data.add(i));
+                        accumulator = op_or(accumulator, newvec);
+                    }
+                }
+                most_significant_bit(or_collapse_to_u32(accumulator))
+            }
+
+            #[$cpufeature]
+            unsafe fn num_bits_sorted(initial: u32, decompressed: &[u32]) -> u8 {
+                assert_eq!(
+                    decompressed.len(),
+                    BLOCK_LEN,
+                    "`decompressed`'s len is not `BLOCK_LEN={}`",
+                    BLOCK_LEN
+                );
+                let initial_vec = set1(initial as i32);
+                let data: *const DataType = decompressed.as_ptr().cast::<DataType>();
+
+                let first = load_unaligned(data);
+                let mut accumulator = compute_delta(load_unaligned(data), initial_vec);
+                let mut previous = first;
+
+                unroll! {
+                    for iter in 0..30 {
+                        let i = iter + 1;
+                        let current = load_unaligned(data.add(i));
+                        let delta = compute_delta(current, previous);
+                        accumulator =  op_or(accumulator, delta);
+                        previous = current;
+                    }
+                }
+                let current = load_unaligned(data.add(31));
+                let delta = compute_delta(current, previous);
+                accumulator = op_or(accumulator, delta);
+                most_significant_bit(or_collapse_to_u32(accumulator))
+            }
+
+            #[$cpufeature]
+            unsafe fn num_bits_strictly_sorted(initial: Option<u32>, decompressed: &[u32]) -> u8 {
+                assert_eq!(
+                    decompressed.len(),
+                    BLOCK_LEN,
+                    "`decompressed`'s len is not `BLOCK_LEN={}`",
+                    BLOCK_LEN
+                );
+                let initial = initial.unwrap_or(u32::MAX);
+                let initial_vec = set1(initial as i32);
+                let one = set1(1);
+                let data: *const DataType = decompressed.as_ptr().cast::<DataType>();
+
+                let first = load_unaligned(data);
+                let mut accumulator = sub(compute_delta(load_unaligned(data), initial_vec), one);
+                let mut previous = first;
+
+                unroll! {
+                    for iter in 0..30 {
+                        let i = iter + 1;
+                        let current = load_unaligned(data.add(i));
+                        let delta = sub(compute_delta(current, previous), one);
+                        accumulator =  op_or(accumulator, delta);
+                        previous = current;
+                    }
+                }
+                let current = load_unaligned(data.add(31));
+                let delta = sub(compute_delta(current, previous), one);
+                accumulator = op_or(accumulator, delta);
+                most_significant_bit(or_collapse_to_u32(accumulator))
+            }
+        }
+
+        #[cfg(any())]
+        mod tests {
+            use super::super::UnsafeBitPacker;
+            use super::UnsafeBitPackerImpl;
+            use crate::Available;
+            use crate::tests::{DeltaKind, test_suite_compress_decompress};
+
+            #[test]
+            fn test_num_bits() {
+                if UnsafeBitPackerImpl::available() {
+                    for num_bits in 0..32 {
+                        for pos in 0..32 {
+                            let mut vals = [0u32; UnsafeBitPackerImpl::BLOCK_LEN];
+                            if num_bits > 0 {
+                                vals[pos] = 1 << (num_bits - 1);
+                            }
+                            assert_eq!(
+                                unsafe { UnsafeBitPackerImpl::num_bits(&vals[..]) },
+                                num_bits
+                            );
+                        }
+                    }
+                }
+            }
+
+            #[test]
+            fn test_bitpacker() {
+                if UnsafeBitPackerImpl::available() {
+                    test_suite_compress_decompress::<UnsafeBitPackerImpl>(DeltaKind::NoDelta);
+                }
+            }
+
+            #[test]
+            fn test_bitpacker_delta() {
+                if UnsafeBitPackerImpl::available() {
+                    test_suite_compress_decompress::<UnsafeBitPackerImpl>(DeltaKind::Delta);
+                }
+            }
+
+            #[test]
+            fn test_bitpacker_strict_delta() {
+                if UnsafeBitPackerImpl::available() {
+                    test_suite_compress_decompress::<UnsafeBitPackerImpl>(DeltaKind::StrictDelta);
+                }
+            }
+        }
+    };
+}
diff --git a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs
new file mode 100644
index 00000000000..80803e50ec8
--- /dev/null
+++ b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+// Lance-owned u32 SIMD bitpacking kernels.
+//
+// This is adapted from the MIT-licensed `bitpacking` crate so Lance can keep the
+// hot FTS posting-list bitpacking implementation inside lance-bitpacking while
+// preserving byte compatibility with the existing 4x format.
+
+#![allow(dead_code)]
+#![allow(unsafe_op_in_unsafe_fn)]
+#![allow(clippy::redundant_pub_crate)]
+#![allow(clippy::upper_case_acronyms)]
+#![allow(clippy::use_self)]
+
+#[macro_use]
+mod macros;
+
+mod bitpacker4x;
+mod bitpacker8x;
+
+pub use bitpacker4x::BitPacker4x;
+pub use bitpacker8x::BitPacker8x;
+
+pub(crate) trait Available {
+    fn available() -> bool;
+}
+
+pub(crate) trait UnsafeBitPacker {
+    const BLOCK_LEN: usize;
+
+    unsafe fn compress(decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize;
+
+    unsafe fn compress_sorted(
+        initial: u32,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize;
+
+    unsafe fn compress_strictly_sorted(
+        initial: Option<u32>,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize;
+
+    unsafe fn decompress(compressed: &[u8], decompressed: &mut [u32], num_bits: u8) -> usize;
+
+    unsafe fn decompress_sorted(
+        initial: u32,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize;
+
+    unsafe fn decompress_strictly_sorted(
+        initial: Option<u32>,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize;
+
+    unsafe fn num_bits(decompressed: &[u32]) -> u8;
+
+    unsafe fn num_bits_sorted(initial: u32, decompressed: &[u32]) -> u8;
+
+    unsafe fn num_bits_strictly_sorted(initial: Option<u32>, decompressed: &[u32]) -> u8;
+}
+
+/// Block bitpacker for fixed-size `u32` blocks.
+///
+/// Implementations own runtime SIMD dispatch and use caller-provided buffers.
+/// Packed bytes are stable for a given implementation and bit width.
+pub trait BitPacker: Sized + Clone + Copy {
+    /// Number of `u32` values in one physical block.
+    const BLOCK_LEN: usize;
+
+    /// Select the best supported implementation for the current CPU.
+    ///
+    /// Lance uses SIMD backends when available and falls back to a scalar
+    /// backend otherwise, matching the existing allocation-free call shape used
+    /// by the upstream `bitpacking` crate.
+    fn new() -> Self;
+
+    /// Compress one full block of raw values into `compressed`.
+    fn compress(&self, decompressed: &[u32], compressed: &mut [u8], num_bits: u8) -> usize;
+
+    /// Delta-compress one full non-decreasing block into `compressed`.
+    fn compress_sorted(
+        &self,
+        initial: u32,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize;
+
+    /// Delta-compress one full strictly increasing block into `compressed`.
+    fn compress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        decompressed: &[u32],
+        compressed: &mut [u8],
+        num_bits: u8,
+    ) -> usize;
+
+    /// Decompress one raw block into `decompressed`.
+    fn decompress(&self, compressed: &[u8], decompressed: &mut [u32], num_bits: u8) -> usize;
+
+    /// Decompress one delta-compressed non-decreasing block.
+    fn decompress_sorted(
+        &self,
+        initial: u32,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize;
+
+    /// Decompress one delta-compressed strictly increasing block.
+    fn decompress_strictly_sorted(
+        &self,
+        initial: Option<u32>,
+        compressed: &[u8],
+        decompressed: &mut [u32],
+        num_bits: u8,
+    ) -> usize;
+
+    /// Return the minimum bit width needed to represent a full raw block.
+    fn num_bits(&self, decompressed: &[u32]) -> u8;
+
+    /// Return the minimum bit width needed to represent deltas in a full block.
+    fn num_bits_sorted(&self, initial: u32, decompressed: &[u32]) -> u8;
+
+    /// Return the minimum bit width needed to represent strict deltas in a full block.
+    fn num_bits_strictly_sorted(&self, initial: Option<u32>, decompressed: &[u32]) -> u8;
+
+    /// Return the byte size of one compressed block at `num_bits`.
+    #[must_use]
+    fn compressed_block_size(num_bits: u8) -> usize {
+        Self::BLOCK_LEN * num_bits as usize / 8
+    }
+}
+
+#[inline]
+fn most_significant_bit(value: u32) -> u8 {
+    (u32::BITS - value.leading_zeros()) as u8
+}
diff --git a/rust/compression/bitpacking/src/lib.rs b/rust/compression/bitpacking/src/lib.rs
index 0101c4a1df0..c6aa6d75ad0 100644
--- a/rust/compression/bitpacking/src/lib.rs
+++ b/rust/compression/bitpacking/src/lib.rs
@@ -16,6 +16,10 @@
 use arrayref::{array_mut_ref, array_ref};
 use core::mem::size_of;
 
+mod bitpacker_internal;
+
+pub use bitpacker_internal::{BitPacker, BitPacker4x, BitPacker8x};
+
 pub const FL_ORDER: [usize; 8] = [0, 4, 2, 6, 1, 5, 3, 7];
 
 pub trait FastLanes: Sized + Copy {
@@ -1709,6 +1713,7 @@ pack_64!(pack_64_64, 64);
 #[cfg(test)]
 mod test {
     use super::*;
+    use bitpacking::{BitPacker as ExternalBitPacker, BitPacker4x as ExternalBitPacker4x};
     use core::array;
     // a fast random number generator
     pub struct XorShift {
@@ -1730,6 +1735,104 @@ mod test {
         }
     }
 
+    fn mask_for_width(width: u8) -> u32 {
+        match width {
+            0 => 0,
+            32 => u32::MAX,
+            _ => (1u32 << width) - 1,
+        }
+    }
+
+    fn raw_bitpacker4x_case(width: u8, seed: u64) -> Vec<u32> {
+        let mask = mask_for_width(width);
+        let mut rng = XorShift::new(seed);
+        (0..BitPacker4x::BLOCK_LEN)
+            .map(|idx| match seed % 4 {
+                0 => 0,
+                1 => mask,
+                2 => idx as u32 & mask,
+                _ => (rng.next() as u32) & mask,
+            })
+            .collect()
+    }
+
+    fn sorted_bitpacker4x_case(width: u8, seed: u64) -> (u32, Vec<u32>) {
+        if width == 0 {
+            return (17, vec![17; BitPacker4x::BLOCK_LEN]);
+        }
+        if width == 32 {
+            return (0, vec![u32::MAX; BitPacker4x::BLOCK_LEN]);
+        }
+
+        let mask = mask_for_width(width).min(127);
+        let mut rng = XorShift::new(seed);
+        let mut current = 17u32;
+        let values = (0..BitPacker4x::BLOCK_LEN)
+            .map(|_| {
+                current += (rng.next() as u32) & mask;
+                current
+            })
+            .collect();
+        (17, values)
+    }
+
+    #[test]
+    fn test_bitpacker4x_raw_compatible_with_external_bitpacking() {
+        let ours = BitPacker4x::new();
+        let external = ExternalBitPacker4x::new();
+
+        for width in 0..=32 {
+            for seed in [0, 1, 2, 123456789] {
+                let values = raw_bitpacker4x_case(width, seed);
+                assert_eq!(ours.num_bits(&values), external.num_bits(&values));
+
+                let mut actual = vec![0u8; BitPacker4x::compressed_block_size(width)];
+                let actual_len = ours.compress(&values, &mut actual, width);
+
+                let mut expected = vec![0u8; ExternalBitPacker4x::compressed_block_size(width)];
+                let expected_len = external.compress(&values, &mut expected, width);
+
+                assert_eq!(actual_len, expected_len);
+                assert_eq!(actual, expected, "width {width} seed {seed}");
+
+                let mut decoded = vec![0u32; BitPacker4x::BLOCK_LEN];
+                let consumed = ours.decompress(&actual, &mut decoded, width);
+                assert_eq!(consumed, actual_len);
+                assert_eq!(decoded, values);
+            }
+        }
+    }
+
+    #[test]
+    fn test_bitpacker4x_sorted_compatible_with_external_bitpacking() {
+        let ours = BitPacker4x::new();
+        let external = ExternalBitPacker4x::new();
+
+        for width in 0..=32 {
+            for seed in [0, 1, 2, 123456789] {
+                let (initial, values) = sorted_bitpacker4x_case(width, seed);
+                assert_eq!(
+                    ours.num_bits_sorted(initial, &values),
+                    external.num_bits_sorted(initial, &values)
+                );
+
+                let mut actual = vec![0u8; BitPacker4x::compressed_block_size(width)];
+                let actual_len = ours.compress_sorted(initial, &values, &mut actual, width);
+
+                let mut expected = vec![0u8; ExternalBitPacker4x::compressed_block_size(width)];
+                let expected_len = external.compress_sorted(initial, &values, &mut expected, width);
+
+                assert_eq!(actual_len, expected_len);
+                assert_eq!(actual, expected, "width {width} seed {seed}");
+
+                let mut decoded = vec![0u32; BitPacker4x::BLOCK_LEN];
+                let consumed = ours.decompress_sorted(initial, &actual, &mut decoded, width);
+                assert_eq!(consumed, actual_len);
+                assert_eq!(decoded, values);
+            }
+        }
+    }
+
     // a macro version of this function generalize u8, u16, u32, u64 takes very long time for a test build, so I
     // write it for each type separately
     fn pack_unpack_u8(bit_width: usize) {
@@ -2025,6 +2128,10 @@ mod test {
             for value in &mut values {
                 *value = rng.next();
             }
+        } else {
+            for value in &mut values {
+                *value = rng.next() % (1 << bit_width);
+            }
         }
         let mut packed = vec![0; 1024 * bit_width / u64::T];
         unsafe {
diff --git a/rust/compression/fsst/Cargo.toml b/rust/compression/fsst/Cargo.toml
index da5d8f01d04..7056896e3b9 100644
--- a/rust/compression/fsst/Cargo.toml
+++ b/rust/compression/fsst/Cargo.toml
@@ -16,7 +16,6 @@ arrow-array.workspace = true
 rand.workspace = true
 
 [dev-dependencies]
-arrow-array.workspace = true
 test-log.workspace = true
 tokio.workspace = true
 
diff --git a/rust/compression/fsst/examples/benchmark.rs b/rust/compression/fsst/examples/benchmark.rs
index c442243e112..f71abeefedd 100644
--- a/rust/compression/fsst/examples/benchmark.rs
+++ b/rust/compression/fsst/examples/benchmark.rs
@@ -56,7 +56,10 @@ fn benchmark(file_path: &str) {
     let mut decompression_out_bufs = vec![];
     let mut decompression_out_offsets_bufs = vec![];
     for _ in 0..TEST_NUM {
-        let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 3];
+        // `decompress` requires the output buffer to be at least 8x the compressed input (a 1-byte
+        // code can expand to an 8-byte symbol). The compressed buffer is at most `BUFFER_SIZE`, so
+        // `BUFFER_SIZE * 8` is a safe upper bound.
+        let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 8];
         let this_decom_out_offsets_buf = vec![0i32; BUFFER_SIZE * 3];
         decompression_out_bufs.push(this_decom_out_buf);
         decompression_out_offsets_bufs.push(this_decom_out_offsets_buf);
diff --git a/rust/compression/fsst/src/fsst.rs b/rust/compression/fsst/src/fsst.rs
index 0a2bf1d03d7..d00a6ed806b 100644
--- a/rust/compression/fsst/src/fsst.rs
+++ b/rust/compression/fsst/src/fsst.rs
@@ -57,6 +57,12 @@ use std::ptr;
 
 #[inline]
 fn fsst_unaligned_load_unchecked(v: *const u8) -> u64 {
+    // SAFETY: the caller must guarantee that `v` points to at least 8 readable bytes. All callers
+    // uphold this: `compress_bulk` loads from a 520-byte stack buffer at an offset < 511 (leaving
+    // >= 8 bytes), `build_symbol_table` guards the load with `word.len() > 7 && curr < word.len() - 7`,
+    // `find_longest_symbol_from_char_slice` copies into a stack `[u8; 8]` before loading, and
+    // `FsstDecoder::init` reads symbols from a `symbol_table` buffer already validated to be
+    // exactly `FSST_SYMBOL_TABLE_SIZE` bytes.
     unsafe { ptr::read_unaligned(v as *const u64) }
 }
 
@@ -812,12 +818,31 @@ fn decompress_bulk<T: OffsetSizeTrait>(
 ) -> io::Result<()> {
     let symbols = decoder.symbols;
     let lens = decoder.lens;
+    // SAFETY invariant shared by every `unsafe` block in this closure:
+    // - `out` is sized to at least 8x `compressed_strs` (checked in `FsstDecoder::init`, which the
+    //   sole public entry point always runs before reaching this function). Each code advances
+    //   `out_curr` by `lens[code]`, which is 1..=8 for a well-formed symbol table, and each
+    //   consumed input byte yields at most 8 output bytes, so `out_curr + 8 <= out.len()` at every
+    //   8-byte write, including the final one. This is why we can `write_unaligned` a full 8-byte
+    //   word per code and advance by only the length.
+    //   NOTE: `lens` is loaded verbatim from the (untrusted) symbol table and is NOT re-validated
+    //   to be <= 8 on decode, and offsets (below) are likewise trusted. A corrupted table or
+    //   offset buffer can violate these bounds; callers must supply structures produced by
+    //   `compress` (or otherwise trusted). Hardening the decoder against corrupt input is a
+    //   separate concern, not addressed here.
+    // - The only unchecked read is `read_unaligned::<u32>`, gated by `in_curr + 4 <= in_end`; the
+    //   scalar paths use bounds-checked indexing. `in_end` is a caller-provided offset into
+    //   `compressed_strs`; the read is sound only if `in_end <= compressed_strs.len()`, which is a
+    //   trusted precondition (holds for encoder-produced offsets; not validated here).
     let mut decompress = |mut in_curr: usize, in_end: usize, out_curr: &mut usize| {
         // Do SIMD operation here by 4 bytes
         while in_curr + 4 <= in_end {
             let next_block;
             let mut code;
             let mut len;
+            // SAFETY: the loop guard proves `in_curr + 4 <= in_end`. Per the closure-level
+            // invariant, `in_end <= compressed_strs.len()` is a trusted precondition (not checked
+            // here), so the 4-byte read is in bounds for well-formed input.
             unsafe {
                 next_block =
                     ptr::read_unaligned(compressed_strs.as_ptr().add(in_curr) as *const u32);
@@ -983,6 +1008,10 @@ fn decompress_bulk<T: OffsetSizeTrait>(
         if in_curr < in_end {
             // last code cannot be an escape code
             let code = compressed_strs[in_curr] as usize;
+            // SAFETY: see the closure-level invariant. This is the final write and has no
+            // subsequent write to cover its slack, so it is the tightest case: `out_curr` is at
+            // most 8*(consumed_input_bytes - 1), and the 8-byte store lands within `out.len()`
+            // precisely because the caller sized `out` to 8x the input.
             unsafe {
                 let src = symbols[code];
                 ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src);
@@ -1188,8 +1217,19 @@ impl<T: OffsetSizeTrait> FsstDecoder<T> {
         }
 
         self.decoder_switch_on = (st_info & (1 << 24)) != 0;
-        // when decoder_switch_on is true, we make sure the out_buf is at least 3 times the size of the in_buf,
-        if self.decoder_switch_on && in_buf.len() * 3 > out_buf.len() {
+        // A single 1-byte code can decode to a symbol of up to MAX_SYMBOL_LENGTH (8) bytes, so the
+        // decoded output can be up to 8x the input. `decompress_bulk` also relies on this bound: it
+        // writes a full 8-byte word per code (advancing only by the symbol length), so the output
+        // buffer must be large enough that even the final write stays in bounds. Require out_buf to
+        // be at least 8x in_buf. `checked_mul` guards against `in_buf.len() * 8` wrapping on 32-bit
+        // targets (an input >= 512 MiB would otherwise bypass the check); treat overflow as too
+        // small.
+        if self.decoder_switch_on
+            && in_buf
+                .len()
+                .checked_mul(8)
+                .is_none_or(|needed| needed > out_buf.len())
+        {
             return Err(io::Error::new(
                 io::ErrorKind::InvalidInput,
                 "output buffer too small for FSST decoder",
@@ -1288,8 +1328,10 @@ pub fn compress<T: OffsetSizeTrait>(
 // the following 32 bits after FSST_MAGIC contains information about FSST encoding, such as decoder_switch_on, suffix_lim, terminator, n_symbols
 // when the decoder_switch_on is off in the in_buf header, `decompress` first make sure the out_buf is at least the same size as the in_buf, then simply copy the
 // input data to the output
-// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 3 times the size of the in_buf, then start decoding the
-// data using the symbol table
+// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 8 times the size of the in_buf, then start decoding the
+// data using the symbol table. The 8x bound is required for correctness: a 1-byte code can expand to
+// an 8-byte symbol, and the decode loop writes a full 8-byte word per code, so a smaller buffer can
+// be written out of bounds.
 // the out_offsets_buf should be at least the same size as the in_offsets_buf, otherwise an error is returned
 // the symbol_table is the same symbol table created by `compression`
 pub fn decompress<T: OffsetSizeTrait>(
@@ -1644,4 +1686,59 @@ But exactly how the acquaintance and friendship came about, we cannot say.";
             );
         }
     }
+
+    // Build a genuinely FSST-compressed (decoder_switch_on) buffer to exercise the decode-side
+    // output-buffer size contract. Returns (symbol_table, compressed_bytes, compressed_offsets).
+    fn compress_paragraph() -> ([u8; FSST_SYMBOL_TABLE_SIZE], Vec<u8>, Vec<i32>) {
+        let test_input = TEST_PARAGRAPH.repeat((1024 * 1024) / TEST_PARAGRAPH.len());
+        let lines_vec = test_input.lines().collect::<Vec<&str>>();
+        let string_array = StringArray::from(lines_vec);
+        let mut compress_output_buf: Vec<u8> = vec![0; string_array.value_data().len()];
+        let mut compress_offset_buf: Vec<i32> = vec![0; string_array.value_offsets().len()];
+        let mut symbol_table = [0; FSST_SYMBOL_TABLE_SIZE];
+        compress(
+            symbol_table.as_mut(),
+            string_array.value_data(),
+            string_array.value_offsets(),
+            &mut compress_output_buf,
+            &mut compress_offset_buf,
+        )
+        .unwrap();
+        (symbol_table, compress_output_buf, compress_offset_buf)
+    }
+
+    // The decoder writes a full 8-byte word per code, so the output buffer must be at least 8x the
+    // compressed input. A buffer sized below 8x must be rejected rather than written out of bounds.
+    #[test_log::test(tokio::test)]
+    async fn test_decompress_rejects_undersized_output_buffer() {
+        let (symbol_table, compressed, compressed_offsets) = compress_paragraph();
+        // Sanity check: this input actually engaged FSST compression (decoder_switch_on).
+        let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap());
+        assert!(st_info & (1 << 24) != 0, "expected decoder_switch_on input");
+
+        // One byte short of the 8x requirement must be rejected.
+        let mut too_small = vec![0u8; compressed.len() * 8 - 1];
+        let mut out_offsets = vec![0i32; compressed_offsets.len()];
+        let err = decompress(
+            &symbol_table,
+            &compressed,
+            &compressed_offsets,
+            &mut too_small,
+            &mut out_offsets,
+        )
+        .unwrap_err();
+        assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
+
+        // Exactly 8x is the tight bound and must succeed.
+        let mut exact = vec![0u8; compressed.len() * 8];
+        let mut out_offsets = vec![0i32; compressed_offsets.len()];
+        decompress(
+            &symbol_table,
+            &compressed,
+            &compressed_offsets,
+            &mut exact,
+            &mut out_offsets,
+        )
+        .unwrap();
+    }
 }
diff --git a/rust/examples/Cargo.toml b/rust/examples/Cargo.toml
index 3a1ce0ea03c..80eff457140 100644
--- a/rust/examples/Cargo.toml
+++ b/rust/examples/Cargo.toml
@@ -38,7 +38,7 @@ arrow-select = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 itertools = { workspace = true }
 futures = { workspace = true }
-lance = { workspace = true, features = ["aws", "azure", "gcp", "oss", "huggingface", "tencent"] }
+lance = { workspace = true, features = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "goosefs"] }
 lance-index = { workspace = true }
 lance-core = { workspace = true }
 lance-linalg = { workspace = true }
@@ -49,6 +49,6 @@ tokio = { workspace = true }
 all_asserts = "2.3.1"
 env_logger = "0.11.7"
 hf-hub = "0.4.2"
-parquet = "58.0.0"
+parquet = { version = "58.0.0", default-features = false, features = ["arrow", "async"] }
 tokenizers = "0.15.2"
 rand.workspace = true
diff --git a/rust/lance-arrow/Cargo.toml b/rust/lance-arrow/Cargo.toml
index afbc9c26ed3..21513e638d8 100644
--- a/rust/lance-arrow/Cargo.toml
+++ b/rust/lance-arrow/Cargo.toml
@@ -21,6 +21,7 @@ arrow-ipc = { workspace = true }
 arrow-ord = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
+bytemuck = { workspace = true }
 bytes = { workspace = true }
 futures = { workspace = true }
 half = { workspace = true }
diff --git a/rust/lance-arrow/src/bfloat16.rs b/rust/lance-arrow/src/bfloat16.rs
index 3049be1117e..74c7f259a40 100644
--- a/rust/lance-arrow/src/bfloat16.rs
+++ b/rust/lance-arrow/src/bfloat16.rs
@@ -7,7 +7,7 @@ use std::fmt::Formatter;
 use std::slice;
 
 use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder};
-use arrow_buffer::MutableBuffer;
+use arrow_buffer::{Buffer, MutableBuffer};
 use arrow_data::ArrayData;
 use arrow_schema::{ArrowError, DataType, Field as ArrowField};
 use half::bf16;
@@ -144,6 +144,11 @@ impl FromIterator<Option<bf16>> for BFloat16Array {
             .len(len)
             .add_buffer(buffer.into())
             .null_bit_buffer(null_buffer);
+        // SAFETY: the value buffer contains exactly `2 * len` bytes (two bytes
+        // pushed per iteration of the loop above, including the zero-fill for
+        // null slots), which matches the `FixedSizeBinary(2)` storage layout.
+        // The null bit buffer, when present, has `len` bits appended above, so
+        // its length covers the array's logical range.
         let array_data = unsafe { array_data.build_unchecked() };
         Self {
             inner: FixedSizeBinaryArray::from(array_data),
@@ -159,17 +164,20 @@ impl FromIterator<bf16> for BFloat16Array {
 
 impl From<Vec<bf16>> for BFloat16Array {
     fn from(data: Vec<bf16>) -> Self {
-        let mut buffer = MutableBuffer::with_capacity(data.len() * 2);
-
-        let bytes = data.iter().flat_map(|val| {
-            let bytes = val.to_bits().to_le_bytes();
-            bytes.to_vec()
-        });
-
-        buffer.extend(bytes);
+        let len = data.len();
+        // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives
+        // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place —
+        // no per-element copy or heap alloc. The crate-root `compile_error!`
+        // pins `target_endian = "little"`, so the resulting bytes match the
+        // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere.
+        let raw: Vec<u16> = bytemuck::cast_vec(data);
         let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
-            .len(data.len())
-            .add_buffer(buffer.into());
+            .len(len)
+            .add_buffer(Buffer::from_vec(raw));
+        // SAFETY: the value buffer contains exactly `2 * len` bytes — one
+        // `u16` per element after the layout-compatible cast — matching the
+        // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so
+        // every element is logically valid.
         let array_data = unsafe { array_data.build_unchecked() };
         Self {
             inner: FixedSizeBinaryArray::from(array_data),
@@ -268,12 +276,64 @@ mod from_arrow {
 impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
     type FloatType = BFloat16Type;
 
+    /// Returns the underlying `bf16` values as a borrowed slice.
+    ///
+    /// # Preconditions
+    ///
+    /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape
+    ///   used by [`BFloat16Array`]). Asserted at entry.
+    /// - The value buffer must be at least 2-byte aligned. Lance's in-tree
+    ///   constructors always satisfy this: value buffers are built either via
+    ///   `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32
+    ///   bytes) or via `Buffer::from_vec::<u16>` (aligned to `align_of::<u16>()`
+    ///   == 2); both meet `bf16`'s 2-byte requirement. Externally-built
+    ///   `FixedSizeBinaryArray`s arriving via FFI, IPC, or
+    ///   `Buffer::from_custom_allocation` are not required by arrow-rs to be
+    ///   aligned beyond a single byte; passing one to this method violates the
+    ///   precondition. A `debug_assert` below catches such inputs in debug and
+    ///   test builds.
+    ///
+    /// # Endianness
+    ///
+    /// `lance-arrow` is gated on `target_endian = "little"` at the crate root,
+    /// so this method always returns values in the same byte order Lance writes
+    /// (see [`BFloat16Array::value`] and the [`FromIterator`] impls).
     fn as_slice(&self) -> &[bf16] {
         assert_eq!(
             self.value_length(),
             2,
             "BFloat16 arrays must use FixedSizeBinary(2) storage"
         );
+        debug_assert_eq!(
+            (self.value_data().as_ptr() as usize) % std::mem::align_of::<bf16>(),
+            0,
+            "BFloat16 value buffer must be at least 2-byte aligned"
+        );
+        // SAFETY:
+        // - The assert above pins `value_size == 2`, so `value_data().len() / 2`
+        //   equals the array's logical element count.
+        //   `FixedSizeBinaryArray::From<ArrayData>` constructs its value buffer
+        //   as `buffers[0].slice_with_length(offset * 2, len * 2)` (arrow-array
+        //   `fixed_size_binary_array.rs`), so `value_data()` already returns
+        //   the offset-adjusted slice. Do not replace `value_data()` with an
+        //   accessor that returns the un-sliced backing buffer.
+        // - `bf16` is `#[repr(transparent)]` over `u16` (size 2, alignment 2);
+        //   every `u16` bit pattern is a valid `bf16`, so any byte content
+        //   yields a defined value — never UB.
+        // - Alignment is the caller's responsibility per the precondition
+        //   documented above. The `debug_assert_eq!` immediately preceding this
+        //   block catches violations in debug and test builds only — release
+        //   builds rely on callers honoring the precondition. arrow-rs
+        //   declares `FixedSizeBinary(n)`'s
+        //   `BufferSpec::FixedWidth { alignment: align_of::<u8>() == 1 }`
+        //   (arrow-data `data.rs`), so arrow-rs alone does not guarantee
+        //   2-byte alignment. Lance's in-tree construction paths build value
+        //   buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant,
+        //   ≥32 bytes) or `Buffer::from_vec::<u16>` (2-byte aligned), both of
+        //   which satisfy `bf16`'s 2-byte requirement.
+        // - The returned slice borrows from `self`; the underlying ref-counted,
+        //   immutable Arrow buffer cannot be mutated or freed for the slice's
+        //   lifetime.
         unsafe {
             slice::from_raw_parts(
                 self.value_data().as_ptr() as *const bf16,
@@ -301,6 +361,16 @@ mod tests {
         assert_eq!(array, array2);
         assert_eq!(array.len(), 3);
 
+        // Pin the raw little-endian bytes emitted by `From<Vec<bf16>>` (rewritten to
+        // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order
+        // regression is caught directly rather than only through Debug formatting.
+        // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040.
+        let inner = array2.clone().into_inner();
+        let raw_bytes: Vec<u8> = (0..inner.len())
+            .flat_map(|i| inner.value(i).to_vec())
+            .collect();
+        assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]);
+
         let expected_fmt = "BFloat16Array\n[\n  1.0,\n  2.0,\n  3.0,\n]";
         assert_eq!(expected_fmt, format!("{:?}", array));
 
diff --git a/rust/lance-arrow/src/deepcopy.rs b/rust/lance-arrow/src/deepcopy.rs
index b747b24d466..afeb853350b 100644
--- a/rust/lance-arrow/src/deepcopy.rs
+++ b/rust/lance-arrow/src/deepcopy.rs
@@ -14,6 +14,15 @@ pub fn deep_copy_buffer(buffer: &Buffer) -> Buffer {
 pub fn deep_copy_nulls(nulls: Option<&NullBuffer>) -> Option<NullBuffer> {
     let nulls = nulls?;
     let bit_buffer = deep_copy_buffer(nulls.inner().inner());
+    // SAFETY: `null_count` is taken from the source `NullBuffer`, which already
+    // upheld `NullBuffer::new_unchecked`'s invariant — the unset-bit count over
+    // the logical bit slice `[bit_offset, bit_offset + bit_len)`. `NullBuffer::slice`
+    // adjusts only `BooleanBuffer::bit_offset` / `bit_len` and never byte-advances
+    // the inner `Buffer`, so `deep_copy_buffer` (which copies the source `Buffer`'s
+    // `as_slice()` view from byte 0) reproduces the exact bit pattern at the same
+    // bit offsets; the unset-bit count is therefore preserved. `BooleanBuffer::new`
+    // panics (does not UB) if `bit_offset + bit_len > 8 * buffer.len()`, and the
+    // copy has the same length, so that check still passes.
     Some(unsafe {
         NullBuffer::new_unchecked(
             BooleanBuffer::new(bit_buffer, nulls.offset(), nulls.len()),
@@ -37,6 +46,19 @@ pub fn deep_copy_array_data(data: &ArrayData) -> ArrayData {
         .iter()
         .map(deep_copy_array_data)
         .collect::<Vec<_>>();
+    // SAFETY: `build_unchecked` inherits `ArrayData::new_unchecked`'s contract —
+    // `(data_type, len, offset, nulls, buffers, child_data)` must form a valid
+    // Arrow array. This call reproduces `data` structurally: `data_type`, `len`,
+    // and `offset` are forwarded unchanged; each buffer is replaced by a byte-
+    // identical copy of its offset-applied `as_slice()` view (the output buffer
+    // is `MutableBuffer`-allocated, at least as aligned as the source); `nulls`
+    // is deep-copied with the same bit offset/length and unset-bit count (see
+    // `deep_copy_nulls`); `child_data` is recursively cloned with the same
+    // guarantee. Every value-level invariant the source upheld — UTF-8 validity,
+    // monotonic offsets, in-bounds dictionary indices, run-end monotonicity,
+    // struct child-length matching — therefore transfers to the copy. If the
+    // source `ArrayData` was itself constructed via `new_unchecked` with an
+    // invalid payload, this function faithfully reproduces that invalidity.
     unsafe {
         ArrayDataBuilder::new(data_type)
             .len(len)
diff --git a/rust/lance-arrow/src/floats.rs b/rust/lance-arrow/src/floats.rs
index 054f4418e0b..ad5f3768980 100644
--- a/rust/lance-arrow/src/floats.rs
+++ b/rust/lance-arrow/src/floats.rs
@@ -184,6 +184,22 @@ pub trait FloatArray<T: ArrowFloatType + ?Sized>: Array + Clone + 'static {
     type FloatType: ArrowFloatType;
 
     /// Returns a reference to the underlying data as a slice.
+    ///
+    /// # Panics
+    ///
+    /// Implementations may panic if the array's storage shape does not match
+    /// the expected element layout. In particular, the `bf16` impl panics if
+    /// `value_length() != 2` (the `FixedSizeBinary(2)` shape required by
+    /// `BFloat16Array`).
+    ///
+    /// # Preconditions
+    ///
+    /// Implementations may impose additional invariants on the underlying
+    /// buffer. The `bf16` impl requires the value buffer to be at least
+    /// 2-byte aligned — satisfied automatically by every in-tree Lance
+    /// constructor, but external callers passing externally-built arrays
+    /// (FFI, IPC, `Buffer::from_custom_allocation`) must ensure alignment.
+    /// See the impl's docstring for details.
     fn as_slice(&self) -> &[T::Native];
 
     /// Construct an array from a vector of values.
diff --git a/rust/lance-arrow/src/ipc.rs b/rust/lance-arrow/src/ipc.rs
index 1c6364c4525..8b6e5cf41fe 100644
--- a/rust/lance-arrow/src/ipc.rs
+++ b/rust/lance-arrow/src/ipc.rs
@@ -270,7 +270,7 @@ pub fn read_ipc_stream_single_at(
 /// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`.
 /// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`.
 fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> {
-    let has_continuation = buf.len() >= 4 && buf[..4] == [0xff; 4];
+    let has_continuation = buf.len() >= 4 && buf[..4] == IPC_CONTINUATION;
     if has_continuation {
         if buf.len() < 8 {
             return Err(ArrowError::ParseError(
@@ -358,6 +358,134 @@ pub fn read_ipc_stream_single(data: &Bytes) -> Result<RecordBatch, ArrowError> {
     }
 }
 
+// ---------------------------------------------------------------------------
+// Aligned IPC sections
+// ---------------------------------------------------------------------------
+
+/// Byte alignment that each IPC section's stream start is padded to.
+///
+/// When several IPC streams are concatenated into one larger blob (e.g. a
+/// cache entry), a section that starts at an arbitrary offset would leave its
+/// array data misaligned. [`FileDecoder`] with `require_alignment = false`
+/// then silently copies each buffer into a freshly aligned allocation on
+/// every read, defeating zero-copy. Padding each section start to a 64-byte
+/// boundary keeps the decoded buffers borrowed directly from the input.
+pub const IPC_SECTION_ALIGNMENT: usize = 64;
+
+/// Number of zero-padding bytes needed to advance `pos` to the next
+/// [`IPC_SECTION_ALIGNMENT`] boundary.
+fn section_padding(pos: usize) -> usize {
+    (IPC_SECTION_ALIGNMENT - (pos % IPC_SECTION_ALIGNMENT)) % IPC_SECTION_ALIGNMENT
+}
+
+/// A [`Write`] adapter that counts the bytes written through it.
+struct CountingWriter<'a> {
+    inner: &'a mut dyn Write,
+    count: usize,
+}
+
+impl Write for CountingWriter<'_> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        let n = self.inner.write(buf)?;
+        self.count += n;
+        Ok(n)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+}
+
+/// Write zero padding so the next byte lands on an [`IPC_SECTION_ALIGNMENT`]
+/// boundary, advancing `pos` past it.
+fn write_section_padding(writer: &mut dyn Write, pos: &mut usize) -> Result<(), ArrowError> {
+    let pad = section_padding(*pos);
+    if pad > 0 {
+        const ZEROS: [u8; IPC_SECTION_ALIGNMENT] = [0u8; IPC_SECTION_ALIGNMENT];
+        writer
+            .write_all(&ZEROS[..pad])
+            .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+        *pos += pad;
+    }
+    Ok(())
+}
+
+/// Write `batch` as a 64-byte-aligned single-batch Arrow IPC section.
+///
+/// `pos` is the absolute byte offset of `writer` within the enclosing blob.
+/// Zero padding is written first so the IPC stream begins on an
+/// [`IPC_SECTION_ALIGNMENT`] boundary, then the stream itself. `pos` is
+/// advanced past both the padding and the stream so the caller can write
+/// further aligned sections.
+///
+/// Paired with [`read_ipc_section_at`]. For the decoded buffers to be borrowed
+/// zero-copy, the blob must ultimately be read back from a buffer whose base
+/// address is at least 64-byte aligned.
+pub fn write_ipc_section(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    batch: &RecordBatch,
+) -> Result<(), ArrowError> {
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream(batch, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read a single [`RecordBatch`] from an aligned IPC section at `offset`.
+///
+/// Skips the alignment padding written by [`write_ipc_section`], then reads
+/// the stream, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section`]).
+pub fn read_ipc_section_at(data: &Bytes, offset: &mut usize) -> Result<RecordBatch, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_single_at(data, offset)
+}
+
+/// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC section.
+///
+/// Like [`write_ipc_section`] but emits every batch from `iter` into one IPC
+/// stream (schema + N batches + EOS). `iter` must yield at least one batch.
+/// Paired with [`read_ipc_section_batches_at`].
+pub fn write_ipc_section_batches<I>(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    iter: I,
+) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream_batches(iter, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read all [`RecordBatch`]es from an aligned multi-batch IPC section at
+/// `offset`, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section_batches`]).
+pub fn read_ipc_section_batches_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_at(data, offset)
+}
+
 #[cfg(test)]
 mod tests {
     use arrow_array::{ArrayRef, record_batch};
@@ -403,4 +531,90 @@ mod tests {
             assert_col_zero_copy(batch.column(1));
         }
     }
+
+    /// Allocate a [`Bytes`] whose base address is 64-byte aligned, modelling a
+    /// backend that reads cache entries into an aligned buffer. A plain
+    /// `Bytes::from(vec)` only guarantees the allocator's alignment for `u8`.
+    fn aligned_bytes(payload: &[u8]) -> Bytes {
+        let mut v = vec![0u8; payload.len() + IPC_SECTION_ALIGNMENT];
+        let pad = section_padding(v.as_ptr() as usize);
+        v[pad..pad + payload.len()].copy_from_slice(payload);
+        Bytes::from(v).slice(pad..pad + payload.len())
+    }
+
+    #[test]
+    fn test_aligned_ipc_sections_are_zero_copy() {
+        // A LargeBinary column exercises the i64-offset buffer whose 8-byte
+        // alignment requirement triggers a realigning memcpy when misaligned.
+        let blocks = arrow_array::LargeBinaryArray::from_vec(vec![&b"hello"[..], b"world"]);
+        let section_a = RecordBatch::try_from_iter([("a", Arc::new(blocks) as ArrayRef)]).unwrap();
+        let section_b = record_batch!(("b", Int64, [10i64, 20, 30, 40, 50])).unwrap();
+
+        let mut buf = Vec::new();
+        // Arbitrary, deliberately non-64-aligned preamble so the first section
+        // must be padded rather than landing at offset 0 by luck.
+        buf.extend_from_slice(&[0xABu8; 7]);
+        let mut pos = buf.len();
+        // The first section's stream begins after padding the 7-byte preamble
+        // up to the next 64-byte boundary.
+        assert_eq!(7 + section_padding(7), IPC_SECTION_ALIGNMENT);
+        write_ipc_section(&mut buf, &mut pos, &section_a).unwrap();
+        write_ipc_section(&mut buf, &mut pos, &section_b).unwrap();
+
+        let data = aligned_bytes(&buf);
+        assert_eq!(
+            section_padding(data.as_ptr() as usize),
+            0,
+            "base not aligned"
+        );
+
+        let mut offset = 7;
+        let read_a = read_ipc_section_at(&data, &mut offset).unwrap();
+        let read_b = read_ipc_section_at(&data, &mut offset).unwrap();
+        assert_eq!(read_a, section_a);
+        assert_eq!(read_b, section_b);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for batch in [&read_a, &read_b] {
+            for buffer in batch.column(0).to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "section buffer at {ptr:#x} was realigned out of the input \
+                     [{data_base:#x}..{data_end:#x}) — misaligned section",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_aligned_multi_batch_section_roundtrip_zero_copy() {
+        // A multi-batch section (e.g. IVF SQ storage chunks) must round-trip
+        // every batch and decode the first batch's buffers zero-copy.
+        let b1 = record_batch!(("v", Int64, [1i64, 2, 3])).unwrap();
+        let b2 = record_batch!(("v", Int64, [4i64, 5])).unwrap();
+        let b3 = record_batch!(("v", Int64, [6i64])).unwrap();
+
+        let mut buf = vec![0xCDu8; 5];
+        let mut pos = buf.len();
+        write_ipc_section_batches(&mut buf, &mut pos, [b1.clone(), b2.clone(), b3.clone()])
+            .unwrap();
+
+        let data = aligned_bytes(&buf);
+        let mut offset = 5;
+        let read = read_ipc_section_batches_at(&data, &mut offset).unwrap();
+        assert_eq!(read, vec![b1, b2, b3]);
+        assert_eq!(offset, buf.len(), "offset should land at section end");
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for buffer in read[0].column(0).to_data().buffers() {
+            let ptr = buffer.as_ptr() as usize;
+            assert!(
+                ptr >= data_base && ptr < data_end,
+                "first batch buffer at {ptr:#x} was realigned out of the input",
+            );
+        }
+    }
 }
diff --git a/rust/lance-arrow/src/lib.rs b/rust/lance-arrow/src/lib.rs
index b993cf00745..2ac962aa1aa 100644
--- a/rust/lance-arrow/src/lib.rs
+++ b/rust/lance-arrow/src/lib.rs
@@ -5,6 +5,16 @@
 //!
 //! To improve Arrow-RS ergonomic
 
+#![warn(clippy::undocumented_unsafe_blocks)]
+
+// lance-arrow reinterprets value bytes as native numeric types in
+// `FloatArray::as_slice` for `bf16` (rust/lance-arrow/src/bfloat16.rs), which
+// requires the host byte order to match the on-disk byte order Lance writes.
+// Lance writes little-endian; building on a big-endian target would silently
+// produce wrong numeric values.
+#[cfg(not(target_endian = "little"))]
+compile_error!("lance-arrow only supports little-endian targets");
+
 use std::sync::Arc;
 use std::{collections::HashMap, ptr::NonNull};
 
@@ -52,6 +62,11 @@ pub const BLOB_V2_EXT_NAME: &str = "lance.blob.v2";
 /// Metadata key for overriding the dedicated blob size threshold (in bytes)
 pub const BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY: &str =
     "lance-encoding:blob-dedicated-size-threshold";
+/// Metadata key for overriding the inline blob size threshold (in bytes)
+pub const BLOB_INLINE_SIZE_THRESHOLD_META_KEY: &str = "lance-encoding:blob-inline-size-threshold";
+/// Metadata key for overriding the maximum size (in bytes) of a packed blob sidecar file
+pub const BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY: &str =
+    "lance-encoding:blob-pack-file-size-threshold";
 
 type Result<T> = std::result::Result<T, ArrowError>;
 
@@ -289,7 +304,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float32Array::from_iter_values(
+                    Arc::new(Float32Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<Int8Array>()
@@ -297,7 +312,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to Int8Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f32)),
+                            .map(|x| x.map(|y| y as f32)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -308,7 +323,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float32Array::from_iter_values(
+                    Arc::new(Float32Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<Int16Array>()
@@ -316,7 +331,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to Int16Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f32)),
+                            .map(|x| x.map(|y| y as f32)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -327,7 +342,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float32Array::from_iter_values(
+                    Arc::new(Float32Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<Int32Array>()
@@ -335,7 +350,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to Int32Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f32)),
+                            .map(|x| x.map(|y| y as f32)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -346,7 +361,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float64Array::from_iter_values(
+                    Arc::new(Float64Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<Int64Array>()
@@ -354,7 +369,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to Int64Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f64)),
+                            .map(|x| x.map(|y| y as f64)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -365,7 +380,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float64Array::from_iter_values(
+                    Arc::new(Float64Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<UInt8Array>()
@@ -373,7 +388,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to UInt8Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f64)),
+                            .map(|x| x.map(|y| y as f64)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -384,7 +399,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                         field.is_nullable(),
                     )),
                     *size,
-                    Arc::new(Float64Array::from_iter_values(
+                    Arc::new(Float64Array::from_iter(
                         self.values()
                             .as_any()
                             .downcast_ref::<UInt32Array>()
@@ -392,7 +407,7 @@ impl FixedSizeListArrayExt for FixedSizeListArray {
                                 "Fail to cast primitive array to UInt32Type".to_string(),
                             ))?
                             .into_iter()
-                            .filter_map(|x| x.map(|y| y as f64)),
+                            .map(|x| x.map(|y| y as f64)),
                     )),
                     self.nulls().cloned(),
                 )),
@@ -1007,13 +1022,7 @@ fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc<dyn Array> {
 fn normalize_validity(
     validity: Option<&arrow_buffer::NullBuffer>,
 ) -> Option<&arrow_buffer::NullBuffer> {
-    validity.and_then(|v| {
-        if v.null_count() == v.len() {
-            None
-        } else {
-            Some(v)
-        }
-    })
+    validity.filter(|v| v.null_count() != v.len())
 }
 
 /// Helper function to merge validity buffers from two struct arrays
@@ -1370,9 +1379,12 @@ fn merge_with_schema(
                         );
                         let merged_validity =
                             merge_struct_validity(left_list.nulls(), right_list.nulls());
+                        // `trimmed_values` starts at the first used value, so offsets
+                        // must be shifted to match or `ListArray::new` panics when the
+                        // input list was sliced (e.g. from a filtered batch).
                         let merged_list = ListArray::new(
                             child_field.clone(),
-                            left_list.offsets().clone(),
+                            left_list.trimmed_offsets(),
                             merged_values,
                             merged_validity,
                         );
@@ -1397,7 +1409,7 @@ fn merge_with_schema(
                             merge_struct_validity(left_list.nulls(), right_list.nulls());
                         let merged_list = LargeListArray::new(
                             child_field.clone(),
-                            left_list.offsets().clone(),
+                            left_list.trimmed_offsets(),
                             merged_values,
                             merged_validity,
                         );
@@ -1561,6 +1573,63 @@ mod tests {
     use arrow_array::{ListArray, StringArray, new_empty_array, new_null_array};
     use arrow_buffer::OffsetBuffer;
 
+    #[test]
+    fn test_convert_to_floating_point_preserves_inner_nulls() {
+        // A FixedSizeList<Int8> with a null inner element must convert to a
+        // FixedSizeList<Float32> with the null kept in place. Dropping it would
+        // shorten the values array and shift every later element (and, when the
+        // remaining count is not a multiple of the list size, panic).
+        let values = Int8Array::from(vec![Some(1), None, Some(3), Some(4)]);
+        let fsl = FixedSizeListArray::new(
+            Arc::new(Field::new("item", DataType::Int8, true)),
+            2,
+            Arc::new(values),
+            None,
+        );
+
+        let converted = fsl.convert_to_floating_point().unwrap();
+
+        assert_eq!(converted.len(), 2);
+        let conv_values = converted
+            .values()
+            .as_any()
+            .downcast_ref::<Float32Array>()
+            .unwrap();
+        assert_eq!(conv_values.len(), 4);
+        assert_eq!(conv_values.value(0), 1.0);
+        assert!(conv_values.is_null(1));
+        assert_eq!(conv_values.value(2), 3.0);
+        assert_eq!(conv_values.value(3), 4.0);
+    }
+
+    #[test]
+    fn test_convert_to_floating_point_preserves_inner_nulls_f64_arm() {
+        // The Float64-producing arms (Int64/UInt8/UInt32) share the same fix as the
+        // Float32 arms; cover one representative (UInt8 -> Float64) so both branch
+        // families are exercised.
+        let values = UInt8Array::from(vec![Some(10u8), None, Some(30), Some(40)]);
+        let fsl = FixedSizeListArray::new(
+            Arc::new(Field::new("item", DataType::UInt8, true)),
+            2,
+            Arc::new(values),
+            None,
+        );
+
+        let converted = fsl.convert_to_floating_point().unwrap();
+
+        assert_eq!(converted.len(), 2);
+        let conv_values = converted
+            .values()
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .unwrap();
+        assert_eq!(conv_values.len(), 4);
+        assert_eq!(conv_values.value(0), 10.0);
+        assert!(conv_values.is_null(1));
+        assert_eq!(conv_values.value(2), 30.0);
+        assert_eq!(conv_values.value(3), 40.0);
+    }
+
     #[test]
     fn test_merge_recursive() {
         let a_array = Int32Array::from(vec![Some(1), Some(2), Some(3)]);
@@ -2308,6 +2377,118 @@ mod tests {
         assert_eq!(merged_array.len(), 2);
     }
 
+    #[test]
+    fn test_merge_with_schema_sliced_list_struct() {
+        test_merge_with_schema_sliced_list_struct_generic::<i32>();
+    }
+
+    #[test]
+    fn test_merge_with_schema_sliced_large_list_struct() {
+        test_merge_with_schema_sliced_list_struct_generic::<i64>();
+    }
+
+    // Regression for #6580: merge_with_schema panicked when the left list was a
+    // sliced view whose offsets did not start at zero (common after a filtered
+    // scan). Cloning those offsets alongside `trimmed_values` produced offsets
+    // larger than the trimmed child, panicking in `(Large)ListArray::new`.
+    fn test_merge_with_schema_sliced_list_struct_generic<O: OffsetSizeTrait>() {
+        let make_list_dtype = |item_field: Arc<Field>| {
+            if O::IS_LARGE {
+                DataType::LargeList(item_field)
+            } else {
+                DataType::List(item_field)
+            }
+        };
+
+        // Build a List<Struct> with two rows of 5 items each, then slice away
+        // the first row so the remaining list's offsets start at 5, not 0.
+        let struct_fields_a = Fields::from(vec![Field::new("a", DataType::Int32, true)]);
+        let left_values = Arc::new(StructArray::new(
+            struct_fields_a.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..10)) as ArrayRef],
+            None,
+        ));
+        let full_list = GenericListArray::<O>::new(
+            Arc::new(Field::new("item", DataType::Struct(struct_fields_a), true)),
+            OffsetBuffer::<O>::from_lengths([5, 5]),
+            left_values,
+            None,
+        );
+        let sliced_left = full_list.slice(1, 1);
+        assert_eq!(sliced_left.offsets()[0].as_usize(), 5);
+        assert_eq!(sliced_left.offsets()[1].as_usize(), 10);
+
+        let struct_fields_b = Fields::from(vec![Field::new("b", DataType::Int32, true)]);
+        let right_values = Arc::new(StructArray::new(
+            struct_fields_b.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(100..105)) as ArrayRef],
+            None,
+        ));
+        let right_list = GenericListArray::<O>::new(
+            Arc::new(Field::new("item", DataType::Struct(struct_fields_b), true)),
+            OffsetBuffer::<O>::from_lengths([5]),
+            right_values,
+            None,
+        );
+
+        let target_item_field = Arc::new(Field::new(
+            "item",
+            DataType::Struct(Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Int32, true),
+            ])),
+            true,
+        ));
+        let target_fields = Fields::from(vec![Field::new(
+            "items",
+            make_list_dtype(target_item_field),
+            true,
+        )]);
+
+        let left_batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                "items",
+                sliced_left.data_type().clone(),
+                true,
+            )])),
+            vec![Arc::new(sliced_left) as ArrayRef],
+        )
+        .unwrap();
+        let right_batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                "items",
+                right_list.data_type().clone(),
+                true,
+            )])),
+            vec![Arc::new(right_list) as ArrayRef],
+        )
+        .unwrap();
+
+        let merged = left_batch
+            .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec()))
+            .unwrap();
+
+        let merged_list = merged
+            .column_by_name("items")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<GenericListArray<O>>()
+            .unwrap();
+        assert_eq!(merged_list.len(), 1);
+        assert_eq!(merged_list.value_length(0).as_usize(), 5);
+        let merged_struct = merged_list.values().as_struct();
+        assert_eq!(merged_struct.num_columns(), 2);
+        let a = merged_struct
+            .column_by_name("a")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        // After shifting offsets to zero, values 5..10 should be first.
+        let a_vals: Vec<i32> = a.iter().map(|v| v.unwrap()).collect();
+        assert_eq!(a_vals, vec![5, 6, 7, 8, 9]);
+    }
+
     #[test]
     fn test_project_by_schema_list_struct_reorder() {
         // Test that project_by_schema correctly reorders fields inside List<Struct>
diff --git a/rust/lance-arrow/src/list.rs b/rust/lance-arrow/src/list.rs
index 0c24fc579da..06b0fc592cf 100644
--- a/rust/lance-arrow/src/list.rs
+++ b/rust/lance-arrow/src/list.rs
@@ -23,6 +23,16 @@ pub trait ListArrayExt {
     /// behaves similarly to `values()` except it slices the array so that it starts at
     /// the first list offset and ends at the last list offset.
     fn trimmed_values(&self) -> Arc<dyn Array>;
+    /// The offset type of the underlying list array.
+    type Offset: OffsetSizeTrait;
+    /// Returns offsets shifted so the first offset is zero, matching
+    /// [`Self::trimmed_values`].
+    ///
+    /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the
+    /// original values buffer, so combining them with trimmed values produces
+    /// offsets that exceed the values length. Use this together with
+    /// `trimmed_values` when constructing a new list array.
+    fn trimmed_offsets(&self) -> OffsetBuffer<Self::Offset>;
 }
 
 impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize> {
@@ -90,6 +100,20 @@ impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize>
             .unwrap_or(0);
         self.values().slice(first_value, last_value - first_value)
     }
+
+    type Offset = OffsetSize;
+
+    fn trimmed_offsets(&self) -> OffsetBuffer<OffsetSize> {
+        let offsets = self.offsets();
+        let Some(&first) = offsets.first() else {
+            return offsets.clone();
+        };
+        if first == OffsetSize::zero() {
+            return offsets.clone();
+        }
+        let shifted: Vec<OffsetSize> = offsets.iter().map(|&o| o - first).collect();
+        OffsetBuffer::new(ScalarBuffer::from(shifted))
+    }
 }
 
 #[cfg(test)]
diff --git a/rust/lance-core/Cargo.toml b/rust/lance-core/Cargo.toml
index 9dff4b001a4..2f6183be8a1 100644
--- a/rust/lance-core/Cargo.toml
+++ b/rust/lance-core/Cargo.toml
@@ -14,6 +14,7 @@ rust-version.workspace = true
 [dependencies]
 arrow-array.workspace = true
 arrow-buffer.workspace = true
+arrow-data.workspace = true
 arrow-schema.workspace = true
 async-trait.workspace = true
 lance-arrow.workspace = true
@@ -21,10 +22,11 @@ byteorder.workspace = true
 bytes.workspace = true
 datafusion-common = { workspace = true, optional = true }
 datafusion-sql = { workspace = true, optional = true }
-deepsize.workspace = true
+lance-derive.workspace = true
 futures.workspace = true
 itertools.workspace = true
 libc.workspace = true
+libm.workspace = true
 moka.workspace = true
 num_cpus = "1.0"
 object_store = { workspace = true }
@@ -39,6 +41,7 @@ tokio.workspace = true
 tokio-stream.workspace = true
 tokio-util.workspace = true
 tracing.workspace = true
+twox-hash.workspace = true
 url.workspace = true
 log.workspace = true
 
@@ -52,6 +55,10 @@ proptest.workspace = true
 rstest.workspace = true
 
 [features]
+# Capture Rust backtraces in error types. When disabled (the default),
+# the backtrace field is zero-sized with no overhead. At runtime, capture
+# is still gated by RUST_BACKTRACE=1.
+backtrace = []
 datafusion = ["dep:datafusion-common", "dep:datafusion-sql"]
 
 [lints]
diff --git a/rust/lance-core/src/cache/backend.rs b/rust/lance-core/src/cache/backend.rs
index 237254c464f..9307868f399 100644
--- a/rust/lance-core/src/cache/backend.rs
+++ b/rust/lance-core/src/cache/backend.rs
@@ -22,6 +22,9 @@ use super::CacheCodec;
 /// A type-erased cache entry.
 pub type CacheEntry = Arc<dyn Any + Send + Sync>;
 
+/// Iterator over cache keys currently known to a backend.
+pub type CacheKeyIterator<'a> = Box<dyn Iterator<Item = InternalCacheKey> + Send + 'a>;
+
 /// Structured cache key passed to [`CacheBackend`] methods.
 ///
 /// CacheBackend impls receive these ready-made from [`LanceCache`](super::LanceCache)
@@ -116,6 +119,15 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug {
     /// Remove all entries.
     async fn clear(&self);
 
+    /// Return an iterator over cache keys currently known to this backend.
+    ///
+    /// Backends that cannot enumerate keys cheaply or accurately should return
+    /// `None`. An empty iterator means key inventory is supported and the
+    /// cache currently has no entries.
+    async fn keys(&self) -> Option<CacheKeyIterator<'_>> {
+        None
+    }
+
     /// Number of entries currently stored (may flush pending operations).
     async fn num_entries(&self) -> usize;
 
diff --git a/rust/lance-core/src/cache/codec.rs b/rust/lance-core/src/cache/codec.rs
index 34e5264bb28..bba54840829 100644
--- a/rust/lance-core/src/cache/codec.rs
+++ b/rust/lance-core/src/cache/codec.rs
@@ -5,12 +5,184 @@
 //!
 //! Implement [`CacheCodecImpl`] on concrete types, then use
 //! [`CacheCodec::from_impl`] to produce a type-erased codec for the cache.
+//!
+//! # Wire format
+//!
+//! Every serialized entry begins with a small hand-framed **envelope** so the
+//! reader can validate it before trusting the body:
+//!
+//! ```text
+//! [magic: 4B = b"LCE1"]
+//! [envelope_version: u8]
+//! [type_id_len: u16 LE][type_id: utf8]   # stable, author-assigned
+//! [type_version: u32 LE]                 # per-type body schema version
+//! <body, written by the type's CacheCodecImpl::serialize>
+//! ```
+//!
+//! The envelope is deliberately *not* protobuf: it is the most
+//! stability-critical part, must parse robustly against arbitrary bytes
+//! (including data written by older, pre-stabilization builds), and never
+//! changes shape. Bodies use protobuf headers, where field-number evolution
+//! pays off.
+//!
+//! # Decode outcome
+//!
+//! Deserialization never propagates a parse failure as a hard error into the
+//! cache path. Anything the reader cannot confidently interpret — absent or
+//! wrong magic, an unknown `envelope_version`, a `type_id` mismatch, an
+//! unsupported `type_version`, or a body decode error — becomes
+//! [`CacheDecode::Miss`]. A backend turns `Miss` into a normal cache miss and
+//! recomputes the value. This is what lets data written by an older format
+//! self-heal: it simply fails the magic check and is regenerated.
 
+use std::io::Write;
 use std::sync::Arc;
 
 use bytes::Bytes;
 
-use crate::Result;
+use crate::{Error, Result};
+
+use super::{CacheEntryReader, CacheEntryWriter};
+
+// ---------------------------------------------------------------------------
+// Envelope
+// ---------------------------------------------------------------------------
+
+/// Magic bytes that prefix every stabilized cache entry.
+///
+/// An ASCII tag (`0x4C 0x43 0x45 0x31`) chosen so it cannot collide with any
+/// pre-stabilization blob: those began with either a small little-endian
+/// length (tens of bytes) or a small tag byte, never these values.
+///
+/// Exported so backends can cheaply identify Lance cache entries (e.g. when
+/// scanning a persistent store at startup) without hardcoding the bytes —
+/// prefer [`has_cache_envelope`] over comparing against this directly.
+pub const MAGIC: [u8; 4] = *b"LCE1";
+
+/// Returns `true` if `data` begins with the cache-entry [`MAGIC`].
+///
+/// A cheap prefix check for backends that need to recognize Lance cache
+/// entries without fully [`deserialize`](CacheCodec::deserialize)-ing them. A
+/// `true` result only means the framing looks like ours; the entry can still
+/// decode to a [`Miss`](CacheDecode::Miss) (e.g. wrong `type_id`).
+pub fn has_cache_envelope(data: &[u8]) -> bool {
+    data.get(..MAGIC.len()) == Some(&MAGIC[..])
+}
+
+/// Version of the envelope framing itself. Bumped only if the outer frame
+/// (magic/version/type_id/type_version layout) ever changes — expected never.
+const ENVELOPE_VERSION: u8 = 1;
+
+/// Parsed envelope borrowed from the input bytes.
+struct ParsedEnvelope<'a> {
+    type_id: &'a str,
+    type_version: u32,
+    /// Offset of the first body byte within the input.
+    body_offset: usize,
+}
+
+/// Parse and validate the envelope at the start of `data`.
+///
+/// Returns `None` for anything that is not a well-formed envelope this build
+/// understands (wrong/absent magic, unknown `envelope_version`, truncation,
+/// non-utf8 `type_id`). Callers translate `None` into [`CacheDecode::Miss`].
+fn parse_envelope(data: &Bytes) -> Option<ParsedEnvelope<'_>> {
+    let bytes = data.as_ref();
+    let mut off = 0usize;
+
+    let magic = bytes.get(off..off + 4)?;
+    if magic != MAGIC {
+        return None;
+    }
+    off += 4;
+
+    if *bytes.get(off)? != ENVELOPE_VERSION {
+        return None;
+    }
+    off += 1;
+
+    let type_id_len = u16::from_le_bytes(bytes.get(off..off + 2)?.try_into().ok()?) as usize;
+    off += 2;
+
+    let type_id = std::str::from_utf8(bytes.get(off..off + type_id_len)?).ok()?;
+    off += type_id_len;
+
+    let type_version = u32::from_le_bytes(bytes.get(off..off + 4)?.try_into().ok()?);
+    off += 4;
+
+    Some(ParsedEnvelope {
+        type_id,
+        type_version,
+        body_offset: off,
+    })
+}
+
+/// Write the envelope for `type_id`/`type_version`, returning the number of
+/// bytes written (the body's starting offset).
+fn write_envelope(writer: &mut dyn Write, type_id: &str, type_version: u32) -> Result<usize> {
+    let type_id_len = u16::try_from(type_id.len()).map_err(|_| {
+        Error::io(format!(
+            "cache codec type_id too long ({} bytes, max {})",
+            type_id.len(),
+            u16::MAX
+        ))
+    })?;
+
+    writer.write_all(&MAGIC)?;
+    writer.write_all(&[ENVELOPE_VERSION])?;
+    writer.write_all(&type_id_len.to_le_bytes())?;
+    writer.write_all(type_id.as_bytes())?;
+    writer.write_all(&type_version.to_le_bytes())?;
+
+    Ok(4 + 1 + 2 + type_id.len() + 4)
+}
+
+// ---------------------------------------------------------------------------
+// CacheDecode — first-class cache-miss outcome
+// ---------------------------------------------------------------------------
+
+/// Why a cache entry could not be decoded into the expected type.
+///
+/// Carried by [`CacheDecode::Miss`] so backends can emit targeted metrics
+/// (e.g. distinguish "evicting due to a stale format" from "type collision")
+/// without re-parsing. Every reason maps to the same behavior — recompute via
+/// the loader — so callers that don't care can ignore it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CacheMissReason {
+    /// Absent or wrong magic, unknown `envelope_version`, truncated framing, or
+    /// a non-utf8 `type_id`. Typically an entry written by a pre-stabilization
+    /// or otherwise foreign build.
+    InvalidEnvelope,
+    /// Well-formed envelope, but its `type_id` names a different entry type than
+    /// the codec reading it.
+    TypeMismatch,
+    /// Written by a newer build whose `type_version` this build does not
+    /// understand and must not attempt to interpret.
+    VersionTooNew,
+    /// Envelope validated, but the body failed to decode (truncation, a
+    /// malformed protobuf header, an IPC error, etc.).
+    BodyError,
+}
+
+/// Outcome of deserializing a cache entry.
+///
+/// `Miss` means the bytes could not be confidently decoded into `T`; the
+/// [`CacheMissReason`] says why. A backend treats any `Miss` exactly like a key
+/// that was never present: recompute via the loader.
+#[derive(Debug)]
+pub enum CacheDecode<T> {
+    Hit(T),
+    Miss(CacheMissReason),
+}
+
+impl<T> CacheDecode<T> {
+    pub fn hit(self) -> Option<T> {
+        match self {
+            Self::Hit(v) => Some(v),
+            Self::Miss(_) => None,
+        }
+    }
+}
 
 // ---------------------------------------------------------------------------
 // CacheCodecImpl — trait for serializable cache entry types
@@ -18,31 +190,40 @@ use crate::Result;
 
 /// Serialization trait for cache entries.
 ///
-/// **Experimental**: the serialized format is not stable and may change
-/// between releases without notice.
+/// **Experimental**: the serialized format is not yet covered by a stability
+/// guarantee and may change between releases. When it does stabilize, the
+/// rules are: `TYPE_ID`, protobuf field numbers, and enum values are
+/// append-only forever; format changes that protobuf cannot express
+/// transparently bump [`CURRENT_VERSION`](Self::CURRENT_VERSION).
 ///
-/// Implement this on concrete types that need to survive serialization
-/// through a persistent cache backend. Then wire it into a [`CacheKey`](super::CacheKey)
-/// via [`CacheCodec::from_impl`]:
+/// Implement this on concrete types that need to survive serialization through
+/// a persistent cache backend, then wire it into a
+/// [`CacheKey`](super::CacheKey) via [`CacheCodec::from_impl`].
 ///
-/// ```ignore
-/// impl CacheCodecImpl for MyData {
-///     fn serialize(&self, w: &mut dyn Write) -> Result<()> { /* ... */ }
-///     fn deserialize(data: &Bytes) -> Result<Self> { /* ... */ }
-/// }
-///
-/// impl CacheKey for MyDataKey {
-///     type ValueType = MyData;
-///     fn codec() -> Option<CacheCodec> {
-///         Some(CacheCodec::from_impl::<MyData>())
-///     }
-///     // ...
-/// }
-/// ```
+/// The envelope (magic/version/type_id/type_version) is written and validated
+/// by the [`CacheCodec`] wrapper. [`serialize`](Self::serialize) writes only
+/// the body — a header followed by sections in a fixed, version-keyed order —
+/// and [`deserialize`](Self::deserialize) reads them back in that same order.
+/// The read sequence mirroring the write sequence for each `type_version` is
+/// the invariant the implementor owns.
 pub trait CacheCodecImpl: Send + Sync {
-    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()>;
+    /// Stable identity for this entry type. **Must not change once shipped.**
+    /// This is a deliberate author-assigned string, not `std::any::type_name`
+    /// (which is not stable across compiler versions).
+    const TYPE_ID: &'static str;
+
+    /// Body schema version this build writes. Bump when the body layout
+    /// changes in a way protobuf field additions cannot express transparently
+    /// (adding/removing/reordering sections, a raw-blob encoding change, etc.).
+    const CURRENT_VERSION: u32;
+
+    /// Write the body: a header, then sections in a fixed order.
+    fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()>;
 
-    fn deserialize(data: &Bytes) -> Result<Self>
+    /// Reconstruct from the body. Branch on
+    /// [`reader.version()`](CacheEntryReader::version) for backward compat;
+    /// sections are read in write order.
+    fn deserialize(reader: &mut CacheEntryReader<'_>) -> Result<Self>
     where
         Self: Sized;
 }
@@ -55,25 +236,31 @@ pub(crate) type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
 
 /// Type-erased codec for serializing and deserializing cache entries.
 ///
-/// `CacheCodec` is two plain function pointers — it is `Copy` and has no
-/// heap allocation. Construct one via [`CacheCodec::from_impl`] for types
-/// that implement [`CacheCodecImpl`], or [`CacheCodec::new`] for custom
-/// cases (e.g. when the orphan rule prevents a direct impl).
+/// `CacheCodec` carries the entry's stable `type_id`/`version` plus two plain
+/// function pointers — it is `Copy` and has no heap allocation. Construct one
+/// via [`CacheCodec::from_impl`] for types that implement [`CacheCodecImpl`],
+/// or [`CacheCodec::new`] for custom cases (e.g. when the orphan rule prevents
+/// a direct impl).
 #[derive(Copy, Clone)]
 pub struct CacheCodec {
-    pub(crate) serialize: fn(&ArcAny, &mut dyn std::io::Write) -> Result<()>,
-    pub(crate) deserialize: fn(&Bytes) -> Result<ArcAny>,
+    type_id: &'static str,
+    version: u32,
+    serialize_body: fn(&ArcAny, &mut CacheEntryWriter<'_>) -> Result<()>,
+    deserialize_body: fn(&mut CacheEntryReader<'_>) -> Result<ArcAny>,
 }
 
 impl std::fmt::Debug for CacheCodec {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("CacheCodec").finish_non_exhaustive()
+        f.debug_struct("CacheCodec")
+            .field("type_id", &self.type_id)
+            .field("version", &self.version)
+            .finish_non_exhaustive()
     }
 }
 
 fn serialize_via_impl<T: CacheCodecImpl + 'static>(
     any: &ArcAny,
-    writer: &mut dyn std::io::Write,
+    writer: &mut CacheEntryWriter<'_>,
 ) -> Result<()> {
     let val = any
         .downcast_ref::<T>()
@@ -81,44 +268,278 @@ fn serialize_via_impl<T: CacheCodecImpl + 'static>(
     val.serialize(writer)
 }
 
-fn deserialize_via_impl<T: CacheCodecImpl + 'static>(data: &Bytes) -> Result<ArcAny> {
-    let val = T::deserialize(data)?;
+fn deserialize_via_impl<T: CacheCodecImpl + 'static>(
+    reader: &mut CacheEntryReader<'_>,
+) -> Result<ArcAny> {
+    let val = T::deserialize(reader)?;
     Ok(Arc::new(val) as ArcAny)
 }
 
 impl CacheCodec {
-    /// Create a `CacheCodec` from plain function pointers.
+    /// Create a `CacheCodec` from explicit body function pointers.
     ///
     /// Prefer [`from_impl`](Self::from_impl) when the value type implements
     /// [`CacheCodecImpl`]. Use this for types where a direct impl isn't
-    /// possible (e.g. orphan rule prevents it).
+    /// possible (e.g. the orphan rule prevents it). `type_id` and `version`
+    /// play the same role as the corresponding [`CacheCodecImpl`] constants.
     pub fn new(
-        serialize: fn(&ArcAny, &mut dyn std::io::Write) -> Result<()>,
-        deserialize: fn(&Bytes) -> Result<ArcAny>,
+        type_id: &'static str,
+        version: u32,
+        serialize_body: fn(&ArcAny, &mut CacheEntryWriter<'_>) -> Result<()>,
+        deserialize_body: fn(&mut CacheEntryReader<'_>) -> Result<ArcAny>,
     ) -> Self {
         Self {
-            serialize,
-            deserialize,
+            type_id,
+            version,
+            serialize_body,
+            deserialize_body,
         }
     }
 
     /// Create a `CacheCodec` from a [`CacheCodecImpl`] implementation.
-    ///
-    /// For **sized** types stored directly in the cache. The codec
-    /// downcasts `&dyn Any` to `&T` for serialization and returns `Arc<T>`
-    /// from deserialization.
     pub fn from_impl<T: CacheCodecImpl + 'static>() -> Self {
         Self {
-            serialize: serialize_via_impl::<T>,
-            deserialize: deserialize_via_impl::<T>,
+            type_id: T::TYPE_ID,
+            version: T::CURRENT_VERSION,
+            serialize_body: serialize_via_impl::<T>,
+            deserialize_body: deserialize_via_impl::<T>,
         }
     }
 
-    pub fn serialize(&self, value: &ArcAny, writer: &mut dyn std::io::Write) -> Result<()> {
-        (self.serialize)(value, writer)
+    /// Serialize `value` into `writer`: envelope first, then the body.
+    pub fn serialize(&self, value: &ArcAny, writer: &mut dyn Write) -> Result<()> {
+        let body_offset = write_envelope(writer, self.type_id, self.version)?;
+        let mut entry_writer = CacheEntryWriter::with_pos(writer, body_offset);
+        (self.serialize_body)(value, &mut entry_writer)
+    }
+
+    /// Deserialize an entry from `data`.
+    ///
+    /// Never fails: any non-fatal failure to interpret the bytes becomes a
+    /// [`CacheDecode::Miss`] with the reason why (see [`CacheMissReason`]).
+    /// Reading from an in-memory [`Bytes`] cannot do I/O, so there is no fault
+    /// channel — a miss is the only non-`Hit` outcome.
+    pub fn deserialize(&self, data: &Bytes) -> CacheDecode<ArcAny> {
+        let Some(envelope) = parse_envelope(data) else {
+            log::debug!("cache entry rejected: missing or invalid envelope");
+            return CacheDecode::Miss(CacheMissReason::InvalidEnvelope);
+        };
+
+        if envelope.type_id != self.type_id {
+            log::debug!(
+                "cache entry type_id mismatch: got {:?}, expected {:?}",
+                envelope.type_id,
+                self.type_id
+            );
+            return CacheDecode::Miss(CacheMissReason::TypeMismatch);
+        }
+
+        // A version newer than this build writes was produced by a newer build
+        // whose body layout we cannot assume to understand. Older/equal versions
+        // are the impl's responsibility to handle (branching on reader.version()).
+        if envelope.type_version > self.version {
+            log::debug!(
+                "cache entry {:?} has unsupported type_version {} (this build writes {})",
+                self.type_id,
+                envelope.type_version,
+                self.version
+            );
+            return CacheDecode::Miss(CacheMissReason::VersionTooNew);
+        }
+
+        let mut reader = CacheEntryReader::new(data, envelope.body_offset, envelope.type_version);
+        match (self.deserialize_body)(&mut reader) {
+            Ok(value) => CacheDecode::Hit(value),
+            Err(e) => {
+                log::debug!(
+                    "cache entry {:?} v{} failed to decode: {e}",
+                    self.type_id,
+                    envelope.type_version
+                );
+                CacheDecode::Miss(CacheMissReason::BodyError)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A trivial codec used to exercise the envelope and miss semantics
+    /// without pulling in arrow-backed payloads.
+    #[derive(Debug, PartialEq)]
+    struct Widget {
+        n: u32,
+    }
+
+    impl CacheCodecImpl for Widget {
+        const TYPE_ID: &'static str = "test.Widget";
+        const CURRENT_VERSION: u32 = 1;
+
+        fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()> {
+            writer.write_raw(&self.n.to_le_bytes())
+        }
+
+        fn deserialize(reader: &mut CacheEntryReader<'_>) -> Result<Self> {
+            let bytes = reader.read_raw()?;
+            let n = u32::from_le_bytes(
+                bytes
+                    .as_ref()
+                    .try_into()
+                    .map_err(|_| Error::io("bad widget".to_string()))?,
+            );
+            Ok(Self { n })
+        }
+    }
+
+    fn serialize_widget(widget: &Widget) -> Bytes {
+        let codec = CacheCodec::from_impl::<Widget>();
+        let any: ArcAny = Arc::new(Widget { n: widget.n });
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+        Bytes::from(buf)
+    }
+
+    /// The miss reason, or `None` if the decode was a hit.
+    fn miss_reason(data: &Bytes) -> Option<CacheMissReason> {
+        match deserialize_widget(data) {
+            CacheDecode::Hit(_) => None,
+            CacheDecode::Miss(reason) => Some(reason),
+        }
     }
 
-    pub fn deserialize(&self, data: &Bytes) -> Result<ArcAny> {
-        (self.deserialize)(data)
+    fn deserialize_widget(data: &Bytes) -> CacheDecode<Widget> {
+        let codec = CacheCodec::from_impl::<Widget>();
+        match codec.deserialize(data) {
+            CacheDecode::Hit(any) => {
+                CacheDecode::Hit(Arc::try_unwrap(any.downcast::<Widget>().unwrap()).unwrap())
+            }
+            CacheDecode::Miss(reason) => CacheDecode::Miss(reason),
+        }
+    }
+
+    #[test]
+    fn envelope_roundtrip_hits() {
+        let bytes = serialize_widget(&Widget { n: 0xDEADBEEF });
+        // Sanity: the entry starts with the magic.
+        assert_eq!(&bytes[..4], b"LCE1");
+        let decoded = deserialize_widget(&bytes).hit().unwrap();
+        assert_eq!(decoded, Widget { n: 0xDEADBEEF });
+    }
+
+    #[test]
+    fn has_cache_envelope_detects_magic() {
+        let bytes = serialize_widget(&Widget { n: 1 });
+        assert!(has_cache_envelope(&bytes));
+        assert!(has_cache_envelope(&MAGIC)); // exactly the magic, nothing after
+        assert!(!has_cache_envelope(b"LCE")); // too short
+        assert!(!has_cache_envelope(b"JUNK and more"));
+        assert!(!has_cache_envelope(&[]));
+    }
+
+    #[test]
+    fn wrong_magic_is_miss() {
+        let mut bytes = serialize_widget(&Widget { n: 7 }).to_vec();
+        bytes[0] = b'X';
+        assert_eq!(
+            miss_reason(&Bytes::from(bytes)),
+            Some(CacheMissReason::InvalidEnvelope)
+        );
+    }
+
+    #[test]
+    fn pre_stabilization_blob_is_miss() {
+        // An old unstable blob led with a small u64 LE length prefix (a JSON
+        // header of tens of bytes) — no magic. It must self-heal to a miss.
+        let mut blob = Vec::new();
+        blob.extend_from_slice(&(42u64).to_le_bytes());
+        blob.extend_from_slice(&[0u8; 42]);
+        assert_eq!(
+            miss_reason(&Bytes::from(blob)),
+            Some(CacheMissReason::InvalidEnvelope)
+        );
+
+        // A different unstable shape led with a small u8 tag (0/1/2).
+        assert_eq!(
+            miss_reason(&Bytes::from(vec![0u8, 1, 2, 3])),
+            Some(CacheMissReason::InvalidEnvelope)
+        );
+    }
+
+    #[test]
+    fn unknown_envelope_version_is_miss() {
+        let mut bytes = serialize_widget(&Widget { n: 7 }).to_vec();
+        bytes[4] = 0xFF; // envelope_version byte
+        assert_eq!(
+            miss_reason(&Bytes::from(bytes)),
+            Some(CacheMissReason::InvalidEnvelope)
+        );
+    }
+
+    #[test]
+    fn type_id_mismatch_is_miss() {
+        // Hand-build an envelope with a foreign type_id but valid framing.
+        let mut buf = Vec::new();
+        write_envelope(&mut buf, "some.OtherType", 1).unwrap();
+        buf.extend_from_slice(&(4u64).to_le_bytes());
+        buf.extend_from_slice(&99u32.to_le_bytes());
+        assert_eq!(
+            miss_reason(&Bytes::from(buf)),
+            Some(CacheMissReason::TypeMismatch)
+        );
+    }
+
+    #[test]
+    fn unsupported_future_type_version_is_miss() {
+        // An entry written by a newer build (higher type_version) must miss
+        // rather than be misread by this build.
+        let mut buf = Vec::new();
+        write_envelope(&mut buf, Widget::TYPE_ID, Widget::CURRENT_VERSION + 1).unwrap();
+        lance_arrow::ipc::write_len_prefixed_bytes(&mut buf, &9u32.to_le_bytes()).unwrap();
+        assert_eq!(
+            miss_reason(&Bytes::from(buf)),
+            Some(CacheMissReason::VersionTooNew)
+        );
+    }
+
+    #[test]
+    fn truncated_envelope_is_miss() {
+        let bytes = serialize_widget(&Widget { n: 7 });
+        for cut in [0, 1, 4, 5, 7, 9] {
+            assert_eq!(
+                miss_reason(&bytes.slice(..cut.min(bytes.len()))),
+                Some(CacheMissReason::InvalidEnvelope),
+                "truncating to {cut} bytes should miss as InvalidEnvelope"
+            );
+        }
+    }
+
+    #[test]
+    fn body_decode_error_is_miss() {
+        // Valid envelope, but the body is too short for the widget.
+        let mut buf = Vec::new();
+        write_envelope(&mut buf, Widget::TYPE_ID, Widget::CURRENT_VERSION).unwrap();
+        buf.extend_from_slice(&(1u64).to_le_bytes());
+        buf.push(0u8);
+        assert_eq!(
+            miss_reason(&Bytes::from(buf)),
+            Some(CacheMissReason::BodyError)
+        );
+    }
+
+    #[test]
+    fn reader_exposes_envelope_version() {
+        // type_version travels through the envelope to reader.version().
+        let mut buf = Vec::new();
+        write_envelope(&mut buf, Widget::TYPE_ID, 7).unwrap();
+        let body_off = buf.len();
+        // A widget body so the codec can decode it.
+        lance_arrow::ipc::write_len_prefixed_bytes(&mut buf, &5u32.to_le_bytes()).unwrap();
+        let data = Bytes::from(buf);
+
+        let mut r = CacheEntryReader::new(&data, body_off, 7);
+        assert_eq!(r.version(), 7);
+        assert_eq!(r.read_raw().unwrap().as_ref(), 5u32.to_le_bytes());
     }
 }
diff --git a/rust/lance-core/src/cache/entry_io.rs b/rust/lance-core/src/cache/entry_io.rs
new file mode 100644
index 00000000000..fe91b11ca7d
--- /dev/null
+++ b/rust/lance-core/src/cache/entry_io.rs
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Streaming readers/writers for cache entry bodies.
+//!
+//! [`CacheCodecImpl`](super::CacheCodecImpl) bodies are written and read
+//! through these wrappers. They keep serialization streaming (no buffering of
+//! the whole entry) and reads zero-copy (sections borrow from the input
+//! [`Bytes`]), while tracking the byte position needed to keep Arrow IPC
+//! sections 64-byte aligned (see [`lance_arrow::ipc`]).
+//!
+//! Body layout primitives:
+//!
+//! ```text
+//! HEADER    : [header_len: u32 LE][header proto bytes]
+//! ARROW_IPC : [pad to 64B][self-delimiting IPC stream]
+//! RAW_BLOB  : [len: u64 LE][bytes]
+//! ```
+
+use std::io::Write;
+
+use arrow_array::RecordBatch;
+use bytes::Bytes;
+use prost::Message;
+
+use crate::{Error, Result};
+
+/// Writes a cache entry body: a header followed by sections, streaming
+/// directly to the underlying writer.
+///
+/// The envelope is written by the [`CacheCodec`](super::CacheCodec) wrapper
+/// before this writer is handed to
+/// [`CacheCodecImpl::serialize`](super::CacheCodecImpl::serialize).
+pub struct CacheEntryWriter<'a> {
+    writer: &'a mut dyn Write,
+    /// Absolute byte offset within the entry, used to align IPC sections.
+    pos: usize,
+}
+
+impl<'a> CacheEntryWriter<'a> {
+    /// Create a writer positioned at the start of an entry (offset 0).
+    ///
+    /// Use this for nested serialization into a standalone buffer. The
+    /// envelope-aware entry point is [`CacheCodec::serialize`](super::CacheCodec::serialize).
+    pub fn new(writer: &'a mut dyn Write) -> Self {
+        Self { writer, pos: 0 }
+    }
+
+    /// Create a writer whose section alignment accounts for `pos` bytes
+    /// already written ahead of the body (i.e. the envelope).
+    pub(crate) fn with_pos(writer: &'a mut dyn Write, pos: usize) -> Self {
+        Self { writer, pos }
+    }
+
+    /// Write a single discriminant byte (e.g. a variant tag).
+    pub fn write_u8(&mut self, value: u8) -> Result<()> {
+        self.writer.write_all(&[value])?;
+        self.pos += 1;
+        Ok(())
+    }
+
+    /// Write a protobuf header as `[len: u32 LE][bytes]`.
+    pub fn write_header<P: Message>(&mut self, header: &P) -> Result<()> {
+        let bytes = header.encode_to_vec();
+        let len = u32::try_from(bytes.len())
+            .map_err(|_| Error::io(format!("cache header too large: {} bytes", bytes.len())))?;
+        self.writer.write_all(&len.to_le_bytes())?;
+        self.writer.write_all(&bytes)?;
+        self.pos += 4 + bytes.len();
+        Ok(())
+    }
+
+    /// Write `batch` as a 64-byte-aligned Arrow IPC section.
+    pub fn write_ipc(&mut self, batch: &RecordBatch) -> Result<()> {
+        lance_arrow::ipc::write_ipc_section(self.writer, &mut self.pos, batch)
+            .map_err(|e| Error::io(e.to_string()))
+    }
+
+    /// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC
+    /// section. The iterator must yield at least one batch.
+    pub fn write_ipc_batches<I>(&mut self, batches: I) -> Result<()>
+    where
+        I: IntoIterator<Item = RecordBatch>,
+    {
+        lance_arrow::ipc::write_ipc_section_batches(self.writer, &mut self.pos, batches)
+            .map_err(|e| Error::io(e.to_string()))
+    }
+
+    /// Write a raw blob as `[len: u64 LE][bytes]`.
+    ///
+    /// Only for byte payloads that already have their own stable, portable
+    /// encoding (e.g. a roaring bitmap, a varint-packed stream).
+    pub fn write_raw(&mut self, bytes: &[u8]) -> Result<()> {
+        lance_arrow::ipc::write_len_prefixed_bytes(self.writer, bytes)
+            .map_err(|e| Error::io(e.to_string()))?;
+        self.pos += 8 + bytes.len();
+        Ok(())
+    }
+
+    /// The underlying writer, for a payload that carries its own framing.
+    ///
+    /// Use this only when the codec writes a self-delimiting or whole-body
+    /// payload — e.g. streaming a roaring bitmap as the entire body, where the
+    /// length prefix of [`write_raw`](Self::write_raw) would be redundant and
+    /// buffering to measure that length would force an extra copy. For
+    /// structured bodies prefer [`write_header`](Self::write_header) /
+    /// [`write_ipc`](Self::write_ipc) / [`write_raw`](Self::write_raw), which
+    /// give you versioning and 64-byte IPC alignment.
+    ///
+    /// Bytes written through this do **not** advance the section-alignment
+    /// position, so it must not be interleaved with [`write_ipc`](Self::write_ipc).
+    pub fn raw_writer(&mut self) -> &mut dyn Write {
+        self.writer
+    }
+}
+
+/// Reads a cache entry body, tracking an offset into the input and exposing
+/// the entry's `type_version` so implementors can branch for backward compat.
+///
+/// All reads are zero-copy: returned [`Bytes`] and the buffers behind decoded
+/// [`RecordBatch`]es borrow from the input allocation.
+pub struct CacheEntryReader<'a> {
+    data: &'a Bytes,
+    offset: usize,
+    version: u32,
+}
+
+impl<'a> CacheEntryReader<'a> {
+    /// Create a reader over `data`, starting at body byte `offset`, for an
+    /// entry written at `version`.
+    pub fn new(data: &'a Bytes, offset: usize, version: u32) -> Self {
+        Self {
+            data,
+            offset,
+            version,
+        }
+    }
+
+    /// The `type_version` from the envelope. Branch on this for backward compat.
+    pub fn version(&self) -> u32 {
+        self.version
+    }
+
+    /// Read a single discriminant byte written by [`CacheEntryWriter::write_u8`].
+    pub fn read_u8(&mut self) -> Result<u8> {
+        let bytes = self.data.as_ref();
+        let v = *bytes
+            .get(self.offset)
+            .ok_or_else(|| Error::io("cache entry: truncated, missing tag byte".to_string()))?;
+        self.offset += 1;
+        Ok(v)
+    }
+
+    /// Read a protobuf header written by [`CacheEntryWriter::write_header`].
+    pub fn read_header<P: Message + Default>(&mut self) -> Result<P> {
+        let bytes = self.data.as_ref();
+        let len_end = self
+            .offset
+            .checked_add(4)
+            .filter(|&e| e <= bytes.len())
+            .ok_or_else(|| Error::io("cache header: truncated length prefix".to_string()))?;
+        let len = u32::from_le_bytes(bytes[self.offset..len_end].try_into().unwrap()) as usize;
+        let data_end = len_end
+            .checked_add(len)
+            .filter(|&e| e <= bytes.len())
+            .ok_or_else(|| Error::io("cache header: truncated body".to_string()))?;
+        let msg = P::decode(&bytes[len_end..data_end])
+            .map_err(|e| Error::io(format!("cache header decode failed: {e}")))?;
+        self.offset = data_end;
+        Ok(msg)
+    }
+
+    /// Read one [`RecordBatch`] from a 64-byte-aligned IPC section.
+    pub fn read_ipc(&mut self) -> Result<RecordBatch> {
+        lance_arrow::ipc::read_ipc_section_at(self.data, &mut self.offset)
+            .map_err(|e| Error::io(e.to_string()))
+    }
+
+    /// Read all [`RecordBatch`]es from a 64-byte-aligned multi-batch IPC
+    /// section written by [`CacheEntryWriter::write_ipc_batches`].
+    pub fn read_ipc_batches(&mut self) -> Result<Vec<RecordBatch>> {
+        lance_arrow::ipc::read_ipc_section_batches_at(self.data, &mut self.offset)
+            .map_err(|e| Error::io(e.to_string()))
+    }
+
+    /// Read a raw blob written by [`CacheEntryWriter::write_raw`], zero-copy.
+    pub fn read_raw(&mut self) -> Result<Bytes> {
+        lance_arrow::ipc::read_len_prefixed_bytes_at(self.data, &mut self.offset)
+            .map_err(|e| Error::io(e.to_string()))
+    }
+
+    /// The not-yet-consumed body bytes as a zero-copy slice.
+    ///
+    /// For a payload that carries its own framing and is parsed with the
+    /// codec's own cursor — the read counterpart of
+    /// [`CacheEntryWriter::raw_writer`]. For structured bodies prefer
+    /// [`read_header`](Self::read_header) / [`read_ipc`](Self::read_ipc) /
+    /// [`read_raw`](Self::read_raw).
+    pub fn body(&self) -> Bytes {
+        self.data.slice(self.offset..)
+    }
+}
diff --git a/rust/lance-core/src/cache/mod.rs b/rust/lance-core/src/cache/mod.rs
index ee6a728ef73..4f93f261bea 100644
--- a/rust/lance-core/src/cache/mod.rs
+++ b/rust/lance-core/src/cache/mod.rs
@@ -47,10 +47,14 @@
 
 pub mod backend;
 pub mod codec;
+mod entry_io;
 mod moka;
 
-pub use backend::{CacheBackend, CacheEntry, InternalCacheKey};
-pub use codec::{CacheCodec, CacheCodecImpl};
+pub use backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey};
+pub use codec::{
+    CacheCodec, CacheCodecImpl, CacheDecode, CacheMissReason, MAGIC, has_cache_envelope,
+};
+pub use entry_io::{CacheEntryReader, CacheEntryWriter};
 pub use moka::MokaCacheBackend;
 
 use std::borrow::Cow;
@@ -63,7 +67,7 @@ use futures::{Future, FutureExt};
 
 use crate::Result;
 
-pub use deepsize::{Context, DeepSizeOf};
+pub use crate::deepsize::{Context, DeepSizeOf};
 
 // ---------------------------------------------------------------------------
 // CacheKey / UnsizedCacheKey — typed key traits for cache users
@@ -245,6 +249,40 @@ impl LanceCache {
         self.cache.size_bytes().await
     }
 
+    /// Return an iterator over keys currently stored under this cache's prefix.
+    ///
+    /// Returns `None` when the backend does not support key inventory. The
+    /// iterator is intended for diagnostics and may be weakly consistent with
+    /// concurrent cache mutations.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use std::{borrow::Cow, sync::Arc};
+    /// # use lance_core::cache::{CacheKey, LanceCache};
+    /// # struct MyKey;
+    /// # impl CacheKey for MyKey {
+    /// #     type ValueType = Vec<i32>;
+    /// #     fn key(&self) -> Cow<'_, str> { Cow::Borrowed("my-key") }
+    /// #     fn type_name() -> &'static str { "VecI32" }
+    /// # }
+    /// # async fn example() {
+    /// let cache = LanceCache::with_capacity(1024);
+    /// cache.insert_with_key(&MyKey, Arc::new(vec![1, 2, 3])).await;
+    ///
+    /// let mut keys = cache.keys().await.expect("Moka supports key inventory");
+    /// assert_eq!(keys.next().unwrap().key(), "my-key");
+    /// # }
+    /// ```
+    pub async fn keys(&self) -> Option<CacheKeyIterator<'_>> {
+        Some(Box::new(
+            self.cache
+                .keys()
+                .await?
+                .filter(|key| key.starts_with(&self.prefix)),
+        ))
+    }
+
     // -- Sized insert/get (internal, shared by sized and unsized paths) --------
 
     async fn insert_with_id<T: DeepSizeOf + Send + Sync + 'static>(
@@ -557,7 +595,7 @@ impl CacheStats {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::collections::HashMap;
+    use std::collections::{BTreeSet, HashMap};
     use std::marker::PhantomData;
 
     struct TestKey<T: 'static> {
@@ -609,6 +647,18 @@ mod tests {
         }
     }
 
+    fn key_fields(keys: &[InternalCacheKey]) -> BTreeSet<(String, String, &'static str)> {
+        keys.iter()
+            .map(|key| {
+                (
+                    key.prefix().to_string(),
+                    key.key().to_string(),
+                    key.type_name(),
+                )
+            })
+            .collect()
+    }
+
     #[tokio::test]
     async fn test_cache_bytes() {
         let item = Arc::new(vec![1, 2, 3]);
@@ -638,6 +688,39 @@ mod tests {
         assert!(cache.size_bytes().await <= capacity);
     }
 
+    #[tokio::test]
+    async fn test_cache_weighs_key_footprint() {
+        // Weighted size charges the key's unique bytes, not just the value.
+        let cache = LanceCache::with_capacity(usize::MAX);
+        let key = "k".repeat(10_000);
+        let value = Arc::new(vec![1_i32]);
+        let expected =
+            std::mem::size_of::<InternalCacheKey>() + key.len() + cache_entry_size(&*value);
+        cache
+            .insert_with_key(&TestKey::<Vec<i32>>::new(&key), value)
+            .await;
+        assert_eq!(cache.size_bytes().await, expected);
+    }
+
+    #[tokio::test]
+    async fn test_cache_shared_prefix_not_charged_per_entry() {
+        // The shared prefix contributes nothing per entry (it isn't freed on a
+        // single eviction); only struct + unique key + value are charged.
+        let cache = LanceCache::with_capacity(usize::MAX).with_key_prefix(&"p".repeat(10_000));
+        for i in 0..100 {
+            cache
+                .insert_with_key(
+                    &TestKey::<Vec<i32>>::new(&i.to_string()),
+                    Arc::new(vec![1_i32]),
+                )
+                .await;
+        }
+        let value_cost = cache_entry_size(&vec![1_i32]);
+        let key_bytes: usize = (0..100).map(|i| i.to_string().len()).sum();
+        let expected = 100 * (std::mem::size_of::<InternalCacheKey>() + value_cost) + key_bytes;
+        assert_eq!(cache.size_bytes().await, expected);
+    }
+
     #[tokio::test]
     async fn test_cache_trait_objects() {
         #[derive(Debug, DeepSizeOf)]
@@ -718,6 +801,99 @@ mod tests {
         assert_eq!(base.stats().await.hits, 1);
     }
 
+    #[tokio::test]
+    async fn test_cache_keys_with_prefixes() {
+        let base = LanceCache::with_capacity(1000);
+        let prefixed = base.with_key_prefix("ns");
+        let nested = prefixed.with_key_prefix("index");
+        let other = base.with_key_prefix("ns-other");
+
+        base.insert_with_key(&TestKey::new("root"), Arc::new(vec![0]))
+            .await;
+        prefixed
+            .insert_with_key(&TestKey::new("child"), Arc::new(vec![1]))
+            .await;
+        nested
+            .insert_with_key(&TestKey::new("nested"), Arc::new(vec![2]))
+            .await;
+        other
+            .insert_with_key(&TestKey::new("other"), Arc::new(vec![3]))
+            .await;
+
+        let base_keys = base.keys().await.unwrap().collect::<Vec<_>>();
+        assert_eq!(
+            key_fields(&base_keys),
+            BTreeSet::from([
+                (
+                    "".to_string(),
+                    "root".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+                (
+                    "ns/".to_string(),
+                    "child".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+                (
+                    "ns/index/".to_string(),
+                    "nested".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+                (
+                    "ns-other/".to_string(),
+                    "other".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+            ])
+        );
+
+        let prefixed_keys = prefixed.keys().await.unwrap().collect::<Vec<_>>();
+        assert_eq!(
+            key_fields(&prefixed_keys),
+            BTreeSet::from([
+                (
+                    "ns/".to_string(),
+                    "child".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+                (
+                    "ns/index/".to_string(),
+                    "nested".to_string(),
+                    TestKey::<Vec<i32>>::type_name()
+                ),
+            ])
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cache_keys_reflect_invalidation_and_clear() {
+        let base = LanceCache::with_capacity(1000);
+        let prefixed = base.with_key_prefix("ns");
+        let other = base.with_key_prefix("other");
+
+        prefixed
+            .insert_with_key(&TestKey::new("child"), Arc::new(vec![1]))
+            .await;
+        other
+            .insert_with_key(&TestKey::new("other"), Arc::new(vec![2]))
+            .await;
+        assert_eq!(base.keys().await.unwrap().count(), 2);
+
+        prefixed.invalidate_prefix("").await;
+        let keys = base.keys().await.unwrap().collect::<Vec<_>>();
+        assert_eq!(
+            key_fields(&keys),
+            BTreeSet::from([(
+                "other/".to_string(),
+                "other".to_string(),
+                TestKey::<Vec<i32>>::type_name()
+            )])
+        );
+
+        base.clear().await;
+        assert_eq!(base.keys().await.unwrap().count(), 0);
+    }
+
     #[tokio::test]
     async fn test_cache_get_or_insert() {
         let cache = LanceCache::with_capacity(1000);
@@ -833,6 +1009,7 @@ mod tests {
                 .await
                 .is_none()
         );
+        assert!(cache.keys().await.is_none());
     }
 
     #[tokio::test]
diff --git a/rust/lance-core/src/cache/moka.rs b/rust/lance-core/src/cache/moka.rs
index 6be7760458a..fd86b064f6b 100644
--- a/rust/lance-core/src/cache/moka.rs
+++ b/rust/lance-core/src/cache/moka.rs
@@ -11,7 +11,7 @@ use futures::Future;
 use crate::Result;
 
 use super::CacheCodec;
-use super::backend::{CacheBackend, CacheEntry, InternalCacheKey};
+use super::backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey};
 
 /// Internal record stored in the moka cache.
 #[derive(Clone, Debug)]
@@ -20,6 +20,12 @@ struct MokaCacheEntry {
     size_bytes: usize,
 }
 
+/// Per-entry key cost for eviction: the struct plus the unique `key` bytes.
+/// Excludes the shared `prefix` `Arc<str>`, which isn't freed per eviction.
+fn key_footprint(key: &InternalCacheKey) -> usize {
+    std::mem::size_of::<InternalCacheKey>() + key.key().len()
+}
+
 /// Default [`CacheBackend`] backed by a [moka](https://crates.io/crates/moka) cache.
 ///
 /// Provides weighted-capacity eviction and concurrent-load deduplication
@@ -40,7 +46,12 @@ impl MokaCacheBackend {
     pub fn with_capacity(capacity: usize) -> Self {
         let cache = moka::future::Cache::builder()
             .max_capacity(capacity as u64)
-            .weigher(|_, v: &MokaCacheEntry| v.size_bytes.try_into().unwrap_or(u32::MAX))
+            .weigher(|key: &InternalCacheKey, entry: &MokaCacheEntry| {
+                key_footprint(key)
+                    .saturating_add(entry.size_bytes)
+                    .try_into()
+                    .unwrap_or(u32::MAX)
+            })
             .support_invalidation_closures()
             .build();
         Self { cache }
@@ -123,6 +134,13 @@ impl CacheBackend for MokaCacheBackend {
         self.cache.run_pending_tasks().await;
     }
 
+    async fn keys(&self) -> Option<CacheKeyIterator<'_>> {
+        self.cache.run_pending_tasks().await;
+        Some(Box::new(
+            self.cache.iter().map(|(key, _)| key.as_ref().clone()),
+        ))
+    }
+
     async fn num_entries(&self) -> usize {
         self.cache.run_pending_tasks().await;
         self.cache.entry_count() as usize
@@ -141,6 +159,9 @@ impl CacheBackend for MokaCacheBackend {
         // Iterate rather than using `weighted_size()` because moka's
         // weighted_size can be stale without `run_pending_tasks()`, which
         // is async and can't be called from this synchronous context.
-        self.cache.iter().map(|(_, v)| v.size_bytes).sum()
+        self.cache
+            .iter()
+            .map(|(key, entry)| key_footprint(key.as_ref()) + entry.size_bytes)
+            .sum()
     }
 }
diff --git a/rust/lance-core/src/container/list.rs b/rust/lance-core/src/container/list.rs
index 4f1593f4de1..9d8205cb398 100644
--- a/rust/lance-core/src/container/list.rs
+++ b/rust/lance-core/src/container/list.rs
@@ -3,7 +3,7 @@
 
 use std::collections::LinkedList;
 
-use deepsize::DeepSizeOf;
+use crate::deepsize::DeepSizeOf;
 
 /// A linked list that grows exponentially. It is used to store a large number of
 /// elements in a memory-efficient way. The list grows by doubling the capacity of
@@ -134,7 +134,7 @@ impl<T> ExpLinkedList<T> {
 }
 
 impl<T: DeepSizeOf> DeepSizeOf for ExpLinkedList<T> {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut crate::deepsize::Context) -> usize {
         self.inner
             .iter()
             .map(|v| v.deep_size_of_children(context))
diff --git a/rust/lance-core/src/datatypes.rs b/rust/lance-core/src/datatypes.rs
index 026e6b0bbe9..2f5c7b0680e 100644
--- a/rust/lance-core/src/datatypes.rs
+++ b/rust/lance-core/src/datatypes.rs
@@ -7,9 +7,9 @@ use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 use std::sync::{Arc, LazyLock};
 
+use crate::deepsize::DeepSizeOf;
 use arrow_array::ArrayRef;
 use arrow_schema::{DataType, Field as ArrowField, Fields, TimeUnit};
-use deepsize::DeepSizeOf;
 use lance_arrow::bfloat16::{BFLOAT16_EXT_NAME, is_bfloat16_field};
 use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
 
@@ -25,6 +25,7 @@ pub use field::{
 pub use schema::{
     BlobHandling, FieldRef, OnMissing, Projectable, Projection, Schema,
     escape_field_path_for_project, format_field_path, parse_field_path,
+    validate_fixed_size_list_dimensions,
 };
 
 pub static BLOB_DESC_FIELDS: LazyLock<Fields> = LazyLock::new(|| {
@@ -116,7 +117,7 @@ impl LogicalType {
         self.0.starts_with("fixed_size_list:struct:")
     }
 
-    fn is_struct(&self) -> bool {
+    pub fn is_struct(&self) -> bool {
         self.0 == "struct"
     }
 
@@ -408,10 +409,10 @@ pub struct Dictionary {
 }
 
 impl DeepSizeOf for Dictionary {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut crate::deepsize::Context) -> usize {
         self.values
             .as_ref()
-            .map(|v| v.get_array_memory_size())
+            .map(|v| (v.as_ref() as &dyn arrow_array::Array).deep_size_of_children(context))
             .unwrap_or(0)
     }
 }
diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs
index b122ce64ac4..d5eb89dccb0 100644
--- a/rust/lance-core/src/datatypes/field.rs
+++ b/rust/lance-core/src/datatypes/field.rs
@@ -10,6 +10,7 @@ use std::{
     sync::Arc,
 };
 
+use crate::deepsize::DeepSizeOf;
 use arrow_array::{
     ArrayRef,
     cast::AsArray,
@@ -18,7 +19,6 @@ use arrow_array::{
     },
 };
 use arrow_schema::{DataType, Field as ArrowField};
-use deepsize::DeepSizeOf;
 use lance_arrow::{
     ARROW_EXT_NAME_KEY, BLOB_META_KEY, BLOB_V2_EXT_NAME, DataTypeExt,
     json::{is_arrow_json_field, is_json_field},
@@ -58,6 +58,8 @@ pub const LANCE_UNENFORCED_CLUSTERING_KEY_POSITION: &str =
 /// The value should be non-negative i32 value. Any negative value will be seen as -1.
 pub const LANCE_FIELD_ID_KEY: &str = "lance:field_id";
 
+const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
+
 fn has_blob_v2_extension(field: &ArrowField) -> bool {
     field
         .metadata()
@@ -269,24 +271,15 @@ impl Field {
     }
 
     pub fn apply_projection(&self, projection: &Projection) -> Option<Self> {
-        // Map fields encode their physical layout as a single child entries
-        // struct (`Struct<key, value>`) whose presence is required for the
-        // parent to be readable — we never want to filter into that subtree.
-        // But the parent field itself is still subject to selection: if the
-        // caller didn't ask for this Map column, drop it like any other
-        // non-selected leaf. Without this early return the unconditional
-        // children clone would keep `children.is_empty() == false` forever
-        // and every Map column in the schema would survive every projection,
-        // pulling tens-of-bytes-per-row of unrelated data through downstream
-        // operators (notably `SortExec` in scalar-index training, where it
-        // was responsible for >100 GiB external-sort spills on real-world
-        // tables).
-        if self.logical_type.is_map() && !projection.contains_field_id(self.id) {
+        // Maps and blob descriptors are atomic physical layouts. Map children
+        // must remain together, while projected blob descriptor children may
+        // have synthetic IDs that cannot be selected independently.
+        let is_atomic_layout = self.logical_type.is_map() || self.is_blob();
+        if is_atomic_layout && !projection.contains_field_id(self.id) {
             return None;
         }
 
-        let children = if self.logical_type.is_map() {
-            // Map field is selected: keep all children intact.
+        let children = if is_atomic_layout {
             self.children.clone()
         } else {
             self.children
@@ -549,6 +542,20 @@ impl Field {
             .get(ARROW_EXT_NAME_KEY)
             .map(|name| name == BLOB_V2_EXT_NAME)
             .unwrap_or(false)
+            || self.is_blob_v2_descriptor()
+    }
+
+    fn is_blob_v2_descriptor(&self) -> bool {
+        self.metadata.contains_key(BLOB_META_KEY)
+            && self.logical_type == BLOB_V2_DESC_LANCE_FIELD.logical_type
+            && self.children.len() == BLOB_V2_DESC_LANCE_FIELD.children.len()
+            && self
+                .children
+                .iter()
+                .zip(BLOB_V2_DESC_LANCE_FIELD.children.iter())
+                .all(|(child, expected)| {
+                    child.name == expected.name && child.data_type() == expected.data_type()
+                })
     }
 
     // Blob columns intentionally have two schema representations:
@@ -575,6 +582,43 @@ impl Field {
         }
     }
 
+    /// Convert a blob field to the materialized binary payload view.
+    ///
+    /// The field keeps its name and id but uses `LargeBinary` with no children.
+    /// Blob v2 fields retain their extension marker internally so scan planning
+    /// can recognize the binary view before exposing a plain Arrow binary field.
+    pub fn binary_blob_mut(&mut self) {
+        if !self.is_blob() {
+            return;
+        }
+        let is_blob_v2 = self.is_blob_v2();
+
+        self.logical_type = LogicalType::try_from(&DataType::LargeBinary)
+            .expect("LargeBinary is always a valid logical type");
+        self.children.clear();
+        self.encoding = Some(Encoding::VarBinary);
+        if is_blob_v2 {
+            self.metadata.remove(BLOB_META_KEY);
+            for key in PACKED_KEYS {
+                self.metadata.remove(key);
+            }
+            self.metadata
+                .insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string());
+        }
+    }
+
+    /// Convert blob v2 fields in this field tree to their descriptor view.
+    pub fn unload_blobs_recursive(&mut self) {
+        if self.is_blob_v2() {
+            self.unloaded_mut();
+            return;
+        }
+
+        for child in &mut self.children {
+            child.unload_blobs_recursive();
+        }
+    }
+
     pub fn project(&self, path_components: &[&str]) -> Result<Self> {
         let mut f = Self {
             name: self.name.clone(),
@@ -800,6 +844,13 @@ impl Field {
         }
 
         if self.is_blob() != other.is_blob() {
+            if ignore_types {
+                return Ok(if self.id >= 0 {
+                    self.clone()
+                } else {
+                    other.clone()
+                });
+            }
             return Err(Error::arrow(format!(
                 "Attempt to intersect blob and non-blob field: {}",
                 self.name
@@ -835,7 +886,7 @@ impl Field {
                 .iter()
                 .filter_map(|c| {
                     if let Some(other_child) = other.child(&c.name) {
-                        let intersection = c.intersection(other_child).ok()?;
+                        let intersection = c.do_intersection(other_child, ignore_types).ok()?;
                         Some(intersection)
                     } else {
                         None
@@ -1026,7 +1077,6 @@ impl Field {
 
     // Check if field has metadata `packed` set to true, this check is case insensitive.
     pub fn is_packed_struct(&self) -> bool {
-        const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
         PACKED_KEYS.iter().any(|key| {
             self.metadata
                 .get(*key)
@@ -1835,6 +1885,14 @@ mod tests {
     #[test]
     fn blob_unloaded_mut_selects_layout_from_metadata() {
         let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let mut binary_field: Field = ArrowField::new("blob", DataType::LargeBinary, true)
+            .with_metadata(metadata.clone())
+            .try_into()
+            .unwrap();
+        binary_field.binary_blob_mut();
+        assert!(binary_field.metadata.contains_key(BLOB_META_KEY));
+        assert!(!binary_field.is_blob_v2());
+
         let mut field: Field = ArrowField::new("blob", DataType::LargeBinary, true)
             .with_metadata(metadata)
             .try_into()
@@ -1842,6 +1900,12 @@ mod tests {
         field.unloaded_mut();
         assert_eq!(field.children.len(), 2);
         assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type);
+        assert!(field.is_blob());
+        assert!(!field.is_blob_v2());
+        field.unloaded_mut();
+        assert_eq!(field.children.len(), 2);
+        assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.is_blob_v2());
 
         let metadata =
             HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]);
@@ -1862,6 +1926,60 @@ mod tests {
         field.unloaded_mut();
         assert_eq!(field.children.len(), 5);
         assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY));
+        assert!(field.is_blob_v2());
+        field.unloaded_mut();
+        assert_eq!(field.children.len(), 5);
+        assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
+        assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY));
+    }
+
+    #[test]
+    fn unload_blobs_recursive_only_unloads_blob_v2() {
+        let legacy_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let blob_v2_metadata =
+            HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]);
+
+        let mut field: Field = ArrowField::new(
+            "parent",
+            DataType::Struct(Fields::from(vec![
+                ArrowField::new("legacy_blob", DataType::LargeBinary, true)
+                    .with_metadata(legacy_metadata),
+                ArrowField::new(
+                    "blob_v2",
+                    DataType::Struct(
+                        vec![
+                            ArrowField::new("data", DataType::LargeBinary, true),
+                            ArrowField::new("uri", DataType::Utf8, true),
+                        ]
+                        .into(),
+                    ),
+                    true,
+                )
+                .with_metadata(blob_v2_metadata),
+            ])),
+            true,
+        )
+        .try_into()
+        .unwrap();
+
+        field.unload_blobs_recursive();
+
+        let legacy_blob = field
+            .children
+            .iter()
+            .find(|f| f.name == "legacy_blob")
+            .unwrap();
+        assert_eq!(
+            legacy_blob.logical_type,
+            LogicalType::try_from(&DataType::LargeBinary).unwrap()
+        );
+        assert_eq!(legacy_blob.children.len(), 0);
+        assert!(legacy_blob.metadata.contains_key(BLOB_META_KEY));
+
+        let blob_v2 = field.children.iter().find(|f| f.name == "blob_v2").unwrap();
+        assert_eq!(blob_v2.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type);
+        assert_eq!(blob_v2.children.len(), 5);
     }
 
     #[test]
@@ -1884,4 +2002,43 @@ mod tests {
             .unwrap();
         assert_eq!(unloaded_projected, unloaded);
     }
+
+    #[test]
+    fn blob_descriptor_projection_preserves_synthetic_children() {
+        let metadata =
+            HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]);
+        let mut blob: Field = ArrowField::new(
+            "blob",
+            DataType::Struct(
+                vec![
+                    ArrowField::new("data", DataType::LargeBinary, true),
+                    ArrowField::new("uri", DataType::Utf8, true),
+                ]
+                .into(),
+            ),
+            true,
+        )
+        .with_metadata(metadata)
+        .try_into()
+        .unwrap();
+        let mut next_id = 0;
+        blob.set_id(-1, &mut next_id);
+
+        let schema = Arc::new(crate::datatypes::Schema {
+            fields: vec![blob],
+            metadata: HashMap::new(),
+        });
+        let descriptor_schema = Projection::full(schema)
+            .with_blob_handling(crate::datatypes::BlobHandling::BlobsDescriptions)
+            .to_bare_schema();
+        assert!(
+            descriptor_schema.fields[0]
+                .children
+                .iter()
+                .all(|child| child.id == -1)
+        );
+
+        let projected = Projection::full(Arc::new(descriptor_schema)).to_bare_schema();
+        assert_eq!(projected.fields[0].children.len(), 5);
+    }
 }
diff --git a/rust/lance-core/src/datatypes/schema.rs b/rust/lance-core/src/datatypes/schema.rs
index 9502f1e45a8..6f9fc61b334 100644
--- a/rust/lance-core/src/datatypes/schema.rs
+++ b/rust/lance-core/src/datatypes/schema.rs
@@ -9,9 +9,9 @@ use std::{
     sync::Arc,
 };
 
+use crate::deepsize::DeepSizeOf;
 use arrow_array::RecordBatch;
-use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
-use deepsize::DeepSizeOf;
+use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use lance_arrow::*;
 
 use super::field::{Field, OnTypeMismatch, SchemaCompareOptions};
@@ -110,6 +110,29 @@ impl<'a> Iterator for SchemaFieldIterPreOrder<'a> {
     }
 }
 
+/// Reject `FixedSizeList` types whose dimension is not a positive integer.
+///
+/// The row count of a fixed-size list is derived by dividing the number of
+/// child items by the dimension, so a zero dimension panics with a
+/// divide-by-zero further down the write path (see issue #5102). A
+/// `FixedSizeList` of a `FixedSizeList` over a primitive collapses into a
+/// single leaf field, so the pre-order field walk never visits the inner list;
+/// recurse through the nested list types here to catch an inner zero dimension.
+///
+/// Shared by [`Schema::validate`] on the write path and the decoder's
+/// field-scheduler builders on the read path.
+pub fn validate_fixed_size_list_dimensions(field_name: &str, data_type: &DataType) -> Result<()> {
+    if let DataType::FixedSizeList(inner, dimension) = data_type {
+        if *dimension <= 0 {
+            return Err(Error::schema(format!(
+                "Field \"{field_name}\" contains a FixedSizeList with dimension {dimension}; dimension must be a positive integer"
+            )));
+        }
+        validate_fixed_size_list_dimensions(field_name, inner.data_type())?;
+    }
+    Ok(())
+}
+
 impl Schema {
     /// The unenforced primary key fields in the schema, ordered by position.
     ///
@@ -346,6 +369,10 @@ impl Schema {
                     field.id, self
                 )));
             }
+            // The row count of a fixed-size list is derived by dividing the
+            // number of items by the dimension, so a zero dimension would
+            // panic with a divide-by-zero further down the write path.
+            validate_fixed_size_list_dimensions(&field.name, &field.data_type())?;
         }
 
         Ok(())
@@ -733,10 +760,6 @@ impl Schema {
         Ok(schema)
     }
 
-    pub fn all_fields_nullable(&self) -> bool {
-        SchemaFieldIterPreOrder::new(self).all(|f| f.nullable)
-    }
-
     /// Returns the properly formatted path from root to the field.
     /// Field names containing dots are quoted (e.g., struct.`field.with.dot`)
     pub fn field_path(&self, field_id: i32) -> Result<String> {
@@ -1048,6 +1071,17 @@ pub enum BlobHandling {
 }
 
 impl BlobHandling {
+    fn should_load_binary(&self, field: &Field) -> bool {
+        if !field.is_blob() {
+            return false;
+        }
+        match self {
+            Self::AllBinary => true,
+            Self::SomeBlobsBinary(set) | Self::SomeBinary(set) => set.contains(&(field.id as u32)),
+            Self::BlobsDescriptions | Self::AllDescriptions => false,
+        }
+    }
+
     fn should_unload(&self, field: &Field) -> bool {
         // Blob v2 columns are Structs, so we need to treat any blob-marked field as unloadable
         // even if the physical data type is not binary-like.
@@ -1063,10 +1097,34 @@ impl BlobHandling {
         }
     }
 
+    /// Whether `field` will be projected as a lightweight blob *description*
+    /// (offset + size) rather than its full binary value under this handling.
+    ///
+    /// A description is tiny and cheap to read eagerly; the full binary value is
+    /// not. Materialization heuristics use this to decide early vs late loading.
+    pub fn returns_description(&self, field: &Field) -> bool {
+        self.should_unload(field)
+    }
+
+    /// Apply this blob handling policy to a projected field tree.
+    ///
+    /// Blob descriptor modes convert blob leaves to descriptor views. Binary
+    /// modes convert selected blob leaves to `LargeBinary`. Non-blob nested
+    /// fields are preserved while their children are handled recursively.
     pub fn unload_if_needed(&self, mut field: Field) -> Field {
+        if self.should_load_binary(&field) {
+            field.binary_blob_mut();
+            return field;
+        }
         if self.should_unload(&field) {
             field.unloaded_mut();
+            return field;
         }
+        field.children = field
+            .children
+            .into_iter()
+            .map(|child| self.unload_if_needed(child))
+            .collect();
         field
     }
 }
@@ -2447,86 +2505,6 @@ mod tests {
         assert!(res.is_none(), "Expected None, got {:?}", res);
     }
 
-    #[test]
-    pub fn test_all_fields_nullable() {
-        let test_cases = vec![
-            (
-                vec![], // empty schema
-                true,
-            ),
-            (
-                vec![
-                    Field::new_arrow("a", DataType::Int32, true).unwrap(),
-                    Field::new_arrow("b", DataType::Utf8, true).unwrap(),
-                ], // basic case
-                true,
-            ),
-            (
-                vec![
-                    Field::new_arrow("a", DataType::Int32, false).unwrap(),
-                    Field::new_arrow("b", DataType::Utf8, true).unwrap(),
-                ],
-                false,
-            ),
-            (
-                // check nested schema, parent is nullable
-                vec![
-                    Field::new_arrow(
-                        "struct",
-                        DataType::Struct(ArrowFields::from(vec![ArrowField::new(
-                            "a",
-                            DataType::Int32,
-                            false,
-                        )])),
-                        true,
-                    )
-                    .unwrap(),
-                ],
-                false,
-            ),
-            (
-                // check nested schema, child is nullable
-                vec![
-                    Field::new_arrow(
-                        "struct",
-                        DataType::Struct(ArrowFields::from(vec![ArrowField::new(
-                            "a",
-                            DataType::Int32,
-                            true,
-                        )])),
-                        false,
-                    )
-                    .unwrap(),
-                ],
-                false,
-            ),
-            (
-                // check nested schema, all is nullable
-                vec![
-                    Field::new_arrow(
-                        "struct",
-                        DataType::Struct(ArrowFields::from(vec![ArrowField::new(
-                            "a",
-                            DataType::Int32,
-                            true,
-                        )])),
-                        true,
-                    )
-                    .unwrap(),
-                ],
-                true,
-            ),
-        ];
-
-        for (fields, expected) in test_cases {
-            let schema = Schema {
-                fields,
-                metadata: Default::default(),
-            };
-            assert_eq!(schema.all_fields_nullable(), expected);
-        }
-    }
-
     #[test]
     fn test_schema_unenforced_primary_key() {
         let cases = vec![
@@ -2825,6 +2803,67 @@ mod tests {
         assert!(paths.contains(&"name".to_string()));
     }
 
+    #[test]
+    fn test_validate_rejects_zero_dimension_fixed_size_list() {
+        // A zero dimension divides-by-zero further down the write path (#5102)
+        let fsl = |dimension: i32| {
+            ArrowDataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", ArrowDataType::Float32, true)),
+                dimension,
+            )
+        };
+
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("vec", fsl(0), true)]);
+        let err = Schema::try_from(&arrow_schema).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("dimension must be a positive integer"),
+            "unexpected error: {}",
+            err
+        );
+
+        // Nested inside a struct is rejected too
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "outer",
+            ArrowDataType::Struct(ArrowFields::from(vec![ArrowField::new(
+                "vec",
+                fsl(0),
+                true,
+            )])),
+            true,
+        )]);
+        let err = Schema::try_from(&arrow_schema).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("dimension must be a positive integer"),
+            "unexpected error: {}",
+            err
+        );
+
+        // A zero-dimension FixedSizeList nested inside a positive-dimension
+        // FixedSizeList collapses into a single leaf field, so the inner
+        // dimension is not visited by the pre-order field walk and must still
+        // be rejected: FixedSizeList(FixedSizeList(Float32, 0), 4).
+        let nested =
+            ArrowDataType::FixedSizeList(Arc::new(ArrowField::new("inner", fsl(0), true)), 4);
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("vec", nested, true)]);
+        let err = Schema::try_from(&arrow_schema).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("dimension must be a positive integer"),
+            "unexpected error: {}",
+            err
+        );
+
+        // A positive dimension still validates, including nested lists
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("vec", fsl(2), true)]);
+        assert!(Schema::try_from(&arrow_schema).is_ok());
+        let nested_ok =
+            ArrowDataType::FixedSizeList(Arc::new(ArrowField::new("inner", fsl(2), true)), 4);
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new("vec", nested_ok, true)]);
+        assert!(Schema::try_from(&arrow_schema).is_ok());
+    }
+
     #[test]
     fn test_schema_unenforced_clustering_key() {
         use crate::datatypes::field::LANCE_UNENFORCED_CLUSTERING_KEY_POSITION;
diff --git a/rust/lance-core/src/deepsize.rs b/rust/lance-core/src/deepsize.rs
new file mode 100644
index 00000000000..350c52b146c
--- /dev/null
+++ b/rust/lance-core/src/deepsize.rs
@@ -0,0 +1,467 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+pub use lance_derive::DeepSizeOf;
+
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::mem::{size_of, size_of_val};
+use std::sync::atomic::{AtomicU64, AtomicUsize};
+use std::sync::{Arc, Mutex, RwLock};
+
+use arrow_array::{Array, RecordBatch};
+use arrow_buffer::ArrowNativeType;
+use arrow_data::ArrayData;
+
+pub struct Context {
+    seen: HashSet<usize>,
+}
+
+impl Default for Context {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Context {
+    pub fn new() -> Self {
+        Self {
+            seen: HashSet::new(),
+        }
+    }
+
+    /// Returns true if this pointer was NOT previously seen (i.e., it's new).
+    pub fn mark_seen(&mut self, ptr: usize) -> bool {
+        self.seen.insert(ptr)
+    }
+}
+
+pub trait DeepSizeOf {
+    fn deep_size_of(&self) -> usize {
+        size_of_val(self) + self.deep_size_of_children(&mut Context::new())
+    }
+
+    fn deep_size_of_children(&self, context: &mut Context) -> usize;
+}
+
+// Primitives — no heap children
+macro_rules! impl_deep_size_primitive {
+    ($($t:ty),*) => {
+        $(
+            impl DeepSizeOf for $t {
+                fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+                    0
+                }
+            }
+        )*
+    };
+}
+
+impl_deep_size_primitive!(
+    u8,
+    u16,
+    u32,
+    u64,
+    u128,
+    usize,
+    i8,
+    i16,
+    i32,
+    i64,
+    i128,
+    isize,
+    f32,
+    f64,
+    bool,
+    ()
+);
+
+impl DeepSizeOf for str {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        0
+    }
+}
+
+impl DeepSizeOf for String {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        self.capacity()
+    }
+}
+
+impl DeepSizeOf for bytes::Bytes {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        if context.mark_seen(self.as_ptr() as usize) {
+            self.len()
+        } else {
+            0
+        }
+    }
+}
+
+impl DeepSizeOf for AtomicU64 {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        0
+    }
+}
+
+impl DeepSizeOf for AtomicUsize {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        0
+    }
+}
+
+impl<T: DeepSizeOf, const N: usize> DeepSizeOf for [T; N] {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.iter()
+            .map(|item| item.deep_size_of_children(context))
+            .sum()
+    }
+}
+
+impl<T: DeepSizeOf> DeepSizeOf for [T] {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        // The slice's own element bytes are accounted for by the owner (e.g. the
+        // `size_of_val` in the `Arc`/`Box` impls); here we only sum the heap
+        // children of each element.
+        self.iter()
+            .map(|item| item.deep_size_of_children(context))
+            .sum()
+    }
+}
+
+impl<T: DeepSizeOf> DeepSizeOf for RwLock<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.read()
+            .map(|val| val.deep_size_of_children(context))
+            .unwrap_or(0)
+    }
+}
+
+impl<T: DeepSizeOf> DeepSizeOf for Mutex<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.lock()
+            .map(|val| val.deep_size_of_children(context))
+            .unwrap_or(0)
+    }
+}
+
+// Tuples
+macro_rules! impl_deep_size_tuple {
+    ($($name:ident),+) => {
+        impl<$($name: DeepSizeOf),+> DeepSizeOf for ($($name,)+) {
+            #[allow(non_snake_case)]
+            fn deep_size_of_children(&self, context: &mut Context) -> usize {
+                let ($($name,)+) = self;
+                0 $(+ $name.deep_size_of_children(context))+
+            }
+        }
+    };
+}
+
+impl_deep_size_tuple!(A, B);
+impl_deep_size_tuple!(A, B, C);
+impl_deep_size_tuple!(A, B, C, D);
+impl_deep_size_tuple!(A, B, C, D, E);
+impl_deep_size_tuple!(A, B, C, D, E, F);
+
+impl<T: DeepSizeOf> DeepSizeOf for Vec<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.capacity() * size_of::<T>()
+            + self
+                .iter()
+                .map(|item| item.deep_size_of_children(context))
+                .sum::<usize>()
+    }
+}
+
+impl<T: DeepSizeOf + ?Sized> DeepSizeOf for Box<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        size_of_val(&**self) + (**self).deep_size_of_children(context)
+    }
+}
+
+impl<T: DeepSizeOf + ?Sized> DeepSizeOf for Arc<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        if context.mark_seen(Self::as_ptr(self) as *const () as usize) {
+            size_of_val(&**self) + (**self).deep_size_of_children(context)
+        } else {
+            0
+        }
+    }
+}
+
+impl<T: DeepSizeOf> DeepSizeOf for Option<T> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        match self {
+            Some(val) => val.deep_size_of_children(context),
+            None => 0,
+        }
+    }
+}
+
+impl<K: DeepSizeOf, V: DeepSizeOf> DeepSizeOf for HashMap<K, V> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        // Each bucket holds a key-value pair plus hash metadata (~1 byte control per bucket).
+        // Robin hood / Swiss table capacity is always a power of 2.
+        let capacity_bytes = self.capacity() * (size_of::<K>() + size_of::<V>() + 1);
+        let children: usize = self
+            .iter()
+            .map(|(k, v)| k.deep_size_of_children(context) + v.deep_size_of_children(context))
+            .sum();
+        capacity_bytes + children
+    }
+}
+
+impl<K: DeepSizeOf> DeepSizeOf for HashSet<K> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        let capacity_bytes = self.capacity() * (size_of::<K>() + 1);
+        let children: usize = self.iter().map(|k| k.deep_size_of_children(context)).sum();
+        capacity_bytes + children
+    }
+}
+
+impl<K: DeepSizeOf, V: DeepSizeOf> DeepSizeOf for BTreeMap<K, V> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        // BTreeMap nodes have ~11 entries each. Rough estimate: per-entry overhead ~3 pointers.
+        let per_entry = size_of::<K>() + size_of::<V>() + 3 * size_of::<usize>();
+        let overhead = self.len() * per_entry;
+        let children: usize = self
+            .iter()
+            .map(|(k, v)| k.deep_size_of_children(context) + v.deep_size_of_children(context))
+            .sum();
+        overhead + children
+    }
+}
+
+impl<K: DeepSizeOf> DeepSizeOf for BTreeSet<K> {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        let per_entry = size_of::<K>() + 3 * size_of::<usize>();
+        let overhead = self.len() * per_entry;
+        let children: usize = self.iter().map(|k| k.deep_size_of_children(context)).sum();
+        overhead + children
+    }
+}
+
+// Arrow types
+
+fn record_array_data(context: &mut Context, data: &ArrayData) -> usize {
+    let mut total = 0;
+    for buffer in data.buffers() {
+        if context.mark_seen(buffer.as_ptr() as usize) {
+            total += buffer.capacity();
+        }
+    }
+    if let Some(nulls) = data.nulls() {
+        let null_buf = nulls.inner().inner();
+        if context.mark_seen(null_buf.as_ptr() as usize) {
+            total += null_buf.capacity();
+        }
+    }
+    for child in data.child_data() {
+        total += record_array_data(context, child);
+    }
+    total
+}
+
+impl DeepSizeOf for dyn Array {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        // `to_data()` only clones Arc refs (no data copy) and allocates a small
+        // ArrayData metadata struct. This lets us walk buffer pointers for dedup.
+        // Cost is O(number_of_buffers), not O(data_size).
+        let data = self.to_data();
+        record_array_data(context, &data)
+    }
+}
+
+impl DeepSizeOf for RecordBatch {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.columns()
+            .iter()
+            .map(|col| col.deep_size_of_children(context))
+            .sum()
+    }
+}
+
+impl<T> DeepSizeOf for arrow_buffer::ScalarBuffer<T>
+where
+    T: ArrowNativeType,
+{
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        // Track the underlying buffer pointer to avoid double-counting shared allocations.
+        // Use capacity() rather than len() * size_of::<T>() because sliced buffers retain
+        // their full original allocation.
+        let buf = self.inner();
+        if context.mark_seen(buf.as_ptr() as usize) {
+            buf.capacity()
+        } else {
+            0
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::{Int32Array, StringArray, StructArray};
+    use arrow_schema::{DataType, Field, Fields, Schema};
+
+    #[test]
+    fn test_basic_record_batch() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+
+        let size = batch.deep_size_of();
+        // Should at least include the buffer for 3 i32s
+        assert!(size >= 3 * size_of::<i32>());
+    }
+
+    #[test]
+    fn test_same_batch_dedup() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],
+        )
+        .unwrap();
+
+        let mut ctx = Context::new();
+        let size_a = batch.deep_size_of_children(&mut ctx);
+        let size_b = batch.deep_size_of_children(&mut ctx);
+
+        // First measurement should report buffer sizes
+        assert!(size_a > 0);
+        // Second measurement of the same batch should add nothing (buffers already seen)
+        assert_eq!(size_b, 0);
+    }
+
+    #[test]
+    fn test_arc_dedup() {
+        let batch = Arc::new(
+            RecordBatch::try_new(
+                Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+                vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+            )
+            .unwrap(),
+        );
+        let clone = Arc::clone(&batch);
+
+        let mut ctx = Context::new();
+        let size_a = batch.deep_size_of_children(&mut ctx);
+        let size_b = clone.deep_size_of_children(&mut ctx);
+
+        assert!(size_a > 0);
+        assert_eq!(size_b, 0);
+    }
+
+    #[test]
+    fn test_multi_column_shared_array() {
+        // Two columns pointing to the same Arc<dyn Array>
+        let array: Arc<dyn Array> = Arc::new(Int32Array::from(vec![10, 20, 30]));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+        ]));
+
+        // Single-column batch for reference
+        let one_col = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![array.clone()],
+        )
+        .unwrap();
+
+        // Two-column batch with the same Arc shared
+        let two_col = RecordBatch::try_new(schema, vec![array.clone(), array]).unwrap();
+
+        let mut ctx1 = Context::new();
+        let size_one = one_col.deep_size_of_children(&mut ctx1);
+
+        let mut ctx2 = Context::new();
+        let size_two = two_col.deep_size_of_children(&mut ctx2);
+
+        // Both should report the same size since the second column's Arc is
+        // already seen and contributes nothing
+        assert_eq!(size_one, size_two);
+    }
+
+    #[test]
+    fn test_nested_struct_array() {
+        let int_array = Int32Array::from(vec![1, 2, 3]);
+        let str_array = StringArray::from(vec!["a", "b", "c"]);
+        let struct_array = StructArray::from(vec![
+            (
+                Arc::new(Field::new("x", DataType::Int32, false)),
+                Arc::new(int_array) as Arc<dyn Array>,
+            ),
+            (
+                Arc::new(Field::new("y", DataType::Utf8, false)),
+                Arc::new(str_array) as Arc<dyn Array>,
+            ),
+        ]);
+
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new(
+                "s",
+                DataType::Struct(Fields::from(vec![
+                    Field::new("x", DataType::Int32, false),
+                    Field::new("y", DataType::Utf8, false),
+                ])),
+                false,
+            )])),
+            vec![Arc::new(struct_array)],
+        )
+        .unwrap();
+
+        let size = batch.deep_size_of();
+        // Should include buffers for both child arrays
+        assert!(size > 3 * size_of::<i32>());
+    }
+
+    #[test]
+    fn test_std_types() {
+        assert_eq!(42u32.deep_size_of(), size_of::<u32>());
+
+        let s = String::from("hello");
+        assert!(s.deep_size_of() >= size_of::<String>() + 5);
+
+        let v = vec![1u32, 2, 3];
+        assert!(v.deep_size_of() >= size_of::<Vec<u32>>() + 3 * size_of::<u32>());
+
+        let a = Arc::new(42u32);
+        let b = Arc::clone(&a);
+        let mut ctx = Context::new();
+        let size_a = a.deep_size_of_children(&mut ctx);
+        let size_b = b.deep_size_of_children(&mut ctx);
+        assert_eq!(size_a, size_of::<u32>());
+        assert_eq!(size_b, 0);
+    }
+
+    #[test]
+    fn test_derive_macro() {
+        use lance_derive::DeepSizeOf;
+
+        #[derive(DeepSizeOf)]
+        struct Outer {
+            count: u64,
+            label: String,
+            inner: Inner,
+        }
+
+        #[derive(DeepSizeOf)]
+        struct Inner {
+            values: Vec<u32>,
+        }
+
+        let val = Outer {
+            count: 7,
+            label: String::from("hello"),
+            inner: Inner {
+                values: vec![1, 2, 3],
+            },
+        };
+
+        let size = val.deep_size_of();
+        // Must be at least the stack size + heap allocations for label + values
+        assert!(size >= std::mem::size_of::<Outer>() + 5 + 3 * std::mem::size_of::<u32>());
+    }
+}
diff --git a/rust/lance-core/src/error.rs b/rust/lance-core/src/error.rs
index 3dcde1fc5b2..a85f08cb741 100644
--- a/rust/lance-core/src/error.rs
+++ b/rust/lance-core/src/error.rs
@@ -8,6 +8,52 @@ use snafu::{IntoError as _, Location, Snafu};
 
 type BoxedError = Box<dyn std::error::Error + Send + Sync + 'static>;
 
+#[cfg(feature = "backtrace")]
+mod backtrace_support {
+    use std::backtrace::Backtrace;
+
+    use snafu::{AsBacktrace, GenerateImplicitData};
+
+    #[derive(Debug)]
+    pub struct MaybeBacktrace(pub Option<Backtrace>);
+
+    impl GenerateImplicitData for MaybeBacktrace {
+        fn generate() -> Self {
+            Self(<Option<Backtrace>>::generate())
+        }
+    }
+
+    impl AsBacktrace for MaybeBacktrace {
+        fn as_backtrace(&self) -> Option<&Backtrace> {
+            self.0.as_ref()
+        }
+    }
+}
+
+#[cfg(not(feature = "backtrace"))]
+mod backtrace_support {
+    use std::backtrace::Backtrace;
+
+    use snafu::{AsBacktrace, GenerateImplicitData};
+
+    #[derive(Debug)]
+    pub struct MaybeBacktrace;
+
+    impl GenerateImplicitData for MaybeBacktrace {
+        fn generate() -> Self {
+            Self
+        }
+    }
+
+    impl AsBacktrace for MaybeBacktrace {
+        fn as_backtrace(&self) -> Option<&Backtrace> {
+            None
+        }
+    }
+}
+
+use backtrace_support::MaybeBacktrace;
+
 /// Error for when a requested field is not found in a schema.
 ///
 /// This error computes suggestions lazily (only when displayed) to avoid
@@ -51,6 +97,28 @@ pub fn box_error(e: impl std::error::Error + Send + Sync + 'static) -> BoxedErro
     Box::new(e)
 }
 
+/// Why a writer is fenced. Both reasons are terminal, but callers must tell them
+/// apart (a peer takeover vs. our own failure) rather than parse the message.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum FenceReason {
+    /// A successor writer claimed a higher epoch; this writer lost ownership.
+    PeerClaimedEpoch,
+    /// Our own WAL persistence failed, so in-memory state may have diverged from
+    /// the durable WAL. The writer must be reopened to replay.
+    PersistenceFailure,
+}
+
+impl std::fmt::Display for FenceReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // Stable strings — surfaced in error messages.
+        let s = match self {
+            Self::PeerClaimedEpoch => "peer claimed epoch",
+            Self::PersistenceFailure => "persistence failure",
+        };
+        f.write_str(s)
+    }
+}
+
 #[derive(Debug, Snafu)]
 #[snafu(visibility(pub))]
 pub enum Error {
@@ -59,18 +127,24 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Dataset already exists: {uri}, {location}"))]
     DatasetAlreadyExists {
         uri: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Append with different schema: {difference}, location: {location}"))]
     SchemaMismatch {
         difference: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Dataset at path {path} was not found: {source}, {location}"))]
     DatasetNotFound {
@@ -78,6 +152,8 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Encountered corrupt file {path}: {source}, {location}"))]
     CorruptFile {
@@ -85,13 +161,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
-        // TODO: add backtrace?
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Not supported: {source}, {location}"))]
     NotSupported {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Commit conflict for version {version}: {source}, {location}"))]
     CommitConflict {
@@ -99,12 +178,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Incompatible transaction: {source}, {location}"))]
     IncompatibleTransaction {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Retryable commit conflict for version {version}: {source}, {location}"))]
     RetryableCommitConflict {
@@ -112,12 +195,16 @@ pub enum Error {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Too many concurrent writers. {message}, {location}"))]
     TooMuchWriteContention {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Operation timed out: {message}, {location}"))]
     Timeout {
@@ -132,54 +219,72 @@ pub enum Error {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("A prerequisite task failed: {message}, {location}"))]
     PrerequisiteFailed {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Unprocessable: {message}, {location}"))]
     Unprocessable {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Arrow): {message}, {location}"))]
     Arrow {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Schema): {message}, {location}"))]
     Schema {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Not found: {uri}, {location}"))]
     NotFound {
         uri: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(IO): {source}, {location}"))]
     IO {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("LanceError(Index): {message}, {location}"))]
     Index {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Lance index not found: {identity}, {location}"))]
     IndexNotFound {
         identity: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Cannot infer storage location from: {message}"))]
     InvalidTableLocation { message: String },
@@ -190,18 +295,24 @@ pub enum Error {
         error: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Cloned error: {message}, {location}"))]
     Cloned {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Query Execution error: {message}, {location}"))]
     Execution {
         message: String,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Ref is invalid: {message}"))]
     InvalidRef { message: String },
@@ -220,12 +331,16 @@ pub enum Error {
         minor_version: u16,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     #[snafu(display("Namespace error: {source}, {location}"))]
     Namespace {
         source: BoxedError,
         #[snafu(implicit)]
         location: Location,
+        #[snafu(implicit)]
+        backtrace: MaybeBacktrace,
     },
     /// External error passed through from user code.
     ///
@@ -238,9 +353,88 @@ pub enum Error {
     /// A requested field was not found in a schema.
     #[snafu(transparent)]
     FieldNotFound { source: FieldNotFoundError },
+
+    #[snafu(display(
+        "Spill disk cap of {cap_bytes} bytes exceeded; currently using {used_bytes} bytes, {location}"
+    ))]
+    DiskCapExceeded {
+        cap_bytes: u64,
+        used_bytes: u64,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    /// A writer has been fenced and must stop (see [`FenceReason`]). The message
+    /// keeps the `Writer fenced` prefix for legacy string consumers; new code
+    /// should match on [`Error::fence_reason`].
+    #[snafu(display("Writer fenced ({reason}): {message}, {location}"))]
+    Fenced {
+        reason: FenceReason,
+        message: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 impl Error {
+    /// Returns the captured Rust backtrace, if available.
+    ///
+    /// Requires the `backtrace` feature to be enabled at compile time
+    /// and `RUST_BACKTRACE=1` at runtime.
+    #[cfg(feature = "backtrace")]
+    pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> {
+        match self {
+            Self::InvalidInput { backtrace, .. }
+            | Self::DatasetAlreadyExists { backtrace, .. }
+            | Self::SchemaMismatch { backtrace, .. }
+            | Self::DatasetNotFound { backtrace, .. }
+            | Self::CorruptFile { backtrace, .. }
+            | Self::NotSupported { backtrace, .. }
+            | Self::CommitConflict { backtrace, .. }
+            | Self::IncompatibleTransaction { backtrace, .. }
+            | Self::RetryableCommitConflict { backtrace, .. }
+            | Self::TooMuchWriteContention { backtrace, .. }
+            | Self::Internal { backtrace, .. }
+            | Self::PrerequisiteFailed { backtrace, .. }
+            | Self::Unprocessable { backtrace, .. }
+            | Self::Arrow { backtrace, .. }
+            | Self::Schema { backtrace, .. }
+            | Self::NotFound { backtrace, .. }
+            | Self::IO { backtrace, .. }
+            | Self::Index { backtrace, .. }
+            | Self::IndexNotFound { backtrace, .. }
+            | Self::Wrapped { backtrace, .. }
+            | Self::Cloned { backtrace, .. }
+            | Self::Execution { backtrace, .. }
+            | Self::VersionConflict { backtrace, .. }
+            | Self::Namespace { backtrace, .. } => {
+                use snafu::AsBacktrace;
+                backtrace.as_backtrace()
+            }
+            // Variants without a backtrace field — listed explicitly so that
+            // adding a new variant with a backtrace field triggers a compiler error.
+            Self::InvalidTableLocation { .. }
+            | Self::Stop
+            | Self::InvalidRef { .. }
+            | Self::RefConflict { .. }
+            | Self::RefNotFound { .. }
+            | Self::Cleanup { .. }
+            | Self::VersionNotFound { .. }
+            | Self::External { .. }
+            | Self::FieldNotFound { .. }
+            | Self::Timeout { .. }
+            | Self::DiskCapExceeded { .. }
+            | Self::Fenced { .. } => None,
+        }
+    }
+
+    /// Returns the captured Rust backtrace, if available.
+    ///
+    /// Always returns `None` when the `backtrace` feature is not enabled.
+    #[cfg(not(feature = "backtrace"))]
+    pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> {
+        None
+    }
+
     #[track_caller]
     pub fn corrupt_file(path: object_store::path::Path, message: impl Into<String>) -> Self {
         CorruptFileSnafu { path }.into_error(message.into().into())
@@ -261,6 +455,36 @@ impl Error {
         IOSnafu.into_error(message.into().into())
     }
 
+    /// A successor writer claimed a higher epoch; this writer lost ownership.
+    #[track_caller]
+    pub fn fenced_by_peer(message: impl Into<String>) -> Self {
+        FencedSnafu {
+            reason: FenceReason::PeerClaimedEpoch,
+            message: message.into(),
+        }
+        .build()
+    }
+
+    /// Our WAL persistence failed; in-memory state may have diverged from the
+    /// durable WAL, so the writer must be reopened to replay.
+    #[track_caller]
+    pub fn writer_poisoned(message: impl Into<String>) -> Self {
+        FencedSnafu {
+            reason: FenceReason::PersistenceFailure,
+            message: message.into(),
+        }
+        .build()
+    }
+
+    /// The [`FenceReason`] if this is [`Error::Fenced`], else `None`. Prefer this
+    /// over matching the error message to decide how to react to a fence.
+    pub fn fence_reason(&self) -> Option<FenceReason> {
+        match self {
+            Self::Fenced { reason, .. } => Some(*reason),
+            _ => None,
+        }
+    }
+
     #[track_caller]
     pub fn io_source(source: BoxedError) -> Self {
         IOSnafu.into_error(source)
@@ -431,6 +655,15 @@ impl Error {
         IncompatibleTransactionSnafu.into_error(source)
     }
 
+    #[track_caller]
+    pub fn disk_cap_exceeded(cap_bytes: u64, used_bytes: u64) -> Self {
+        DiskCapExceededSnafu {
+            cap_bytes,
+            used_bytes,
+        }
+        .build()
+    }
+
     /// Create an External error from a boxed error source.
     pub fn external(source: BoxedError) -> Self {
         Self::External { source }
@@ -512,6 +745,17 @@ impl From<&ArrowError> for Error {
 impl From<std::io::Error> for Error {
     #[track_caller]
     fn from(e: std::io::Error) -> Self {
+        // A lance `Error` may have been wrapped in an `io::Error` (e.g. via
+        // `io::Error::other(Error::...)`) to cross an `AsyncWrite`/`AsyncRead`
+        // boundary. Recover it so typed errors such as `DiskCapExceeded`
+        // survive the round-trip instead of collapsing into an opaque `IO`.
+        if e.get_ref().is_some_and(|inner| inner.is::<Self>()) {
+            return *e
+                .into_inner()
+                .expect("checked Some above")
+                .downcast::<Self>()
+                .expect("checked type above");
+        }
         Self::io_source(box_error(e))
     }
 }
@@ -519,7 +763,11 @@ impl From<std::io::Error> for Error {
 impl From<object_store::Error> for Error {
     #[track_caller]
     fn from(e: object_store::Error) -> Self {
-        Self::io_source(box_error(e))
+        match e {
+            // source intentionally dropped; Error::NotFound carries only the path
+            object_store::Error::NotFound { path, .. } => Self::not_found(path),
+            other => Self::io_source(box_error(other)),
+        }
     }
 }
 
@@ -704,6 +952,41 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_caller_location_capture_not_found() {
+        let current_fn = get_caller_location();
+        let f: Box<dyn Fn() -> Result<()>> = Box::new(|| {
+            Err(object_store::Error::NotFound {
+                path: "some/path".to_string(),
+                source: "not found".into(),
+            })?;
+            Ok(())
+        });
+        match f().unwrap_err() {
+            Error::NotFound { location, .. } => {
+                // +2 is the beginning of object_store::Error::NotFound...
+                assert_eq!(location.line(), current_fn.line() + 2, "{}", location)
+            }
+            #[allow(unreachable_patterns)]
+            other => panic!("expected NotFound, got {:?}", other),
+        }
+    }
+
+    #[test]
+    fn test_object_store_not_found_converts_to_not_found() {
+        let os_err = object_store::Error::NotFound {
+            path: "test/path".to_string(),
+            source: "no such file".into(),
+        };
+        let lance_err: Error = os_err.into();
+        match lance_err {
+            Error::NotFound { uri, .. } => {
+                assert_eq!(uri, "test/path");
+            }
+            other => panic!("Expected NotFound, got {:?}", other),
+        }
+    }
+
     #[derive(Debug)]
     struct MyCustomError {
         code: i32,
@@ -718,6 +1001,33 @@ mod test {
 
     impl std::error::Error for MyCustomError {}
 
+    #[test]
+    fn test_io_error_recovers_wrapped_lance_error() {
+        // A lance Error wrapped in io::Error::other should round-trip back to
+        // the original variant rather than collapsing into Error::IO.
+        let io_err = std::io::Error::other(Error::disk_cap_exceeded(100, 50));
+        let recovered: Error = io_err.into();
+        match recovered {
+            Error::DiskCapExceeded {
+                cap_bytes,
+                used_bytes,
+                ..
+            } => {
+                assert_eq!(cap_bytes, 100);
+                assert_eq!(used_bytes, 50);
+            }
+            other => panic!("expected DiskCapExceeded, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn test_io_error_without_lance_error_stays_io() {
+        // A plain io::Error (no wrapped lance Error) should become Error::IO.
+        let io_err = std::io::Error::new(std::io::ErrorKind::NotFound, "missing");
+        let converted: Error = io_err.into();
+        assert!(matches!(converted, Error::IO { .. }));
+    }
+
     #[test]
     fn test_external_error_creation() {
         let custom_err = MyCustomError {
@@ -929,4 +1239,68 @@ mod test {
             _ => panic!("Expected InvalidInput variant, got {:?}", recovered),
         }
     }
+
+    #[test]
+    fn test_backtrace_accessor() {
+        // Verify that backtrace() returns the expected result based on feature state
+        let err = Error::io("test backtrace");
+        let bt = err.backtrace();
+        #[cfg(feature = "backtrace")]
+        {
+            // With the backtrace feature enabled, whether a backtrace is captured
+            // depends on the RUST_BACKTRACE env var at runtime. We just verify
+            // the accessor doesn't panic and returns a valid Option.
+            let _ = bt;
+        }
+        #[cfg(not(feature = "backtrace"))]
+        {
+            // Without the backtrace feature, this must always be None.
+            assert!(bt.is_none());
+        }
+    }
+
+    #[test]
+    fn test_backtrace_captured_when_feature_enabled() {
+        // Test that backtrace is actually captured when the feature is on and
+        // RUST_BACKTRACE=1 is set in the environment before the process starts.
+        //
+        // NOTE: std::backtrace::Backtrace caches the RUST_BACKTRACE env check,
+        // so set_var at runtime does not reliably enable capture. This test
+        // verifies the accessor works correctly in both cases:
+        // - If RUST_BACKTRACE=1 was set before the test binary started, we get Some.
+        // - If not, we get None (even with the feature on), which is expected.
+        #[cfg(feature = "backtrace")]
+        {
+            let err = Error::io("backtrace capture test");
+            if std::env::var("RUST_BACKTRACE").is_ok() {
+                assert!(
+                    err.backtrace().is_some(),
+                    "Expected a backtrace when RUST_BACKTRACE=1 and backtrace feature is enabled"
+                );
+            }
+            // When RUST_BACKTRACE is not set, backtrace() may return None even
+            // with the feature enabled — this is correct runtime gating behavior.
+        }
+        #[cfg(not(feature = "backtrace"))]
+        {
+            let err = Error::io("backtrace capture test");
+            assert!(err.backtrace().is_none());
+        }
+    }
+
+    #[test]
+    fn test_backtrace_returns_none_for_variants_without_location() {
+        let err = Error::InvalidTableLocation {
+            message: "test".to_string(),
+        };
+        assert!(err.backtrace().is_none());
+
+        let err = Error::InvalidRef {
+            message: "test".to_string(),
+        };
+        assert!(err.backtrace().is_none());
+
+        let err = Error::Stop;
+        assert!(err.backtrace().is_none());
+    }
 }
diff --git a/rust/lance-core/src/lib.rs b/rust/lance-core/src/lib.rs
index 173c7d0ceaa..32fb34ad5fe 100644
--- a/rust/lance-core/src/lib.rs
+++ b/rust/lance-core/src/lib.rs
@@ -2,18 +2,22 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 #![cfg_attr(coverage, feature(coverage_attribute))]
 
+// Allow the derive macro to reference `lance_core::deepsize` from within this crate.
+extern crate self as lance_core;
+
 use arrow_schema::{DataType, Field as ArrowField};
 use std::sync::LazyLock;
 
 pub mod cache;
 pub mod container;
 pub mod datatypes;
+pub mod deepsize;
 pub mod error;
 pub mod levenshtein;
 pub mod traits;
 pub mod utils;
 
-pub use error::{ArrowResult, Error, Result, box_error};
+pub use error::{ArrowResult, Error, FenceReason, Result, box_error};
 
 /// Wildcard to indicate all non-system columns
 pub const WILDCARD: &str = "*";
diff --git a/rust/lance-core/src/utils.rs b/rust/lance-core/src/utils.rs
index a7ac74a5b27..51d72c3dae3 100644
--- a/rust/lance-core/src/utils.rs
+++ b/rust/lance-core/src/utils.rs
@@ -7,12 +7,15 @@ pub mod assume;
 pub mod backoff;
 pub mod bit;
 pub mod blob;
+pub mod bloomfilter;
 pub mod cpu;
 pub mod deletion;
 pub mod futures;
 pub mod hash;
+pub mod io_stats;
 pub mod parse;
 pub mod path;
+pub mod row_addr_remap;
 pub mod tempfile;
 pub mod testing;
 pub mod tokio;
diff --git a/rust/lance-core/src/utils/bloomfilter.rs b/rust/lance-core/src/utils/bloomfilter.rs
new file mode 100644
index 00000000000..46cc272a694
--- /dev/null
+++ b/rust/lance-core/src/utils/bloomfilter.rs
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Generic bloom filter primitives.
+//!
+//! These are storage-agnostic data structures with no Lance semantics, used by
+//! higher-level crates (e.g. the bloom filter scalar index in `lance-index`).
+
+pub mod as_bytes;
+pub mod sbbf;
diff --git a/rust/lance-index/src/scalar/bloomfilter/as_bytes.rs b/rust/lance-core/src/utils/bloomfilter/as_bytes.rs
similarity index 98%
rename from rust/lance-index/src/scalar/bloomfilter/as_bytes.rs
rename to rust/lance-core/src/utils/bloomfilter/as_bytes.rs
index 22df8d6af7c..86b9632ce39 100644
--- a/rust/lance-index/src/scalar/bloomfilter/as_bytes.rs
+++ b/rust/lance-core/src/utils/bloomfilter/as_bytes.rs
@@ -7,7 +7,7 @@
 //! similar to parquet::data_type::AsBytes but without the external dependency.
 
 /// Trait to convert primitive types to byte slices
-/// Reference: https://arrow.apache.org/rust/src/parquet/data_type.rs.html
+/// Reference: <https://arrow.apache.org/rust/src/parquet/data_type.rs.html>
 pub trait AsBytes {
     /// Convert the value to a byte slice
     fn as_bytes(&self) -> impl AsRef<[u8]>;
diff --git a/rust/lance-index/src/scalar/bloomfilter/sbbf.rs b/rust/lance-core/src/utils/bloomfilter/sbbf.rs
similarity index 99%
rename from rust/lance-index/src/scalar/bloomfilter/sbbf.rs
rename to rust/lance-core/src/utils/bloomfilter/sbbf.rs
index cbb4eb76b12..06df2641008 100644
--- a/rust/lance-index/src/scalar/bloomfilter/sbbf.rs
+++ b/rust/lance-core/src/utils/bloomfilter/sbbf.rs
@@ -28,7 +28,7 @@
 //! removed from Lance.
 //! <https://github.com/apache/arrow-rs/issues/8277>
 
-use crate::scalar::bloomfilter::as_bytes::AsBytes;
+use super::as_bytes::AsBytes;
 use libm::lgamma;
 use std::error::Error;
 use std::fmt;
diff --git a/rust/lance-core/src/utils/cpu.rs b/rust/lance-core/src/utils/cpu.rs
index 4e7ab01871d..c4d5a976cbb 100644
--- a/rust/lance-core/src/utils/cpu.rs
+++ b/rust/lance-core/src/utils/cpu.rs
@@ -1,14 +1,29 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::fmt;
 use std::sync::LazyLock;
 
-/// A level of SIMD support for some feature
+/// A level of SIMD support for some feature.
+///
+/// `#[non_exhaustive]` so future tiers (e.g. AVX-512 BF16, AMX) can be added
+/// without breaking external `match` consumers.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
 pub enum SimdSupport {
     None,
     Neon,
     Sse,
+    /// AVX (256-bit float ops) but no FMA and no AVX2.
+    /// Intel Sandy Bridge / Ivy Bridge.
+    Avx,
+    /// AVX + FMA but no AVX2.
+    /// AMD Piledriver / Steamroller / FX-7500.
+    AvxFma,
+    /// AVX2 + FMA. Intel Haswell / AMD Excavator and later.
+    ///
+    /// Selecting this tier asserts FMA is present: the kernels it dispatches to
+    /// are `#[target_feature(enable = "avx,fma")]`.
     Avx2,
     Avx512,
     Avx512FP16,
@@ -16,6 +31,132 @@ pub enum SimdSupport {
     Lasx,
 }
 
+impl fmt::Display for SimdSupport {
+    /// Formats the tier name in lowercase, matching pyarrow's
+    /// `runtime_info().simd_level` convention.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let name = match self {
+            Self::None => "none",
+            Self::Neon => "neon",
+            Self::Sse => "sse",
+            Self::Avx => "avx",
+            Self::AvxFma => "avx_fma",
+            Self::Avx2 => "avx2",
+            Self::Avx512 => "avx512",
+            Self::Avx512FP16 => "avx512_fp16",
+            Self::Lsx => "lsx",
+            Self::Lasx => "lasx",
+        };
+        f.write_str(name)
+    }
+}
+
+/// Snapshot of the SIMD tier lance dispatches to on the current host, plus the
+/// raw CPU features detected for diagnostic purposes.
+///
+/// Mirrors the role of `pyarrow.runtime_info()`: a single, cheap call users can
+/// make to verify which SIMD tier the runtime selected and what underlying
+/// features the host advertises. Obtain one with [`simd_info()`].
+#[derive(Debug, Clone)]
+pub struct SimdInfo {
+    /// The SIMD tier lance dispatches to at runtime on this host.
+    pub tier: SimdSupport,
+    /// The architecture name (e.g. "x86_64", "aarch64", "loongarch64").
+    pub target_arch: &'static str,
+    /// Raw CPU feature flags detected on this host (x86_64 only; empty on
+    /// other architectures). Each entry is a feature name like "avx2",
+    /// "fma", "avx512f", "popcnt", etc.
+    pub host_features: Vec<&'static str>,
+}
+
+/// Returns a snapshot of the SIMD tier lance is using on this host along with
+/// the raw CPU feature flags that drove the decision.
+///
+/// Useful for performance debugging and giving users a way to verify which
+/// dispatch tier they are hitting without rebuilding lance. See [`SimdInfo`]
+/// for the meaning of each field and [`SimdSupport`] for the tier values.
+///
+/// # Examples
+///
+/// ```
+/// use lance_core::utils::cpu::simd_info;
+///
+/// let info = simd_info();
+/// println!("dispatching to {} on {}", info.tier, info.target_arch);
+/// ```
+pub fn simd_info() -> SimdInfo {
+    SimdInfo {
+        tier: *SIMD_SUPPORT,
+        target_arch: std::env::consts::ARCH,
+        host_features: detect_host_features(),
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+fn detect_host_features() -> Vec<&'static str> {
+    // Each call must be inline: `is_x86_feature_detected!` does its own custom
+    // input parsing and rejects feature names received via a `macro_rules!`
+    // `:literal` metavariable on some toolchains.
+    let mut features = Vec::with_capacity(17);
+    if is_x86_feature_detected!("sse2") {
+        features.push("sse2");
+    }
+    if is_x86_feature_detected!("sse3") {
+        features.push("sse3");
+    }
+    if is_x86_feature_detected!("ssse3") {
+        features.push("ssse3");
+    }
+    if is_x86_feature_detected!("sse4.1") {
+        features.push("sse4.1");
+    }
+    if is_x86_feature_detected!("sse4.2") {
+        features.push("sse4.2");
+    }
+    if is_x86_feature_detected!("popcnt") {
+        features.push("popcnt");
+    }
+    if is_x86_feature_detected!("avx") {
+        features.push("avx");
+    }
+    if is_x86_feature_detected!("avx2") {
+        features.push("avx2");
+    }
+    if is_x86_feature_detected!("fma") {
+        features.push("fma");
+    }
+    if is_x86_feature_detected!("f16c") {
+        features.push("f16c");
+    }
+    if is_x86_feature_detected!("bmi1") {
+        features.push("bmi1");
+    }
+    if is_x86_feature_detected!("bmi2") {
+        features.push("bmi2");
+    }
+    if is_x86_feature_detected!("avx512f") {
+        features.push("avx512f");
+    }
+    if is_x86_feature_detected!("avx512bw") {
+        features.push("avx512bw");
+    }
+    if is_x86_feature_detected!("avx512cd") {
+        features.push("avx512cd");
+    }
+    if is_x86_feature_detected!("avx512dq") {
+        features.push("avx512dq");
+    }
+    if is_x86_feature_detected!("avx512vl") {
+        features.push("avx512vl");
+    }
+    features
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+fn detect_host_features() -> Vec<&'static str> {
+    Vec::new()
+}
+
 /// Support for SIMD operations
 pub static SIMD_SUPPORT: LazyLock<SimdSupport> = LazyLock::new(|| {
     #[cfg(all(target_arch = "aarch64", any(target_os = "ios", target_os = "tvos")))]
@@ -42,8 +183,19 @@ pub static SIMD_SUPPORT: LazyLock<SimdSupport> = LazyLock::new(|| {
             } else {
                 SimdSupport::Avx512
             }
-        } else if is_x86_feature_detected!("avx2") {
+        } else if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") {
+            // FMA is checked explicitly: every kernel selected for this tier is
+            // `#[target_feature(enable = "avx,fma")]`, and AVX2 does not imply
+            // FMA in the ISA. Every shipping AVX2 part has FMA, so this only
+            // guards against a host that would otherwise take an FMA kernel
+            // without FMA.
             SimdSupport::Avx2
+        } else if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") {
+            // AMD Piledriver / Steamroller / FX-7500: 256-bit float ops + FMA but no AVX2.
+            SimdSupport::AvxFma
+        } else if is_x86_feature_detected!("avx") {
+            // Intel Sandy Bridge / Ivy Bridge: 256-bit float ops without FMA.
+            SimdSupport::Avx
         } else {
             SimdSupport::None
         }
@@ -138,3 +290,68 @@ mod aarch64 {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[test]
+    fn simd_info_exposes_tier() {
+        let info = simd_info();
+        assert_eq!(info.target_arch, std::env::consts::ARCH);
+        // Tier should match the detected SIMD support.
+        assert_eq!(info.tier, *SIMD_SUPPORT);
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn simd_info_features_include_baseline() {
+        let info = simd_info();
+        // The x86_64 ABI mandates SSE2, so it must always be present on this
+        // architecture.
+        assert!(info.host_features.contains(&"sse2"));
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[test]
+    fn simd_info_features_empty_off_x86_64() {
+        let info = simd_info();
+        assert!(info.host_features.is_empty());
+    }
+
+    /// The `Avx2` and `AvxFma` tiers both dispatch to kernels declared
+    /// `#[target_feature(enable = "avx,fma")]`, so neither may be selected on a
+    /// host without FMA. AVX2 does not imply FMA in the ISA, so the detection
+    /// checks it explicitly. (`Avx512*` is excluded: its kernels declare
+    /// `avx512f`, which is what `has_avx512` verifies.)
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn avx_fma_tiers_are_only_selected_when_fma_is_detected() {
+        if matches!(*SIMD_SUPPORT, SimdSupport::Avx2 | SimdSupport::AvxFma) {
+            assert!(
+                is_x86_feature_detected!("fma"),
+                "tier {} dispatches to avx,fma kernels but the host has no FMA",
+                *SIMD_SUPPORT
+            );
+        }
+    }
+
+    #[rstest]
+    #[case::none(SimdSupport::None, "none")]
+    #[case::neon(SimdSupport::Neon, "neon")]
+    #[case::sse(SimdSupport::Sse, "sse")]
+    #[case::avx(SimdSupport::Avx, "avx")]
+    #[case::avx_fma(SimdSupport::AvxFma, "avx_fma")]
+    #[case::avx2(SimdSupport::Avx2, "avx2")]
+    #[case::avx512(SimdSupport::Avx512, "avx512")]
+    #[case::avx512_fp16(SimdSupport::Avx512FP16, "avx512_fp16")]
+    #[case::lsx(SimdSupport::Lsx, "lsx")]
+    #[case::lasx(SimdSupport::Lasx, "lasx")]
+    fn simd_support_display_matches_lowercase_convention(
+        #[case] tier: SimdSupport,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(tier.to_string(), expected);
+    }
+}
diff --git a/rust/lance-core/src/utils/deletion.rs b/rust/lance-core/src/utils/deletion.rs
index 5ddfc3348e5..c7f8b142464 100644
--- a/rust/lance-core/src/utils/deletion.rs
+++ b/rust/lance-core/src/utils/deletion.rs
@@ -3,8 +3,8 @@
 
 use std::{collections::HashSet, ops::Range, sync::Arc};
 
+use crate::deepsize::{Context, DeepSizeOf};
 use arrow_array::BooleanArray;
-use deepsize::{Context, DeepSizeOf};
 use roaring::RoaringBitmap;
 
 /// Threshold for when a DeletionVector::Set should be promoted to a DeletionVector::Bitmap.
@@ -296,7 +296,7 @@ impl From<RoaringBitmap> for DeletionVector {
 #[cfg_attr(coverage, coverage(off))]
 mod test {
     use super::*;
-    use deepsize::DeepSizeOf;
+    use crate::deepsize::DeepSizeOf;
     use rstest::rstest;
 
     fn set_dv(vals: impl IntoIterator<Item = u32>) -> DeletionVector {
diff --git a/rust/lance-core/src/utils/io_stats.rs b/rust/lance-core/src/utils/io_stats.rs
new file mode 100644
index 00000000000..e2169d71ae3
--- /dev/null
+++ b/rust/lance-core/src/utils/io_stats.rs
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::ops::Range;
+
+/// A sink that records I/O requests as they are submitted to storage.
+///
+/// This lives in `lance-core` so that the encoding layer (`lance-encoding`) and
+/// the I/O layer (`lance-io`) can both refer to it without depending on one
+/// another.  It lets a caller attach a lightweight counter to a file reader and
+/// measure the exact bytes/IOPS performed for a bounded scope (e.g. a single
+/// query); see `lance_io::scheduler::IoStats` for the concrete implementation.
+///
+/// # When to use this
+///
+/// Lance also exposes two *process-wide, cumulative* I/O accounting facilities:
+/// the global scheduler counters (`lance_io::scheduler::iops_counter` /
+/// `bytes_read_counter`) and the object-store `IOTracker` wrapper used in tests.
+/// Both aggregate every read in the process and cannot attribute I/O to a single
+/// bounded scope.  Prefer an `IoStatsRecorder` when you need the *exact* I/O of
+/// one operation (e.g. a single query): attach it to a reader with
+/// `with_io_stats`, then read the snapshot when the scope ends.  It re-uses the
+/// reader's cached metadata, so measuring costs no extra file opens and does not
+/// disturb the global counters.
+pub trait IoStatsRecorder: std::fmt::Debug + Send + Sync {
+    /// Record one completed request, given the byte ranges as actually
+    /// submitted to storage (i.e. after any coalescing/splitting), so the
+    /// counts reflect physical I/O.
+    fn record_request(&self, ranges: &[Range<u64>]);
+}
diff --git a/rust/lance-core/src/utils/row_addr_remap.rs b/rust/lance-core/src/utils/row_addr_remap.rs
new file mode 100644
index 00000000000..6f5a6f2aae5
--- /dev/null
+++ b/rust/lance-core/src/utils/row_addr_remap.rs
@@ -0,0 +1,413 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Compact row-address remapping for compaction.
+//!
+//! Compaction rewrites rows into new fragments, so indices that store physical
+//! row addresses need an old-address to new-address mapping without building an
+//! O(total rows) `HashMap<u64, Option<u64>>`.
+//!
+//! Layout:
+//!
+//! * Old rows: `old_fragment_id -> (old_offsets, old_rows_before)`
+//!     * `old_offsets`: rewritten old row offsets in this old fragment.
+//!     * `old_rows_before`: rewritten row count before this old fragment.
+//! * New rows: ordered new-fragment ranges
+//!   `(fragment_id, new_rows_before, physical_rows)`
+//!     * `new_rows_before`: rewritten row count before this new fragment.
+//!
+//! Lookup:
+//!
+//! * An address whose fragment was not rewritten returns `None`.
+//! * For an address whose fragment was rewritten:
+//!     * Read `(old_offsets, old_rows_before)` from the old-row layout.
+//!     * If `offset` is not in `old_offsets`, return `Some(None)` because the
+//!       row was deleted.
+//!     * Otherwise, `old_offsets.rank(offset) - 1` is this row's 0-based
+//!       position among rewritten old rows in this old fragment. Add
+//!       `old_rows_before` to get `k`, the row's 0-based position among all
+//!       rewritten old rows.
+//!     * In the new-row layout, find the range
+//!       `(fragment_id, new_rows_before, physical_rows)` where
+//!       `new_rows_before <= k < new_rows_before + physical_rows`.
+//!     * The new address is `(fragment_id, k - new_rows_before)`.
+//!
+//! Ordering:
+//!
+//! Compact remap does not store each old-to-new row mapping. It computes `k`
+//! from the old-row layout, then maps it to the k-th row written to the new
+//! fragments. This requires the reader-to-writer pipeline to preserve row order.
+//!
+//! * `old_frag_ids` must match the order old fragments are read. Within each
+//!   old fragment, rewritten rows are interpreted by ascending old row offset.
+//! * `new_frags` must match the order new rows are written.
+//! * Current compaction satisfies this because it scans selected fragments in
+//!   order and writes the resulting stream without reordering rows.
+
+use crate::utils::address::RowAddress;
+use crate::{Error, Result};
+use roaring::{RoaringBitmap, RoaringTreemap};
+use std::collections::HashMap;
+
+/// A queryable row-address remapping with the exact semantics of
+/// `HashMap<u64, Option<u64>>::get(&addr).copied()`:
+///
+/// * `None` — the address is not affected by this remap (keep it unchanged)
+/// * `Some(None)` — the row was deleted
+/// * `Some(Some(addr))` — the row moved to `addr`
+#[derive(Clone)]
+pub enum RowAddrRemap {
+    /// Compact, `O(#fragments)` remap built from per-group rewritten-row
+    /// bitmaps and new-fragment layouts.
+    Compact(CompactRowAddrRemap),
+    /// Full materialized old-to-new address map. Uses `O(#rows)` memory.
+    Direct(HashMap<u64, Option<u64>>),
+}
+
+impl RowAddrRemap {
+    pub fn compact(groups: impl IntoIterator<Item = GroupInput>) -> Result<Self> {
+        Ok(Self::Compact(CompactRowAddrRemap::new(groups)?))
+    }
+
+    /// Build a remap from a fully materialized old-to-new address map.
+    pub fn direct(map: HashMap<u64, Option<u64>>) -> Self {
+        Self::Direct(map)
+    }
+
+    /// An empty remap that leaves every address unchanged.
+    pub fn empty() -> Self {
+        Self::Direct(HashMap::new())
+    }
+
+    /// Look up `addr`. See [`RowAddrRemap`] for the tri-state return semantics.
+    #[inline]
+    pub fn get(&self, addr: u64) -> Option<Option<u64>> {
+        match self {
+            Self::Compact(c) => c.get(addr),
+            Self::Direct(m) => m.get(&addr).copied(),
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Compact(c) => c.is_empty(),
+            Self::Direct(m) => m.is_empty(),
+        }
+    }
+
+    pub fn affected_fragments(&self) -> RoaringBitmap {
+        match self {
+            Self::Compact(c) => RoaringBitmap::from_iter(c.frag_to_group.keys().copied()),
+            Self::Direct(m) => RoaringBitmap::from_iter(m.keys().map(|addr| (addr >> 32) as u32)),
+        }
+    }
+
+    pub fn fully_deleted_fragments(&self) -> Option<RoaringBitmap> {
+        match self {
+            Self::Compact(c) => c.fully_deleted_fragments(),
+            Self::Direct(m) => {
+                if m.values().all(|v| v.is_none()) {
+                    Some(RoaringBitmap::from_iter(
+                        m.keys().map(|addr| (addr >> 32) as u32),
+                    ))
+                } else {
+                    None
+                }
+            }
+        }
+    }
+}
+
+/// Input describing one rewrite group: the old row addresses that were
+/// rewritten plus the fragment layout before/after the rewrite.
+pub struct GroupInput {
+    /// Old row addresses that were read and re-written into the new fragments.
+    pub rewritten_old_row_addrs: RoaringTreemap,
+    /// Old fragment ids covered by this group.
+    pub old_frag_ids: Vec<u32>,
+    /// New fragments produced by this group, as `(fragment_id, physical_rows)`,
+    pub new_frags: Vec<(u32, u32)>,
+}
+
+#[derive(Clone)]
+struct GroupRemap {
+    /// Old fragment id -> (rewritten old row offsets in that fragment,
+    /// rewritten row count before this fragment in the group).
+    frags: HashMap<u32, (RoaringBitmap, u64)>,
+    /// New fragment ranges as `(fragment_id, rewritten_rows_before, physical_rows)`,
+    /// used to map a rewritten row's group-local index to its new address via binary search.
+    new_frag_row_ranges: Vec<(u32, u64, u32)>,
+}
+
+impl GroupRemap {
+    fn new(input: GroupInput) -> Result<Self> {
+        // `compute_new_addr` maps a rewritten row's group-local index to a new
+        // address by accumulating `physical_rows` in `new_frags` order, so that
+        // order must be the order rows were written. New fragment ids are
+        // reserved monotonically in write order (see `reserve_fragment_ids` in
+        // compaction), so ascending id is a proxy for write order; reject any
+        // input that violates it before it can silently misplace addresses.
+        let mut new_frag_row_ranges = Vec::with_capacity(input.new_frags.len());
+        let mut rewritten_rows_before = 0u64;
+        let mut prev_frag_id: Option<u32> = None;
+        for (frag_id, physical_rows) in input.new_frags {
+            if physical_rows == 0 {
+                continue;
+            }
+            if let Some(prev) = prev_frag_id
+                && frag_id <= prev
+            {
+                return Err(Error::invalid_input(format!(
+                    "compaction new fragments must be in ascending id (write) order, but fragment {frag_id} follows {prev}",
+                )));
+            }
+            prev_frag_id = Some(frag_id);
+            new_frag_row_ranges.push((frag_id, rewritten_rows_before, physical_rows));
+            rewritten_rows_before += physical_rows as u64;
+        }
+        let total_new_rows = rewritten_rows_before;
+
+        let mut per_frag: HashMap<u32, RoaringBitmap> = input
+            .rewritten_old_row_addrs
+            .bitmaps()
+            .map(|(frag_id, bitmap)| (frag_id, bitmap.clone()))
+            .collect();
+        let mut frags = HashMap::new();
+        let mut rewritten_rows_before = 0u64;
+        for &frag_id in &input.old_frag_ids {
+            // A fragment with no rewritten rows (fully deleted) contributes
+            // nothing to the rewritten row sequence.
+            if let Some(bitmap) = per_frag.remove(&frag_id) {
+                let num_rewritten_rows = bitmap.len();
+                frags.insert(frag_id, (bitmap, rewritten_rows_before));
+                rewritten_rows_before += num_rewritten_rows;
+            }
+        }
+        // Rewritten old row addresses must reference only fragments listed in `old_frag_ids`.
+        if !per_frag.is_empty() {
+            return Err(Error::invalid_input(format!(
+                "compaction rewritten old row addresses reference fragments {:?} not in the rewrite group's old fragments {:?}",
+                per_frag.keys().collect::<Vec<_>>(),
+                input.old_frag_ids,
+            )));
+        }
+
+        // Rewritten old rows are mapped positionally onto the new rows, so the
+        // two counts must match exactly
+        let total_rewritten_old_rows = input.rewritten_old_row_addrs.len();
+        if total_new_rows != total_rewritten_old_rows {
+            return Err(Error::invalid_input(format!(
+                "compaction rewrote {total_rewritten_old_rows} old rows from fragments {:?} but the new fragments hold {total_new_rows} rows",
+                input.old_frag_ids,
+            )));
+        }
+
+        Ok(Self {
+            frags,
+            new_frag_row_ranges,
+        })
+    }
+
+    fn compute_new_addr(&self, rewritten_row_index: u64) -> u64 {
+        let idx =
+            match self
+                .new_frag_row_ranges
+                .binary_search_by(|(_, rewritten_rows_before, _)| {
+                    rewritten_rows_before.cmp(&rewritten_row_index)
+                }) {
+                Ok(i) => i,
+                Err(i) => i - 1,
+            };
+        let (frag_id, rewritten_rows_before, _rows) = self.new_frag_row_ranges[idx];
+        let offset = (rewritten_row_index - rewritten_rows_before) as u32;
+        u64::from(RowAddress::new_from_parts(frag_id, offset))
+    }
+
+    /// Compute the new address for an old row in this group.
+    /// Returns `None` if the old row was not rewritten.
+    #[inline]
+    fn get(&self, frag: u32, offset: u32) -> Option<u64> {
+        match self.frags.get(&frag) {
+            Some((bitmap, rewritten_rows_before)) if bitmap.contains(offset) => {
+                let rewritten_row_index = rewritten_rows_before + bitmap.rank(offset) - 1;
+                Some(self.compute_new_addr(rewritten_row_index))
+            }
+            _ => None,
+        }
+    }
+}
+
+/// Compact remap backed by per-group rewritten row bitmaps + new-fragment layouts.
+#[derive(Clone)]
+pub struct CompactRowAddrRemap {
+    groups: Vec<GroupRemap>,
+    /// Old fragment id -> index into `groups`. Size is O(#fragments), not rows.
+    frag_to_group: HashMap<u32, usize>,
+}
+
+impl CompactRowAddrRemap {
+    fn new(groups: impl IntoIterator<Item = GroupInput>) -> Result<Self> {
+        let mut frag_to_group = HashMap::new();
+        let mut group_remaps = Vec::new();
+        for input in groups {
+            let gi = group_remaps.len();
+            for &frag_id in &input.old_frag_ids {
+                frag_to_group.insert(frag_id, gi);
+            }
+            group_remaps.push(GroupRemap::new(input)?);
+        }
+        Ok(Self {
+            groups: group_remaps,
+            frag_to_group,
+        })
+    }
+
+    #[inline]
+    pub fn get(&self, addr: u64) -> Option<Option<u64>> {
+        let frag = (addr >> 32) as u32;
+        // Not in any rewrite group -> unaffected by this remap.
+        let gi = *self.frag_to_group.get(&frag)?;
+        Some(self.groups[gi].get(frag, addr as u32))
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.groups.is_empty()
+    }
+
+    fn fully_deleted_fragments(&self) -> Option<RoaringBitmap> {
+        // A group with any rewritten row moved at least one row.
+        if self.groups.iter().any(|g| !g.frags.is_empty()) {
+            return None;
+        }
+        Some(RoaringBitmap::from_iter(self.frag_to_group.keys().copied()))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn addr(frag: u32, offset: u32) -> u64 {
+        u64::from(RowAddress::new_from_parts(frag, offset))
+    }
+
+    #[test]
+    fn test_compact_lookup() {
+        // Group A: out-of-order old frags [4, 3], split new frags (11 empty),
+        // some deletions. frag 4 (5 rows) keeps 0,2,4; frag 3 keeps 0,1, so the
+        // rewritten rows (4,0)(4,2)(4,4)(3,0)(3,1) go to new frags 10(2), 12(3).
+        // Group B is a fully-deleted fragment.
+        let group_a = GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter([
+                addr(4, 0),
+                addr(4, 2),
+                addr(4, 4),
+                addr(3, 0),
+                addr(3, 1),
+            ]),
+            old_frag_ids: vec![4, 3],
+            new_frags: vec![(10, 2), (11, 0), (12, 3)],
+        };
+        let group_b = GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::new(),
+            old_frag_ids: vec![7],
+            new_frags: vec![],
+        };
+        let remap = RowAddrRemap::compact([group_a, group_b]).unwrap();
+
+        // Moves, in rewrite order; frag 4 comes first despite the larger id.
+        assert_eq!(remap.get(addr(4, 0)), Some(Some(addr(10, 0))));
+        assert_eq!(remap.get(addr(4, 2)), Some(Some(addr(10, 1))));
+        // Rank 2 skips the zero-row new fragment 11 and lands in fragment 12.
+        assert_eq!(remap.get(addr(4, 4)), Some(Some(addr(12, 0))));
+        assert_eq!(remap.get(addr(3, 0)), Some(Some(addr(12, 1))));
+        assert_eq!(remap.get(addr(3, 1)), Some(Some(addr(12, 2))));
+        // Deleted offsets inside a rewritten fragment.
+        assert_eq!(remap.get(addr(4, 1)), Some(None));
+        assert_eq!(remap.get(addr(4, 3)), Some(None));
+        // Covered but fully-deleted fragment -> Some(None), not None.
+        assert_eq!(remap.get(addr(7, 0)), Some(None));
+        // Fragment in no group -> unaffected.
+        assert_eq!(remap.get(addr(9, 0)), None);
+        assert!(!remap.is_empty());
+    }
+
+    #[test]
+    fn test_fragment_sets() {
+        // No rewritten rows at all: every covered fragment is fully deleted.
+        let dead = RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::new(),
+            old_frag_ids: vec![3, 7],
+            new_frags: vec![],
+        }])
+        .unwrap();
+        assert_eq!(
+            dead.fully_deleted_fragments(),
+            Some(RoaringBitmap::from_iter([3u32, 7u32]))
+        );
+        assert_eq!(
+            dead.affected_fragments(),
+            RoaringBitmap::from_iter([3u32, 7u32])
+        );
+
+        // At least one rewritten row -> not fully deleted, but both covered
+        // fragments (including the fully-deleted frag 1) are still affected.
+        let alive = RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0)]),
+            old_frag_ids: vec![0, 1],
+            new_frags: vec![(10, 1)],
+        }])
+        .unwrap();
+        assert!(alive.fully_deleted_fragments().is_none());
+        assert_eq!(
+            alive.affected_fragments(),
+            RoaringBitmap::from_iter([0u32, 1u32])
+        );
+    }
+
+    #[test]
+    fn test_compact_rejects_rewritten_addrs_outside_old_frags() {
+        // Rewritten addresses reference frag 5, not in old_frag_ids. The count
+        // still matches (2 == 2), so only the per-fragment split catches it.
+        let input = GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0), addr(5, 0)]),
+            old_frag_ids: vec![0],
+            new_frags: vec![(10, 2)],
+        };
+        assert!(RowAddrRemap::compact([input]).is_err());
+    }
+
+    #[test]
+    fn test_compact_rejects_new_frags_out_of_write_order() {
+        // New fragments out of ascending id (write) order would make
+        // `compute_new_addr` accumulate rows in the wrong order, silently
+        // misplacing addresses. A zero-row fragment between them is ignored.
+        let input = GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0), addr(0, 1)]),
+            old_frag_ids: vec![0],
+            new_frags: vec![(12, 1), (11, 1)],
+        };
+        assert!(RowAddrRemap::compact([input]).is_err());
+    }
+
+    #[test]
+    fn test_direct_and_empty() {
+        // Direct covers arbitrary maps the compact form can't express.
+        let mut map = HashMap::new();
+        map.insert(addr(2, 0), Some(addr(9, 9)));
+        map.insert(addr(5, 1), None);
+        let remap = RowAddrRemap::direct(map);
+        assert_eq!(remap.get(addr(2, 0)), Some(Some(addr(9, 9))));
+        assert_eq!(remap.get(addr(5, 1)), Some(None));
+        assert_eq!(remap.get(addr(2, 1)), None);
+        // affected_fragments over an explicit map: the fragment of every key.
+        assert_eq!(
+            remap.affected_fragments(),
+            RoaringBitmap::from_iter([2u32, 5u32])
+        );
+
+        let empty = RowAddrRemap::empty();
+        assert!(empty.is_empty());
+        assert_eq!(empty.get(addr(0, 0)), None);
+    }
+}
diff --git a/rust/lance-core/src/utils/tokio.rs b/rust/lance-core/src/utils/tokio.rs
index 46c9475665b..fd33fc6b216 100644
--- a/rust/lance-core/src/utils/tokio.rs
+++ b/rust/lance-core/src/utils/tokio.rs
@@ -66,11 +66,15 @@ static RUNTIME_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false);
 
 static ATFORK_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false);
 
-fn global_cpu_runtime() -> &'static mut Runtime {
+fn global_cpu_runtime() -> &'static Runtime {
     loop {
         let ptr = CPU_RUNTIME.load(Ordering::SeqCst);
         if !ptr.is_null() {
-            return unsafe { &mut *ptr };
+            // SAFETY: `ptr` was produced by `Box::into_raw` below and is only ever
+            // reset to null by `atfork_tokio_child` in the forked child (single-
+            // threaded, async-signal context). The `Box` is never reclaimed, so the
+            // `Runtime` lives for the rest of the process.
+            return unsafe { &*ptr };
         }
         if !RUNTIME_INSTALLED.fetch_or(true, Ordering::SeqCst) {
             break;
@@ -82,7 +86,9 @@ fn global_cpu_runtime() -> &'static mut Runtime {
     }
     let new_ptr = Box::into_raw(Box::new(create_runtime()));
     CPU_RUNTIME.store(new_ptr, Ordering::SeqCst);
-    unsafe { &mut *new_ptr }
+    // SAFETY: `new_ptr` was just obtained from `Box::into_raw`, so it is non-null,
+    // aligned, and points to a live `Runtime` that is never reclaimed.
+    unsafe { &*new_ptr }
 }
 
 /// After a fork() operation, force re-creation of the BackgroundExecutor. Note: this function
@@ -108,6 +114,43 @@ fn install_atfork() {}
 ///
 /// This can also be used to convert a big chunk of synchronous work into a future
 /// so that it can be run in parallel with something like StreamExt::buffered()
+///
+/// # Only hand over substantial CPU work
+///
+/// Dispatching to the pool has real overhead (a `spawn_blocking` hop plus a oneshot
+/// channel round trip). As a rule of thumb the closure should be expected to do at
+/// least ~100µs of CPU work; below that the thread-pool overhead is likely to
+/// outweigh any parallelism benefit, and the work is better left inline.
+///
+/// # The task must never wait on anything
+///
+/// The CPU pool is sized to [`get_num_compute_intensive_cpus`], which is
+/// `max(1, num_cpus - LANCE_IO_CORE_RESERVATION)`. On a big host that is plenty of
+/// workers (e.g. 62 on a 64-core box), but in resource-constrained environments it can
+/// collapse to a **single blocking thread** — on machines with `<= 3` visible CPUs
+/// (1-vCPU VMs, CI runners, CPU-limited Kubernetes pods) the pool has exactly one
+/// worker. A closure passed to `spawn_cpu` occupies one of these threads for its entire
+/// lifetime, including any time it spends *parked*. So the closure must only consume
+/// CPU and return; it must
+/// **never** block, wait, or park. Concretely, the closure must not, directly or
+/// transitively:
+///
+/// * **No channels** — no blocking send/recv (`send_blocking`, blocking `recv`, etc.).
+///   A full/empty channel parks the thread, and whatever would drain/fill the channel
+///   may need the same pool to run.
+/// * **No I/O** — no file, network, or object-store reads/writes, and no disk spills.
+///   I/O parks the thread while making no progress on CPU work.
+/// * **No locks** — no acquiring a contended lock (or any lock that is held across an
+///   `.await` elsewhere). Waiting for the lock parks the thread.
+/// * **No `block_on` / `.blocking_*`** — never drive or wait on another async task
+///   from inside the closure.
+///
+/// If any of these hold, the parked thread can starve the exact work that would
+/// unblock it, deadlocking the whole pool with no timeout and no error — a silent
+/// hang at 0% CPU. (See <https://github.com/lancedb/lance/pull/7423>.) When work
+/// needs to wait on a channel/lock/I/O, keep the waiting in an async task and only
+/// hand the pure-CPU portion to `spawn_cpu`, e.g. build each batch with `spawn_cpu`
+/// and dispatch it with `tx.send(batch).await` in the surrounding async code.
 pub fn spawn_cpu<
     E: std::error::Error + Send + 'static,
     F: FnOnce() -> std::result::Result<R, E> + Send + 'static,
diff --git a/rust/lance-datafusion/src/exec.rs b/rust/lance-datafusion/src/exec.rs
index 5d7c5465132..07c2a33debc 100644
--- a/rust/lance-datafusion/src/exec.rs
+++ b/rust/lance-datafusion/src/exec.rs
@@ -6,6 +6,7 @@
 use std::{
     collections::HashMap,
     fmt::{self, Formatter},
+    num::NonZero,
     sync::{Arc, Mutex, OnceLock},
     time::Duration,
 };
@@ -14,7 +15,6 @@ use chrono::{DateTime, Utc};
 
 use arrow_array::RecordBatch;
 use arrow_schema::Schema as ArrowSchema;
-use datafusion::physical_plan::metrics::MetricType;
 use datafusion::{
     catalog::streaming::StreamingTable,
     dataframe::DataFrame,
@@ -28,6 +28,7 @@ use datafusion::{
     physical_plan::{
         DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream,
         analyze::AnalyzeExec,
+        coalesce_partitions::CoalescePartitionsExec,
         display::DisplayableExecutionPlan,
         execution_plan::{Boundedness, CardinalityEffect, EmissionType},
         metrics::MetricValue,
@@ -35,6 +36,7 @@ use datafusion::{
         streaming::PartitionStream,
     },
 };
+use datafusion::{execution::memory_pool::TrackConsumersPool, physical_plan::metrics::MetricType};
 use datafusion_common::{DataFusionError, Statistics};
 use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
 
@@ -151,10 +153,6 @@ impl ExecutionPlan for OneShotExec {
         "OneShotExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.schema.clone()
     }
@@ -242,10 +240,6 @@ impl ExecutionPlan for TracedExec {
         "TracedExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -309,7 +303,7 @@ impl std::fmt::Debug for LanceExecutionOptions {
     }
 }
 
-const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 100 * 1024 * 1024;
+const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 150 * 1024 * 1024;
 const DEFAULT_LANCE_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB
 
 impl LanceExecutionOptions {
@@ -365,12 +359,21 @@ pub fn new_session_context(options: &LanceExecutionOptions) -> SessionContext {
         session_config = session_config.with_target_partitions(target_partition);
     }
     if options.use_spilling() {
+        // The default 10MB sort spill reservation seems to be too small for many common cases.
+        //
+        // There currently is no reasonable guidance provided by DataFusion for setting this value.
+        // We bump this to 40MB but try a smaller value if the mem pool is small.
+        let sort_spill_reservation_bytes =
+            (options.mem_pool_size() / 3).min(40 * 1024 * 1024) as usize;
+        session_config =
+            session_config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes);
         let disk_manager_builder = DiskManagerBuilder::default()
             .with_max_temp_directory_size(options.max_temp_directory_size());
         runtime_env_builder = runtime_env_builder
             .with_disk_manager_builder(disk_manager_builder)
-            .with_memory_pool(Arc::new(FairSpillPool::new(
-                options.mem_pool_size() as usize
+            .with_memory_pool(Arc::new(TrackConsumersPool::new(
+                FairSpillPool::new(options.mem_pool_size() as usize),
+                NonZero::try_from(16).unwrap(),
             )));
     }
     let runtime_env = runtime_env_builder.build_arc().unwrap();
@@ -606,9 +609,15 @@ pub fn execute_plan(
 
     let session_ctx = get_session_context(&options);
 
-    // NOTE: we are only executing the first partition here. Therefore, if
-    // the plan has more than one partition, we will be missing data.
-    assert_eq!(plan.properties().partitioning.partition_count(), 1);
+    // Coalesce to a single partition if the optimizer left more than one.
+    // EnforceDistribution may remove RepartitionExec(1) nodes when the parent
+    // declares UnspecifiedDistribution, leaving multi-partition plans here.
+    let plan: Arc<dyn ExecutionPlan> = if plan.properties().partitioning.partition_count() == 1 {
+        plan
+    } else {
+        Arc::new(CoalescePartitionsExec::new(plan))
+    };
+
     let stream = plan.execute(0, get_task_context(&session_ctx, &options))?;
 
     let schema = stream.schema();
@@ -633,7 +642,8 @@ pub async fn analyze_plan(
     let analyze = Arc::new(AnalyzeExec::new(
         true,
         true,
-        vec![MetricType::SUMMARY],
+        vec![MetricType::Summary],
+        None,
         plan,
         schema,
     ));
@@ -887,10 +897,6 @@ impl ExecutionPlan for StrictBatchSizeExec {
         "StrictBatchSizeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
@@ -931,7 +937,7 @@ impl ExecutionPlan for StrictBatchSizeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion_common::Result<Statistics> {
+    ) -> datafusion_common::Result<std::sync::Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
@@ -993,10 +999,6 @@ impl ExecutionPlan for HardCapBatchSizeExec {
         "HardCapBatchSizeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.input.properties()
     }
@@ -1058,7 +1060,7 @@ impl ExecutionPlan for HardCapBatchSizeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion_common::Result<Statistics> {
+    ) -> datafusion_common::Result<std::sync::Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
diff --git a/rust/lance-datafusion/src/expr.rs b/rust/lance-datafusion/src/expr.rs
index 79650f6775e..a0da34ba2bb 100644
--- a/rust/lance-datafusion/src/expr.rs
+++ b/rust/lance-datafusion/src/expr.rs
@@ -17,6 +17,18 @@ const MS_PER_DAY: i64 = 86400000;
 // will always yield "x = 7_u64" regardless of the type of the column "x".  As a result, we
 // need to do that literal coercion ourselves.
 pub fn safe_coerce_scalar(value: &ScalarValue, ty: &DataType) -> Option<ScalarValue> {
+    // A dictionary target coerces the value to the dictionary's value type and
+    // re-wraps it as a dictionary literal. Only an untyped `ScalarValue::Null`
+    // keeps its untyped form, matching the behavior for all other targets; a
+    // *typed* null (e.g. `Utf8(None)`) is coerced and wrapped like any other
+    // value so it produces a `Dictionary(..)` literal that matches the column.
+    if let DataType::Dictionary(key_type, value_type) = ty {
+        if matches!(value, ScalarValue::Null) {
+            return Some(value.clone());
+        }
+        let inner = safe_coerce_scalar(value, value_type)?;
+        return Some(ScalarValue::Dictionary(key_type.clone(), Box::new(inner)));
+    }
     match value {
         ScalarValue::Int8(val) => match ty {
             DataType::Int8 => Some(value.clone()),
@@ -436,6 +448,9 @@ pub fn safe_coerce_scalar(value: &ScalarValue, ty: &DataType) -> Option<ScalarVa
             DataType::BinaryView => Some(value.clone()),
             _ => None,
         },
+        // A dictionary-encoded literal (e.g. produced by DataFusion's dictionary
+        // cast in the scalar-index path) coerces by unwrapping its underlying value.
+        ScalarValue::Dictionary(_, inner) => safe_coerce_scalar(inner, ty),
         _ => None,
     }
 }
@@ -775,4 +790,97 @@ mod tests {
             Some(ScalarValue::BinaryView(Some(vec![1, 2, 3])))
         );
     }
+
+    #[test]
+    fn test_dictionary_coerce() {
+        let dict_ty = DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8));
+
+        // A string literal coerces to a dictionary target by wrapping the
+        // coerced value in a dictionary scalar.
+        assert_eq!(
+            safe_coerce_scalar(&ScalarValue::Utf8(Some("com".to_string())), &dict_ty),
+            Some(ScalarValue::Dictionary(
+                Box::new(DataType::Int16),
+                Box::new(ScalarValue::Utf8(Some("com".to_string()))),
+            ))
+        );
+
+        // The inner value is coerced through to the dictionary value type, so a
+        // LargeUtf8 literal lands as a Utf8 value inside the dictionary.
+        assert_eq!(
+            safe_coerce_scalar(&ScalarValue::LargeUtf8(Some("com".to_string())), &dict_ty),
+            Some(ScalarValue::Dictionary(
+                Box::new(DataType::Int16),
+                Box::new(ScalarValue::Utf8(Some("com".to_string()))),
+            ))
+        );
+
+        // A dictionary literal round-trips back to its value type.
+        assert_eq!(
+            safe_coerce_scalar(
+                &ScalarValue::Dictionary(
+                    Box::new(DataType::Int16),
+                    Box::new(ScalarValue::Utf8(Some("com".to_string()))),
+                ),
+                &DataType::Utf8,
+            ),
+            Some(ScalarValue::Utf8(Some("com".to_string())))
+        );
+
+        // A dictionary literal coerces to a dictionary target, adopting the
+        // target's key type.
+        assert_eq!(
+            safe_coerce_scalar(
+                &ScalarValue::Dictionary(
+                    Box::new(DataType::Int32),
+                    Box::new(ScalarValue::Utf8(Some("com".to_string()))),
+                ),
+                &dict_ty,
+            ),
+            Some(ScalarValue::Dictionary(
+                Box::new(DataType::Int16),
+                Box::new(ScalarValue::Utf8(Some("com".to_string()))),
+            ))
+        );
+
+        // An untyped null keeps its untyped form for a dictionary target, just
+        // like for every other target type.
+        assert_eq!(
+            safe_coerce_scalar(&ScalarValue::Null, &dict_ty),
+            Some(ScalarValue::Null)
+        );
+
+        // A *typed* null (e.g. an API-built `Utf8(None)` literal, or an IN value
+        // already typed as Utf8) is still wrapped in the dictionary type so it
+        // matches the dictionary column. Returning a bare `Utf8(None)` here would
+        // leave `resolve_value` with a literal whose type does not line up with
+        // the column, breaking planning/evaluation the same way non-null strings
+        // used to break.
+        assert_eq!(
+            safe_coerce_scalar(&ScalarValue::Utf8(None), &dict_ty),
+            Some(ScalarValue::Dictionary(
+                Box::new(DataType::Int16),
+                Box::new(ScalarValue::Utf8(None)),
+            ))
+        );
+
+        // The inner null is coerced through to the dictionary value type as well,
+        // so a LargeUtf8 typed null lands as a Utf8 null inside the dictionary.
+        assert_eq!(
+            safe_coerce_scalar(&ScalarValue::LargeUtf8(None), &dict_ty),
+            Some(ScalarValue::Dictionary(
+                Box::new(DataType::Int16),
+                Box::new(ScalarValue::Utf8(None)),
+            ))
+        );
+
+        // A value that cannot be coerced to the dictionary value type fails.
+        assert_eq!(
+            safe_coerce_scalar(
+                &ScalarValue::Utf8(Some("com".to_string())),
+                &DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Int32)),
+            ),
+            None
+        );
+    }
 }
diff --git a/rust/lance-datafusion/src/logical_expr.rs b/rust/lance-datafusion/src/logical_expr.rs
index ab0936d31da..db9abd7e204 100644
--- a/rust/lance-datafusion/src/logical_expr.rs
+++ b/rust/lance-datafusion/src/logical_expr.rs
@@ -51,14 +51,10 @@ pub fn resolve_column_type(expr: &Expr, schema: &Schema) -> Option<DataType> {
                 field_path.push(c.name.as_str());
                 break;
             }
-            Expr::ScalarFunction(udf) => {
-                if udf.name() == GetFieldFunc::default().name() {
-                    let name = get_as_string_scalar_opt(&udf.args[1])?;
-                    field_path.push(name);
-                    current_expr = &udf.args[0];
-                } else {
-                    return None;
-                }
+            Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => {
+                let name = get_as_string_scalar_opt(&udf.args[1])?;
+                field_path.push(name);
+                current_expr = &udf.args[0];
             }
             _ => return None,
         }
@@ -285,8 +281,10 @@ pub fn field_path_to_expr(field_path: &str) -> Result<Expr> {
         )));
     }
 
-    // Build the column expression, handling nested fields
-    let mut expr = col(&parts[0]);
+    // Build the column expression, handling nested fields.
+    let mut expr = Expr::Column(datafusion::common::Column::new_unqualified(
+        parts[0].clone(),
+    ));
     for part in &parts[1..] {
         expr = expr.field_newstyle(part);
     }
@@ -301,8 +299,26 @@ mod tests {
     use super::*;
 
     use arrow_schema::{Field, Schema as ArrowSchema};
+    use datafusion::common::Column;
     use datafusion_functions::core::expr_ext::FieldAccessor;
 
+    #[test]
+    fn test_field_path_to_expr_preserves_case_sensitive_root_column() {
+        let expr = field_path_to_expr("VECTOR").unwrap();
+
+        assert_eq!(expr, Expr::Column(Column::new_unqualified("VECTOR")));
+    }
+
+    #[test]
+    fn test_field_path_to_expr_preserves_case_sensitive_escaped_nested_path() {
+        let expr = field_path_to_expr("Parent.`Child.With.Dot`").unwrap();
+
+        assert_eq!(
+            expr,
+            Expr::Column(Column::new_unqualified("Parent")).field_newstyle("Child.With.Dot")
+        );
+    }
+
     #[test]
     fn test_resolve_large_utf8() {
         let arrow_schema = ArrowSchema::new(vec![Field::new("a", DataType::LargeUtf8, false)]);
@@ -463,4 +479,58 @@ mod tests {
             _ => unreachable!("Expected BinaryExpr"),
         }
     }
+
+    #[test]
+    fn test_resolve_typed_null_against_dictionary_column() {
+        // A dictionary-encoded string column, e.g. a categorical field.
+        let dict_ty = DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8));
+        let arrow_schema = ArrowSchema::new(vec![Field::new("etld", dict_ty, true)]);
+        let schema = Schema::try_from(&arrow_schema).unwrap();
+
+        // A typed null must be wrapped in the dictionary type, not left as a bare
+        // `Utf8(None)` literal sitting next to a `Dictionary(...)` column.
+        let expected_null = Expr::Literal(
+            ScalarValue::Dictionary(Box::new(DataType::Int16), Box::new(ScalarValue::Utf8(None))),
+            None,
+        );
+
+        // `etld = <typed null>` built directly via the API, as opposed to coming
+        // through SQL parsing.
+        let expr = Expr::BinaryExpr(BinaryExpr {
+            left: Box::new(Expr::Column("etld".to_string().into())),
+            op: Operator::Eq,
+            right: Box::new(Expr::Literal(ScalarValue::Utf8(None), None)),
+        });
+        match resolve_expr(&expr, &schema).unwrap() {
+            Expr::BinaryExpr(be) => assert_eq!(be.right.as_ref(), &expected_null),
+            other => unreachable!("Expected BinaryExpr, got {other:?}"),
+        }
+
+        // `etld IN ('a', <typed null>)` — a typed value mixed with a typed null,
+        // both already typed as Utf8. Every list element is wrapped in the
+        // dictionary type.
+        let expr = Expr::in_list(
+            Expr::Column("etld".to_string().into()),
+            vec![
+                Expr::Literal(ScalarValue::Utf8(Some("a".to_string())), None),
+                Expr::Literal(ScalarValue::Utf8(None), None),
+            ],
+            false,
+        );
+        let expected = Expr::in_list(
+            Expr::Column("etld".to_string().into()),
+            vec![
+                Expr::Literal(
+                    ScalarValue::Dictionary(
+                        Box::new(DataType::Int16),
+                        Box::new(ScalarValue::Utf8(Some("a".to_string()))),
+                    ),
+                    None,
+                ),
+                expected_null,
+            ],
+            false,
+        );
+        assert_eq!(resolve_expr(&expr, &schema).unwrap(), expected);
+    }
 }
diff --git a/rust/lance-datafusion/src/planner.rs b/rust/lance-datafusion/src/planner.rs
index 3fd1841ada7..f72e9fcfacb 100644
--- a/rust/lance-datafusion/src/planner.rs
+++ b/rust/lance-datafusion/src/planner.rs
@@ -72,10 +72,6 @@ impl CastListF16Udf {
 }
 
 impl ScalarUDFImpl for CastListF16Udf {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn name(&self) -> &str {
         "_cast_list_f16"
     }
@@ -197,6 +193,13 @@ impl ContextProvider for LanceContextProvider {
         self.state.window_functions().get(name).cloned()
     }
 
+    fn get_higher_order_meta(
+        &self,
+        name: &str,
+    ) -> Option<Arc<datafusion::logical_expr::HigherOrderUDF>> {
+        self.state.higher_order_functions().get(name).cloned()
+    }
+
     fn get_function_meta(&self, f: &str) -> Option<Arc<ScalarUDF>> {
         match f {
             // TODO: cast should go thru CAST syntax instead of UDF
@@ -227,6 +230,14 @@ impl ContextProvider for LanceContextProvider {
         self.state.window_functions().keys().cloned().collect()
     }
 
+    fn higher_order_function_names(&self) -> Vec<String> {
+        self.state
+            .higher_order_functions()
+            .keys()
+            .cloned()
+            .collect()
+    }
+
     fn get_expr_planners(&self) -> &[Arc<dyn ExprPlanner>] {
         &self.expr_planners
     }
@@ -329,10 +340,82 @@ impl Planner {
         })
     }
 
+    fn is_logical_binary_op(op: &BinaryOperator) -> bool {
+        matches!(op, BinaryOperator::And | BinaryOperator::Or)
+    }
+
+    fn is_same_logical_binary_op(left: &BinaryOperator, right: &BinaryOperator) -> bool {
+        matches!(
+            (left, right),
+            (BinaryOperator::And, BinaryOperator::And) | (BinaryOperator::Or, BinaryOperator::Or)
+        )
+    }
+
+    fn flatten_logical_binary_exprs<'a>(
+        left: &'a SQLExpr,
+        op: &BinaryOperator,
+        right: &'a SQLExpr,
+    ) -> Vec<&'a SQLExpr> {
+        let mut leaves = Vec::new();
+        let mut stack = vec![right, left];
+
+        while let Some(expr) = stack.pop() {
+            match expr {
+                SQLExpr::BinaryOp {
+                    left,
+                    op: child_op,
+                    right,
+                } if Self::is_same_logical_binary_op(op, child_op) => {
+                    stack.push(right.as_ref());
+                    stack.push(left.as_ref());
+                }
+                _ => leaves.push(expr),
+            }
+        }
+
+        leaves
+    }
+
+    fn balanced_binary_expr(mut exprs: VecDeque<Expr>, op: Operator) -> Result<Expr> {
+        if exprs.is_empty() {
+            return Err(Error::invalid_input("Binary expression has no operands"));
+        }
+
+        while exprs.len() > 1 {
+            let mut next = VecDeque::with_capacity(exprs.len().div_ceil(2));
+            while let Some(left) = exprs.pop_front() {
+                if let Some(right) = exprs.pop_front() {
+                    next.push_back(Expr::BinaryExpr(BinaryExpr::new(
+                        Box::new(left),
+                        op,
+                        Box::new(right),
+                    )));
+                } else {
+                    next.push_back(left);
+                }
+            }
+            exprs = next;
+        }
+
+        exprs
+            .pop_front()
+            .ok_or_else(|| Error::invalid_input("Binary expression has no operands"))
+    }
+
     fn binary_expr(&self, left: &SQLExpr, op: &BinaryOperator, right: &SQLExpr) -> Result<Expr> {
+        let df_op = self.binary_op(op)?;
+        if Self::is_logical_binary_op(op) {
+            let leaves = Self::flatten_logical_binary_exprs(left, op, right);
+            let mut exprs = VecDeque::with_capacity(leaves.len());
+            for leaf in leaves {
+                exprs.push_back(self.parse_sql_expr(leaf)?);
+            }
+            return Self::balanced_binary_expr(exprs, df_op);
+        }
+
         Ok(Expr::BinaryExpr(BinaryExpr::new(
             Box::new(self.parse_sql_expr(left)?),
-            self.binary_op(op)?,
+            df_op,
             Box::new(self.parse_sql_expr(right)?),
         )))
     }
@@ -431,7 +514,7 @@ impl Planner {
             }
             _ => Err(Error::invalid_input(format!(
                 "Unsupported function args: {:?}",
-                &func.args
+                func.args
             ))),
         }
     }
@@ -675,10 +758,10 @@ impl Planner {
                 data_type, value, ..
             }) => {
                 let value = value.clone().into_string().expect_ok()?;
-                Ok(Expr::Cast(datafusion::logical_expr::Cast {
-                    expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)),
-                    data_type: self.parse_type(data_type)?,
-                }))
+                Ok(Expr::Cast(datafusion::logical_expr::Cast::new(
+                    Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)),
+                    self.parse_type(data_type)?,
+                )))
             }
             SQLExpr::IsFalse(expr) => Ok(Expr::IsFalse(Box::new(self.parse_sql_expr(expr)?))),
             SQLExpr::IsNotFalse(expr) => Ok(Expr::IsNotFalse(Box::new(self.parse_sql_expr(expr)?))),
@@ -711,7 +794,10 @@ impl Planner {
                 Box::new(self.parse_sql_expr(expr)?),
                 Box::new(self.parse_sql_expr(pattern)?),
                 match escape_char {
-                    Some(Value::SingleQuotedString(char)) => char.chars().next(),
+                    Some(ValueWithSpan {
+                        value: Value::SingleQuotedString(char),
+                        ..
+                    }) => char.chars().next(),
                     Some(value) => {
                         return Err(Error::invalid_input(format!(
                             "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}",
@@ -733,7 +819,10 @@ impl Planner {
                 Box::new(self.parse_sql_expr(expr)?),
                 Box::new(self.parse_sql_expr(pattern)?),
                 match escape_char {
-                    Some(Value::SingleQuotedString(char)) => char.chars().next(),
+                    Some(ValueWithSpan {
+                        value: Value::SingleQuotedString(char),
+                        ..
+                    }) => char.chars().next(),
                     Some(value) => {
                         return Err(Error::invalid_input(format!(
                             "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}",
@@ -766,15 +855,15 @@ impl Planner {
             } => match kind {
                 datafusion::sql::sqlparser::ast::CastKind::TryCast
                 | datafusion::sql::sqlparser::ast::CastKind::SafeCast => {
-                    Ok(Expr::TryCast(datafusion::logical_expr::TryCast {
-                        expr: Box::new(self.parse_sql_expr(expr)?),
-                        data_type: self.parse_type(data_type)?,
-                    }))
+                    Ok(Expr::TryCast(datafusion::logical_expr::TryCast::new(
+                        Box::new(self.parse_sql_expr(expr)?),
+                        self.parse_type(data_type)?,
+                    )))
                 }
-                _ => Ok(Expr::Cast(datafusion::logical_expr::Cast {
-                    expr: Box::new(self.parse_sql_expr(expr)?),
-                    data_type: self.parse_type(data_type)?,
-                })),
+                _ => Ok(Expr::Cast(datafusion::logical_expr::Cast::new(
+                    Box::new(self.parse_sql_expr(expr)?),
+                    self.parse_type(data_type)?,
+                ))),
             },
             SQLExpr::JsonAccess { .. } => Err(Error::invalid_input("JSON access is not supported")),
             SQLExpr::CompoundFieldAccess { root, access_chain } => {
@@ -919,9 +1008,10 @@ impl Planner {
 
         // DataFusion needs the coerce and simplify passes to be applied before
         // expressions can be handled by the physical planner.
-        let simplify_context = SimplifyContext::default()
+        let simplify_context = SimplifyContext::builder()
             .with_schema(df_schema.clone())
-            .with_query_execution_start_time(Some(Utc::now()));
+            .with_query_execution_start_time(Some(Utc::now()))
+            .build();
         let simplifier =
             datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
 
@@ -990,13 +1080,9 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor {
                 self.columns.insert(path);
                 self.current_path.clear();
             }
-            Expr::ScalarFunction(udf) => {
-                if udf.name() == GetFieldFunc::default().name() {
-                    if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) {
-                        self.current_path.push_front(name.to_string())
-                    } else {
-                        self.current_path.clear();
-                    }
+            Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => {
+                if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) {
+                    self.current_path.push_front(name.to_string())
                 } else {
                     self.current_path.clear();
                 }
@@ -1012,7 +1098,6 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor {
 
 #[cfg(test)]
 mod tests {
-    use std::any::Any;
 
     use crate::logical_expr::ExprExt;
 
@@ -1102,6 +1187,22 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_parse_deep_logical_filter() {
+        let planner = Planner::new(Arc::new(Schema::empty()));
+
+        for op in ["AND", "OR"] {
+            let filter = std::iter::repeat_n("true", 1000)
+                .collect::<Vec<_>>()
+                .join(&format!(" {op} "));
+
+            let expr = planner.parse_filter(&filter).unwrap();
+            let optimized = planner.optimize_expr(expr).unwrap();
+
+            assert_eq!(optimized, lit(true));
+        }
+    }
+
     #[derive(Debug, Eq, PartialEq, Hash)]
     struct StrictFloat64Udf {
         signature: Signature,
@@ -1116,10 +1217,6 @@ mod tests {
     }
 
     impl ScalarUDFImpl for StrictFloat64Udf {
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-
         fn name(&self) -> &str {
             "strict_float64"
         }
@@ -1521,7 +1618,7 @@ mod tests {
 
             match expr {
                 Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() {
-                    Expr::Cast(Cast { expr, data_type }) => {
+                    Expr::Cast(Cast { expr, field }) => {
                         match expr.as_ref() {
                             Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => {
                                 assert_eq!(value_str, expected_value_str);
@@ -1531,7 +1628,7 @@ mod tests {
                             }
                             _ => panic!("Expected cast to be applied to literal"),
                         }
-                        assert_eq!(data_type, expected_data_type);
+                        assert_eq!(field.data_type(), expected_data_type);
                     }
                     _ => panic!("Expected right to be a cast"),
                 },
@@ -1572,14 +1669,14 @@ mod tests {
 
             match expr {
                 Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() {
-                    Expr::Cast(Cast { expr, data_type }) => {
+                    Expr::Cast(Cast { expr, field }) => {
                         match expr.as_ref() {
                             Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => {
                                 assert_eq!(value_str, expected_value_str);
                             }
                             _ => panic!("Expected cast to be applied to literal"),
                         }
-                        assert_eq!(data_type, expected_data_type);
+                        assert_eq!(field.data_type(), expected_data_type);
                     }
                     _ => panic!("Expected right to be a cast"),
                 },
diff --git a/rust/lance-datafusion/src/sql.rs b/rust/lance-datafusion/src/sql.rs
index b9049aa0214..67ce2ea24a2 100644
--- a/rust/lance-datafusion/src/sql.rs
+++ b/rust/lance-datafusion/src/sql.rs
@@ -45,18 +45,16 @@ pub(crate) fn parse_sql_filter(filter: &str) -> Result<Expr> {
     let sql = format!("SELECT 1 FROM t WHERE {filter}");
     let statement = parse_statement(&sql)?;
 
-    let selection = if let Statement::Query(query) = &statement {
-        if let SetExpr::Select(s) = query.body.as_ref() {
-            s.selection.as_ref()
-        } else {
-            None
-        }
+    if let Statement::Query(query) = statement
+        && let SetExpr::Select(select) = *query.body
+        && let Some(expr) = select.selection
+    {
+        Ok(expr)
     } else {
-        None
-    };
-    let expr =
-        selection.ok_or_else(|| Error::invalid_input(format!("Filter is not valid: {filter}")))?;
-    Ok(expr.clone())
+        Err(Error::invalid_input(format!(
+            "Filter is not valid: {filter}"
+        )))
+    }
 }
 
 /// Parse a SQL expression to Expression. This is more lenient than parse_sql_filter
@@ -65,22 +63,16 @@ pub(crate) fn parse_sql_expr(expr: &str) -> Result<Expr> {
     let sql = format!("SELECT {expr} FROM t");
     let statement = parse_statement(&sql)?;
 
-    let selection = if let Statement::Query(query) = &statement {
-        if let SetExpr::Select(s) = query.body.as_ref() {
-            if let SelectItem::UnnamedExpr(expr) = &s.projection[0] {
-                Some(expr)
-            } else {
-                None
-            }
-        } else {
-            None
-        }
+    if let Statement::Query(query) = statement
+        && let SetExpr::Select(select) = *query.body
+        && let Some(SelectItem::UnnamedExpr(expr)) = select.projection.into_iter().next()
+    {
+        Ok(expr)
     } else {
-        None
-    };
-    let expr = selection
-        .ok_or_else(|| Error::invalid_input(format!("Expression is not valid: {expr}")))?;
-    Ok(expr.clone())
+        Err(Error::invalid_input(format!(
+            "Expression is not valid: {expr}"
+        )))
+    }
 }
 
 fn parse_statement(statement: &str) -> Result<Statement> {
diff --git a/rust/lance-datafusion/src/substrait.rs b/rust/lance-datafusion/src/substrait.rs
index 1c465fcae4a..f38e6b79cf9 100644
--- a/rust/lance-datafusion/src/substrait.rs
+++ b/rust/lance-datafusion/src/substrait.rs
@@ -164,6 +164,9 @@ fn remap_expr_references(expr: &mut Expression, mapping: &HashMap<usize, usize>)
         RexType::WindowFunction(_) | RexType::Subquery(_) => Err(Error::invalid_input(
             "Window functions or subqueries not allowed in filter expression",
         )),
+        RexType::Lambda(_) | RexType::LambdaInvocation(_) => Err(Error::invalid_input(
+            "Lambda expressions not allowed in filter expression",
+        )),
         // Pass through operators, nested children may have field references
         RexType::ScalarFunction(func) => {
             #[allow(deprecated)]
@@ -551,7 +554,7 @@ mod tests {
         },
         expression_reference::ExprType,
         extensions::{
-            SimpleExtensionDeclaration, SimpleExtensionUri, SimpleExtensionUrn,
+            SimpleExtensionDeclaration, SimpleExtensionUrn,
             simple_extension_declaration::{ExtensionFunction, MappingType},
         },
         function_argument::ArgType,
@@ -576,13 +579,6 @@ mod tests {
                 git_hash: "".to_string(),
                 producer: "unit-test".to_string(),
             }),
-            #[expect(deprecated)]
-            extension_uris: vec![
-                SimpleExtensionUri {
-                    extension_uri_anchor: 1,
-                    uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_comparison.yaml".to_string(),
-                }
-            ],
             extension_urns: vec![
                 SimpleExtensionUrn {
                     extension_urn_anchor: 1,
@@ -592,8 +588,6 @@ mod tests {
             extensions: vec![
                 SimpleExtensionDeclaration {
                     mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-                        #[expect(deprecated)]
-                        extension_uri_reference: 1,
                         extension_urn_reference: 1,
                         function_anchor: 1,
                         name: "lt".to_string(),
@@ -881,9 +875,7 @@ mod tests {
     fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration {
         SimpleExtensionDeclaration {
             mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-                #[allow(deprecated)]
-                extension_uri_reference: 1,
-                extension_urn_reference: 0,
+                extension_urn_reference: 1,
                 function_anchor: anchor,
                 name: name.to_string(),
             })),
@@ -919,10 +911,9 @@ mod tests {
                 git_hash: String::new(),
                 producer: "lance-test".to_string(),
             }),
-            #[allow(deprecated)]
-            extension_uris: vec![SimpleExtensionUri {
-                extension_uri_anchor: 1,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
+            extension_urns: vec![SimpleExtensionUrn {
+                extension_urn_anchor: 1,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
             }],
             extensions,
             relations: vec![PlanRel {
@@ -935,7 +926,6 @@ mod tests {
             }],
             advanced_extensions: None,
             expected_type_urls: vec![],
-            extension_urns: vec![],
             parameter_bindings: vec![],
             type_aliases: vec![],
         };
diff --git a/rust/lance-datagen/Cargo.toml b/rust/lance-datagen/Cargo.toml
index eae1e3086b6..83b5aba3689 100644
--- a/rust/lance-datagen/Cargo.toml
+++ b/rust/lance-datagen/Cargo.toml
@@ -21,7 +21,6 @@ hex = "0.4.3"
 rand = { workspace = true }
 rand_distr = { workspace = true }
 rand_xoshiro = { workspace = true }
-random_word = { version = "0.5", features = ["en"] }
 
 [dev-dependencies]
 criterion = { workspace = true }
diff --git a/rust/lance-datagen/src/generator.rs b/rust/lance-datagen/src/generator.rs
index 3756e354bea..39da4734619 100644
--- a/rust/lance-datagen/src/generator.rs
+++ b/rust/lance-datagen/src/generator.rs
@@ -21,7 +21,6 @@ use arrow_schema::{ArrowError, DataType, Field, Fields, IntervalUnit, Schema, Sc
 use futures::{StreamExt, stream::BoxStream};
 use rand::{Rng, RngCore, SeedableRng, distr::Uniform};
 use rand_distr::Zipf;
-use random_word;
 
 use self::array::rand_with_distribution;
 
@@ -1172,24 +1171,223 @@ impl ArrayGenerator for BinaryPrefixPlusCounterGenerator {
     }
 }
 
-// Common English stop words placed at the front to be sampled more frequently
+// Common English stop words placed at the front to be sampled more frequently.
 const STOP_WORDS: &[&str] = &[
     "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it",
     "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these",
     "they", "this", "to", "was", "will", "with",
 ];
 
+const ENGLISH_WORDS: &[&str] = &[
+    "ability",
+    "able",
+    "about",
+    "above",
+    "accept",
+    "access",
+    "account",
+    "across",
+    "action",
+    "active",
+    "activity",
+    "actual",
+    "address",
+    "adjust",
+    "admin",
+    "advance",
+    "agent",
+    "align",
+    "allow",
+    "amount",
+    "analysis",
+    "answer",
+    "application",
+    "archive",
+    "array",
+    "asset",
+    "async",
+    "attribute",
+    "available",
+    "balance",
+    "batch",
+    "binary",
+    "bitmap",
+    "block",
+    "branch",
+    "buffer",
+    "build",
+    "cache",
+    "capacity",
+    "catalog",
+    "change",
+    "chunk",
+    "client",
+    "cluster",
+    "column",
+    "commit",
+    "common",
+    "compare",
+    "compile",
+    "compute",
+    "condition",
+    "config",
+    "connect",
+    "content",
+    "context",
+    "control",
+    "convert",
+    "copy",
+    "core",
+    "count",
+    "create",
+    "current",
+    "cursor",
+    "data",
+    "dataset",
+    "decode",
+    "default",
+    "delete",
+    "delta",
+    "depend",
+    "derive",
+    "design",
+    "detail",
+    "detect",
+    "device",
+    "direct",
+    "display",
+    "document",
+    "domain",
+    "drive",
+    "dynamic",
+    "encode",
+    "engine",
+    "error",
+    "event",
+    "example",
+    "execute",
+    "expand",
+    "expect",
+    "export",
+    "extend",
+    "feature",
+    "field",
+    "filter",
+    "final",
+    "finish",
+    "format",
+    "fragment",
+    "future",
+    "generate",
+    "global",
+    "group",
+    "handle",
+    "header",
+    "index",
+    "input",
+    "insert",
+    "inspect",
+    "instance",
+    "integer",
+    "internal",
+    "item",
+    "join",
+    "kernel",
+    "large",
+    "layer",
+    "layout",
+    "length",
+    "level",
+    "limit",
+    "linear",
+    "local",
+    "logical",
+    "lookup",
+    "manage",
+    "manifest",
+    "memory",
+    "merge",
+    "metric",
+    "model",
+    "module",
+    "namespace",
+    "native",
+    "node",
+    "normal",
+    "number",
+    "object",
+    "offset",
+    "option",
+    "output",
+    "package",
+    "page",
+    "parallel",
+    "parse",
+    "partition",
+    "pattern",
+    "physical",
+    "plan",
+    "policy",
+    "prefix",
+    "prepare",
+    "primary",
+    "process",
+    "profile",
+    "project",
+    "property",
+    "query",
+    "range",
+    "reader",
+    "record",
+    "region",
+    "registry",
+    "request",
+    "resolve",
+    "resource",
+    "result",
+    "return",
+    "row",
+    "runtime",
+    "scalar",
+    "scan",
+    "schema",
+    "search",
+    "segment",
+    "select",
+    "session",
+    "setting",
+    "source",
+    "stable",
+    "stage",
+    "state",
+    "static",
+    "storage",
+    "stream",
+    "string",
+    "struct",
+    "table",
+    "target",
+    "task",
+    "thread",
+    "token",
+    "trace",
+    "transform",
+    "type",
+    "update",
+    "upload",
+    "value",
+    "vector",
+    "version",
+    "view",
+    "write",
+    "writer",
+];
+
 /// Word list with stop words at the front for Zipf sampling, computed once.
 static SENTENCE_WORDS: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
-    let all_words = random_word::all(random_word::Lang::En);
-    let mut words = Vec::with_capacity(STOP_WORDS.len() + all_words.len());
+    let mut words = Vec::with_capacity(STOP_WORDS.len() + ENGLISH_WORDS.len());
     words.extend(STOP_WORDS.iter().copied());
-    words.extend(
-        all_words
-            .iter()
-            .filter(|w| !STOP_WORDS.contains(w))
-            .copied(),
-    );
+    words.extend(ENGLISH_WORDS.iter().copied());
     words
 });
 
@@ -1279,7 +1477,7 @@ struct RandomWordGenerator {
 
 impl RandomWordGenerator {
     pub fn new(is_large: bool) -> Self {
-        let words = random_word::all(random_word::Lang::En);
+        let words = ENGLISH_WORDS;
         Self { words, is_large }
     }
 }
@@ -3190,9 +3388,9 @@ mod tests {
         assert_eq!(
             *genn.generate(RowCount::from(3), &mut rng).unwrap(),
             arrow_array::BinaryArray::from_iter_values([
-                vec![174, 178],
-                vec![64, 122, 207, 248],
-                vec![124, 3, 58]
+                vec![111, 9, 80],
+                vec![86, 118, 13, 209],
+                vec![68, 33, 202]
             ])
         );
     }
diff --git a/rust/lance-derive/Cargo.toml b/rust/lance-derive/Cargo.toml
new file mode 100644
index 00000000000..4bb99d3ac93
--- /dev/null
+++ b/rust/lance-derive/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "lance-derive"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+readme.workspace = true
+description = "Derive macros for Lance"
+keywords.workspace = true
+categories.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1.0.67"
+quote = "1.0.33"
+syn = { version = "2.0.37", features = ["full"] }
+
+[lints]
+workspace = true
diff --git a/rust/lance-derive/src/lib.rs b/rust/lance-derive/src/lib.rs
new file mode 100644
index 00000000000..d0486133ddc
--- /dev/null
+++ b/rust/lance-derive/src/lib.rs
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{Data, DeriveInput, Fields, parse_macro_input};
+
+/// Derive macro for the `DeepSizeOf` trait.
+///
+/// Generates an implementation that sums the `deep_size_of_children` of all
+/// fields (for structs) or the active variant's fields (for enums).
+#[proc_macro_derive(DeepSizeOf)]
+pub fn derive_deep_size_of(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    let name = &input.ident;
+    let generics = &input.generics;
+
+    // Add DeepSizeOf bounds to all type parameters
+    let mut bounded_generics = generics.clone();
+    for param in &mut bounded_generics.params {
+        if let syn::GenericParam::Type(ref mut type_param) = *param {
+            type_param
+                .bounds
+                .push(syn::parse_quote!(lance_core::deepsize::DeepSizeOf));
+        }
+    }
+    let (impl_generics, _, where_clause) = bounded_generics.split_for_impl();
+    let (_, ty_generics, _) = generics.split_for_impl();
+
+    let body = match &input.data {
+        Data::Struct(data) => generate_struct_body(&data.fields),
+        Data::Enum(data) => {
+            let arms: Vec<_> = data
+                .variants
+                .iter()
+                .map(|variant| {
+                    let variant_ident = &variant.ident;
+                    match &variant.fields {
+                        Fields::Unit => {
+                            quote! { Self::#variant_ident => 0 }
+                        }
+                        Fields::Unnamed(fields) => {
+                            let bindings: Vec<_> = (0..fields.unnamed.len())
+                                .map(|i| {
+                                    syn::Ident::new(
+                                        &format!("__field_{}", i),
+                                        proc_macro2::Span::call_site(),
+                                    )
+                                })
+                                .collect();
+                            let sum = bindings.iter().map(|b| {
+                                quote! { lance_core::deepsize::DeepSizeOf::deep_size_of_children(#b, __context) }
+                            });
+                            quote! {
+                                Self::#variant_ident(#(#bindings),*) => {
+                                    0 #(+ #sum)*
+                                }
+                            }
+                        }
+                        Fields::Named(fields) => {
+                            let field_names: Vec<_> =
+                                fields.named.iter().map(|f| &f.ident).collect();
+                            let sum = field_names.iter().map(|f| {
+                                quote! { lance_core::deepsize::DeepSizeOf::deep_size_of_children(#f, __context) }
+                            });
+                            quote! {
+                                Self::#variant_ident { #(#field_names),* } => {
+                                    0 #(+ #sum)*
+                                }
+                            }
+                        }
+                    }
+                })
+                .collect();
+            quote! {
+                match self {
+                    #(#arms),*
+                }
+            }
+        }
+        Data::Union(_) => {
+            return syn::Error::new_spanned(&input, "DeepSizeOf cannot be derived for unions")
+                .to_compile_error()
+                .into();
+        }
+    };
+
+    let expanded = quote! {
+        impl #impl_generics lance_core::deepsize::DeepSizeOf for #name #ty_generics #where_clause {
+            fn deep_size_of_children(&self, __context: &mut lance_core::deepsize::Context) -> usize {
+                #body
+            }
+        }
+    };
+
+    TokenStream::from(expanded)
+}
+
+fn generate_struct_body(fields: &Fields) -> proc_macro2::TokenStream {
+    match fields {
+        Fields::Named(fields) => {
+            let field_sizes = fields.named.iter().map(|f| {
+                let name = &f.ident;
+                quote! { lance_core::deepsize::DeepSizeOf::deep_size_of_children(&self.#name, __context) }
+            });
+            quote! { 0 #(+ #field_sizes)* }
+        }
+        Fields::Unnamed(fields) => {
+            let field_sizes = (0..fields.unnamed.len()).map(|i| {
+                let index = syn::Index::from(i);
+                quote! { lance_core::deepsize::DeepSizeOf::deep_size_of_children(&self.#index, __context) }
+            });
+            quote! { 0 #(+ #field_sizes)* }
+        }
+        Fields::Unit => {
+            quote! { 0 }
+        }
+    }
+}
diff --git a/rust/lance-encoding/src/compression.rs b/rust/lance-encoding/src/compression.rs
index e5d617e44ef..8d2d28f21b0 100644
--- a/rust/lance-encoding/src/compression.rs
+++ b/rust/lance-encoding/src/compression.rs
@@ -26,7 +26,10 @@ use crate::{
     },
     data::{DataBlock, FixedWidthDataBlock, VariableWidthBlock},
     encodings::{
-        logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor},
+        logical::primitive::{
+            fullzip::PerValueCompressor,
+            miniblock::{MAX_MINIBLOCK_VALUES, MiniBlockCompressor},
+        },
         physical::{
             binary::{
                 BinaryBlockDecompressor, BinaryMiniBlockDecompressor, BinaryMiniBlockEncoder,
@@ -51,7 +54,10 @@ use crate::{
                 PackedStructVariablePerValueEncoder, VariablePackedStructFieldDecoder,
                 VariablePackedStructFieldKind,
             },
-            rle::{RleDecompressor, RleEncoder},
+            rle::{
+                RleChildDecompressor, RleDecompressor, RleEncoder, RunLengthWidth,
+                rle_encoded_size, select_run_length_width,
+            },
             value::{ValueDecompressor, ValueEncoder},
         },
     },
@@ -78,6 +84,7 @@ const DEFAULT_RLE_COMPRESSION_THRESHOLD: f64 = 0.5;
 
 // Minimum block size (32kb) to trigger general block compression
 const MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION: u64 = 32 * 1024;
+const RLE_BLOCK_HEADER_BYTES: u128 = std::mem::size_of::<u64>() as u128;
 
 /// Trait for compression algorithms that compress an entire block of data into one opaque
 /// and self-described chunk.
@@ -164,7 +171,9 @@ fn try_bss_for_mini_block(
 
 fn try_rle_for_mini_block(
     data: &FixedWidthDataBlock,
+    version: LanceFileVersion,
     params: &CompressionFieldParams,
+    use_rle_v2: bool,
 ) -> Option<Box<dyn MiniBlockCompressor>> {
     let bits = data.bits_per_value;
     if !matches!(bits, 8 | 16 | 32 | 64) {
@@ -188,43 +197,88 @@ fn try_rle_for_mini_block(
         return None;
     }
 
-    // Estimate the encoded size.
-    //
-    // RLE stores (value, run_length) pairs. Run lengths are u8 and long runs are split into
-    // multiple entries of up to 255 values. We don't know the run length distribution here,
-    // so we conservatively account for splitting with an upper bound.
     let num_values = data.num_values;
-    let estimated_pairs = (run_count.saturating_add(num_values / 255)).min(num_values);
-
     let raw_bytes = (num_values as u128) * (type_size as u128);
-    let rle_bytes = (estimated_pairs as u128) * ((type_size + 1) as u128);
+    let (run_length_width, rle_bytes) = if use_rle_v2 {
+        estimate_rle_width_and_size_from_data(data, Some(*MAX_MINIBLOCK_VALUES)).ok()?
+    } else {
+        (
+            RunLengthWidth::U8,
+            estimate_rle_size_for_width_from_data(
+                data,
+                Some(*MAX_MINIBLOCK_VALUES),
+                RunLengthWidth::U8,
+            )
+            .ok()?,
+        )
+    };
+
+    let use_child_encodings = version.resolve() >= LanceFileVersion::V2_3;
+    let child_compression = if use_child_encodings {
+        rle_child_compression_config(params)
+    } else {
+        None
+    };
+    let use_child_bitpacking = use_child_encodings;
+    let rle_encoder = || {
+        if use_child_encodings {
+            RleEncoder::with_child_encoding(
+                run_length_width,
+                child_compression,
+                child_compression,
+                use_child_bitpacking,
+            )
+        } else {
+            RleEncoder::with_run_length_width(run_length_width)
+        }
+    };
+
+    #[cfg(feature = "bitpacking")]
+    let bitpack_bytes = estimate_inline_bitpacking_bytes(data).map(u128::from);
+    #[cfg(not(feature = "bitpacking"))]
+    let bitpack_bytes = None::<u128>;
+
+    let mut selected_rle_bytes = rle_bytes;
+    let should_estimate_child_size = use_child_encodings
+        && (child_compression.is_some() || cfg!(feature = "bitpacking"))
+        && (rle_bytes >= raw_bytes || bitpack_bytes.is_some_and(|bytes| bytes < rle_bytes));
+    if should_estimate_child_size {
+        selected_rle_bytes = rle_encoder().selected_payload_size(data).ok()?;
+    }
 
-    if rle_bytes < raw_bytes {
-        #[cfg(feature = "bitpacking")]
+    if selected_rle_bytes < raw_bytes {
+        if let Some(bitpack_bytes) = bitpack_bytes
+            && bitpack_bytes < selected_rle_bytes
         {
-            if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data)
-                && (bitpack_bytes as u128) < rle_bytes
-            {
-                return None;
-            }
+            return None;
         }
-        return Some(Box::new(RleEncoder::new()));
+        return Some(Box::new(rle_encoder()));
     }
     None
 }
 
+fn rle_child_compression_config(params: &CompressionFieldParams) -> Option<CompressionConfig> {
+    let raw = params.compression.as_deref()?;
+    if matches!(raw, "none" | "fsst") {
+        return None;
+    }
+    let scheme = CompressionScheme::from_str(raw).ok()?;
+    Some(CompressionConfig::new(scheme, params.compression_level))
+}
+
 fn try_rle_for_block(
     data: &FixedWidthDataBlock,
     version: LanceFileVersion,
     params: &CompressionFieldParams,
-) -> Option<(Box<dyn BlockCompressor>, CompressiveEncoding)> {
+    use_rle_v2: bool,
+) -> Result<Option<(Box<dyn BlockCompressor>, CompressiveEncoding)>> {
     if version < LanceFileVersion::V2_2 {
-        return None;
+        return Ok(None);
     }
 
     let bits = data.bits_per_value;
     if !matches!(bits, 8 | 16 | 32 | 64) {
-        return None;
+        return Ok(None);
     }
 
     let run_count = data.expect_single_stat::<UInt64Type>(Stat::RunCount);
@@ -232,15 +286,71 @@ fn try_rle_for_block(
         .rle_threshold
         .unwrap_or(DEFAULT_RLE_COMPRESSION_THRESHOLD);
 
-    if (run_count as f64) < (data.num_values as f64) * threshold {
-        let compressor = Box::new(RleEncoder::new());
-        let encoding = ProtobufUtils21::rle(
-            ProtobufUtils21::flat(bits, None),
-            ProtobufUtils21::flat(/*bits_per_value=*/ 8, None),
-        );
-        return Some((compressor, encoding));
+    let passes_threshold = match params.rle_threshold {
+        Some(_) => (run_count as f64) < (data.num_values as f64) * threshold,
+        None => true,
+    };
+
+    if !passes_threshold {
+        return Ok(None);
     }
-    None
+
+    let raw_bytes = (data.num_values as u128) * ((bits / 8) as u128);
+    let (run_length_width, rle_payload_bytes) = if use_rle_v2 {
+        estimate_rle_width_and_size_from_data(data, None)?
+    } else {
+        (
+            RunLengthWidth::U8,
+            estimate_rle_size_for_width_from_data(data, None, RunLengthWidth::U8)?,
+        )
+    };
+    let rle_bytes = rle_payload_bytes.saturating_add(RLE_BLOCK_HEADER_BYTES);
+
+    if rle_bytes >= raw_bytes {
+        return Ok(None);
+    }
+
+    #[cfg(feature = "bitpacking")]
+    {
+        if let Some(bitpack_bytes) = estimate_block_bitpacking_bytes(data)
+            && bitpack_bytes < rle_bytes
+        {
+            return Ok(None);
+        }
+    }
+
+    let compressor = Box::new(RleEncoder::with_run_length_width(run_length_width));
+    let encoding = ProtobufUtils21::rle(
+        ProtobufUtils21::flat(bits, None),
+        ProtobufUtils21::flat(run_length_width.bits_per_value(), None),
+    );
+    Ok(Some((compressor, encoding)))
+}
+
+fn estimate_rle_width_and_size_from_data(
+    data: &FixedWidthDataBlock,
+    max_segment_values: Option<u64>,
+) -> Result<(RunLengthWidth, u128)> {
+    select_run_length_width(
+        &data.data,
+        data.num_values,
+        data.bits_per_value,
+        max_segment_values,
+    )
+}
+
+fn estimate_rle_size_for_width_from_data(
+    data: &FixedWidthDataBlock,
+    max_segment_values: Option<u64>,
+    run_length_width: RunLengthWidth,
+) -> Result<u128> {
+    rle_encoded_size(
+        &data.data,
+        data.num_values,
+        data.bits_per_value,
+        max_segment_values,
+        run_length_width,
+    )
 }
 
 fn try_bitpack_for_mini_block(_data: &FixedWidthDataBlock) -> Option<Box<dyn MiniBlockCompressor>> {
@@ -325,6 +435,67 @@ fn try_bitpack_for_block(
     }
 }
 
+#[cfg(feature = "bitpacking")]
+fn estimate_block_bitpacking_bytes(data: &FixedWidthDataBlock) -> Option<u128> {
+    let bits = data.bits_per_value;
+    if !matches!(bits, 8 | 16 | 32 | 64) || data.num_values == 0 {
+        return None;
+    }
+
+    let bit_widths = data.expect_stat(Stat::BitWidth);
+    let widths = bit_widths.as_primitive::<UInt64Type>();
+    let max_bit_width = *widths.values().iter().max()?;
+    let word_bytes = (bits / 8) as u128;
+
+    let bitpacked_words = if data.num_values <= 1024 {
+        1 + (1024u128 * (max_bit_width as u128)) / (bits as u128)
+    } else {
+        estimate_out_of_line_bitpacking_words(data.num_values, max_bit_width, bits)?
+    };
+    let bitpacked_bytes = bitpacked_words.saturating_mul(word_bytes);
+    if bitpacked_bytes >= data.data_size() as u128 {
+        return None;
+    }
+
+    Some(bitpacked_bytes)
+}
+
+#[cfg(feature = "bitpacking")]
+fn estimate_out_of_line_bitpacking_words(
+    num_values: u64,
+    compressed_bits_per_value: u64,
+    bits_per_value: u64,
+) -> Option<u128> {
+    let num_values = usize::try_from(num_values).ok()?;
+    let compressed_bits_per_value = usize::try_from(compressed_bits_per_value).ok()?;
+    let bits_per_value = usize::try_from(bits_per_value).ok()?;
+    if compressed_bits_per_value >= bits_per_value {
+        return None;
+    }
+
+    let elems_per_chunk = 1024usize;
+    let num_chunks = num_values.div_ceil(elems_per_chunk);
+    let words_per_chunk = (elems_per_chunk * compressed_bits_per_value).div_ceil(bits_per_value);
+    let last_chunk_is_runt = !num_values.is_multiple_of(elems_per_chunk);
+
+    if !last_chunk_is_runt {
+        return Some((num_chunks * words_per_chunk) as u128);
+    }
+
+    let num_whole_chunks = num_chunks - 1;
+    let remaining_items = num_values - num_whole_chunks * elems_per_chunk;
+    let tail_bit_savings = bits_per_value - compressed_bits_per_value;
+    let padding_cost = compressed_bits_per_value * (elems_per_chunk - remaining_items);
+    let tail_pack_savings = tail_bit_savings * remaining_items;
+    let tail_words = if padding_cost < tail_pack_savings {
+        words_per_chunk
+    } else {
+        remaining_items
+    };
+
+    Some((num_whole_chunks * words_per_chunk + tail_words) as u128)
+}
+
 fn maybe_wrap_general_for_mini_block(
     inner: Box<dyn MiniBlockCompressor>,
     params: &CompressionFieldParams,
@@ -393,6 +564,10 @@ impl DefaultCompressionStrategy {
         self
     }
 
+    fn use_rle_v2(&self) -> bool {
+        self.version.resolve() >= LanceFileVersion::V2_3
+    }
+
     /// Parse compression parameters from field metadata
     fn parse_field_metadata(field: &Field, version: &LanceFileVersion) -> CompressionFieldParams {
         let mut params = CompressionFieldParams::default();
@@ -456,7 +631,7 @@ impl DefaultCompressionStrategy {
         }
 
         let base = try_bss_for_mini_block(data, params)
-            .or_else(|| try_rle_for_mini_block(data, params))
+            .or_else(|| try_rle_for_mini_block(data, self.version, params, self.use_rle_v2()))
             .or_else(|| try_bitpack_for_mini_block(data))
             .unwrap_or_else(|| Box::new(ValueEncoder::default()));
 
@@ -620,6 +795,13 @@ impl CompressionStrategy for DefaultCompressionStrategy {
                 if (max_len > 32 * 1024 || per_value_requested)
                     && data_size >= FSST_LEAST_INPUT_SIZE as u64
                 {
+                    if compression == Some("zstd") {
+                        let config = CompressionConfig::new(
+                            CompressionScheme::Zstd,
+                            field_params.compression_level,
+                        );
+                        return Ok(Box::new(CompressedBufferEncoder::try_new(config)?));
+                    }
                     return Ok(Box::new(CompressedBufferEncoder::default()));
                 }
 
@@ -664,7 +846,7 @@ impl CompressionStrategy for DefaultCompressionStrategy {
         match data {
             DataBlock::FixedWidth(fixed_width) => {
                 if let Some((compressor, encoding)) =
-                    try_rle_for_block(fixed_width, self.version, &field_params)
+                    try_rle_for_block(fixed_width, self.version, &field_params, self.use_rle_v2())?
                 {
                     return Ok((compressor, encoding));
                 }
@@ -814,10 +996,10 @@ impl DecompressionStrategy for DefaultDecompressionStrategy {
                 // compression.
                 Ok(Box::new(ValueDecompressor::from_fsl(fsl)))
             }
-            Compression::Rle(rle) => {
-                let bits_per_value = validate_rle_compression(rle)?;
-                Ok(Box::new(RleDecompressor::new(bits_per_value)))
-            }
+            Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(
+                rle,
+                decompression_strategy,
+            )?)),
             Compression::ByteStreamSplit(bss) => {
                 let Compression::Flat(values) =
                     bss.values.as_ref().unwrap().compression.as_ref().unwrap()
@@ -1004,16 +1186,15 @@ impl DecompressionStrategy for DefaultDecompressionStrategy {
 
                 Ok(Box::new(general_decompressor))
             }
-            Compression::Rle(rle) => {
-                let bits_per_value = validate_rle_compression(rle)?;
-                Ok(Box::new(RleDecompressor::new(bits_per_value)))
-            }
+            Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(rle, self)?)),
             _ => todo!(),
         }
     }
 }
-/// Validates RLE compression format and extracts bits_per_value
-fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<u64> {
+fn create_rle_decompressor(
+    rle: &crate::format::pb21::Rle,
+    decompression_strategy: &dyn DecompressionStrategy,
+) -> Result<RleDecompressor> {
     let values = rle
         .values
         .as_ref()
@@ -1023,34 +1204,162 @@ fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<u64> {
         .as_ref()
         .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths encoding"))?;
 
-    let values = values
-        .compression
-        .as_ref()
-        .ok_or_else(|| Error::invalid_input("RLE compression missing values compression"))?;
-    let Compression::Flat(values) = values else {
+    let values = create_rle_child_decompressor(values, "values", decompression_strategy)?;
+    let run_lengths =
+        create_rle_child_decompressor(run_lengths, "run lengths", decompression_strategy)?;
+
+    if !matches!(values.bits_per_value(), 8 | 16 | 32 | 64) {
+        return Err(Error::invalid_input(format!(
+            "RLE compression only supports 8, 16, 32, or 64-bit values, got {}",
+            values.bits_per_value()
+        )));
+    }
+
+    let run_length_width =
+        RunLengthWidth::from_bits(run_lengths.bits_per_value()).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "RLE compression only supports 8, 16, or 32-bit run lengths, got {}",
+                run_lengths.bits_per_value()
+            ))
+        })?;
+
+    if values.requires_num_values() && run_lengths.requires_num_values() {
         return Err(Error::invalid_input(
-            "RLE compression only supports flat values",
+            "RLE values and run lengths child encodings cannot both require the run count",
         ));
-    };
+    }
+
+    if values.is_identity() && run_lengths.is_identity() {
+        return Ok(RleDecompressor::with_run_length_width(
+            values.bits_per_value(),
+            run_length_width,
+        ));
+    }
+
+    Ok(RleDecompressor::with_child_decompressors(
+        values.bits_per_value(),
+        run_length_width,
+        values,
+        run_lengths,
+    ))
+}
 
-    let run_lengths = run_lengths
+fn create_rle_child_decompressor(
+    encoding: &CompressiveEncoding,
+    role: &str,
+    decompression_strategy: &dyn DecompressionStrategy,
+) -> Result<RleChildDecompressor> {
+    let compression = encoding
         .compression
         .as_ref()
-        .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths compression"))?;
-    let Compression::Flat(run_lengths) = run_lengths else {
-        return Err(Error::invalid_input(
-            "RLE compression only supports flat run lengths",
-        ));
-    };
+        .ok_or_else(|| Error::invalid_input(format!("RLE {role} missing child compression")))?;
+    let (bits_per_value, requires_num_values, needs_decompressor) =
+        validate_rle_child_compression(compression, role)?;
 
-    if run_lengths.bits_per_value != 8 {
-        return Err(Error::invalid_input(format!(
-            "RLE compression only supports 8-bit run lengths, got {}",
-            run_lengths.bits_per_value
-        )));
+    if needs_decompressor {
+        Ok(RleChildDecompressor::block(
+            bits_per_value,
+            decompression_strategy.create_block_decompressor(encoding)?,
+            requires_num_values,
+        ))
+    } else {
+        Ok(RleChildDecompressor::flat(bits_per_value))
+    }
+}
+
+fn validate_rle_child_compression(
+    compression: &Compression,
+    role: &str,
+) -> Result<(u64, bool, bool)> {
+    match compression {
+        Compression::Flat(flat) => Ok((flat.bits_per_value, false, false)),
+        Compression::General(general) => {
+            general.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} general child missing compression config"
+                ))
+            })?;
+            let values = general.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!("RLE {role} general child missing inner encoding"))
+            })?;
+            let inner = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} general child missing inner compression"
+                ))
+            })?;
+            let (bits_per_value, requires_num_values) =
+                validate_rle_block_child_inner(inner, role)?;
+            Ok((bits_per_value, requires_num_values, true))
+        }
+        Compression::OutOfLineBitpacking(out_of_line) => {
+            let values = out_of_line.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values encoding"
+                ))
+            })?;
+            let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values compression"
+                ))
+            })?
+            else {
+                return Err(Error::invalid_input(format!(
+                    "RLE {role} bitpacking child only supports flat values"
+                )));
+            };
+            Ok((out_of_line.uncompressed_bits_per_value, true, true))
+        }
+        other => Err(Error::invalid_input(format!(
+            "RLE {role} only supports flat, general, or out-of-line bitpacking child encodings, got {}",
+            compression_name(other)
+        ))),
     }
+}
+
+fn validate_rle_block_child_inner(compression: &Compression, role: &str) -> Result<(u64, bool)> {
+    match compression {
+        Compression::Flat(flat) => Ok((flat.bits_per_value, false)),
+        Compression::OutOfLineBitpacking(out_of_line) => {
+            let values = out_of_line.values.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values encoding"
+                ))
+            })?;
+            let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "RLE {role} bitpacking child missing values compression"
+                ))
+            })?
+            else {
+                return Err(Error::invalid_input(format!(
+                    "RLE {role} bitpacking child only supports flat values"
+                )));
+            };
+            Ok((out_of_line.uncompressed_bits_per_value, true))
+        }
+        other => Err(Error::invalid_input(format!(
+            "RLE {role} general child only supports flat or out-of-line bitpacking inner encodings, got {}",
+            compression_name(other)
+        ))),
+    }
+}
 
-    Ok(values.bits_per_value)
+fn compression_name(compression: &Compression) -> &'static str {
+    match compression {
+        Compression::Flat(_) => "flat",
+        Compression::Variable(_) => "variable",
+        Compression::Fsst(_) => "fsst",
+        Compression::OutOfLineBitpacking(_) => "out-of-line bitpacking",
+        Compression::InlineBitpacking(_) => "inline bitpacking",
+        Compression::General(_) => "general",
+        Compression::Constant(_) => "constant",
+        Compression::Dictionary(_) => "dictionary",
+        Compression::ByteStreamSplit(_) => "byte stream split",
+        Compression::PackedStruct(_) => "packed struct",
+        Compression::FixedSizeList(_) => "fixed-size list",
+        Compression::VariablePackedStruct(_) => "variable packed struct",
+        Compression::Rle(_) => "rle",
+    }
 }
 
 #[cfg(test)]
@@ -1139,6 +1448,37 @@ mod tests {
         DataBlock::FixedWidth(block)
     }
 
+    fn rle_run_length_bits(encoding: &CompressiveEncoding) -> u64 {
+        let Compression::Rle(rle) = encoding.compression.as_ref().unwrap() else {
+            panic!("expected RLE encoding");
+        };
+        let Compression::Flat(run_lengths) = rle
+            .run_lengths
+            .as_ref()
+            .unwrap()
+            .compression
+            .as_ref()
+            .unwrap()
+        else {
+            panic!("expected flat run lengths");
+        };
+        run_lengths.bits_per_value
+    }
+
+    fn expect_rle_encoding(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle {
+        match encoding.compression.as_ref().unwrap() {
+            Compression::Rle(rle) => rle,
+            Compression::General(general) => {
+                let inner = general.values.as_ref().unwrap();
+                let Compression::Rle(rle) = inner.compression.as_ref().unwrap() else {
+                    panic!("expected wrapped RLE encoding");
+                };
+                rle
+            }
+            other => panic!("expected RLE encoding, got {}", compression_name(other)),
+        }
+    }
+
     fn create_variable_width_block(
         bits_per_offset: u8,
         num_values: u64,
@@ -1290,6 +1630,60 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_rle_block_accounts_for_header_before_selecting() {
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let field = create_test_field("small_constant", DataType::Int32);
+        let values = vec![42i32; 2];
+        let mut block = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 2,
+            block_info: BlockInfo::default(),
+        };
+        block.compute_stat();
+        let data = DataBlock::FixedWidth(block);
+
+        let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap();
+
+        assert!(format!("{compressor:?}").contains("ValueEncoder"));
+        assert!(matches!(
+            encoding.compression.as_ref(),
+            Some(Compression::Flat(_))
+        ));
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_block_prefers_bitpacking_when_smaller() {
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let field = create_test_field("levels", DataType::UInt16);
+
+        let mut values = Vec::with_capacity(2048);
+        for run_idx in 0..1024 {
+            values.extend(std::iter::repeat_n((run_idx % 2) as u16, 2));
+        }
+        let mut block = FixedWidthDataBlock {
+            bits_per_value: 16,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 2048,
+            block_info: BlockInfo::default(),
+        };
+        block.compute_stat();
+        let data = DataBlock::FixedWidth(block);
+
+        let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(
+            debug_str.contains("OutOfLineBitpacking"),
+            "expected OutOfLineBitpacking, got: {debug_str}"
+        );
+        assert!(matches!(
+            encoding.compression.as_ref(),
+            Some(Compression::OutOfLineBitpacking(_))
+        ));
+    }
+
     #[test]
     #[cfg(feature = "bitpacking")]
     fn test_low_cardinality_prefers_bitpacking_over_rle() {
@@ -1505,6 +1899,30 @@ mod tests {
         );
     }
 
+    #[test]
+    #[cfg(feature = "zstd")]
+    fn test_compression_level_honored_for_large_per_value() {
+        let mut params = CompressionParams::new();
+        params.columns.insert(
+            "html".to_string(),
+            CompressionFieldParams {
+                compression: Some("zstd".to_string()),
+                compression_level: Some(19),
+                ..Default::default()
+            },
+        );
+        let strategy = DefaultCompressionStrategy::with_params(params);
+        let field = create_test_field("html", DataType::Utf8);
+        let large = create_variable_width_block(32, 64, 40 * 1024);
+
+        let per_value = strategy.create_per_value(&field, &large).unwrap();
+        let debug = format!("{per_value:?}");
+        assert!(
+            debug.contains("ZstdBufferCompressor") && debug.contains("compression_level: 19"),
+            "expected zstd level 19 to reach the per-value compressor, got: {debug}"
+        );
+    }
+
     #[test]
     fn test_parameter_merge_priority() {
         let mut params = CompressionParams::new();
@@ -1659,6 +2077,277 @@ mod tests {
         assert!(debug_str.contains("RleEncoder"));
     }
 
+    #[test]
+    fn test_rle_v2_miniblock_selects_u16_run_lengths() {
+        let mut metadata = HashMap::new();
+        metadata.insert(RLE_THRESHOLD_META_KEY.to_string(), "1.0".to_string());
+        metadata.insert(BSS_META_KEY.to_string(), "off".to_string());
+        let mut field = create_test_field("test_column", DataType::Int32);
+        field.metadata = metadata;
+
+        let values = vec![7i32; 1000];
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 1000,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 16);
+    }
+
+    #[test]
+    fn test_rle_v2_miniblock_keeps_u8_run_lengths_before_v2_3() {
+        for version in [LanceFileVersion::V2_1, LanceFileVersion::V2_2] {
+            let mut metadata = HashMap::new();
+            metadata.insert(RLE_THRESHOLD_META_KEY.to_string(), "1.0".to_string());
+            metadata.insert(BSS_META_KEY.to_string(), "off".to_string());
+            let mut field = create_test_field("test_column", DataType::Int32);
+            field.metadata = metadata;
+
+            let values = vec![7i32; 1000];
+            let mut data = FixedWidthDataBlock {
+                bits_per_value: 32,
+                data: LanceBuffer::reinterpret_vec(values),
+                num_values: 1000,
+                block_info: BlockInfo::default(),
+            };
+            data.compute_stat();
+            let data = DataBlock::FixedWidth(data);
+
+            let strategy = DefaultCompressionStrategy::new().with_version(version);
+            let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+            let (_compressed, encoding) = compressor.compress(data).unwrap();
+            assert_eq!(rle_run_length_bits(&encoding), 8, "version={version}");
+        }
+    }
+
+    #[test]
+    fn test_rle_v2_uses_selected_width_cost_before_bitpacking() {
+        let mut metadata = HashMap::new();
+        metadata.insert(RLE_THRESHOLD_META_KEY.to_string(), "1.0".to_string());
+        metadata.insert(BSS_META_KEY.to_string(), "off".to_string());
+        let mut field = create_test_field("test_column", DataType::Int32);
+        field.metadata = metadata;
+
+        let values = vec![0i32; 4096];
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 4096,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(debug_str.contains("RleEncoder"));
+
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 16);
+    }
+
+    #[test]
+    fn test_rle_v2_sorted_dictionary_indices_select_u16_run_lengths() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+
+        let mut values = Vec::with_capacity(1_200);
+        for value in 0..4 {
+            values.extend(std::iter::repeat_n(value, 300));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 1_200,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 16);
+    }
+
+    #[test]
+    fn test_rle_v2_short_runs_keep_u8_run_lengths() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+
+        let mut values = Vec::with_capacity(1_280);
+        for value in 0..10 {
+            values.extend(std::iter::repeat_n(value, 128));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 1_280,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 8);
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_released_versions_keep_flat_children_when_compression_requested() {
+        for version in [LanceFileVersion::V2_1, LanceFileVersion::V2_2] {
+            let mut params = CompressionParams::new();
+            params.columns.insert(
+                "dict_indices".to_string(),
+                CompressionFieldParams {
+                    compression: Some(
+                        if cfg!(feature = "lz4") { "lz4" } else { "zstd" }.to_string(),
+                    ),
+                    rle_threshold: Some(1.0),
+                    bss: Some(BssMode::Off),
+                    ..Default::default()
+                },
+            );
+            let strategy = DefaultCompressionStrategy::with_params(params).with_version(version);
+            let field = create_test_field("dict_indices", DataType::UInt32);
+
+            let mut values = Vec::with_capacity(8192 * 4);
+            for value in 0..8192u32 {
+                values.extend(std::iter::repeat_n(value, 4));
+            }
+            let mut data = FixedWidthDataBlock {
+                bits_per_value: 32,
+                data: LanceBuffer::reinterpret_vec(values),
+                num_values: 8192 * 4,
+                block_info: BlockInfo::default(),
+            };
+            data.compute_stat();
+            let data = DataBlock::FixedWidth(data);
+
+            let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+            let (_compressed, encoding) = compressor.compress(data).unwrap();
+            let rle = expect_rle_encoding(&encoding);
+
+            assert!(
+                matches!(
+                    rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+                    Compression::Flat(_)
+                ),
+                "version={version}"
+            );
+            assert!(
+                matches!(
+                    rle.run_lengths
+                        .as_ref()
+                        .unwrap()
+                        .compression
+                        .as_ref()
+                        .unwrap(),
+                    Compression::Flat(_)
+                ),
+                "version={version}"
+            );
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_strategy_bitpacks_child_values_when_smaller() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+
+        let mut values = Vec::with_capacity(8192 * 4);
+        for value in 0..8192 {
+            values.extend(std::iter::repeat_n(value, 4));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 8192 * 4,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(debug_str.contains("RleEncoder"));
+
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        let Compression::Rle(rle) = encoding.compression.as_ref().unwrap() else {
+            panic!("expected RLE encoding");
+        };
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            Compression::Flat(_)
+        ));
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_keeps_child_bitpacked_rle_when_smaller_than_inline_bitpacking() {
+        let field = create_test_field("int_score", DataType::UInt64);
+
+        let mut values = Vec::with_capacity(8192 * 8);
+        for run_idx in 0..8192 {
+            let value = match run_idx % 3 {
+                0 => 3u64,
+                1 => 4u64,
+                _ => 5u64,
+            };
+            values.extend(std::iter::repeat_n(value, 8));
+        }
+        let mut data = FixedWidthDataBlock {
+            bits_per_value: 64,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 8192 * 8,
+            block_info: BlockInfo::default(),
+        };
+        data.compute_stat();
+        let data = DataBlock::FixedWidth(data);
+
+        let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3);
+        let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap();
+        let debug_str = format!("{compressor:?}");
+        assert!(
+            debug_str.contains("RleEncoder"),
+            "expected RLE to beat inline bitpacking after child selection, got: {debug_str}"
+        );
+
+        let (_compressed, encoding) = compressor.compress(data).unwrap();
+        let rle = expect_rle_encoding(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            Compression::Flat(_)
+        ));
+    }
+
     #[test]
     fn test_field_metadata_override_params() {
         // Set up params with one configuration
@@ -1875,6 +2564,60 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_rle_v2_block_selects_u32_run_lengths() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+        let expected_values = vec![42i32; 70_000];
+        let mut block = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(expected_values.clone()),
+            num_values: expected_values.len() as u64,
+            block_info: BlockInfo::default(),
+        };
+        block.compute_stat();
+        let data = DataBlock::FixedWidth(block);
+
+        let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new())
+            .with_version(LanceFileVersion::V2_3);
+        let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 32);
+
+        let compressed = compressor.compress(data).unwrap();
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_block_decompressor(&encoding)
+            .unwrap();
+        let decoded = decompressor
+            .decompress(compressed, expected_values.len() as u64)
+            .unwrap();
+
+        match decoded {
+            DataBlock::FixedWidth(block) => {
+                let values = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(values.as_ref(), expected_values);
+            }
+            _ => panic!("expected fixed-width block"),
+        }
+    }
+
+    #[test]
+    fn test_rle_v2_block_keeps_u8_run_lengths_for_v2_2() {
+        let field = create_test_field("dict_indices", DataType::Int32);
+        let values = vec![42i32; 70_000];
+        let mut block = FixedWidthDataBlock {
+            bits_per_value: 32,
+            data: LanceBuffer::reinterpret_vec(values),
+            num_values: 70_000,
+            block_info: BlockInfo::default(),
+        };
+        block.compute_stat();
+        let data = DataBlock::FixedWidth(block);
+
+        let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new())
+            .with_version(LanceFileVersion::V2_2);
+        let (_compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap();
+        assert_eq!(rle_run_length_bits(&encoding), 8);
+    }
+
     #[test]
     fn test_rle_block_used_for_version_v2_2() {
         let field = create_test_field("test_repdef", DataType::UInt16);
diff --git a/rust/lance-encoding/src/decoder.rs b/rust/lance-encoding/src/decoder.rs
index 59886d337d1..aea3575dcb1 100644
--- a/rust/lance-encoding/src/decoder.rs
+++ b/rust/lance-encoding/src/decoder.rs
@@ -225,11 +225,14 @@ use futures::future::{BoxFuture, MaybeDone, maybe_done};
 use futures::stream::{self, BoxStream};
 use futures::{FutureExt, StreamExt};
 use lance_arrow::DataTypeExt;
-use lance_core::cache::LanceCache;
-use lance_core::datatypes::{BLOB_DESC_LANCE_FIELD, Field, Schema};
+use lance_core::cache::{Context, DeepSizeOf, LanceCache};
+use lance_core::datatypes::{
+    BLOB_DESC_LANCE_FIELD, Field, Schema, validate_fixed_size_list_dimensions,
+};
 use lance_core::utils::futures::{FinallyStreamExt, StreamOnDropExt};
 use lance_core::utils::parse::parse_env_as_bool;
 use log::{debug, trace, warn};
+use prost::Message;
 use tokio::sync::mpsc::error::SendError;
 use tokio::sync::mpsc::{self, unbounded_channel};
 
@@ -297,6 +300,15 @@ pub enum PageEncoding {
     Structural(pb21::PageLayout),
 }
 
+impl DeepSizeOf for PageEncoding {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        match self {
+            Self::Legacy(encoding) => encoding.encoded_len() * 4,
+            Self::Structural(encoding) => encoding.encoded_len() * 4,
+        }
+    }
+}
+
 impl PageEncoding {
     pub fn as_legacy(&self) -> &pb::ArrayEncoding {
         match self {
@@ -334,6 +346,13 @@ pub struct PageInfo {
     pub buffer_offsets_and_sizes: Arc<[(u64, u64)]>,
 }
 
+impl DeepSizeOf for PageInfo {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.encoding.deep_size_of_children(context)
+            + self.buffer_offsets_and_sizes.deep_size_of_children(context)
+    }
+}
+
 /// Metadata describing a column in a file
 ///
 /// This is typically created by reading the metadata section of a Lance file
@@ -348,6 +367,14 @@ pub struct ColumnInfo {
     pub encoding: pb::ColumnEncoding,
 }
 
+impl DeepSizeOf for ColumnInfo {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        self.page_infos.deep_size_of_children(context)
+            + self.buffer_offsets_and_sizes.deep_size_of_children(context)
+            + self.encoding.encoded_len() * 4
+    }
+}
+
 impl ColumnInfo {
     /// Create a new instance
     pub fn new(
@@ -723,6 +750,7 @@ impl CoreFieldDecoderStrategy {
         column_infos: &mut ColumnInfoIter,
     ) -> Result<Box<dyn StructuralFieldScheduler>> {
         let data_type = field.data_type();
+        validate_fixed_size_list_dimensions(&field.name, &data_type)?;
         if Self::is_structural_primitive(&data_type) {
             let column_info = column_infos.expect_next()?;
             let scheduler = Box::new(StructuralPrimitiveFieldScheduler::try_new(
@@ -832,6 +860,7 @@ impl CoreFieldDecoderStrategy {
         buffers: FileBuffers,
     ) -> Result<Box<dyn crate::previous::decoder::FieldScheduler>> {
         let data_type = field.data_type();
+        validate_fixed_size_list_dimensions(&field.name, &data_type)?;
         if Self::is_primitive_legacy(&data_type) {
             let column_info = column_infos.expect_next()?;
             let scheduler = self.create_primitive_scheduler(field, column_info, buffers)?;
@@ -1261,7 +1290,7 @@ impl DecodeBatchScheduler {
     /// * `ranges` - The ranges of rows to load
     /// * `sink` - A channel to send the decode tasks
     /// * `scheduler` An I/O scheduler to issue I/O requests
-    #[instrument(skip_all)]
+    #[instrument(level = "debug", skip_all)]
     pub fn schedule_ranges(
         &mut self,
         ranges: &[Range<u64>],
@@ -1297,7 +1326,7 @@ impl DecodeBatchScheduler {
     /// * `range` - The range of rows to load
     /// * `sink` - A channel to send the decode tasks
     /// * `scheduler` An I/O scheduler to issue I/O requests
-    #[instrument(skip_all)]
+    #[instrument(level = "debug", skip_all)]
     pub fn schedule_range(
         &mut self,
         range: Range<u64>,
@@ -1479,34 +1508,41 @@ impl BatchDecodeStream {
 
     pub fn into_stream(self) -> BoxStream<'static, ReadBatchTask> {
         let stream = futures::stream::unfold(self, |mut slf| async move {
-            let next_task = slf.next_batch_task().await;
-            let next_task = next_task.transpose().map(|next_task| {
-                let num_rows = next_task.as_ref().map(|t| t.num_rows).unwrap_or(0);
-                let emitted_batch_size_warning = slf.emitted_batch_size_warning.clone();
-                let task = async move {
-                    let next_task = next_task?;
-                    // Real decode work happens inside into_batch, which can block the current
-                    // thread for a long time. By spawning it as a new task, we allow Tokio's
-                    // worker threads to keep making progress.
-                    let (batch, _data_size) =
-                        tokio::spawn(
-                            async move { next_task.into_batch(emitted_batch_size_warning) },
-                        )
+            let next_task = match slf.next_batch_task().await {
+                Ok(Some(next_task)) => next_task,
+                Ok(None) => return None,
+                Err(err) => {
+                    slf.rows_remaining = 0;
+                    return Some((
+                        ReadBatchTask {
+                            task: async move { Err(err) }.boxed(),
+                            num_rows: 0,
+                        },
+                        slf,
+                    ));
+                }
+            };
+            let num_rows = next_task.num_rows;
+            let emitted_batch_size_warning = slf.emitted_batch_size_warning.clone();
+            let task = async move {
+                // Real decode work happens inside into_batch, which can block the current
+                // thread for a long time. By spawning it as a new task, we allow Tokio's
+                // worker threads to keep making progress.
+                let (batch, _data_size) =
+                    tokio::spawn(async move { next_task.into_batch(emitted_batch_size_warning) })
                         .await
                         .map_err(|err| Error::wrapped(err.into()))??;
-                    Ok(batch)
-                };
-                (task, num_rows)
-            });
-            next_task.map(|(task, num_rows)| {
-                // This should be true since batch size is u32
-                debug_assert!(num_rows <= u32::MAX as u64);
-                let next_task = ReadBatchTask {
+                Ok(batch)
+            };
+            // This should be true since batch size is u32
+            debug_assert!(num_rows <= u32::MAX as u64);
+            Some((
+                ReadBatchTask {
                     task: task.boxed(),
                     num_rows: num_rows as u32,
-                };
-                (next_task, slf)
-            })
+                },
+                slf,
+            ))
         });
         stream.boxed()
     }
@@ -1611,7 +1647,7 @@ impl<T: RootDecoderType> BatchDecodeIterator<T> {
     ///
     /// Note that `scheduled_need` is cumulative.  E.g. this method
     /// should be called with 5, 10, 15 and not 5, 5, 5
-    #[instrument(skip_all)]
+    #[instrument(level = "debug", skip_all)]
     fn wait_for_io(&mut self, scheduled_need: u64, to_take: u64) -> Result<u64> {
         while self.rows_scheduled < scheduled_need && !self.messages.is_empty() {
             let message = self.messages.pop_front().unwrap()?;
@@ -1884,54 +1920,60 @@ impl StructuralBatchDecodeStream {
 
     pub fn into_stream(self) -> BoxStream<'static, ReadBatchTask> {
         let stream = futures::stream::unfold(self, |mut slf| async move {
-            let next_task = slf.next_batch_task().await;
-            let next_task = next_task.transpose().map(|next_task| {
-                let num_rows = next_task.as_ref().map(|t| t.num_rows).unwrap_or(0);
-                let emitted_batch_size_warning = slf.emitted_batch_size_warning.clone();
-                let bytes_per_row_feedback = slf.bytes_per_row_feedback.clone();
-                // Capture the per-stream policy once so every emitted batch task follows the
-                // same throughput-vs-overhead choice made by the scheduler.
-                let spawn_batch_decode_tasks = slf.spawn_batch_decode_tasks;
-                let task = async move {
-                    let next_task = next_task?;
-                    let (batch, data_size) = if spawn_batch_decode_tasks {
-                        tokio::spawn(
-                            async move { next_task.into_batch(emitted_batch_size_warning) },
-                        )
+            let next_task = match slf.next_batch_task().await {
+                Ok(Some(next_task)) => next_task,
+                Ok(None) => return None,
+                Err(err) => {
+                    slf.rows_remaining = 0;
+                    return Some((
+                        ReadBatchTask {
+                            task: async move { Err(err) }.boxed(),
+                            num_rows: 0,
+                        },
+                        slf,
+                    ));
+                }
+            };
+            let num_rows = next_task.num_rows;
+            let emitted_batch_size_warning = slf.emitted_batch_size_warning.clone();
+            let bytes_per_row_feedback = slf.bytes_per_row_feedback.clone();
+            // Capture the per-stream policy once so every emitted batch task follows the
+            // same throughput-vs-overhead choice made by the scheduler.
+            let spawn_batch_decode_tasks = slf.spawn_batch_decode_tasks;
+            let task = async move {
+                let (batch, data_size) = if spawn_batch_decode_tasks {
+                    tokio::spawn(async move { next_task.into_batch(emitted_batch_size_warning) })
                         .await
                         .map_err(|err| Error::wrapped(err.into()))??
+                } else {
+                    next_task.into_batch(emitted_batch_size_warning)?
+                };
+                let num_rows = batch.num_rows() as u64;
+                if let Some(bpr) = data_size.checked_div(num_rows) {
+                    let prev = bytes_per_row_feedback.load(Ordering::Relaxed);
+                    let next = if prev == 0 || bpr >= prev {
+                        // First batch or actual size is larger than estimate:
+                        // adopt immediately to avoid OOM.
+                        bpr
                     } else {
-                        next_task.into_batch(emitted_batch_size_warning)?
+                        // Actual size is smaller: degrade gradually toward
+                        // the true value to avoid over-correcting on a
+                        // single anomalous batch.
+                        (prev + bpr) / 2
                     };
-                    let num_rows = batch.num_rows() as u64;
-                    if num_rows > 0 {
-                        let bpr = data_size / num_rows;
-                        let prev = bytes_per_row_feedback.load(Ordering::Relaxed);
-                        let next = if prev == 0 || bpr >= prev {
-                            // First batch or actual size is larger than estimate:
-                            // adopt immediately to avoid OOM.
-                            bpr
-                        } else {
-                            // Actual size is smaller: degrade gradually toward
-                            // the true value to avoid over-correcting on a
-                            // single anomalous batch.
-                            (prev + bpr) / 2
-                        };
-                        bytes_per_row_feedback.store(next.max(1), Ordering::Relaxed);
-                    }
-                    Ok(batch)
-                };
-                (task, num_rows)
-            });
-            next_task.map(|(task, num_rows)| {
-                // This should be true since batch size is u32
-                debug_assert!(num_rows <= u32::MAX as u64);
-                let next_task = ReadBatchTask {
+                    bytes_per_row_feedback.store(next.max(1), Ordering::Relaxed);
+                }
+                Ok(batch)
+            };
+            // This should be true since batch size is u32
+            debug_assert!(num_rows <= u32::MAX as u64);
+            Some((
+                ReadBatchTask {
                     task: task.boxed(),
                     num_rows: num_rows as u32,
-                };
-                (next_task, slf)
-            })
+                },
+                slf,
+            ))
         });
         stream.boxed()
     }
@@ -2886,6 +2928,200 @@ pub async fn decode_batch(
 // test coalesce indices to ranges
 mod tests {
     use super::*;
+    use crate::previous::decoder::{DecoderReady, LogicalPageDecoder};
+    use std::collections::VecDeque;
+
+    #[derive(Debug)]
+    struct FailingPageDecoder {
+        page_data_type: DataType,
+        total_rows: u64,
+        load_error_message: &'static str,
+    }
+
+    impl FailingPageDecoder {
+        fn new(
+            page_data_type: DataType,
+            total_rows: u64,
+            load_error_message: &'static str,
+        ) -> Self {
+            Self {
+                page_data_type,
+                total_rows,
+                load_error_message,
+            }
+        }
+    }
+
+    impl LogicalPageDecoder for FailingPageDecoder {
+        fn wait_for_loaded(&'_ mut self, _rows_needed: u64) -> BoxFuture<'_, Result<()>> {
+            let load_error_message = self.load_error_message;
+            async move { Err(Error::io(load_error_message)) }.boxed()
+        }
+
+        fn rows_loaded(&self) -> u64 {
+            0
+        }
+
+        fn num_rows(&self) -> u64 {
+            self.total_rows
+        }
+
+        fn rows_drained(&self) -> u64 {
+            0
+        }
+
+        fn drain(&mut self, requested_rows: u64) -> Result<NextDecodeTask> {
+            Err(Error::internal(format!(
+                "failing page decoder should not be drained after load error \
+                 (requested_rows={})",
+                requested_rows
+            )))
+        }
+
+        fn data_type(&self) -> &DataType {
+            &self.page_data_type
+        }
+    }
+
+    #[test]
+    fn test_read_zero_dimension_fsl_errors_instead_of_panicking() {
+        // Simulates reading a column whose stored schema declares a
+        // zero-dimension FixedSizeList, as old writers (before #5102) could
+        // persist. The read plan is built by the field-scheduler factories,
+        // which run the dimension guard before touching any column data, so
+        // an empty column iterator is sufficient to reach the guard. The read
+        // must surface a clean error rather than a divide-by-zero panic.
+        use arrow_schema::Field as ArrowField;
+
+        let zero_dim = DataType::FixedSizeList(
+            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+            0,
+        );
+        let field = Field::try_from(&ArrowField::new("vec", zero_dim, true)).unwrap();
+        let strategy = CoreFieldDecoderStrategy::default();
+
+        let mut structural_columns = ColumnInfoIter::new(vec![], &[]);
+        let err = strategy
+            .create_structural_field_scheduler(&field, &mut structural_columns)
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("dimension must be a positive integer"),
+            "unexpected error: {}",
+            err
+        );
+
+        let mut legacy_columns = ColumnInfoIter::new(vec![], &[]);
+        let err = strategy
+            .create_legacy_field_scheduler(
+                &field,
+                &mut legacy_columns,
+                FileBuffers {
+                    positions_and_sizes: &[],
+                },
+            )
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("dimension must be a positive integer"),
+            "unexpected error: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_legacy_stream_stops_on_load_error() {
+        use arrow_schema::Field as ArrowField;
+
+        let rows_per_batch = 1;
+        let total_rows = 2;
+        let scheduled_rows = 1;
+        let page_rows = 1;
+        let batch_readahead = 2;
+        let load_error_message = "simulated page load failure";
+        let fields = Fields::from(vec![ArrowField::new("vector", DataType::Float32, true)]);
+        let root_decoder = SimpleStructDecoder::new(fields, total_rows);
+        let (tx, rx) = unbounded_channel();
+
+        tx.send(Ok(DecoderMessage {
+            scheduled_so_far: scheduled_rows,
+            decoders: vec![MessageType::DecoderReady(DecoderReady {
+                decoder: Box::new(FailingPageDecoder::new(
+                    DataType::Float32,
+                    page_rows,
+                    load_error_message,
+                )),
+                path: VecDeque::from([0]),
+            })],
+        }))
+        .unwrap();
+        drop(tx);
+
+        let stream =
+            BatchDecodeStream::new(rx, rows_per_batch, total_rows, root_decoder).into_stream();
+        let mut batches = stream.map(|task| task.task).buffered(batch_readahead);
+
+        let err = batches
+            .next()
+            .await
+            .expect("stream should emit the legacy page-load error")
+            .unwrap_err();
+        assert!(
+            err.to_string().contains(load_error_message),
+            "unexpected error: {}",
+            err
+        );
+        assert!(
+            batches.next().await.is_none(),
+            "stream should stop after the legacy page-load error"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_structural_stream_stops_on_load_error() {
+        let rows_per_batch = 1;
+        let total_rows = 2;
+        let scheduled_rows = 1;
+        let batch_readahead = 2;
+        let load_error_message = "simulated page load failure";
+        let fields = Fields::from(vec![ArrowField::new("vector", DataType::Float32, true)]);
+        let root_decoder = StructuralStructDecoder::new(fields, false, /*is_root=*/ true).unwrap();
+        let (tx, rx) = unbounded_channel();
+        let failed_page = async move { Err(Error::io(load_error_message)) }.boxed();
+
+        tx.send(Ok(DecoderMessage {
+            scheduled_so_far: scheduled_rows,
+            decoders: vec![MessageType::UnloadedPage(UnloadedPageShard(failed_page))],
+        }))
+        .unwrap();
+        drop(tx);
+
+        let stream = StructuralBatchDecodeStream::new(
+            rx,
+            rows_per_batch,
+            total_rows,
+            root_decoder,
+            /*spawn_batch_decode_tasks=*/ true,
+            None,
+        )
+        .into_stream();
+        let mut batches = stream.map(|task| task.task).buffered(batch_readahead);
+
+        let err = batches
+            .next()
+            .await
+            .expect("stream should emit the page-load error")
+            .unwrap_err();
+        assert!(
+            err.to_string().contains(load_error_message),
+            "unexpected error: {}",
+            err
+        );
+        assert!(
+            batches.next().await.is_none(),
+            "stream should stop after the page-load error"
+        );
+    }
 
     #[test]
     fn test_coalesce_indices_to_ranges_with_single_index() {
diff --git a/rust/lance-encoding/src/encodings/logical/blob.rs b/rust/lance-encoding/src/encodings/logical/blob.rs
index cad2112bafe..d1798b1ef69 100644
--- a/rust/lance-encoding/src/encodings/logical/blob.rs
+++ b/rust/lance-encoding/src/encodings/logical/blob.rs
@@ -267,16 +267,11 @@ impl FieldEncoder for BlobV2StructuralEncoder {
         &mut self,
         array: ArrayRef,
         external_buffers: &mut OutOfLineBuffers,
-        mut repdef: RepDefBuilder,
+        repdef: RepDefBuilder,
         row_number: u64,
         num_rows: u64,
     ) -> Result<Vec<EncodeTask>> {
         let struct_arr = array.as_struct();
-        if let Some(validity) = struct_arr.nulls() {
-            repdef.add_validity_bitmap(validity.clone());
-        } else {
-            repdef.add_no_null(struct_arr.len());
-        }
 
         let kind_col = struct_arr
             .column_by_name("kind")
@@ -403,7 +398,7 @@ impl FieldEncoder for BlobV2StructuralEncoder {
         let descriptor_array = Arc::new(StructArray::try_new(
             BLOB_V2_DESC_FIELDS.clone(),
             children,
-            None,
+            struct_arr.nulls().cloned(),
         )?) as ArrayRef;
 
         self.descriptor_encoder.maybe_encode(
@@ -535,6 +530,32 @@ mod tests {
         .await;
     }
 
+    #[tokio::test]
+    async fn test_blob_round_trip_empty_values() {
+        // Empty values share size == 0 with nulls in the descriptor layout
+        // and schedule no read; each must decode to zero-length bytes without
+        // consuming the read result of a following non-empty blob. Empties
+        // are placed before payloads so a misassignment corrupts the output
+        // instead of only exhausting the read iterator.
+        let blob_metadata =
+            HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]);
+
+        let val1: &[u8] = &vec![1u8; 1024];
+        let val2: &[u8] = &vec![2u8; 10240];
+        let empty: &[u8] = &[];
+        let array = Arc::new(LargeBinaryArray::from(vec![
+            Some(empty),
+            Some(val1),
+            None,
+            Some(empty),
+            Some(val2),
+            None,
+            Some(empty),
+        ]));
+
+        check_round_trip_encoding_of_data(vec![array], &TestCases::default(), blob_metadata).await;
+    }
+
     #[tokio::test]
     async fn test_blob_v2_external_round_trip() {
         let blob_metadata = HashMap::from([(
diff --git a/rust/lance-encoding/src/encodings/logical/list.rs b/rust/lance-encoding/src/encodings/logical/list.rs
index 153238719ae..250eb476671 100644
--- a/rust/lance-encoding/src/encodings/logical/list.rs
+++ b/rust/lance-encoding/src/encodings/logical/list.rs
@@ -428,6 +428,96 @@ mod tests {
         check_basic_random(field).await;
     }
 
+    /// Regression test: a `List<List<Float32>>` column written as MULTIPLE
+    /// batches (chunks) whose flattened leaf values cross a value-page boundary
+    /// fails to decode with "Max offset N exceeds length of values M" (Arrow
+    /// error raised by `ListArray::try_new` in `StructuralListDecodeTask::decode`).
+    ///
+    /// The trigger (verified against the production file and pylance 7.0.0b12 /
+    /// 7.0.0 / 9.0.0-beta.10) requires ALL of:
+    ///   1. >= 2 list layers (`List<List<..>>`),
+    ///   2. a leaf large enough to be chunked into multiple value pages,
+    ///   3. the column written as more than one batch.
+    /// A single batch of the identical data round-trips fine — which is why the
+    /// earlier single-chunk version of this test (and the small `test_nested_list`
+    /// cases) did not catch it. Found in production on the gaming TransNet
+    /// `dino_embedding_per_frame` column (rectangular 3 x 768 float per row).
+    ///
+    /// Each element of the `vec![..]` passed to `check_round_trip_encoding_of_data`
+    /// is encoded as a separate batch (its own `RepDefBuilder`), so we split the
+    /// rows into two chunks to exercise the multi-batch repdef accumulation path.
+    #[rstest]
+    #[test_log::test(tokio::test)]
+    async fn test_multipage_nested_float_list(
+        #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)]
+        structural_encoding: &str,
+    ) {
+        use arrow_array::Float32Array;
+
+        // Production shape: 3 inner lists per row, 768 floats each.
+        let inner_per_row: usize = 3;
+        let inner_len: usize = 768;
+        // Two chunks (batches) -> two pages; a read batch that spans the page
+        // boundary is where the multi-page outer-offset bug triggered. A single
+        // [2731] chunk (one page) decodes fine, which is why this needs >= 2.
+        let chunk_rows: &[usize] = &[1366, 1365];
+
+        let make_chunk = |start_row: usize, num_rows: usize| -> Arc<dyn Array> {
+            let total_inner = num_rows * inner_per_row;
+            let total_values = total_inner * inner_len;
+            let values = Float32Array::from(
+                (0..total_values)
+                    .map(|i| (start_row + i) as f32)
+                    .collect::<Vec<_>>(),
+            );
+            let inner_offsets = ScalarBuffer::<i32>::from(
+                (0..=total_inner)
+                    .map(|i| (i * inner_len) as i32)
+                    .collect::<Vec<_>>(),
+            );
+            let inner_list = ListArray::new(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                OffsetBuffer::new(inner_offsets),
+                Arc::new(values),
+                None,
+            );
+            let outer_offsets = ScalarBuffer::<i32>::from(
+                (0..=num_rows)
+                    .map(|i| (i * inner_per_row) as i32)
+                    .collect::<Vec<_>>(),
+            );
+            Arc::new(ListArray::new(
+                Arc::new(Field::new(
+                    "item",
+                    DataType::List(Arc::new(Field::new("item", DataType::Float32, true))),
+                    true,
+                )),
+                OffsetBuffer::new(outer_offsets),
+                Arc::new(inner_list),
+                None,
+            ))
+        };
+
+        let mut start = 0;
+        let chunks: Vec<Arc<dyn Array>> = chunk_rows
+            .iter()
+            .map(|&n| {
+                let c = make_chunk(start, n);
+                start += n;
+                c
+            })
+            .collect();
+
+        let mut field_metadata = HashMap::new();
+        field_metadata.insert(
+            STRUCTURAL_ENCODING_META_KEY.to_string(),
+            structural_encoding.into(),
+        );
+
+        let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1);
+        check_round_trip_encoding_of_data(chunks, &test_cases, field_metadata).await;
+    }
+
     #[test_log::test(tokio::test)]
     async fn test_list_struct_list() {
         let struct_type = DataType::Struct(Fields::from(vec![Field::new(
diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs
index 78bc45d4d9a..6e5cb4cd66b 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive.rs
@@ -58,7 +58,7 @@ use crate::{
 use crate::{
     repdef::{
         CompositeRepDefUnraveler, ControlWordIterator, ControlWordParser, DefinitionInterpretation,
-        RepDefSlicer, SerializedRepDefs, StructuralPagePlan, build_control_word_iterator,
+        MiniBlockRepDefBudget, RepDefSlicer, SerializedRepDefs, build_control_word_iterator,
     },
     utils::accumulation::AccumulationQueue,
 };
@@ -3660,7 +3660,18 @@ impl StructuralFieldDecoder for StructuralPrimitiveFieldDecoder {
         let mut remaining = num_rows;
         let mut tasks = Vec::new();
         while remaining > 0 {
-            let cur_page = self.page_decoders.front_mut().unwrap();
+            let queued_pages = self.page_decoders.len();
+            let Some(cur_page) = self.page_decoders.front_mut() else {
+                return Err(Error::internal(format!(
+                    "Primitive decoder missing page decoder while draining field '{}' (data_type={:?}, requested_rows={}, remaining_rows={}, rows_drained_in_current={}, queued_pages={})",
+                    self.field.name(),
+                    self.field.data_type(),
+                    num_rows,
+                    remaining,
+                    self.rows_drained_in_current,
+                    queued_pages
+                )));
+            };
             let num_in_page = cur_page.num_rows() - self.rows_drained_in_current;
             let to_take = num_in_page.min(remaining);
 
@@ -3701,12 +3712,7 @@ struct SerializedFullZip {
 //
 // If we directly record the size in bytes with 12 bits we would be limited to
 // 4KiB which is too small.  Since we know each mini-block consists of 8 byte
-// words we can store the # of words instead which gives us 32KiB.  We want
-// at least 24KiB so we can handle even the worst case of
-// - 4Ki values compressed into an 8186 byte buffer
-// - 4 bytes to describe rep & def lengths
-// - 16KiB of rep & def buffer (this will almost never happen but life is easier if we
-//   plan for it)
+// words we can store the # of words instead which gives us 32KiB.
 //
 // Second, each chunk in a mini-block is aligned to 8 bytes.  This allows multi-byte
 // values like offsets to be stored in a mini-block and safely read back out.  It also
@@ -3781,7 +3787,7 @@ struct DictEncodingBudget {
     max_encoded_size: usize,
 }
 
-// A primitive page after optional structural splitting.
+// A primitive page after applying the dense mini-block rep/def budget.
 struct PrimitivePageData {
     // Arrow leaf arrays that contain this page's visible values.
     arrays: Vec<ArrayRef>,
@@ -3791,8 +3797,8 @@ struct PrimitivePageData {
     row_number: u64,
     // Number of top-level rows in this page.
     num_rows: u64,
-    // Present when one top-level row is too large for one miniblock rep/def chunk.
-    unsplittable_miniblock_levels: Option<u64>,
+    // Present when one top-level row is too large for one mini-block rep/def page.
+    single_row_miniblock_repdef_levels: Option<u64>,
 }
 
 // Immutable encoder state shared by per-page encode tasks.
@@ -3906,9 +3912,9 @@ impl PrimitiveStructuralEncoder {
     // 0xA)  All blocks except the last must have power-of-two number of values.
     // This not only makes metadata smaller but it makes decoding easier since
     // batch sizes are typically a power of 2.  4 bits would allow us to express
-    // up to 16Ki values but we restrict this further to 4Ki values.
+    // up to 32Ki values.
     //
-    // This means blocks can have 1 to 4Ki values and 8 - 32Ki bytes.
+    // This means blocks can have 1 to 32Ki values and 8 - 32Ki bytes.
     //
     // All metadata words are serialized (as little endian) into a single buffer
     // of metadata values.
@@ -4007,7 +4013,13 @@ impl PrimitiveStructuralEncoder {
                 }
             } else {
                 for &buffer_size in &chunk.buffer_sizes {
-                    data_buffer.extend_from_slice(&(buffer_size as u16).to_le_bytes());
+                    let buffer_size = u16::try_from(buffer_size).map_err(|_| {
+                        Error::internal(format!(
+                            "Mini-block buffer size ({} bytes) too large for 16-bit metadata",
+                            buffer_size
+                        ))
+                    })?;
+                    data_buffer.extend_from_slice(&buffer_size.to_le_bytes());
                 }
             }
 
@@ -4041,15 +4053,28 @@ impl PrimitiveStructuralEncoder {
 
             let chunk_bytes = data_buffer.len() - start_pos;
             let max_chunk_size = if support_large_chunk {
-                4 * 1024 * 1024 * 1024 // 4GB limit with u32 metadata
+                1_u64 << 31 // 28 bits of 8-byte words in u32 metadata
             } else {
                 32 * 1024 // 32KiB limit with u16 metadata
             };
-            assert!(chunk_bytes <= max_chunk_size);
-            assert!(chunk_bytes > 0);
-            assert_eq!(chunk_bytes % 8, 0);
-            // 4Ki values max
-            assert!(chunk.log_num_values <= 12);
+            if chunk_bytes == 0 || chunk_bytes as u64 > max_chunk_size {
+                return Err(Error::internal(format!(
+                    "Mini-block chunk size {} bytes exceeds the {} byte metadata limit",
+                    chunk_bytes, max_chunk_size
+                )));
+            }
+            if chunk_bytes % MINIBLOCK_ALIGNMENT != 0 {
+                return Err(Error::internal(format!(
+                    "Mini-block chunk size {} bytes is not aligned to {} bytes",
+                    chunk_bytes, MINIBLOCK_ALIGNMENT
+                )));
+            }
+            if chunk.log_num_values > 15 {
+                return Err(Error::internal(format!(
+                    "Mini-block log_num_values {} exceeds the 4-bit metadata limit",
+                    chunk.log_num_values
+                )));
+            }
             // We subtract 1 here from chunk_bytes because we want to be able to express
             // a size of 32KiB and not (32Ki - 8)B which is what we'd get otherwise with
             // 0xFFF
@@ -5081,13 +5106,19 @@ impl PrimitiveStructuralEncoder {
         let max_encoded_size = (data_size as f64 * threshold_ratio) as u64;
         let max_encoded_size = usize::try_from(max_encoded_size).ok()?;
 
-        // Avoid probing dictionary encoding on data that appears to be near-unique.
-        if Self::sample_is_near_unique(
-            data_block,
-            DEFAULT_SAMPLE_SIZE,
-            DEFAULT_SAMPLE_UNIQUE_RATIO,
-        )? {
-            return None;
+        // Avoid probing dictionary encoding on data that appears to be near-unique
+        // or likely to exceed the dictionary budget.
+        if let Some(sample_unique_ratio) =
+            Self::sample_unique_ratio(data_block, DEFAULT_SAMPLE_SIZE)?
+        {
+            if sample_unique_ratio >= DEFAULT_SAMPLE_UNIQUE_RATIO {
+                return None;
+            }
+
+            let projected_cardinality = (sample_unique_ratio * num_values as f64).ceil() as u64;
+            if projected_cardinality > threshold_cardinality {
+                return None;
+            }
         }
 
         let max_dict_entries = u32::try_from(threshold_cardinality.min(i32::MAX as u64)).ok()?;
@@ -5097,66 +5128,79 @@ impl PrimitiveStructuralEncoder {
         })
     }
 
-    /// Probe whether a page looks near-unique before attempting dictionary encoding.
+    /// Samples whether a page looks near-unique before attempting dictionary encoding.
     ///
-    /// The probe uses deterministic stride sampling (not RNG sampling), which keeps
+    /// The probe uses deterministic block sampling (not RNG sampling), which keeps
     /// the check cheap and reproducible across runs. The result is only a gate for
     /// whether we try dictionary encoding, not a cardinality statistic.
-    fn sample_is_near_unique(
-        data_block: &DataBlock,
-        max_samples: usize,
-        unique_ratio_threshold: f64,
-    ) -> Option<bool> {
+    /// Returns `Some(None)` when there are too few reliable samples or the block type does not
+    /// support dictionary encoding. Returns `None` for malformed data.
+    fn sample_unique_ratio(data_block: &DataBlock, max_samples: usize) -> Option<Option<f64>> {
         use std::collections::HashSet;
 
-        if unique_ratio_threshold <= 0.0 || unique_ratio_threshold > 1.0 {
-            return None;
-        }
+        const NUM_SAMPLE_BLOCKS: usize = 32;
+        const MIN_RELIABLE_SAMPLES: usize = 1024;
 
         let num_values = usize::try_from(data_block.num_values()).ok()?;
         if num_values == 0 {
-            return Some(false);
+            return Some(None);
         }
 
         let sample_count = num_values.min(max_samples).max(1);
-        // Uniform stride sampling across the page.
-        let step = (num_values / sample_count).max(1);
+        if sample_count < MIN_RELIABLE_SAMPLES {
+            return Some(None);
+        }
 
-        match data_block {
+        let block_count = NUM_SAMPLE_BLOCKS.min(sample_count).min(num_values).max(1);
+        let samples_per_block = (sample_count / block_count).max(1);
+        let mut indices = Vec::with_capacity(sample_count);
+        for block_idx in 0..block_count {
+            let block_start = block_idx * num_values / block_count;
+            let next_block_start = ((block_idx + 1) * num_values / block_count).min(num_values);
+            let block_len = next_block_start.saturating_sub(block_start);
+            let samples_in_block = samples_per_block.min(block_len);
+            indices.extend((0..samples_in_block).map(|offset| block_start + offset));
+        }
+
+        if indices.len() < MIN_RELIABLE_SAMPLES {
+            return Some(None);
+        }
+
+        let ratio = match data_block {
             DataBlock::FixedWidth(fixed) => match fixed.bits_per_value {
                 64 => {
                     let values = fixed.data.borrow_to_typed_slice::<u64>();
                     let values = values.as_ref();
-                    let mut unique: HashSet<u64> = HashSet::with_capacity(sample_count.min(1024));
-                    for idx in (0..num_values).step_by(step).take(sample_count) {
+                    let mut unique: HashSet<u64> =
+                        HashSet::with_capacity(indices.len().min(MIN_RELIABLE_SAMPLES));
+                    for idx in indices.iter().copied() {
                         unique.insert(values.get(idx).copied()?);
                     }
-                    let ratio = unique.len() as f64 / sample_count as f64;
-                    // Avoid overreacting to tiny pages with too few samples.
-                    Some(sample_count >= 1024 && ratio >= unique_ratio_threshold)
+                    unique.len() as f64 / indices.len() as f64
                 }
                 128 => {
                     let values = fixed.data.borrow_to_typed_slice::<u128>();
                     let values = values.as_ref();
-                    let mut unique: HashSet<u128> = HashSet::with_capacity(sample_count.min(1024));
-                    for idx in (0..num_values).step_by(step).take(sample_count) {
+                    let mut unique: HashSet<u128> =
+                        HashSet::with_capacity(indices.len().min(MIN_RELIABLE_SAMPLES));
+                    for idx in indices.iter().copied() {
                         unique.insert(values.get(idx).copied()?);
                     }
-                    let ratio = unique.len() as f64 / sample_count as f64;
-                    Some(sample_count >= 1024 && ratio >= unique_ratio_threshold)
+                    unique.len() as f64 / indices.len() as f64
                 }
-                _ => Some(false),
+                _ => return Some(None),
             },
             DataBlock::VariableWidth(var) => {
                 use xxhash_rust::xxh3::xxh3_64;
 
                 // Hash variable-width slices instead of storing borrowed slice keys.
-                let mut unique: HashSet<u64> = HashSet::with_capacity(sample_count.min(1024));
+                let mut unique: HashSet<u64> =
+                    HashSet::with_capacity(indices.len().min(MIN_RELIABLE_SAMPLES));
                 match var.bits_per_offset {
                     32 => {
                         let offsets_ref = var.offsets.borrow_to_typed_slice::<u32>();
                         let offsets: &[u32] = offsets_ref.as_ref();
-                        for i in (0..num_values).step_by(step).take(sample_count) {
+                        for i in indices.iter().copied() {
                             let start = usize::try_from(*offsets.get(i)?).ok()?;
                             let end = usize::try_from(*offsets.get(i + 1)?).ok()?;
                             if start > end || end > var.data.len() {
@@ -5168,7 +5212,7 @@ impl PrimitiveStructuralEncoder {
                     64 => {
                         let offsets_ref = var.offsets.borrow_to_typed_slice::<u64>();
                         let offsets: &[u64] = offsets_ref.as_ref();
-                        for i in (0..num_values).step_by(step).take(sample_count) {
+                        for i in indices.iter().copied() {
                             let start = usize::try_from(*offsets.get(i)?).ok()?;
                             let end = usize::try_from(*offsets.get(i + 1)?).ok()?;
                             if start > end || end > var.data.len() {
@@ -5177,13 +5221,14 @@ impl PrimitiveStructuralEncoder {
                             unique.insert(xxh3_64(&var.data[start..end]));
                         }
                     }
-                    _ => return Some(false),
+                    _ => return Some(None),
                 }
-                let ratio = unique.len() as f64 / sample_count as f64;
-                Some(sample_count >= 1024 && ratio >= unique_ratio_threshold)
+                unique.len() as f64 / indices.len() as f64
             }
-            _ => Some(false),
-        }
+            _ => return Some(None),
+        };
+
+        Some(Some(ratio))
     }
 
     fn slice_repdef(repdef: &SerializedRepDefs, range: Range<usize>) -> SerializedRepDefs {
@@ -5247,33 +5292,33 @@ impl PrimitiveStructuralEncoder {
         Ok(sliced)
     }
 
-    fn split_structural_pages_for_miniblock_budget(
+    fn split_pages_for_miniblock_repdef_budget(
         arrays: Vec<ArrayRef>,
         repdef: SerializedRepDefs,
-        plan: StructuralPagePlan,
+        budget: MiniBlockRepDefBudget,
         row_number: u64,
         num_rows: u64,
     ) -> Result<Vec<PrimitivePageData>> {
-        if plan == StructuralPagePlan::Fits {
+        if budget == MiniBlockRepDefBudget::WithinBudget {
             return Ok(vec![PrimitivePageData {
                 arrays,
                 repdef,
                 row_number,
                 num_rows,
-                unsplittable_miniblock_levels: None,
+                single_row_miniblock_repdef_levels: None,
             }]);
         }
-        if let StructuralPagePlan::UnsplittableOverBudget(num_levels) = plan {
+        if let MiniBlockRepDefBudget::SingleRowOverBudget(num_levels) = budget {
             return Ok(vec![PrimitivePageData {
                 arrays,
                 repdef,
                 row_number,
                 num_rows,
-                unsplittable_miniblock_levels: Some(num_levels),
+                single_row_miniblock_repdef_levels: Some(num_levels),
             }]);
         }
 
-        let StructuralPagePlan::Split(splits) = plan else {
+        let MiniBlockRepDefBudget::RequiresPageSplit(splits) = budget else {
             unreachable!();
         };
 
@@ -5286,7 +5331,7 @@ impl PrimitiveStructuralEncoder {
                 repdef,
                 row_number: row_number + split.row_start,
                 num_rows: split.num_rows,
-                unsplittable_miniblock_levels: None,
+                single_row_miniblock_repdef_levels: None,
             });
         }
         Ok(pages)
@@ -5308,7 +5353,7 @@ impl PrimitiveStructuralEncoder {
             repdef,
             row_number,
             num_rows,
-            unsplittable_miniblock_levels,
+            single_row_miniblock_repdef_levels,
         } = page;
         let num_values = arrays.iter().map(|arr| arr.len() as u64).sum();
 
@@ -5391,7 +5436,7 @@ impl PrimitiveStructuralEncoder {
             );
         }
 
-        if let Some(num_levels) = unsplittable_miniblock_levels {
+        if let Some(num_levels) = single_row_miniblock_repdef_levels {
             let requested_encoding = encoding_metadata
                 .get(STRUCTURAL_ENCODING_META_KEY)
                 .map(|requested| requested.to_lowercase());
@@ -5618,16 +5663,17 @@ impl PrimitiveStructuralEncoder {
         let num_values = arrays.iter().map(|arr| arr.len() as u64).sum();
         let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity());
         let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty());
-        let (repdef, structural_plan) = RepDefBuilder::serialize_with_structural_plan(
-            repdefs,
-            miniblock::max_repdef_levels_per_chunk,
-            num_rows,
-            num_values,
-        )?;
-        let pages = Self::split_structural_pages_for_miniblock_budget(
+        let (repdef, miniblock_repdef_budget) =
+            RepDefBuilder::serialize_with_miniblock_repdef_budget(
+                repdefs,
+                miniblock::max_repdef_levels_per_chunk,
+                num_rows,
+                num_values,
+            )?;
+        let pages = Self::split_pages_for_miniblock_repdef_budget(
             arrays,
             repdef,
-            structural_plan,
+            miniblock_repdef_budget,
             row_number,
             num_rows,
         )?;
@@ -5748,8 +5794,9 @@ mod tests {
     use super::{
         ChunkInstructions, DataBlock, DecodeMiniBlockTask, FixedPerValueDecompressor,
         FixedWidthDataBlock, FullZipCacheableState, FullZipDecodeDetails, FullZipReadSource,
-        FullZipRepIndexDetails, FullZipScheduler, MiniBlockRepIndex, PerValueDecompressor,
-        PreambleAction, StructuralPageScheduler, VariableFullZipDecoder,
+        FullZipRepIndexDetails, FullZipScheduler, MiniBlockChunk, MiniBlockCompressed,
+        MiniBlockRepIndex, PerValueDecompressor, PreambleAction, StructuralPageScheduler,
+        VariableFullZipDecoder,
     };
     use crate::buffer::LanceBuffer;
     use crate::compression::DefaultDecompressionStrategy;
@@ -5759,9 +5806,9 @@ mod tests {
         STRUCTURAL_ENCODING_MINIBLOCK,
     };
     use crate::data::BlockInfo;
-    use crate::decoder::PageEncoding;
+    use crate::decoder::{PageEncoding, StructuralFieldDecoder};
     use crate::encodings::logical::primitive::{
-        ChunkDrainInstructions, PrimitiveStructuralEncoder,
+        ChunkDrainInstructions, PrimitiveStructuralEncoder, StructuralPrimitiveFieldDecoder,
     };
     use crate::format::ProtobufUtils21;
     use crate::format::pb21;
@@ -5770,7 +5817,7 @@ mod tests {
     use crate::testing::{TestCases, check_round_trip_encoding_of_data};
     use crate::version::LanceFileVersion;
     use arrow_array::{ArrayRef, Int8Array, StringArray};
-    use arrow_schema::DataType;
+    use arrow_schema::{DataType, Field as ArrowField};
     use std::collections::HashMap;
     use std::{collections::VecDeque, sync::Arc};
 
@@ -5794,6 +5841,33 @@ mod tests {
         assert!((!PrimitiveStructuralEncoder::is_narrow(&block)));
     }
 
+    #[test]
+    fn test_primitive_decoder_empty_page_queue_returns_error() {
+        let field = Arc::new(ArrowField::new("vector", DataType::Float32, true));
+        let mut decoder = StructuralPrimitiveFieldDecoder::new(&field, false);
+
+        let err = decoder.drain(1).unwrap_err();
+        assert!(
+            matches!(&err, lance_core::Error::Internal { .. }),
+            "expected internal error, got: {err:?}"
+        );
+        let message = err.to_string();
+        for expected in [
+            "Primitive decoder missing page decoder",
+            "field 'vector'",
+            "data_type=Float32",
+            "requested_rows=1",
+            "remaining_rows=1",
+            "rows_drained_in_current=0",
+            "queued_pages=0",
+        ] {
+            assert!(
+                message.contains(expected),
+                "expected error to contain {expected:?}, got: {message}"
+            );
+        }
+    }
+
     #[test]
     fn test_fullzip_fixed_rejects_non_byte_aligned_values() {
         let fixed = FixedWidthDataBlock {
@@ -6947,7 +7021,7 @@ mod tests {
     #[tokio::test]
     async fn test_binary_large_minichunk_size_over_max_miniblock_values() {
         let mut string_data = Vec::new();
-        // 128kb/chunk / 6 bytes (t_9999) = 21845 > max 4096 items per chunk
+        // 128kb/chunk / 6 bytes (t_9999) = 21845 items per chunk
         for i in 0..10000 {
             string_data.push(Some(format!("t_{}", i)));
         }
@@ -7364,6 +7438,24 @@ mod tests {
         DataBlock::from_array(Arc::new(array) as ArrayRef)
     }
 
+    fn create_sorted_string_array(num_values: u64, cardinality: u64) -> ArrayRef {
+        use arrow_array::StringArray;
+
+        assert!(cardinality <= num_values && cardinality > 0);
+
+        let mut values = Vec::with_capacity(num_values as usize);
+        for i in 0..num_values {
+            let value_idx = i * cardinality / num_values;
+            values.push(format!("value_{:016}", value_idx));
+        }
+
+        Arc::new(StringArray::from(values)) as ArrayRef
+    }
+
+    fn create_sorted_variable_width_block(num_values: u64, cardinality: u64) -> DataBlock {
+        DataBlock::from_array(create_sorted_string_array(num_values, cardinality))
+    }
+
     #[test]
     fn test_should_dictionary_encode() {
         use crate::constants::DICT_SIZE_RATIO_META_KEY;
@@ -7390,6 +7482,93 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_block_sampling_detects_low_cardinality_in_short_sorted_runs() {
+        let sample_count: usize = 4096;
+        let num_values: u64 = 200_000;
+        let cardinality: u64 = 8_000;
+        let run_length = num_values / cardinality;
+        let stride = num_values as usize / sample_count;
+        assert!(
+            stride > run_length as usize,
+            "test must construct the stride > run_length case"
+        );
+
+        let block = create_sorted_variable_width_block(num_values, cardinality);
+        let sample_unique_ratio =
+            PrimitiveStructuralEncoder::sample_unique_ratio(&block, sample_count).unwrap();
+
+        assert!(
+            sample_unique_ratio.is_some_and(|ratio| ratio < 0.98),
+            "sorted low-cardinality data must not be classified as near-unique"
+        );
+    }
+
+    #[test]
+    fn test_should_dictionary_encode_sorted_low_cardinality() {
+        use crate::constants::DICT_SIZE_RATIO_META_KEY;
+        use lance_core::datatypes::Field as LanceField;
+
+        let block = create_sorted_variable_width_block(200_000, 8_000);
+
+        let mut metadata = HashMap::new();
+        metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.8".to_string());
+        let arrow_field =
+            arrow_schema::Field::new("test", DataType::Utf8, false).with_metadata(metadata);
+        let field = LanceField::try_from(&arrow_field).unwrap();
+
+        let result = PrimitiveStructuralEncoder::should_dictionary_encode(
+            &block,
+            &field,
+            LanceFileVersion::V2_2,
+        );
+
+        assert!(
+            result.is_some(),
+            "sorted low-cardinality data should reach dictionary encoding"
+        );
+    }
+
+    #[test]
+    fn test_should_not_dictionary_encode_sorted_high_cardinality_short_runs() {
+        use crate::constants::DICT_SIZE_RATIO_META_KEY;
+        use lance_core::datatypes::Field as LanceField;
+
+        let num_values = 200_002;
+        let cardinality = 100_001;
+        let block = create_sorted_variable_width_block(num_values, cardinality);
+
+        let mut metadata = HashMap::new();
+        metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.8".to_string());
+        let arrow_field =
+            arrow_schema::Field::new("test", DataType::Utf8, false).with_metadata(metadata);
+        let field = LanceField::try_from(&arrow_field).unwrap();
+
+        let result = PrimitiveStructuralEncoder::should_dictionary_encode(
+            &block,
+            &field,
+            LanceFileVersion::V2_2,
+        );
+
+        assert!(
+            result.is_none(),
+            "sorted high-cardinality short runs should not trigger a full dictionary probe"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_encode_sorted_low_cardinality_uses_dictionary_layout() {
+        use crate::constants::DICT_SIZE_RATIO_META_KEY;
+
+        let mut metadata = HashMap::new();
+        metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.8".to_string());
+        let field = arrow_schema::Field::new("test", DataType::Utf8, false).with_metadata(metadata);
+        let array = create_sorted_string_array(200_000, 8_000);
+
+        let page = encode_first_page(field, array, LanceFileVersion::V2_2).await;
+        let _ = dictionary_encoding_from_page(&page);
+    }
+
     #[test]
     fn test_should_not_dictionary_encode_unsupported_bits() {
         use crate::constants::DICT_SIZE_RATIO_META_KEY;
@@ -7441,6 +7620,36 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_v2_1_miniblock_serializes_log_num_values_15() {
+        let miniblocks = MiniBlockCompressed {
+            data: vec![LanceBuffer::from(vec![1_u8; 16])],
+            chunks: vec![
+                MiniBlockChunk {
+                    buffer_sizes: vec![8],
+                    log_num_values: 15,
+                },
+                MiniBlockChunk {
+                    buffer_sizes: vec![8],
+                    log_num_values: 0,
+                },
+            ],
+            num_values: 32_769,
+        };
+
+        let serialized =
+            PrimitiveStructuralEncoder::serialize_miniblocks(miniblocks, None, None, false)
+                .unwrap();
+
+        let chunk_metadata = serialized.metadata.borrow_to_typed_slice::<u16>();
+        assert_eq!(chunk_metadata.len(), 2);
+        assert_eq!(
+            chunk_metadata[0] & 0x0F,
+            15,
+            "V2.1 metadata should use all 4 bits for log_num_values"
+        );
+    }
+
     async fn encode_first_page(
         field: arrow_schema::Field,
         array: ArrayRef,
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
index 614dcb81ac2..52a7039b2b6 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs
@@ -205,9 +205,9 @@ struct BlobCacheableState {
 }
 
 impl DeepSizeOf for BlobCacheableState {
-    fn deep_size_of_children(&self, context: &mut lance_core::cache::Context) -> usize {
-        self.positions.get_array_memory_size()
-            + self.sizes.get_array_memory_size()
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        (self.positions.as_ref() as &dyn arrow_array::Array).deep_size_of_children(context)
+            + (self.sizes.as_ref() as &dyn arrow_array::Array).deep_size_of_children(context)
             + self.inner_state.deep_size_of_children(context)
     }
 }
@@ -258,7 +258,9 @@ impl BlobPageScheduler {
             let bytes = read_fut.await?;
             let mut bytes_iter = bytes.into_iter();
             for blob in loaded_blobs.iter_mut() {
-                if blob.def == 0 {
+                // Empty values have def == 0 too but scheduled no read; their
+                // bytes were set at scheduling time.
+                if blob.def == 0 && blob.bytes.is_none() {
                     blob.set_bytes(bytes_iter.next().expect_ok()?);
                 }
             }
@@ -364,7 +366,17 @@ impl StructuralPageScheduler for BlobPageScheduler {
                 if size == 0 {
                     let rep = (position & 0xFFFF) as u16;
                     let def = ((position >> 16) & 0xFFFF) as u16;
-                    loaded_blobs.push(LoadedBlob::new(rep, def));
+                    let mut blob = LoadedBlob::new(rep, def);
+                    if def == 0 {
+                        // A size-0 descriptor with definition level 0 is a
+                        // valid, empty value (nulls carry their non-zero
+                        // packed rep/def levels in `position`). No read is
+                        // scheduled for it, so it gets its zero-length bytes
+                        // here rather than consuming another blob's read
+                        // result in the load task.
+                        blob.set_bytes(Bytes::new());
+                    }
+                    loaded_blobs.push(blob);
                 } else {
                     loaded_blobs.push(LoadedBlob::new(0, 0));
                     ranges_to_read.push(position..(position + size));
diff --git a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
index de3227b2a39..1cf3b9bf581 100644
--- a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
+++ b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs
@@ -19,13 +19,14 @@ use lance_core::Result;
 pub const MAX_MINIBLOCK_BYTES: u64 = 8 * 1024 - 6;
 
 const DEFAULT_MAX_MINIBLOCK_VALUES: u64 = 4096;
+const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768;
 
 fn parse_max_miniblock_values() -> u64 {
     let val = std::env::var("LANCE_MINIBLOCK_MAX_VALUES")
         .ok()
         .and_then(|v| v.parse().ok())
         .unwrap_or(DEFAULT_MAX_MINIBLOCK_VALUES);
-    val.clamp(1, DEFAULT_MAX_MINIBLOCK_VALUES)
+    val.clamp(1, MAX_CONFIGURABLE_MINIBLOCK_VALUES)
 }
 
 pub static MAX_MINIBLOCK_VALUES: std::sync::LazyLock<u64> =
@@ -58,9 +59,9 @@ pub struct MiniBlockCompressed {
 /// and contain a power-of-two number of values (except for the last chunk)
 ///
 /// By default we limit a chunk to 4Ki values and slightly less than
-/// 8KiB of compressed data.  This means that even in the extreme case
-/// where we have 4 bytes of rep/def then we will have at most 24KiB of
-/// data (values, repetition, and definition) per mini-block.
+/// 8KiB of compressed value data.  The byte budget remains the primary
+/// constraint, so only encodings that compress many values into that
+/// budget can use larger value counts when explicitly configured.
 ///
 /// The maximum number of values per chunk can be configured via the
 /// `LANCE_MINIBLOCK_MAX_VALUES` environment variable.  This is only
@@ -77,8 +78,8 @@ pub struct MiniBlockChunk {
     // then this should be 0 (the number of values will be calculated by subtracting the
     // size of all other chunks from the total size of the page)
     //
-    // For example, 1 would mean there are 2 values in the chunk and 12 would mean there
-    // are 4Ki values in the chunk.
+    // For example, 1 would mean there are 2 values in the chunk and 15 would mean there
+    // are 32Ki values in the chunk.
     //
     // This must be <= log2(MAX_MINIBLOCK_VALUES) (i.e. <= 12 at the default of 4096)
     pub log_num_values: u8,
@@ -135,6 +136,14 @@ mod tests {
         unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
     }
 
+    #[test]
+    #[serial]
+    fn test_parse_can_raise_to_32k() {
+        unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "32768") };
+        assert_eq!(parse_max_miniblock_values(), 32768);
+        unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
+    }
+
     #[test]
     #[serial]
     fn test_parse_clamps_zero_to_one() {
@@ -147,7 +156,10 @@ mod tests {
     #[serial]
     fn test_parse_clamps_above_max() {
         unsafe { std::env::set_var("LANCE_MINIBLOCK_MAX_VALUES", "99999") };
-        assert_eq!(parse_max_miniblock_values(), DEFAULT_MAX_MINIBLOCK_VALUES);
+        assert_eq!(
+            parse_max_miniblock_values(),
+            MAX_CONFIGURABLE_MINIBLOCK_VALUES
+        );
         unsafe { std::env::remove_var("LANCE_MINIBLOCK_MAX_VALUES") };
     }
 
diff --git a/rust/lance-encoding/src/encodings/physical/bitpacking.rs b/rust/lance-encoding/src/encodings/physical/bitpacking.rs
index 8ebdcc13c56..be0b747e7dc 100644
--- a/rust/lance-encoding/src/encodings/physical/bitpacking.rs
+++ b/rust/lance-encoding/src/encodings/physical/bitpacking.rs
@@ -241,6 +241,15 @@ impl MiniBlockDecompressor for InlineBitpacking {
     fn decompress(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
         assert_eq!(data.len(), 1);
         let data = data.into_iter().next().unwrap();
+        if num_values == 0 {
+            // Empty mini-blocks have no inline bit-width header to decode.
+            return Ok(DataBlock::FixedWidth(FixedWidthDataBlock {
+                data: LanceBuffer::empty(),
+                bits_per_value: self.uncompressed_bit_width,
+                num_values: 0,
+                block_info: BlockInfo::new(),
+            }));
+        }
         match self.uncompressed_bit_width {
             8 => Self::unchunk::<u8>(data, num_values),
             16 => Self::unchunk::<u16>(data, num_values),
@@ -528,15 +537,36 @@ mod test {
 
     use arrow_array::{Array, Int8Array, Int64Array};
     use arrow_schema::DataType;
+    use rstest::rstest;
 
-    use super::{ELEMS_PER_CHUNK, bitpack_out_of_line, unpack_out_of_line};
+    use super::{ELEMS_PER_CHUNK, InlineBitpacking, bitpack_out_of_line, unpack_out_of_line};
     use crate::{
         buffer::LanceBuffer,
-        data::{BlockInfo, FixedWidthDataBlock},
+        compression::MiniBlockDecompressor,
+        data::{BlockInfo, DataBlock, FixedWidthDataBlock},
         testing::{TestCases, check_round_trip_encoding_of_data},
         version::LanceFileVersion,
     };
 
+    #[rstest]
+    #[case::u8(8)]
+    #[case::u16(16)]
+    #[case::u32(32)]
+    #[case::u64(64)]
+    fn test_inline_bitpacking_decompress_empty_miniblock(#[case] bit_width: u64) {
+        let decompressor = InlineBitpacking::new(bit_width);
+        let decompressed =
+            MiniBlockDecompressor::decompress(&decompressor, vec![LanceBuffer::empty()], 0)
+                .unwrap();
+
+        let DataBlock::FixedWidth(block) = decompressed else {
+            panic!("Expected FixedWidth block");
+        };
+        assert_eq!(block.bits_per_value, bit_width);
+        assert_eq!(block.num_values, 0);
+        assert_eq!(block.data.len(), 0);
+    }
+
     #[test_log::test(tokio::test)]
     async fn test_miniblock_bitpack() {
         let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1);
diff --git a/rust/lance-encoding/src/encodings/physical/general.rs b/rust/lance-encoding/src/encodings/physical/general.rs
index 4d58f72e71a..53c61928870 100644
--- a/rust/lance-encoding/src/encodings/physical/general.rs
+++ b/rust/lance-encoding/src/encodings/physical/general.rs
@@ -161,7 +161,7 @@ mod tests {
             // Small data with RLE - should not compress due to size threshold
             TestCase {
                 name: "small_rle_data",
-                inner_encoder: Box::new(RleEncoder),
+                inner_encoder: Box::new(RleEncoder::new()),
                 compression: CompressionConfig {
                     scheme: CompressionScheme::Lz4,
                     level: None,
@@ -173,7 +173,7 @@ mod tests {
             // Large repeated data with RLE + LZ4
             TestCase {
                 name: "large_rle_lz4",
-                inner_encoder: Box::new(RleEncoder),
+                inner_encoder: Box::new(RleEncoder::new()),
                 compression: CompressionConfig {
                     scheme: CompressionScheme::Lz4,
                     level: None,
@@ -185,7 +185,7 @@ mod tests {
             // Large repeated data with RLE + Zstd
             TestCase {
                 name: "large_rle_zstd",
-                inner_encoder: Box::new(RleEncoder),
+                inner_encoder: Box::new(RleEncoder::new()),
                 compression: CompressionConfig {
                     scheme: CompressionScheme::Zstd,
                     level: Some(3),
@@ -403,7 +403,7 @@ mod tests {
         // Test that small buffers don't get compressed
         let small_test = TestCase {
             name: "small_buffer_no_compression",
-            inner_encoder: Box::new(RleEncoder),
+            inner_encoder: Box::new(RleEncoder::new()),
             compression: CompressionConfig {
                 scheme: CompressionScheme::Lz4,
                 level: None,
@@ -496,7 +496,7 @@ mod tests {
         // RLE produces 2 buffers (values and lengths), test that both are handled correctly
         let data = create_repeated_i32_block(vec![1; 100]);
         let compressor = GeneralMiniBlockCompressor::new(
-            Box::new(RleEncoder),
+            Box::new(RleEncoder::new()),
             CompressionConfig {
                 scheme: CompressionScheme::Lz4,
                 level: None,
@@ -519,7 +519,7 @@ mod tests {
         // Test case 1: 32-bit RLE data
         let test_32 = TestCase {
             name: "rle_32bit_with_general_wrapper",
-            inner_encoder: Box::new(RleEncoder),
+            inner_encoder: Box::new(RleEncoder::new()),
             compression: CompressionConfig {
                 scheme: CompressionScheme::Lz4,
                 level: None,
@@ -532,7 +532,7 @@ mod tests {
         // For 32-bit RLE, the compression strategy should automatically wrap it
         // Let's directly test the compressor
         let compressor = GeneralMiniBlockCompressor::new(
-            Box::new(RleEncoder),
+            Box::new(RleEncoder::new()),
             CompressionConfig {
                 scheme: CompressionScheme::Lz4,
                 level: None,
@@ -589,7 +589,7 @@ mod tests {
         let block_64 = DataBlock::from_array(array_64);
 
         let compressor_64 = GeneralMiniBlockCompressor::new(
-            Box::new(RleEncoder),
+            Box::new(RleEncoder::new()),
             CompressionConfig {
                 scheme: CompressionScheme::Lz4,
                 level: None,
diff --git a/rust/lance-encoding/src/encodings/physical/packed.rs b/rust/lance-encoding/src/encodings/physical/packed.rs
index 3ade6a70818..ad2221dffed 100644
--- a/rust/lance-encoding/src/encodings/physical/packed.rs
+++ b/rust/lance-encoding/src/encodings/physical/packed.rs
@@ -323,7 +323,7 @@ impl PerValueCompressor for PackedStructVariablePerValueEncoder {
         let mut field_data = Vec::with_capacity(self.fields.len());
         let mut field_metadata = Vec::with_capacity(self.fields.len());
 
-        for (field, child_block) in self.fields.iter().zip(struct_block.children.into_iter()) {
+        for (field, child_block) in self.fields.iter().zip(struct_block.children) {
             let compressor = crate::compression::CompressionStrategy::create_per_value(
                 &self.strategy,
                 field,
@@ -688,7 +688,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor {
         }
 
         let mut children = Vec::with_capacity(self.fields.len());
-        for (field, accumulator) in self.fields.iter().zip(accumulators.into_iter()) {
+        for (field, accumulator) in self.fields.iter().zip(accumulators) {
             match (field, accumulator) {
                 (
                     VariablePackedStructFieldDecoder {
diff --git a/rust/lance-encoding/src/encodings/physical/rle.rs b/rust/lance-encoding/src/encodings/physical/rle.rs
index 88e27bf954f..145b9c43779 100644
--- a/rust/lance-encoding/src/encodings/physical/rle.rs
+++ b/rust/lance-encoding/src/encodings/physical/rle.rs
@@ -10,7 +10,7 @@
 //! RLE uses a dual-buffer format to store compressed data:
 //!
 //! - **Values Buffer**: Stores unique values in their original data type
-//! - **Lengths Buffer**: Stores the repeat count for each value as u8
+//! - **Lengths Buffer**: Stores the repeat count for each value as u8, u16, or u32
 //!
 //! ### Example
 //!
@@ -18,13 +18,13 @@
 //!
 //! Encoded as:
 //! - Values buffer: `[1, 2, 3]` (3 × 4 bytes for i32)
-//! - Lengths buffer: `[3, 2, 4]` (3 × 1 byte for u8)
+//! - Lengths buffer: `[3, 2, 4]` (3 × 1 byte for u8 in compatibility mode)
 //!
 //! ### Long Run Handling
 //!
-//! When a run exceeds 255 values, it is split into multiple runs of 255
-//! followed by a final run with the remainder. For example, a run of 1000
-//! identical values becomes 4 runs: [255, 255, 255, 235].
+//! In compatibility mode, when a run exceeds 255 values, it is split into multiple
+//! runs of 255 followed by a final run with the remainder. RLE v2 can use u16 or
+//! u32 run lengths to reduce this splitting.
 //!
 //! ## Supported Types
 //!
@@ -65,18 +65,319 @@ use crate::encodings::logical::primitive::miniblock::{
     MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed,
     MiniBlockCompressor,
 };
+use crate::encodings::physical::block::{CompressionConfig, GeneralBufferCompressor};
 use crate::format::ProtobufUtils21;
 use crate::format::pb21::CompressiveEncoding;
 
 use lance_core::{Error, Result};
 
+/// Width used to encode RLE run lengths.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum RunLengthWidth {
+    /// Compatibility mode. Runs longer than 255 values are split.
+    U8,
+    /// RLE v2 mode for runs up to 65,535 values per entry.
+    U16,
+    /// RLE v2 mode for runs up to 4,294,967,295 values per entry.
+    U32,
+}
+
+impl RunLengthWidth {
+    pub(crate) fn from_bits(bits_per_value: u64) -> Option<Self> {
+        match bits_per_value {
+            8 => Some(Self::U8),
+            16 => Some(Self::U16),
+            32 => Some(Self::U32),
+            _ => None,
+        }
+    }
+
+    pub(crate) fn bits_per_value(self) -> u64 {
+        match self {
+            Self::U8 => 8,
+            Self::U16 => 16,
+            Self::U32 => 32,
+        }
+    }
+
+    fn bytes_per_value(self) -> usize {
+        match self {
+            Self::U8 => 1,
+            Self::U16 => 2,
+            Self::U32 => 4,
+        }
+    }
+
+    fn max_run_length(self) -> u64 {
+        match self {
+            Self::U8 => u8::MAX as u64,
+            Self::U16 => u16::MAX as u64,
+            Self::U32 => u32::MAX as u64,
+        }
+    }
+
+    fn write_length(self, length: u64, dst: &mut Vec<u8>) {
+        match self {
+            Self::U8 => dst.push(length as u8),
+            Self::U16 => dst.extend_from_slice(&(length as u16).to_le_bytes()),
+            Self::U32 => dst.extend_from_slice(&(length as u32).to_le_bytes()),
+        }
+    }
+
+    fn read_length(self, bytes: &[u8]) -> u64 {
+        match self {
+            Self::U8 => bytes[0] as u64,
+            Self::U16 => u16::from_le_bytes([bytes[0], bytes[1]]) as u64,
+            Self::U32 => u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64,
+        }
+    }
+}
+
+const RUN_LENGTH_WIDTHS: [RunLengthWidth; 3] =
+    [RunLengthWidth::U8, RunLengthWidth::U16, RunLengthWidth::U32];
+
+/// Select the lowest-cost run length width from precomputed entry counts.
+pub(crate) fn select_run_length_width_from_entries(
+    entries: &[u64],
+    bits_per_value: u64,
+) -> Result<(RunLengthWidth, u128)> {
+    if entries.len() != RUN_LENGTH_WIDTHS.len() {
+        return Err(Error::invalid_input_source(
+            format!(
+                "RLE run length entry statistics must have {} values, got {}",
+                RUN_LENGTH_WIDTHS.len(),
+                entries.len()
+            )
+            .into(),
+        ));
+    }
+
+    if !matches!(bits_per_value, 8 | 16 | 32 | 64) {
+        return Err(Error::invalid_input_source(
+            format!("RLE encoding bits_per_value must be 8, 16, 32, or 64, got {bits_per_value}")
+                .into(),
+        ));
+    }
+
+    let mut best_width = RUN_LENGTH_WIDTHS[0];
+    let mut best_cost = rle_encoded_size_from_entries(entries[0], bits_per_value, best_width);
+    for (&width, &entry_count) in RUN_LENGTH_WIDTHS.iter().zip(entries.iter()).skip(1) {
+        let cost = rle_encoded_size_from_entries(entry_count, bits_per_value, width);
+        if cost < best_cost {
+            best_width = width;
+            best_cost = cost;
+        }
+    }
+
+    Ok((best_width, best_cost))
+}
+
+pub(crate) fn rle_encoded_size_from_entries(
+    entry_count: u64,
+    bits_per_value: u64,
+    run_length_width: RunLengthWidth,
+) -> u128 {
+    let bytes_per_value = (bits_per_value / 8) as u128;
+    let bytes_per_length = run_length_width.bytes_per_value() as u128;
+    (entry_count as u128) * (bytes_per_value + bytes_per_length)
+}
+
+pub(crate) fn run_length_width_index(run_length_width: RunLengthWidth) -> usize {
+    match run_length_width {
+        RunLengthWidth::U8 => 0,
+        RunLengthWidth::U16 => 1,
+        RunLengthWidth::U32 => 2,
+    }
+}
+
+pub(crate) fn select_run_length_width(
+    data: &LanceBuffer,
+    num_values: u64,
+    bits_per_value: u64,
+    max_segment_values: Option<u64>,
+) -> Result<(RunLengthWidth, u128)> {
+    let entries = collect_run_length_entries(data, num_values, bits_per_value, max_segment_values)?;
+    select_run_length_width_from_entries(&entries, bits_per_value)
+}
+
+pub(crate) fn rle_encoded_size(
+    data: &LanceBuffer,
+    num_values: u64,
+    bits_per_value: u64,
+    max_segment_values: Option<u64>,
+    run_length_width: RunLengthWidth,
+) -> Result<u128> {
+    let entries = collect_run_length_entries(data, num_values, bits_per_value, max_segment_values)?;
+    let width_idx = run_length_width_index(run_length_width);
+    Ok(rle_encoded_size_from_entries(
+        entries[width_idx],
+        bits_per_value,
+        run_length_width,
+    ))
+}
+
+fn collect_run_length_entries(
+    data: &LanceBuffer,
+    num_values: u64,
+    bits_per_value: u64,
+    max_segment_values: Option<u64>,
+) -> Result<[u64; 3]> {
+    let num_values = usize::try_from(num_values).map_err(|_| {
+        Error::invalid_input_source(
+            format!("RLE num_values does not fit in usize: {num_values}").into(),
+        )
+    })?;
+
+    macro_rules! collect_entries {
+        ($ty:ty) => {{
+            let type_size = std::mem::size_of::<$ty>();
+            let expected_bytes = num_values.checked_mul(type_size).ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE input byte length overflow: {num_values} values of {type_size} bytes"
+                    )
+                    .into(),
+                )
+            })?;
+            if data.len() != expected_bytes {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE input data size mismatch: {} bytes for {} values of {} bytes",
+                        data.len(),
+                        num_values,
+                        type_size
+                    )
+                    .into(),
+                ));
+            }
+            let values = data.borrow_to_typed_slice::<$ty>();
+            let values = values.get(..num_values).ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE data has {} values but {} were expected",
+                        values.len(),
+                        num_values
+                    )
+                    .into(),
+                )
+            })?;
+            Ok(collect_run_length_entries_from_slice(
+                values,
+                max_segment_values,
+            ))
+        }};
+    }
+
+    match bits_per_value {
+        8 => collect_entries!(u8),
+        16 => collect_entries!(u16),
+        32 => collect_entries!(u32),
+        64 => collect_entries!(u64),
+        _ => Err(Error::invalid_input_source(
+            format!("RLE encoding bits_per_value must be 8, 16, 32, or 64, got {bits_per_value}")
+                .into(),
+        )),
+    }
+}
+
+fn collect_run_length_entries_from_slice<T: PartialEq + Copy>(
+    values: &[T],
+    max_segment_values: Option<u64>,
+) -> [u64; 3] {
+    if values.is_empty() {
+        return [0; 3];
+    }
+
+    let mut entries = [0u64; 3];
+    let mut prev = values[0];
+    let mut current_length = 1u64;
+
+    for &value in &values[1..] {
+        if value != prev {
+            accumulate_run_length_entries(current_length, max_segment_values, &mut entries);
+            prev = value;
+            current_length = 1;
+        } else {
+            current_length += 1;
+        }
+    }
+    accumulate_run_length_entries(current_length, max_segment_values, &mut entries);
+
+    entries
+}
+
+pub(crate) fn accumulate_run_length_entries(
+    run_length: u64,
+    max_segment_values: Option<u64>,
+    entries: &mut [u64; 3],
+) {
+    let max_segment_values = max_segment_values.unwrap_or(run_length).max(1);
+    let mut remaining = run_length;
+    while remaining > 0 {
+        let segment = remaining.min(max_segment_values);
+        for (idx, width) in RUN_LENGTH_WIDTHS.iter().enumerate() {
+            let entry_count = segment.div_ceil(width.max_run_length());
+            entries[idx] = entries[idx].saturating_add(entry_count);
+        }
+        remaining -= segment;
+    }
+}
+
 /// RLE encoder for miniblock format
-#[derive(Debug, Default)]
-pub struct RleEncoder;
+#[derive(Debug)]
+pub struct RleEncoder {
+    run_length_width: RunLengthWidth,
+    values_compression: Option<CompressionConfig>,
+    run_lengths_compression: Option<CompressionConfig>,
+    use_child_bitpacking: bool,
+}
+
+#[derive(Clone)]
+struct RleChildCandidate {
+    encoding: CompressiveEncoding,
+    data: LanceBuffer,
+    chunk_sizes: Vec<u32>,
+    size: usize,
+    requires_num_values: bool,
+}
+
+impl Default for RleEncoder {
+    fn default() -> Self {
+        Self::new()
+    }
+}
 
 impl RleEncoder {
     pub fn new() -> Self {
-        Self
+        Self {
+            run_length_width: RunLengthWidth::U8,
+            values_compression: None,
+            run_lengths_compression: None,
+            use_child_bitpacking: false,
+        }
+    }
+
+    pub(crate) fn with_run_length_width(run_length_width: RunLengthWidth) -> Self {
+        Self {
+            run_length_width,
+            values_compression: None,
+            run_lengths_compression: None,
+            use_child_bitpacking: false,
+        }
+    }
+
+    pub(crate) fn with_child_encoding(
+        run_length_width: RunLengthWidth,
+        values_compression: Option<CompressionConfig>,
+        run_lengths_compression: Option<CompressionConfig>,
+        use_child_bitpacking: bool,
+    ) -> Self {
+        Self {
+            run_length_width,
+            values_compression,
+            run_lengths_compression,
+            use_child_bitpacking,
+        }
     }
 
     fn encode_data(
@@ -89,17 +390,23 @@ impl RleEncoder {
             return Ok((Vec::new(), Vec::new()));
         }
 
+        let num_values = usize::try_from(num_values).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE num_values does not fit in usize: {num_values}").into(),
+            )
+        })?;
         let bytes_per_value = (bits_per_value / 8) as usize;
+        let bytes_per_length = self.run_length_width.bytes_per_value();
 
         // Pre-allocate global buffers with estimated capacity
         // Assume average compression ratio of ~10:1 (10 values per run)
-        let estimated_runs = num_values as usize / 10;
+        let estimated_runs = num_values / 10;
         let mut all_values = Vec::with_capacity(estimated_runs * bytes_per_value);
-        let mut all_lengths = Vec::with_capacity(estimated_runs);
+        let mut all_lengths = Vec::with_capacity(estimated_runs * bytes_per_length);
         let mut chunks = Vec::new();
 
         let mut offset = 0usize;
-        let mut values_remaining = num_values as usize;
+        let mut values_remaining = num_values;
 
         while values_remaining > 0 {
             let values_start = all_values.len();
@@ -134,11 +441,26 @@ impl RleEncoder {
                     &mut all_values,
                     &mut all_lengths,
                 ),
-                _ => unreachable!("RLE encoding bits_per_value must be 8, 16, 32 or 64"),
+                _ => {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE encoding bits_per_value must be 8, 16, 32, or 64, got {bits_per_value}"
+                        )
+                        .into(),
+                    ));
+                }
             };
 
             if values_processed == 0 {
-                break;
+                // A non-final chunk needs at least two values because log_num_values == 0
+                // identifies the final chunk. Report an error instead of returning partial data.
+                return Err(Error::internal(format!(
+                    "RLE encoder made no progress: values_remaining={values_remaining}, \
+                     offset={offset}, data_len={}, bits_per_value={bits_per_value}, \
+                     max_miniblock_values={}",
+                    data.len(),
+                    *MAX_MINIBLOCK_VALUES
+                )));
             }
 
             let log_num_values = if is_last_chunk {
@@ -175,22 +497,75 @@ impl RleEncoder {
         ))
     }
 
-    /// Encodes a chunk of data using RLE compression with dynamic boundary detection.
-    ///
-    /// This function processes values sequentially, detecting runs (sequences of identical values)
-    /// and encoding them as (value, length) pairs. It dynamically determines whether this chunk
-    /// should be the last chunk based on how many values were processed.
-    ///
-    /// # Key Features:
-    /// - Tracks byte usage to ensure we don't exceed MAX_MINIBLOCK_BYTES
-    /// - Maintains power-of-2 checkpoints for non-last chunks
-    /// - Splits long runs (>255) into multiple entries
-    /// - Dynamically determines if this is the last chunk
-    ///
-    /// # Returns:
-    /// - num_runs: Number of runs encoded
-    /// - values_processed: Number of input values processed
-    /// - is_last_chunk: Whether this chunk processed all remaining values
+    fn encode_block_data(
+        &self,
+        data: &LanceBuffer,
+        num_values: u64,
+        bits_per_value: u64,
+    ) -> Result<Vec<LanceBuffer>> {
+        match bits_per_value {
+            8 => self.encode_block_data_generic::<u8>(data, num_values),
+            16 => self.encode_block_data_generic::<u16>(data, num_values),
+            32 => self.encode_block_data_generic::<u32>(data, num_values),
+            64 => self.encode_block_data_generic::<u64>(data, num_values),
+            _ => Err(Error::invalid_input_source(
+                format!(
+                    "RLE encoding bits_per_value must be 8, 16, 32, or 64, got {bits_per_value}"
+                )
+                .into(),
+            )),
+        }
+    }
+
+    fn encode_block_data_generic<T>(
+        &self,
+        data: &LanceBuffer,
+        num_values: u64,
+    ) -> Result<Vec<LanceBuffer>>
+    where
+        T: bytemuck::Pod + PartialEq + Copy + ArrowNativeType,
+    {
+        let num_values = usize::try_from(num_values).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE num_values does not fit in usize: {num_values}").into(),
+            )
+        })?;
+        let type_size = std::mem::size_of::<T>();
+        let expected_bytes = num_values.checked_mul(type_size).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("RLE input byte length overflow: {num_values} values of {type_size} bytes")
+                    .into(),
+            )
+        })?;
+        if data.len() != expected_bytes {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "RLE input data size mismatch: {} bytes for {} values of {} bytes",
+                    data.len(),
+                    num_values,
+                    type_size
+                )
+                .into(),
+            ));
+        }
+        if num_values == 0 {
+            return Ok(vec![LanceBuffer::empty(), LanceBuffer::empty()]);
+        }
+
+        let values_ref = data.borrow_to_typed_slice::<T>();
+        let values = values_ref.as_ref();
+        let estimated_runs = num_values / 10;
+        let mut all_values = Vec::with_capacity(estimated_runs * type_size);
+        let mut all_lengths =
+            Vec::with_capacity(estimated_runs * self.run_length_width.bytes_per_value());
+        self.encode_values(values, &mut all_values, &mut all_lengths);
+        Ok(vec![
+            LanceBuffer::from(all_values),
+            LanceBuffer::from(all_lengths),
+        ])
+    }
+
+    /// Encodes the largest valid mini-block prefix from `offset`.
     fn encode_chunk_rolling<T>(
         &self,
         data: &LanceBuffer,
@@ -203,7 +578,6 @@ impl RleEncoder {
         T: bytemuck::Pod + PartialEq + Copy + std::fmt::Debug + ArrowNativeType,
     {
         let type_size = std::mem::size_of::<T>();
-
         let chunk_start = offset * type_size;
         let max_by_count = *MAX_MINIBLOCK_VALUES as usize;
         let max_values = values_remaining.min(max_by_count);
@@ -217,112 +591,101 @@ impl RleEncoder {
         let chunk_buffer = data.slice_with_length(chunk_start, chunk_len);
         let typed_data_ref = chunk_buffer.borrow_to_typed_slice::<T>();
         let typed_data: &[T] = typed_data_ref.as_ref();
+        let max_values = max_values.min(typed_data.len());
 
         if typed_data.is_empty() {
             return (0, 0, false);
         }
 
-        // Record starting positions for this chunk
         let values_start = all_values.len();
+        let all_remaining_values_fit = values_remaining <= max_by_count;
+        let encoded_size = self.encoded_size(&typed_data[..max_values]);
+        let (values_to_encode, is_last_chunk) = if all_remaining_values_fit
+            && encoded_size <= MAX_MINIBLOCK_BYTES as usize
+        {
+            (max_values, true)
+        } else if let Some(values_to_encode) = self.largest_power_of_two_prefix::<T>(typed_data) {
+            (values_to_encode, false)
+        } else {
+            return (0, 0, false);
+        };
 
-        let mut current_value = typed_data[0];
-        let mut current_length = 1u64;
-        let mut bytes_used = 0usize;
-        let mut total_values_encoded = 0usize; // Track total encoded values
+        self.encode_values(&typed_data[..values_to_encode], all_values, all_lengths);
 
-        // Power-of-2 checkpoints for ensuring non-last chunks have valid sizes.
-        //
-        // We start from a slightly larger minimum checkpoint for smaller types since
-        // they encode more compactly and are less likely to hit MAX_MINIBLOCK_BYTES.
-        let min_checkpoint_log2 = match type_size {
-            1 => 8, // 256
-            2 => 7, // 128
-            _ => 6, // 64
-        };
-        let max_checkpoint_log2 = (values_remaining.min(*MAX_MINIBLOCK_VALUES as usize))
-            .next_power_of_two()
-            .ilog2();
-        let mut checkpoint_log2 = min_checkpoint_log2;
+        let num_runs = (all_values.len() - values_start) / type_size;
+        (num_runs, values_to_encode, is_last_chunk)
+    }
+
+    fn largest_power_of_two_prefix<T>(&self, values: &[T]) -> Option<usize>
+    where
+        T: bytemuck::Pod + PartialEq + Copy,
+    {
+        let max_prefix = values.len().min(*MAX_MINIBLOCK_VALUES as usize);
+        let mut prefix = 1usize << max_prefix.ilog2();
+        while prefix > 1 {
+            if self.encoded_size(&values[..prefix]) <= MAX_MINIBLOCK_BYTES as usize {
+                return Some(prefix);
+            }
+            prefix >>= 1;
+        }
+        None
+    }
 
-        // Save state at checkpoints so we can roll back if needed
-        let mut last_checkpoint_state = None;
+    fn encoded_size<T>(&self, values: &[T]) -> usize
+    where
+        T: bytemuck::Pod + PartialEq + Copy,
+    {
+        if values.is_empty() {
+            return 0;
+        }
+
+        let mut current_value = values[0];
+        let mut current_length = 1u64;
+        let mut encoded_size = 0usize;
 
-        for &value in typed_data[1..].iter() {
+        for &value in values.iter().skip(1) {
             if value == current_value {
                 current_length += 1;
             } else {
-                // Calculate space needed (may need multiple u8s if run > 255)
-                let run_chunks = current_length.div_ceil(255) as usize;
-                let bytes_needed = run_chunks * (type_size + 1);
-
-                // Stop if adding this run would exceed byte limit
-                if bytes_used + bytes_needed > MAX_MINIBLOCK_BYTES as usize {
-                    if let Some((val_pos, len_pos, _, checkpoint_values)) = last_checkpoint_state {
-                        // Roll back to last power-of-2 checkpoint
-                        all_values.truncate(val_pos);
-                        all_lengths.truncate(len_pos);
-                        let num_runs = (val_pos - values_start) / type_size;
-                        return (num_runs, checkpoint_values, false);
-                    }
-                    break;
-                }
-
-                bytes_used += self.add_run(&current_value, current_length, all_values, all_lengths);
-                total_values_encoded += current_length as usize;
+                encoded_size += self.run_size::<T>(current_length);
                 current_value = value;
                 current_length = 1;
             }
-
-            // Check if we reached a power-of-2 checkpoint.
-            while checkpoint_log2 <= max_checkpoint_log2 {
-                let checkpoint_values = 1usize << checkpoint_log2;
-                if checkpoint_values > values_remaining || total_values_encoded < checkpoint_values
-                {
-                    break;
-                }
-                last_checkpoint_state = Some((
-                    all_values.len(),
-                    all_lengths.len(),
-                    bytes_used,
-                    checkpoint_values,
-                ));
-                checkpoint_log2 += 1;
-            }
         }
+        encoded_size += self.run_size::<T>(current_length);
+        encoded_size
+    }
 
-        // After the loop, we always have a pending run that needs to be added
-        // unless we've exceeded the byte limit
-        if current_length > 0 {
-            let run_chunks = current_length.div_ceil(255) as usize;
-            let bytes_needed = run_chunks * (type_size + 1);
+    fn run_size<T>(&self, length: u64) -> usize
+    where
+        T: bytemuck::Pod,
+    {
+        let type_size = std::mem::size_of::<T>();
+        let run_chunks = length.div_ceil(self.run_length_width.max_run_length()) as usize;
+        run_chunks * (type_size + self.run_length_width.bytes_per_value())
+    }
 
-            if bytes_used + bytes_needed <= MAX_MINIBLOCK_BYTES as usize {
-                let _ = self.add_run(&current_value, current_length, all_values, all_lengths);
-                total_values_encoded += current_length as usize;
-            }
+    fn encode_values<T>(&self, values: &[T], all_values: &mut Vec<u8>, all_lengths: &mut Vec<u8>)
+    where
+        T: bytemuck::Pod + PartialEq + Copy,
+    {
+        if values.is_empty() {
+            return;
         }
 
-        // Determine if we've processed all remaining values
-        let is_last_chunk = total_values_encoded == values_remaining;
+        let mut current_value = values[0];
+        let mut current_length = 1u64;
 
-        // Non-last chunks must have power-of-2 values for miniblock format
-        if !is_last_chunk {
-            if total_values_encoded.is_power_of_two() {
-                // Already at power-of-2 boundary
-            } else if let Some((val_pos, len_pos, _, checkpoint_values)) = last_checkpoint_state {
-                // Roll back to last valid checkpoint
-                all_values.truncate(val_pos);
-                all_lengths.truncate(len_pos);
-                let num_runs = (val_pos - values_start) / type_size;
-                return (num_runs, checkpoint_values, false);
+        for &value in values.iter().skip(1) {
+            if value == current_value {
+                current_length += 1;
             } else {
-                // No valid checkpoint, can't create a valid chunk
-                return (0, 0, false);
+                self.add_run(&current_value, current_length, all_values, all_lengths);
+                current_value = value;
+                current_length = 1;
             }
         }
-
-        let num_runs = (all_values.len() - values_start) / type_size;
-        (num_runs, total_values_encoded, is_last_chunk)
+        self.add_run(&current_value, current_length, all_values, all_lengths);
     }
 
     fn add_run<T>(
@@ -337,24 +700,338 @@ impl RleEncoder {
     {
         let value_bytes = bytemuck::bytes_of(value);
         let type_size = std::mem::size_of::<T>();
-        let num_full_chunks = (length / 255) as usize;
-        let remainder = (length % 255) as u8;
+        let max_run_length = self.run_length_width.max_run_length();
+        let num_full_chunks = (length / max_run_length) as usize;
+        let remainder = length % max_run_length;
 
         let total_chunks = num_full_chunks + if remainder > 0 { 1 } else { 0 };
         all_values.reserve(total_chunks * type_size);
-        all_lengths.reserve(total_chunks);
+        all_lengths.reserve(total_chunks * self.run_length_width.bytes_per_value());
 
         for _ in 0..num_full_chunks {
             all_values.extend_from_slice(value_bytes);
-            all_lengths.push(255);
+            self.run_length_width
+                .write_length(max_run_length, all_lengths);
         }
 
         if remainder > 0 {
             all_values.extend_from_slice(value_bytes);
-            all_lengths.push(remainder);
+            self.run_length_width.write_length(remainder, all_lengths);
+        }
+
+        total_chunks * (type_size + self.run_length_width.bytes_per_value())
+    }
+
+    fn flat_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+    ) -> RleChildCandidate {
+        RleChildCandidate {
+            encoding: ProtobufUtils21::flat(bits_per_value, None),
+            data: buffers[buffer_index].clone(),
+            chunk_sizes: chunks
+                .iter()
+                .map(|chunk| chunk.buffer_sizes[buffer_index])
+                .collect(),
+            size: buffers[buffer_index].len(),
+            requires_num_values: false,
+        }
+    }
+
+    fn general_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+        compression: CompressionConfig,
+    ) -> Result<Option<RleChildCandidate>> {
+        if buffers.is_empty() || buffers[buffer_index].is_empty() {
+            return Ok(None);
+        };
+
+        let compressor = GeneralBufferCompressor::get_compressor(compression)?;
+        let original = &buffers[buffer_index];
+        let mut compressed = Vec::new();
+        let mut offset = 0usize;
+        let mut total_original_size = 0usize;
+        let mut compressed_sizes = Vec::with_capacity(chunks.len());
+
+        for chunk in chunks.iter() {
+            let chunk_size = chunk.buffer_sizes[buffer_index] as usize;
+            let end = offset.checked_add(chunk_size).ok_or_else(|| {
+                Error::invalid_input_source("RLE child buffer offset overflow".into())
+            })?;
+            if end > original.len() {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}",
+                        buffer_index,
+                        end,
+                        original.len()
+                    )
+                    .into(),
+                ));
+            }
+
+            let start = compressed.len();
+            compressor.compress(&original.as_ref()[offset..end], &mut compressed)?;
+            let compressed_size = compressed.len() - start;
+            let compressed_size = u32::try_from(compressed_size).map_err(|_| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} compressed chunk is too large: {} bytes",
+                        buffer_index, compressed_size
+                    )
+                    .into(),
+                )
+            })?;
+            compressed_sizes.push(compressed_size);
+            total_original_size += chunk_size;
+            offset = end;
+        }
+
+        if compressed.len() >= total_original_size {
+            return Ok(None);
         }
 
-        total_chunks * (type_size + 1)
+        let encoding =
+            ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(bits_per_value, None))?;
+        Ok(Some(
+            RleChildCandidate {
+                encoding,
+                data: LanceBuffer::from(compressed),
+                chunk_sizes: compressed_sizes,
+                size: 0,
+                requires_num_values: false,
+            }
+            .with_size_from_data(),
+        ))
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn bitpacked_child_candidate(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+    ) -> Result<Option<RleChildCandidate>> {
+        let original = &buffers[buffer_index];
+        if original.is_empty() {
+            return Ok(None);
+        }
+        let packed_bits = Self::required_bits(original, bits_per_value)?;
+        if packed_bits >= bits_per_value {
+            return Ok(None);
+        }
+
+        let compressor = crate::encodings::physical::bitpacking::OutOfLineBitpacking::new(
+            packed_bits,
+            bits_per_value,
+        );
+        let mut packed = Vec::new();
+        let mut offset = 0usize;
+        let mut packed_sizes = Vec::with_capacity(chunks.len());
+        let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE child bit width is too large: {bits_per_value}").into(),
+            )
+        })?;
+
+        for chunk in chunks {
+            let chunk_size = chunk.buffer_sizes[buffer_index] as usize;
+            let end = offset.checked_add(chunk_size).ok_or_else(|| {
+                Error::invalid_input_source("RLE child buffer offset overflow".into())
+            })?;
+            if end > original.len() {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}",
+                        buffer_index,
+                        end,
+                        original.len()
+                    )
+                    .into(),
+                ));
+            }
+            if bytes_per_value == 0 || !chunk_size.is_multiple_of(bytes_per_value) {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} chunk has invalid size {} for {} bits per value",
+                        buffer_index, chunk_size, bits_per_value
+                    )
+                    .into(),
+                ));
+            }
+
+            let child_values = (chunk_size / bytes_per_value) as u64;
+            let block = DataBlock::FixedWidth(FixedWidthDataBlock {
+                bits_per_value,
+                data: original.slice_with_length(offset, chunk_size),
+                num_values: child_values,
+                block_info: BlockInfo::default(),
+            });
+            let chunk_packed = BlockCompressor::compress(&compressor, block)?;
+            let packed_size = u32::try_from(chunk_packed.len()).map_err(|_| {
+                Error::invalid_input_source(
+                    format!(
+                        "RLE child buffer {} bitpacked chunk is too large: {} bytes",
+                        buffer_index,
+                        chunk_packed.len()
+                    )
+                    .into(),
+                )
+            })?;
+            packed_sizes.push(packed_size);
+            packed.extend_from_slice(chunk_packed.as_ref());
+            offset = end;
+        }
+
+        if packed.len() >= original.len() {
+            return Ok(None);
+        }
+
+        Ok(Some(
+            RleChildCandidate {
+                encoding: ProtobufUtils21::out_of_line_bitpacking(
+                    bits_per_value,
+                    ProtobufUtils21::flat(packed_bits, None),
+                ),
+                data: LanceBuffer::from(packed),
+                chunk_sizes: packed_sizes,
+                size: 0,
+                requires_num_values: true,
+            }
+            .with_size_from_data(),
+        ))
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn required_bits(buffer: &LanceBuffer, bits_per_value: u64) -> Result<u64> {
+        let max_value = match bits_per_value {
+            8 => buffer.as_ref().iter().map(|value| *value as u64).max(),
+            16 => buffer
+                .as_ref()
+                .chunks_exact(2)
+                .map(|value| u16::from_le_bytes(value.try_into().unwrap()) as u64)
+                .max(),
+            32 => buffer
+                .as_ref()
+                .chunks_exact(4)
+                .map(|value| u32::from_le_bytes(value.try_into().unwrap()) as u64)
+                .max(),
+            64 => buffer
+                .as_ref()
+                .chunks_exact(8)
+                .map(|value| u64::from_le_bytes(value.try_into().unwrap()))
+                .max(),
+            _ => {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE child bitpacking only supports 8, 16, 32, or 64-bit values, got {bits_per_value}"
+                    )
+                    .into(),
+                ));
+            }
+        }
+        .unwrap_or(0);
+        Ok((u64::BITS - max_value.leading_zeros()).max(1) as u64)
+    }
+
+    fn child_candidates(
+        buffers: &[LanceBuffer],
+        chunks: &[MiniBlockChunk],
+        buffer_index: usize,
+        bits_per_value: u64,
+        compression: Option<CompressionConfig>,
+        use_child_bitpacking: bool,
+    ) -> Result<Vec<RleChildCandidate>> {
+        #[cfg(not(feature = "bitpacking"))]
+        let _ = use_child_bitpacking;
+        let mut candidates = vec![Self::flat_child_candidate(
+            buffers,
+            chunks,
+            buffer_index,
+            bits_per_value,
+        )];
+        if let Some(compression) = compression
+            && let Some(candidate) = Self::general_child_candidate(
+                buffers,
+                chunks,
+                buffer_index,
+                bits_per_value,
+                compression,
+            )?
+        {
+            candidates.push(candidate);
+        }
+        #[cfg(feature = "bitpacking")]
+        {
+            if use_child_bitpacking
+                && let Some(candidate) =
+                    Self::bitpacked_child_candidate(buffers, chunks, buffer_index, bits_per_value)?
+            {
+                candidates.push(candidate);
+            }
+        }
+        Ok(candidates)
+    }
+
+    fn select_child_candidates(
+        values: Vec<RleChildCandidate>,
+        run_lengths: Vec<RleChildCandidate>,
+    ) -> (RleChildCandidate, RleChildCandidate) {
+        let mut best: Option<(usize, usize, usize)> = None;
+        for (value_idx, value) in values.iter().enumerate() {
+            for (length_idx, length) in run_lengths.iter().enumerate() {
+                if value.requires_num_values && length.requires_num_values {
+                    continue;
+                }
+                let size = value.size + length.size;
+                if best.is_none_or(|(_, _, best_size)| size < best_size) {
+                    best = Some((value_idx, length_idx, size));
+                }
+            }
+        }
+        let (value_idx, length_idx, _) =
+            best.expect("flat RLE child candidates should always be selectable");
+        (values[value_idx].clone(), run_lengths[length_idx].clone())
+    }
+
+    pub(crate) fn selected_payload_size(&self, data: &FixedWidthDataBlock) -> Result<u128> {
+        let (all_buffers, chunks) =
+            self.encode_data(&data.data, data.num_values, data.bits_per_value)?;
+        if all_buffers.is_empty() {
+            return Ok(0);
+        }
+
+        let values_candidates = Self::child_candidates(
+            &all_buffers,
+            &chunks,
+            0,
+            data.bits_per_value,
+            self.values_compression,
+            self.use_child_bitpacking,
+        )?;
+        let run_lengths_candidates = Self::child_candidates(
+            &all_buffers,
+            &chunks,
+            1,
+            self.run_length_width.bits_per_value(),
+            self.run_lengths_compression,
+            self.use_child_bitpacking,
+        )?;
+        let (values, run_lengths) =
+            Self::select_child_candidates(values_candidates, run_lengths_candidates);
+        Ok((values.size as u128).saturating_add(run_lengths.size as u128))
+    }
+}
+
+impl RleChildCandidate {
+    fn with_size_from_data(mut self) -> Self {
+        self.size = self.data.len();
+        self
     }
 }
 
@@ -367,17 +1044,53 @@ impl MiniBlockCompressor for RleEncoder {
 
                 let (all_buffers, chunks) =
                     self.encode_data(&fixed_width.data, num_values, bits_per_value)?;
+                if all_buffers.is_empty() {
+                    let compressed = MiniBlockCompressed {
+                        data: all_buffers,
+                        chunks,
+                        num_values,
+                    };
+                    let encoding = ProtobufUtils21::rle(
+                        ProtobufUtils21::flat(bits_per_value, None),
+                        ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None),
+                    );
+                    return Ok((compressed, encoding));
+                }
+
+                let values_candidates = Self::child_candidates(
+                    &all_buffers,
+                    &chunks,
+                    0,
+                    bits_per_value,
+                    self.values_compression,
+                    self.use_child_bitpacking,
+                )?;
+                let run_lengths_candidates = Self::child_candidates(
+                    &all_buffers,
+                    &chunks,
+                    1,
+                    self.run_length_width.bits_per_value(),
+                    self.run_lengths_compression,
+                    self.use_child_bitpacking,
+                )?;
+                let (values, run_lengths) =
+                    Self::select_child_candidates(values_candidates, run_lengths_candidates);
+                let chunks = chunks
+                    .into_iter()
+                    .enumerate()
+                    .map(|(idx, chunk)| MiniBlockChunk {
+                        buffer_sizes: vec![values.chunk_sizes[idx], run_lengths.chunk_sizes[idx]],
+                        log_num_values: chunk.log_num_values,
+                    })
+                    .collect();
 
                 let compressed = MiniBlockCompressed {
-                    data: all_buffers,
+                    data: vec![values.data, run_lengths.data],
                     chunks,
                     num_values,
                 };
 
-                let encoding = ProtobufUtils21::rle(
-                    ProtobufUtils21::flat(bits_per_value, None),
-                    ProtobufUtils21::flat(/*bits_per_value=*/ 8, None),
-                );
+                let encoding = ProtobufUtils21::rle(values.encoding, run_lengths.encoding);
 
                 Ok((compressed, encoding))
             }
@@ -396,36 +1109,192 @@ impl BlockCompressor for RleEncoder {
                 let num_values = fixed_width.num_values;
                 let bits_per_value = fixed_width.bits_per_value;
 
-                let (all_buffers, _) =
-                    self.encode_data(&fixed_width.data, num_values, bits_per_value)?;
+                let all_buffers =
+                    self.encode_block_data(&fixed_width.data, num_values, bits_per_value)?;
 
                 let values_size = all_buffers[0].len() as u64;
 
-                let mut combined = Vec::new();
-                combined.extend_from_slice(&values_size.to_le_bytes());
-                combined.extend_from_slice(&all_buffers[0]);
-                combined.extend_from_slice(&all_buffers[1]);
-                Ok(LanceBuffer::from(combined))
+                let mut combined = Vec::new();
+                combined.extend_from_slice(&values_size.to_le_bytes());
+                combined.extend_from_slice(&all_buffers[0]);
+                combined.extend_from_slice(&all_buffers[1]);
+                Ok(LanceBuffer::from(combined))
+            }
+            _ => Err(Error::invalid_input_source(
+                "RLE encoding only supports FixedWidth data blocks".into(),
+            )),
+        }
+    }
+}
+
+/// RLE decompressor for miniblock format
+#[derive(Debug)]
+pub struct RleDecompressor {
+    bits_per_value: u64,
+    run_length_width: RunLengthWidth,
+    values: RleChildDecompressor,
+    run_lengths: RleChildDecompressor,
+}
+
+#[derive(Debug)]
+pub(crate) struct RleChildDecompressor {
+    bits_per_value: u64,
+    inner: RleChildDecompressorInner,
+}
+
+#[derive(Debug)]
+enum RleChildDecompressorInner {
+    Flat,
+    Block {
+        decompressor: Box<dyn BlockDecompressor>,
+        requires_num_values: bool,
+    },
+}
+
+impl RleChildDecompressor {
+    pub(crate) fn flat(bits_per_value: u64) -> Self {
+        Self {
+            bits_per_value,
+            inner: RleChildDecompressorInner::Flat,
+        }
+    }
+
+    pub(crate) fn block(
+        bits_per_value: u64,
+        decompressor: Box<dyn BlockDecompressor>,
+        requires_num_values: bool,
+    ) -> Self {
+        Self {
+            bits_per_value,
+            inner: RleChildDecompressorInner::Block {
+                decompressor,
+                requires_num_values,
+            },
+        }
+    }
+
+    pub(crate) fn bits_per_value(&self) -> u64 {
+        self.bits_per_value
+    }
+
+    pub(crate) fn requires_num_values(&self) -> bool {
+        match &self.inner {
+            RleChildDecompressorInner::Flat => false,
+            RleChildDecompressorInner::Block {
+                requires_num_values,
+                ..
+            } => *requires_num_values,
+        }
+    }
+
+    pub(crate) fn is_identity(&self) -> bool {
+        matches!(self.inner, RleChildDecompressorInner::Flat)
+    }
+
+    fn decode(
+        &self,
+        data: LanceBuffer,
+        num_values: Option<u64>,
+        label: &str,
+    ) -> Result<LanceBuffer> {
+        match &self.inner {
+            RleChildDecompressorInner::Flat => Ok(data),
+            RleChildDecompressorInner::Block {
+                decompressor,
+                requires_num_values,
+            } => {
+                let num_values = if *requires_num_values {
+                    num_values.ok_or_else(|| {
+                        Error::invalid_input_source(
+                            format!("RLE {label} child compression requires the run count").into(),
+                        )
+                    })?
+                } else {
+                    num_values.unwrap_or(0)
+                };
+                let decoded = decompressor.decompress(data, num_values)?;
+                self.extract_fixed_width(decoded, num_values, label)
+            }
+        }
+    }
+
+    fn extract_fixed_width(
+        &self,
+        data: DataBlock,
+        expected_num_values: u64,
+        label: &str,
+    ) -> Result<LanceBuffer> {
+        match data {
+            DataBlock::FixedWidth(block) => {
+                if block.bits_per_value != self.bits_per_value {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE {label} child decoded {}-bit values, expected {}",
+                            block.bits_per_value, self.bits_per_value
+                        )
+                        .into(),
+                    ));
+                }
+                if expected_num_values != 0 && block.num_values != expected_num_values {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE {label} child decoded {} values, expected {}",
+                            block.num_values, expected_num_values
+                        )
+                        .into(),
+                    ));
+                }
+                Ok(block.data)
             }
             _ => Err(Error::invalid_input_source(
-                "RLE encoding only supports FixedWidth data blocks".into(),
+                format!("RLE {label} child decoded to a non fixed-width block").into(),
             )),
         }
     }
 }
 
-/// RLE decompressor for miniblock format
-#[derive(Debug)]
-pub struct RleDecompressor {
-    bits_per_value: u64,
-}
-
 impl RleDecompressor {
     pub fn new(bits_per_value: u64) -> Self {
-        Self { bits_per_value }
+        Self {
+            bits_per_value,
+            run_length_width: RunLengthWidth::U8,
+            values: RleChildDecompressor::flat(bits_per_value),
+            run_lengths: RleChildDecompressor::flat(RunLengthWidth::U8.bits_per_value()),
+        }
+    }
+
+    pub(crate) fn with_run_length_width(
+        bits_per_value: u64,
+        run_length_width: RunLengthWidth,
+    ) -> Self {
+        Self {
+            bits_per_value,
+            run_length_width,
+            values: RleChildDecompressor::flat(bits_per_value),
+            run_lengths: RleChildDecompressor::flat(run_length_width.bits_per_value()),
+        }
+    }
+
+    pub(crate) fn with_child_decompressors(
+        bits_per_value: u64,
+        run_length_width: RunLengthWidth,
+        values: RleChildDecompressor,
+        run_lengths: RleChildDecompressor,
+    ) -> Self {
+        Self {
+            bits_per_value,
+            run_length_width,
+            values,
+            run_lengths,
+        }
     }
 
-    fn decode_data(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
+    fn decode_data(
+        &self,
+        data: Vec<LanceBuffer>,
+        num_values: u64,
+        clamp_overflow: bool,
+    ) -> Result<DataBlock> {
         if num_values == 0 {
             return Ok(DataBlock::FixedWidth(FixedWidthDataBlock {
                 bits_per_value: self.bits_per_value,
@@ -445,15 +1314,46 @@ impl RleDecompressor {
             ));
         }
 
-        let values_buffer = &data[0];
-        let lengths_buffer = &data[1];
+        let mut data_iter = data.into_iter();
+        let values_buffer = data_iter.next().unwrap();
+        let lengths_buffer = data_iter.next().unwrap();
+        let (values_buffer, lengths_buffer) =
+            self.decode_child_buffers(values_buffer, lengths_buffer)?;
 
         let decoded_data = match self.bits_per_value {
-            8 => self.decode_generic::<u8>(values_buffer, lengths_buffer, num_values)?,
-            16 => self.decode_generic::<u16>(values_buffer, lengths_buffer, num_values)?,
-            32 => self.decode_generic::<u32>(values_buffer, lengths_buffer, num_values)?,
-            64 => self.decode_generic::<u64>(values_buffer, lengths_buffer, num_values)?,
-            _ => unreachable!("RLE decoding bits_per_value must be 8, 16, 32, 64, or 128"),
+            8 => self.decode_generic::<u8>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            16 => self.decode_generic::<u16>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            32 => self.decode_generic::<u32>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            64 => self.decode_generic::<u64>(
+                &values_buffer,
+                &lengths_buffer,
+                num_values,
+                clamp_overflow,
+            )?,
+            _ => {
+                return Err(Error::invalid_input_source(
+                    format!(
+                        "RLE decoding bits_per_value must be 8, 16, 32, or 64, got {}",
+                        self.bits_per_value
+                    )
+                    .into(),
+                ));
+            }
         };
 
         Ok(DataBlock::FixedWidth(FixedWidthDataBlock {
@@ -464,16 +1364,80 @@ impl RleDecompressor {
         }))
     }
 
+    fn decode_child_buffers(
+        &self,
+        values_buffer: LanceBuffer,
+        lengths_buffer: LanceBuffer,
+    ) -> Result<(LanceBuffer, LanceBuffer)> {
+        let values_requires_num_runs = self.values.requires_num_values();
+        let lengths_requires_num_runs = self.run_lengths.requires_num_values();
+        if values_requires_num_runs && lengths_requires_num_runs {
+            return Err(Error::invalid_input_source(
+                "RLE values and run lengths child compression both require the run count".into(),
+            ));
+        }
+
+        if values_requires_num_runs {
+            let lengths_buffer = self
+                .run_lengths
+                .decode(lengths_buffer, None, "run lengths")?;
+            let num_runs = Self::num_child_values(
+                &lengths_buffer,
+                self.run_lengths.bits_per_value(),
+                "run lengths",
+            )?;
+            let values_buffer = self
+                .values
+                .decode(values_buffer, Some(num_runs), "values")?;
+            Ok((values_buffer, lengths_buffer))
+        } else if lengths_requires_num_runs {
+            let values_buffer = self.values.decode(values_buffer, None, "values")?;
+            let num_runs =
+                Self::num_child_values(&values_buffer, self.values.bits_per_value(), "values")?;
+            let lengths_buffer =
+                self.run_lengths
+                    .decode(lengths_buffer, Some(num_runs), "run lengths")?;
+            Ok((values_buffer, lengths_buffer))
+        } else {
+            let values_buffer = self.values.decode(values_buffer, None, "values")?;
+            let lengths_buffer = self
+                .run_lengths
+                .decode(lengths_buffer, None, "run lengths")?;
+            Ok((values_buffer, lengths_buffer))
+        }
+    }
+
+    fn num_child_values(buffer: &LanceBuffer, bits_per_value: u64, label: &str) -> Result<u64> {
+        let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE {label} child bit width is too large: {bits_per_value}").into(),
+            )
+        })?;
+        if bytes_per_value == 0 || !buffer.len().is_multiple_of(bytes_per_value) {
+            return Err(Error::invalid_input_source(
+                format!(
+                    "RLE {label} child decoded to {} bytes, not divisible by {}",
+                    buffer.len(),
+                    bytes_per_value
+                )
+                .into(),
+            ));
+        }
+        Ok((buffer.len() / bytes_per_value) as u64)
+    }
+
     fn decode_generic<T>(
         &self,
         values_buffer: &LanceBuffer,
         lengths_buffer: &LanceBuffer,
         num_values: u64,
+        clamp_overflow: bool,
     ) -> Result<LanceBuffer>
     where
         T: bytemuck::Pod + Copy + std::fmt::Debug + ArrowNativeType,
     {
         let type_size = std::mem::size_of::<T>();
+        let length_size = self.run_length_width.bytes_per_value();
 
         if values_buffer.is_empty() || lengths_buffer.is_empty() {
             if num_values == 0 {
@@ -485,19 +1449,22 @@ impl RleDecompressor {
             }
         }
 
-        if !values_buffer.len().is_multiple_of(type_size) || lengths_buffer.is_empty() {
+        if !values_buffer.len().is_multiple_of(type_size)
+            || !lengths_buffer.len().is_multiple_of(length_size)
+        {
             return Err(Error::invalid_input_source(format!(
-                "Invalid buffer sizes for RLE {} decoding: values {} bytes (not divisible by {}), lengths {} bytes",
+                "Invalid buffer sizes for RLE {} decoding: values {} bytes (not divisible by {}), lengths {} bytes (not divisible by {})",
                 std::any::type_name::<T>(),
                 values_buffer.len(),
                 type_size,
-                lengths_buffer.len()
+                lengths_buffer.len(),
+                length_size
             )
             .into()));
         }
 
         let num_runs = values_buffer.len() / type_size;
-        let num_length_entries = lengths_buffer.len();
+        let num_length_entries = lengths_buffer.len() / length_size;
         if num_runs != num_length_entries {
             return Err(Error::invalid_input_source(
                 format!(
@@ -510,26 +1477,55 @@ impl RleDecompressor {
 
         let values_ref = values_buffer.borrow_to_typed_slice::<T>();
         let values: &[T] = values_ref.as_ref();
-        let lengths: &[u8] = lengths_buffer.as_ref();
-
-        let expected_value_count = num_values as usize;
-        let mut decoded: Vec<T> = Vec::with_capacity(expected_value_count);
-
-        for (value, &length) in values.iter().zip(lengths.iter()) {
-            if decoded.len() == expected_value_count {
-                break;
-            }
+        let lengths = lengths_buffer.as_ref();
 
+        let expected_value_count = usize::try_from(num_values).map_err(|_| {
+            Error::invalid_input_source(
+                format!("RLE num_values does not fit in usize: {num_values}").into(),
+            )
+        })?;
+        // Legacy miniblock encoders rolled back to a power-of-2 checkpoint after a run
+        // had already crossed it, so a chunk's run lengths can sum past its declared
+        // value count (the excess values are re-encoded at the start of the next chunk).
+        // The pre-run-length-width decoder truncated the excess, so miniblock decoding
+        // clamps rather than rejects to keep those files readable. Block payloads never
+        // legitimately overflow, so they decode strictly.
+        let mut decoded: Vec<T> = Vec::new();
+        decoded
+            .try_reserve_exact(expected_value_count)
+            .map_err(|_| {
+                Error::invalid_input_source(
+                    format!("RLE decoding cannot allocate {expected_value_count} values").into(),
+                )
+            })?;
+        for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) {
+            let length = self.run_length_width.read_length(length_bytes);
             if length == 0 {
                 return Err(Error::invalid_input_source(
                     "RLE decoding encountered a zero run length".into(),
                 ));
             }
-
+            let length = usize::try_from(length).map_err(|_| {
+                Error::invalid_input_source(
+                    format!("RLE run length does not fit in usize: {length}").into(),
+                )
+            })?;
             let remaining = expected_value_count - decoded.len();
-            let write_len = (length as usize).min(remaining);
-
-            decoded.resize(decoded.len() + write_len, *value);
+            if length > remaining {
+                if !clamp_overflow {
+                    return Err(Error::invalid_input_source(
+                        format!(
+                            "RLE decoding overflowed expected value count: produced at least {}, expected {}",
+                            decoded.len() + length,
+                            expected_value_count
+                        )
+                        .into(),
+                    ));
+                }
+                decoded.resize(expected_value_count, *value);
+                break;
+            }
+            decoded.resize(decoded.len() + length, *value);
         }
 
         if decoded.len() != expected_value_count {
@@ -554,7 +1550,7 @@ impl RleDecompressor {
 
 impl MiniBlockDecompressor for RleDecompressor {
     fn decompress(&self, data: Vec<LanceBuffer>, num_values: u64) -> Result<DataBlock> {
-        self.decode_data(data, num_values)
+        self.decode_data(data, num_values, true)
     }
 }
 
@@ -591,17 +1587,24 @@ impl BlockDecompressor for RleDecompressor {
         let values_buffer = data.slice_with_length(values_start, values_size);
         let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start);
 
-        self.decode_data(vec![values_buffer, lengths_buffer], num_values)
+        self.decode_data(vec![values_buffer, lengths_buffer], num_values, false)
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy};
     use crate::data::DataBlock;
     use crate::encodings::logical::primitive::miniblock::MAX_MINIBLOCK_VALUES;
-    use crate::{buffer::LanceBuffer, compression::BlockDecompressor};
+    use crate::encodings::physical::block::{CompressionConfig, CompressionScheme};
+    use crate::{
+        buffer::LanceBuffer,
+        compression::{BlockCompressor, BlockDecompressor},
+    };
     use arrow_array::Int32Array;
+    use rstest::rstest;
+
     // ========== Core Functionality Tests ==========
 
     #[test]
@@ -641,6 +1644,371 @@ mod tests {
         assert_eq!(lengths_buffer.len(), 6);
     }
 
+    #[test]
+    fn test_rle_v2_u16_miniblock_encoding() {
+        let encoder = RleEncoder::with_run_length_width(RunLengthWidth::U16);
+
+        let data = vec![42i32; 1000];
+        let array = Int32Array::from(data);
+        let (compressed, encoding) =
+            MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap();
+
+        assert_eq!(compressed.data[0].len(), 4);
+        assert_eq!(compressed.data[1].len(), 2);
+        assert_eq!(compressed.data[1].as_ref(), &1000u16.to_le_bytes());
+
+        let rle = match encoding.compression.as_ref().unwrap() {
+            crate::format::pb21::compressive_encoding::Compression::Rle(rle) => rle,
+            other => panic!("expected RLE encoding, got {other:?}"),
+        };
+        let run_lengths = rle.run_lengths.as_ref().unwrap();
+        let flat = match run_lengths.compression.as_ref().unwrap() {
+            crate::format::pb21::compressive_encoding::Compression::Flat(flat) => flat,
+            other => panic!("expected flat run lengths, got {other:?}"),
+        };
+        assert_eq!(flat.bits_per_value, 16);
+
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let decompressed = MiniBlockDecompressor::decompress(
+            &decompressor,
+            compressed.data,
+            compressed.num_values,
+        )
+        .unwrap();
+        match decompressed {
+            DataBlock::FixedWidth(block) => {
+                let values = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(values.as_ref(), vec![42i32; 1000]);
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_values_child() {
+        let compression = test_general_compression();
+        let encoder =
+            RleEncoder::with_child_encoding(RunLengthWidth::U8, Some(compression), None, false);
+        let array = Int32Array::from(repeating_runs(1024, 4));
+        let (compressed, encoding) =
+            MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded =
+            MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4)
+                .unwrap();
+        assert_decoded_i32_eq(decoded, &repeating_runs(1024, 4));
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_run_lengths_child() {
+        let compression = test_general_compression();
+        let encoder =
+            RleEncoder::with_child_encoding(RunLengthWidth::U8, None, Some(compression), false);
+        let expected = repeating_runs(1024, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded =
+            MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4)
+                .unwrap();
+        assert_decoded_i32_eq(decoded, &expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacked_run_lengths_child() {
+        use crate::encodings::physical::bitpacking::OutOfLineBitpacking;
+
+        let expected = repeating_runs(1024, 4);
+        let (compressed, _) = MiniBlockCompressor::compress(
+            &RleEncoder::new(),
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+        let run_lengths = compressed.data[1].clone();
+        let num_runs = run_lengths.len() as u64;
+        let run_lengths_block = DataBlock::FixedWidth(FixedWidthDataBlock {
+            bits_per_value: 8,
+            data: run_lengths,
+            num_values: num_runs,
+            block_info: BlockInfo::default(),
+        });
+        let bitpacked_run_lengths =
+            BlockCompressor::compress(&OutOfLineBitpacking::new(3, 8), run_lengths_block).unwrap();
+        let encoding = ProtobufUtils21::rle(
+            ProtobufUtils21::flat(32, None),
+            ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)),
+        );
+
+        let decompressor = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap();
+        let decoded = MiniBlockDecompressor::decompress(
+            decompressor.as_ref(),
+            vec![compressed.data[0].clone(), bitpacked_run_lengths],
+            expected.len() as u64,
+        )
+        .unwrap();
+        assert_decoded_i32_eq(decoded, &expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_rejects_two_count_dependent_child_encodings() {
+        let encoding = ProtobufUtils21::rle(
+            ProtobufUtils21::out_of_line_bitpacking(32, ProtobufUtils21::flat(3, None)),
+            ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)),
+        );
+
+        let err = DefaultDecompressionStrategy::default()
+            .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default())
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("cannot both require the run count")
+        );
+    }
+
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_general_compression() -> CompressionConfig {
+        if cfg!(feature = "zstd") {
+            CompressionConfig::new(CompressionScheme::Zstd, Some(3))
+        } else {
+            CompressionConfig::new(CompressionScheme::Lz4, None)
+        }
+    }
+
+    fn repeating_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        for run in 0..num_runs {
+            values.extend(std::iter::repeat_n((run % 8) as i32, run_length));
+        }
+        values
+    }
+
+    fn expect_rle(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle {
+        match encoding.compression.as_ref().unwrap() {
+            crate::format::pb21::compressive_encoding::Compression::Rle(rle) => rle,
+            other => panic!("expected RLE encoding, got {other:?}"),
+        }
+    }
+
+    fn assert_decoded_i32_eq(decoded: DataBlock, expected: &[i32]) {
+        match decoded {
+            DataBlock::FixedWidth(block) => {
+                let values = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(values.as_ref(), expected);
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+    }
+
+    #[test]
+    #[cfg(any(feature = "lz4", feature = "zstd"))]
+    fn test_rle_miniblock_compressed_children_multiple_chunks() {
+        let compression = test_general_compression();
+        let encoder = RleEncoder::with_child_encoding(
+            RunLengthWidth::U8,
+            Some(compression),
+            Some(compression),
+            false,
+        );
+        let expected = repeating_runs(8192, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        assert!(compressed.chunks.len() > 1);
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::General(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacks_values_child_when_smaller() {
+        let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true);
+        let expected = monotonic_runs(2048, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    #[test]
+    #[cfg(feature = "bitpacking")]
+    fn test_rle_miniblock_bitpacks_run_lengths_when_values_do_not_shrink() {
+        let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true);
+        let expected = high_entropy_runs(2048, 4);
+        let (compressed, encoding) = MiniBlockCompressor::compress(
+            &encoder,
+            DataBlock::from_array(Int32Array::from(expected.clone())),
+        )
+        .unwrap();
+
+        let rle = expect_rle(&encoding);
+        assert!(matches!(
+            rle.values.as_ref().unwrap().compression.as_ref().unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::Flat(_)
+        ));
+        assert!(matches!(
+            rle.run_lengths
+                .as_ref()
+                .unwrap()
+                .compression
+                .as_ref()
+                .unwrap(),
+            crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_)
+        ));
+
+        let decoded = decompress_i32_chunks(&compressed, &encoding);
+        assert_eq!(decoded, expected);
+    }
+
+    fn decompress_i32_chunks(
+        compressed: &MiniBlockCompressed,
+        encoding: &CompressiveEncoding,
+    ) -> Vec<i32> {
+        let strategy = DefaultDecompressionStrategy::default();
+        let decompressor = strategy
+            .create_miniblock_decompressor(encoding, &strategy)
+            .unwrap();
+        let mut offsets = vec![0usize; compressed.data.len()];
+        let mut values_processed = 0u64;
+        let mut decoded_values = Vec::new();
+
+        for chunk in &compressed.chunks {
+            let chunk_values = chunk.num_values(values_processed, compressed.num_values);
+            let mut chunk_buffers = Vec::with_capacity(chunk.buffer_sizes.len());
+            for (idx, size) in chunk.buffer_sizes.iter().enumerate() {
+                let size = *size as usize;
+                chunk_buffers.push(compressed.data[idx].slice_with_length(offsets[idx], size));
+                offsets[idx] += size;
+            }
+
+            let decoded = decompressor
+                .decompress(chunk_buffers, chunk_values)
+                .unwrap();
+            match decoded {
+                DataBlock::FixedWidth(block) => {
+                    let values = block.data.borrow_to_typed_slice::<i32>();
+                    decoded_values.extend_from_slice(values.as_ref());
+                }
+                _ => panic!("Expected FixedWidth block"),
+            }
+            values_processed += chunk_values;
+        }
+
+        assert_eq!(values_processed, compressed.num_values);
+        decoded_values
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn monotonic_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        for run in 0..num_runs {
+            values.extend(std::iter::repeat_n(run as i32, run_length));
+        }
+        values
+    }
+
+    #[cfg(feature = "bitpacking")]
+    fn high_entropy_runs(num_runs: usize, run_length: usize) -> Vec<i32> {
+        let mut values = Vec::with_capacity(num_runs * run_length);
+        let mut state = 7u64;
+        for _ in 0..num_runs {
+            state = state
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407);
+            values.extend(std::iter::repeat_n((state >> 32) as i32, run_length));
+        }
+        values
+    }
+
+    #[test]
+    fn test_select_run_length_width_prefers_u16_for_long_runs() {
+        let mut entries = [0u64; 3];
+        accumulate_run_length_entries(300, Some(*MAX_MINIBLOCK_VALUES), &mut entries);
+        let (width, _) = select_run_length_width_from_entries(&entries, 32).unwrap();
+        assert_eq!(width, RunLengthWidth::U16);
+    }
+
     // ========== Round-trip Tests for Different Types ==========
 
     #[test]
@@ -726,8 +2094,73 @@ mod tests {
         }
     }
 
+    #[rstest]
+    #[case::u8_lengths(RunLengthWidth::U8)]
+    #[case::u16_lengths(RunLengthWidth::U16)]
+    #[case::u32_lengths(RunLengthWidth::U32)]
+    fn test_miniblock_chunk_counts_match_encoded_runs(#[case] run_length_width: RunLengthWidth) {
+        // This pattern crosses the 2,048-value boundary in the middle of a two-value run.
+        let levels = (0..4098)
+            .map(|index| if index % 3 == 0 { 1u16 } else { 0u16 })
+            .collect::<Vec<_>>();
+        let num_values = levels.len() as u64;
+        let encoder = RleEncoder::with_run_length_width(run_length_width);
+        let (buffers, chunks) = encoder
+            .encode_data(
+                &LanceBuffer::reinterpret_vec(levels),
+                num_values,
+                u16::BITS as u64,
+            )
+            .unwrap();
+
+        assert_eq!(buffers.len(), 2);
+        let bytes_per_length = run_length_width.bytes_per_value();
+        let mut values_offset = 0usize;
+        let mut lengths_offset = 0usize;
+        let mut values_processed = 0u64;
+
+        for chunk in &chunks {
+            let values_size = chunk.buffer_sizes[0] as usize;
+            let lengths_size = chunk.buffer_sizes[1] as usize;
+            let lengths_end = lengths_offset + lengths_size;
+            let chunk_lengths = &buffers[1].as_ref()[lengths_offset..lengths_end];
+            let length_chunks = chunk_lengths.chunks_exact(bytes_per_length);
+            assert!(length_chunks.remainder().is_empty());
+            let num_runs = length_chunks.len();
+            let encoded_values = length_chunks
+                .map(|bytes| run_length_width.read_length(bytes))
+                .sum::<u64>();
+            let declared_values = chunk.num_values(values_processed, num_values);
+
+            assert_eq!(values_size, num_runs * size_of::<u16>());
+            assert_eq!(encoded_values, declared_values);
+
+            values_offset += values_size;
+            lengths_offset = lengths_end;
+            values_processed += declared_values;
+        }
+
+        assert_eq!(values_processed, num_values);
+        assert_eq!(values_offset, buffers[0].len());
+        assert_eq!(lengths_offset, buffers[1].len());
+    }
+
     // ========== Error Handling Tests ==========
 
+    #[test]
+    fn test_encoder_rejects_zero_progress() {
+        let error = RleEncoder::new()
+            .encode_data(&LanceBuffer::empty(), 1, u16::BITS as u64)
+            .unwrap_err();
+
+        assert!(
+            matches!(&error, Error::Internal { .. }),
+            "expected internal error, got: {error:?}"
+        );
+        assert!(error.to_string().contains("made no progress"));
+        assert!(error.to_string().contains("values_remaining=1"));
+    }
+
     #[test]
     fn test_invalid_buffer_count() {
         let decompressor = RleDecompressor::new(32);
@@ -760,6 +2193,118 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_u16_length_buffer_must_be_aligned() {
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let values = LanceBuffer::from(vec![1, 0, 0, 0]);
+        let lengths = LanceBuffer::from(vec![5]);
+        let result = MiniBlockDecompressor::decompress(&decompressor, vec![values, lengths], 5);
+        assert!(matches!(&result, Err(Error::InvalidInput { .. })));
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("not divisible by 2")
+        );
+    }
+
+    #[test]
+    fn test_rle_rejects_underflow_and_zero_lengths_and_clamps_overflow() {
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let value = LanceBuffer::from(1i32.to_le_bytes().to_vec());
+
+        let underflow = MiniBlockDecompressor::decompress(
+            &decompressor,
+            vec![
+                value.clone(),
+                LanceBuffer::from(4u16.to_le_bytes().to_vec()),
+            ],
+            5,
+        )
+        .unwrap_err();
+        assert!(underflow.to_string().contains("produced 4 values"));
+
+        let overflow = MiniBlockDecompressor::decompress(
+            &decompressor,
+            vec![
+                value.clone(),
+                LanceBuffer::from(6u16.to_le_bytes().to_vec()),
+            ],
+            5,
+        )
+        .unwrap();
+        match overflow {
+            DataBlock::FixedWidth(block) => {
+                assert_eq!(block.num_values, 5);
+                let decoded = block.data.borrow_to_typed_slice::<i32>();
+                assert_eq!(decoded.as_ref(), &[1i32; 5]);
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+
+        let zero = MiniBlockDecompressor::decompress(
+            &decompressor,
+            vec![value, LanceBuffer::from(0u16.to_le_bytes().to_vec())],
+            5,
+        )
+        .unwrap_err();
+        assert!(zero.to_string().contains("zero run length"));
+    }
+
+    #[test]
+    fn test_block_rle_rejects_overflow() {
+        // Block payloads have no chunk boundaries, so run lengths summing past
+        // num_values can only be corruption and must stay a hard error.
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let values = 1i32.to_le_bytes();
+        let lengths = 6u16.to_le_bytes();
+        let mut payload = Vec::new();
+        payload.extend_from_slice(&(values.len() as u64).to_le_bytes());
+        payload.extend_from_slice(&values);
+        payload.extend_from_slice(&lengths);
+
+        let error = BlockDecompressor::decompress(&decompressor, LanceBuffer::from(payload), 5)
+            .unwrap_err();
+        assert!(matches!(&error, Error::InvalidInput { .. }));
+        assert!(
+            error
+                .to_string()
+                .contains("overflowed expected value count")
+        );
+    }
+
+    #[test]
+    fn test_rle_truncates_legacy_chunk_boundary_overflow() {
+        // Legacy encoders emitted chunks declaring 2048 values whose final run crossed
+        // the checkpoint boundary (e.g. run lengths summing to 2080); the excess values
+        // are duplicated at the start of the next chunk and must be ignored here.
+        let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16);
+        let mut values = Vec::new();
+        values.extend_from_slice(&7i32.to_le_bytes());
+        values.extend_from_slice(&8i32.to_le_bytes());
+        let mut lengths = Vec::new();
+        lengths.extend_from_slice(&2000u16.to_le_bytes());
+        lengths.extend_from_slice(&80u16.to_le_bytes());
+
+        let decoded = MiniBlockDecompressor::decompress(
+            &decompressor,
+            vec![LanceBuffer::from(values), LanceBuffer::from(lengths)],
+            2048,
+        )
+        .unwrap();
+        match decoded {
+            DataBlock::FixedWidth(block) => {
+                assert_eq!(block.num_values, 2048);
+                let decoded = block.data.borrow_to_typed_slice::<i32>();
+                let decoded = decoded.as_ref();
+                assert_eq!(decoded.len(), 2048);
+                assert!(decoded[..2000].iter().all(|&v| v == 7));
+                assert!(decoded[2000..].iter().all(|&v| v == 8));
+            }
+            _ => panic!("Expected FixedWidth block"),
+        }
+    }
+
     #[test]
     fn test_empty_data_handling() {
         let encoder = RleEncoder::new();
@@ -1098,6 +2643,30 @@ mod tests {
         ); // 20% variety
         let arr = Arc::new(Int32Array::from(values)) as Arc<dyn Array>;
         check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await;
+
+        #[cfg(any(feature = "lz4", feature = "zstd"))]
+        {
+            let mut metadata = HashMap::new();
+            metadata.insert(
+                "lance-encoding:rle-threshold".to_string(),
+                "0.8".to_string(),
+            );
+            metadata.insert("lance-encoding:bss".to_string(), "off".to_string());
+            metadata.insert(
+                "lance-encoding:compression".to_string(),
+                if cfg!(feature = "zstd") {
+                    "zstd".to_string()
+                } else {
+                    "lz4".to_string()
+                },
+            );
+            let mut values = Vec::with_capacity(2048 * 4);
+            for run in 0..2048 {
+                values.extend(std::iter::repeat_n(i32::MIN + (run % 8), 4));
+            }
+            let arr = Arc::new(Int32Array::from(values)) as Arc<dyn Array>;
+            check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await;
+        }
     }
 
     /// Generator that produces repetitive patterns suitable for RLE
diff --git a/rust/lance-encoding/src/encodings/physical/value.rs b/rust/lance-encoding/src/encodings/physical/value.rs
index c49bbd3efbd..606f49b699a 100644
--- a/rust/lance-encoding/src/encodings/physical/value.rs
+++ b/rust/lance-encoding/src/encodings/physical/value.rs
@@ -27,7 +27,7 @@ pub struct ValueEncoder {}
 
 impl ValueEncoder {
     /// Use the largest chunk we can smaller than 4KiB
-    fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> (u64, u64) {
+    fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> Result<(u64, u64)> {
         let mut size_bytes = 2 * bytes_per_word;
         let (mut log_num_vals, mut num_vals) = match values_per_word {
             1 => (1, 2),
@@ -35,8 +35,14 @@ impl ValueEncoder {
             _ => unreachable!(),
         };
 
-        // If the type is so wide that we can't even fit 2 values we shouldn't be here
-        assert!(size_bytes < MAX_MINIBLOCK_BYTES);
+        if size_bytes >= MAX_MINIBLOCK_BYTES {
+            let num_values = 2 * values_per_word;
+            return Err(Error::invalid_input(format!(
+                "Value is too wide for miniblock encoding: {} values require {} bytes but a \
+                 miniblock chunk is limited to {} bytes.",
+                num_values, size_bytes, MAX_MINIBLOCK_BYTES
+            )));
+        }
 
         while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= *MAX_MINIBLOCK_VALUES {
             log_num_vals += 1;
@@ -44,10 +50,10 @@ impl ValueEncoder {
             num_vals *= 2;
         }
 
-        (log_num_vals, num_vals)
+        Ok((log_num_vals, num_vals))
     }
 
-    fn chunk_data(data: FixedWidthDataBlock) -> MiniBlockCompressed {
+    fn chunk_data(data: FixedWidthDataBlock) -> Result<MiniBlockCompressed> {
         // Usually there are X bytes per value.  However, when working with boolean
         // or FSL<boolean> we might have some number of bits per value that isn't
         // divisible by 8.  In this case, to avoid chunking in the middle of a byte
@@ -60,7 +66,7 @@ impl ValueEncoder {
 
         // Aim for 4KiB chunks
         let (log_vals_per_chunk, vals_per_chunk) =
-            Self::find_log_vals_per_chunk(bytes_per_word, values_per_word);
+            Self::find_log_vals_per_chunk(bytes_per_word, values_per_word)?;
         let num_chunks = bit_util::ceil(data.num_values as usize, vals_per_chunk as usize);
         debug_assert_eq!(vals_per_chunk % values_per_word, 0);
         let bytes_per_chunk = bytes_per_word * (vals_per_chunk / values_per_word);
@@ -99,11 +105,11 @@ impl ValueEncoder {
 
         debug_assert_eq!(chunks.len(), num_chunks);
 
-        MiniBlockCompressed {
+        Ok(MiniBlockCompressed {
             chunks,
             data: vec![data_buffer],
             num_values: data.num_values,
-        }
+        })
     }
 }
 
@@ -177,7 +183,7 @@ impl ValueEncoder {
         data: FixedWidthDataBlock,
         layers: Vec<MiniblockFslLayer>,
         num_rows: u64,
-    ) -> (MiniBlockCompressed, CompressiveEncoding) {
+    ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> {
         // Count size to calculate rows per chunk
         let mut ceil_bytes_validity = 0;
         let mut cum_dim = 1;
@@ -198,7 +204,7 @@ impl ValueEncoder {
         };
         let est_bytes_per_word = (ceil_bytes_validity * vals_per_word) + cum_bytes_per_word;
         let (log_rows_per_chunk, rows_per_chunk) =
-            Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word);
+            Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word)?;
 
         let num_chunks = num_rows.div_ceil(rows_per_chunk) as usize;
 
@@ -258,17 +264,17 @@ impl ValueEncoder {
             .chain(std::iter::once(data.data))
             .collect::<Vec<_>>();
 
-        (
+        Ok((
             MiniBlockCompressed {
                 chunks,
                 data: buffers,
                 num_values: num_rows,
             },
             encoding,
-        )
+        ))
     }
 
-    fn miniblock_fsl(data: DataBlock) -> (MiniBlockCompressed, CompressiveEncoding) {
+    fn miniblock_fsl(data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> {
         let num_rows = data.num_values();
         let fsl = data.as_fixed_size_list().unwrap();
         let mut layers = Vec::new();
@@ -469,9 +475,9 @@ impl MiniBlockCompressor for ValueEncoder {
         match chunk {
             DataBlock::FixedWidth(fixed_width) => {
                 let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None);
-                Ok((Self::chunk_data(fixed_width), encoding))
+                Ok((Self::chunk_data(fixed_width)?, encoding))
             }
-            DataBlock::FixedSizeList(_) => Ok(Self::miniblock_fsl(chunk)),
+            DataBlock::FixedSizeList(_) => Self::miniblock_fsl(chunk),
             _ => Err(Error::invalid_input_source(
                 format!(
                     "Cannot compress a data block of type {} with ValueEncoder",
@@ -989,6 +995,59 @@ mod tests {
         assert_eq!(decompressed.as_ref(), &sample_list);
     }
 
+    fn wide_fixed_size_binary() -> ArrayRef {
+        let wide_value = vec![0xABu8; 5000];
+        Arc::new(
+            arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                std::iter::repeat_n(Some(wide_value.as_slice()), 4),
+                5000,
+            )
+            .unwrap(),
+        )
+    }
+
+    fn wide_fixed_size_list_bool() -> ArrayRef {
+        // A wide FSL<Boolean> is sub-byte, so it chunks eight values per word and the
+        // smallest unit is 16 values rather than 2.
+        let dimension = 4095;
+        let values = arrow_array::BooleanArray::from(vec![false; dimension * 2]);
+        let field = Arc::new(Field::new("item", DataType::Boolean, true));
+        Arc::new(FixedSizeListArray::new(
+            field,
+            dimension as i32,
+            Arc::new(values),
+            None,
+        ))
+    }
+
+    #[rstest::rstest]
+    #[case::fixed_size_binary(wide_fixed_size_binary(), 2)]
+    #[case::fixed_size_list_bool(wide_fixed_size_list_bool(), 16)]
+    fn test_wide_value_miniblock_returns_error(
+        #[case] array: ArrayRef,
+        #[case] expected_min_values: u64,
+    ) {
+        let starting_data = DataBlock::from_array(array);
+
+        let encoder = ValueEncoder::default();
+        let result = MiniBlockCompressor::compress(&encoder, starting_data);
+
+        let err = result.expect_err("wide values should not be encodable as miniblock");
+        assert!(
+            matches!(err, lance_core::Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+        let msg = err.to_string();
+        assert!(
+            msg.contains("too wide for miniblock encoding"),
+            "unexpected error message: {msg}"
+        );
+        assert!(
+            msg.contains(&format!("{expected_min_values} values require")),
+            "unexpected error message: {msg}"
+        );
+    }
+
     #[test]
     fn test_fsl_value_compression_per_value() {
         let sample_list = create_simple_fsl();
diff --git a/rust/lance-encoding/src/lib.rs b/rust/lance-encoding/src/lib.rs
index cb4062d3220..a58e0a14c59 100644
--- a/rust/lance-encoding/src/lib.rs
+++ b/rust/lance-encoding/src/lib.rs
@@ -86,6 +86,22 @@ pub trait EncodingsIo: std::fmt::Debug + Send + Sync {
     fn with_bypass_backpressure(&self) -> Option<Arc<dyn EncodingsIo>> {
         None
     }
+
+    /// Returns a version of this I/O service that additionally records the I/O it
+    /// performs into `stats`, on top of any global accounting.  This is the seam
+    /// used to measure exact per-scope (e.g. per-query) I/O without re-opening
+    /// files: wrap a reader's I/O service, perform the reads, then inspect the
+    /// recorder.
+    ///
+    /// Returns `None` if this implementation does not support per-scope I/O
+    /// statistics (e.g. in-memory or test schedulers), in which case the caller
+    /// should fall back to using self (and no statistics are recorded).
+    fn with_io_stats(
+        &self,
+        _stats: Arc<dyn lance_core::utils::io_stats::IoStatsRecorder>,
+    ) -> Option<Arc<dyn EncodingsIo>> {
+        None
+    }
 }
 
 /// An implementation of EncodingsIo that serves data from an in-memory buffer
diff --git a/rust/lance-encoding/src/previous/encodings/logical/blob.rs b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
index 13fa3b346cb..20f6afc5b36 100644
--- a/rust/lance-encoding/src/previous/encodings/logical/blob.rs
+++ b/rust/lance-encoding/src/previous/encodings/logical/blob.rs
@@ -318,7 +318,7 @@ impl BlobFieldEncoder {
             .nulls()
             .cloned()
             .unwrap_or(NullBuffer::new_valid(binarray.len()));
-        for (w, is_valid) in binarray.value_offsets().windows(2).zip(nulls.into_iter()) {
+        for (w, is_valid) in binarray.value_offsets().windows(2).zip(&nulls) {
             if is_valid {
                 let start = w[0] as u64;
                 let end = w[1] as u64;
diff --git a/rust/lance-encoding/src/repdef.rs b/rust/lance-encoding/src/repdef.rs
index bdff69e0e4d..711ab732928 100644
--- a/rust/lance-encoding/src/repdef.rs
+++ b/rust/lance-encoding/src/repdef.rs
@@ -122,9 +122,9 @@ use crate::buffer::LanceBuffer;
 
 pub type LevelBuffer = Vec<u16>;
 
-/// A contiguous top-level-row range that can be encoded as one structural page.
+/// A top-level-row range whose dense rep/def stream fits one mini-block page.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct StructuralPageSplit {
+pub(crate) struct MiniBlockRepDefSplit {
     /// Top-level row offset, relative to the original unsplit page.
     pub(crate) row_start: u64,
     /// Number of top-level rows in this split.
@@ -137,15 +137,15 @@ pub(crate) struct StructuralPageSplit {
     pub(crate) num_values: u64,
 }
 
-/// Planner result for structural page budget handling.
+/// Dense mini-block rep/def budget result for one accumulated page.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) enum StructuralPagePlan {
-    /// The original page can be encoded as-is.
-    Fits,
-    /// The original page should be split on top-level row boundaries.
-    Split(Vec<StructuralPageSplit>),
-    /// One top-level row is larger than the requested structural page budget.
-    UnsplittableOverBudget(u64),
+pub(crate) enum MiniBlockRepDefBudget {
+    /// The dense rep/def stream fits one mini-block structural page.
+    WithinBudget,
+    /// The dense rep/def stream fits after splitting on top-level row boundaries.
+    RequiresPageSplit(Vec<MiniBlockRepDefSplit>),
+    /// A single top-level row has this many rep/def levels and exceeds the budget.
+    SingleRowOverBudget(u64),
 }
 
 // As we build def levels we add this to special values to indicate that they
@@ -806,21 +806,20 @@ impl SerializerContext {
         max_levels_per_page: Option<u64>,
         num_rows: u64,
         num_values: u64,
-    ) -> Result<StructuralPagePlan> {
+    ) -> Result<MiniBlockRepDefBudget> {
         // Extremely sparse lists can have many rep/def levels for very few
         // visible leaf values.  If this ratio becomes too skewed then a
-        // miniblock structural chunk can exceed its packed rep/def metadata
-        // budget even though the value buffers are small.  We detect that case
-        // while normalizing special def levels and split the structural page on
-        // top-level row boundaries so each emitted page stays within the
-        // miniblock structural budget.
+        // mini-block rep/def chunk can exceed its packed metadata budget even
+        // though the value buffers are small.  We detect that case while
+        // normalizing special def levels and split on top-level row boundaries
+        // so each emitted dense mini-block page stays within the budget.
         if self.def_levels.is_empty() {
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         if self.rep_levels.is_empty() {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         if self.rep_levels.len() != self.def_levels.len() {
@@ -833,12 +832,12 @@ impl SerializerContext {
 
         let Some(max_levels_per_page) = max_levels_per_page else {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         };
 
         if num_values == 0 {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         let max_schema_rep = def_meaning.iter().filter(|level| level.is_list()).count() as u16;
@@ -847,7 +846,7 @@ impl SerializerContext {
 
         if !should_plan {
             self.normalize_specials();
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
 
         let max_visible_level = max_visible_level.unwrap();
@@ -855,7 +854,7 @@ impl SerializerContext {
         let mut counted_rows = 0u64;
         let mut counted_values = 0u64;
         let mut saw_structural_overhead = false;
-        let mut unsplittable_over_budget = None;
+        let mut single_row_over_budget_levels = None;
 
         let mut current_row_level_start = None;
         let mut current_row_num_values = 0u64;
@@ -876,14 +875,14 @@ impl SerializerContext {
                 saw_structural_overhead |= row_has_structural_overhead;
 
                 if row_has_structural_overhead && row_num_levels > max_levels_per_page {
-                    unsplittable_over_budget = Some(row_num_levels);
+                    single_row_over_budget_levels = Some(row_num_levels);
                 }
 
                 if current_page_num_rows > 0
                     && (current_page_has_structural_overhead || row_has_structural_overhead)
                     && current_page_num_levels + row_num_levels > max_levels_per_page
                 {
-                    splits.push(StructuralPageSplit {
+                    splits.push(MiniBlockRepDefSplit {
                         row_start: current_page_row_start,
                         num_rows: current_page_num_rows,
                         level_range: current_page_level_start..current_page_level_end,
@@ -966,14 +965,14 @@ impl SerializerContext {
             )));
         }
         if !saw_structural_overhead {
-            return Ok(StructuralPagePlan::Fits);
+            return Ok(MiniBlockRepDefBudget::WithinBudget);
         }
-        if let Some(row_num_levels) = unsplittable_over_budget {
-            return Ok(StructuralPagePlan::UnsplittableOverBudget(row_num_levels));
+        if let Some(row_num_levels) = single_row_over_budget_levels {
+            return Ok(MiniBlockRepDefBudget::SingleRowOverBudget(row_num_levels));
         }
 
         if current_page_num_rows > 0 {
-            splits.push(StructuralPageSplit {
+            splits.push(MiniBlockRepDefSplit {
                 row_start: current_page_row_start,
                 num_rows: current_page_num_rows,
                 level_range: current_page_level_start..current_page_level_end,
@@ -983,9 +982,9 @@ impl SerializerContext {
         }
 
         if splits.len() > 1 {
-            Ok(StructuralPagePlan::Split(splits))
+            Ok(MiniBlockRepDefBudget::RequiresPageSplit(splits))
         } else {
-            Ok(StructuralPagePlan::Fits)
+            Ok(MiniBlockRepDefBudget::WithinBudget)
         }
     }
 
@@ -1023,12 +1022,12 @@ impl SerializerContext {
         )
     }
 
-    fn build_with_structural_plan(
+    fn build_with_miniblock_repdef_budget(
         mut self,
         max_levels_per_page: Option<u64>,
         num_rows: u64,
         num_values: u64,
-    ) -> Result<(SerializedRepDefs, StructuralPagePlan)> {
+    ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> {
         if self.current_len == 0 {
             return Ok((
                 SerializedRepDefs::new_with_fixed_size_list_levels(
@@ -1037,7 +1036,7 @@ impl SerializerContext {
                     self.def_meaning,
                     self.has_fsl,
                 ),
-                StructuralPagePlan::Fits,
+                MiniBlockRepDefBudget::WithinBudget,
             ));
         }
 
@@ -1046,7 +1045,7 @@ impl SerializerContext {
             .into_iter()
             .rev()
             .collect::<Vec<_>>();
-        let plan = self.normalize_specials_and_plan_splits(
+        let budget = self.normalize_specials_and_plan_splits(
             &def_meaning,
             max_levels_per_page,
             num_rows,
@@ -1071,7 +1070,7 @@ impl SerializerContext {
                 def_meaning,
                 self.has_fsl,
             ),
-            plan,
+            budget,
         ))
     }
 }
@@ -1412,15 +1411,15 @@ impl RepDefBuilder {
         Self::serialize_builders(builders).0.build()
     }
 
-    /// Converts gathered structural buffers into rep/def levels and an encode-time plan.
-    pub(crate) fn serialize_with_structural_plan(
+    /// Converts gathered structural buffers into rep/def levels and a mini-block budget result.
+    pub(crate) fn serialize_with_miniblock_repdef_budget(
         builders: Vec<Self>,
         max_levels_for_bits: impl FnOnce(u64) -> u64,
         num_rows: u64,
         num_values: u64,
-    ) -> Result<(SerializedRepDefs, StructuralPagePlan)> {
+    ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> {
         let (context, bits_per_level) = Self::serialize_builders(builders);
-        context.build_with_structural_plan(
+        context.build_with_miniblock_repdef_budget(
             bits_per_level.map(max_levels_for_bits),
             num_rows,
             num_values,
@@ -1743,7 +1742,11 @@ impl RepDefUnraveler {
             }
             let num_new_lists = offsets.len() - old_offsets_len;
             offsets.push(to_offset(curlen)?);
-            rep_levels.truncate(offsets.len() - 1);
+            // Truncate to the number of lists THIS unraveler produced (write_idx),
+            // not `offsets.len() - 1` — the latter includes offsets contributed by
+            // earlier unravelers in a multi-page read, which would leave too many
+            // rep levels for the next (outer) layer and over-count its lists.
+            rep_levels.truncate(write_idx);
             if let Some(validity) = validity {
                 // Even though we don't have validity it is possible another unraveler did and so we need
                 // to push all valids
@@ -2959,6 +2962,40 @@ mod tests {
         assert_eq!(val, None);
     }
 
+    #[test]
+    fn test_repdef_nested_list_multibatch_matches_single() {
+        // Single builder: List<List<i32>>, 3 rows.
+        //   outer [0,2,3,5] -> rows have 2,1,2 inner lists
+        //   inner [0,1,3,5,7,9] -> 5 inner lists, lengths 1,2,2,2,2 (9 leaf)
+        let mut single = RepDefBuilder::default();
+        single.add_offsets(offsets_64(&[0, 2, 3, 5]), None);
+        single.add_offsets(offsets_64(&[0, 1, 3, 5, 7, 9]), None);
+        single.add_no_null(9);
+        let single_rep = RepDefBuilder::serialize(vec![single])
+            .repetition_levels
+            .unwrap();
+
+        // Same logical data split into two batches:
+        //   batch0 = rows 0,1 : outer [0,2,3], inner [0,1,3,5] (3 inner, 5 leaf)
+        //   batch1 = row 2    : outer [0,2],   inner [0,2,4]   (2 inner, 4 leaf)
+        let mut b0 = RepDefBuilder::default();
+        b0.add_offsets(offsets_64(&[0, 2, 3]), None);
+        b0.add_offsets(offsets_64(&[0, 1, 3, 5]), None);
+        b0.add_no_null(5);
+        let mut b1 = RepDefBuilder::default();
+        b1.add_offsets(offsets_64(&[0, 2]), None);
+        b1.add_offsets(offsets_64(&[0, 2, 4]), None);
+        b1.add_no_null(4);
+        let multi_rep = RepDefBuilder::serialize(vec![b0, b1])
+            .repetition_levels
+            .unwrap();
+
+        assert_eq!(
+            *single_rep, *multi_rep,
+            "multi-batch nested-list rep levels must equal single-batch"
+        );
+    }
+
     #[test]
     fn test_only_empty_lists() {
         let mut builder = RepDefBuilder::default();
diff --git a/rust/lance-encoding/src/version.rs b/rust/lance-encoding/src/version.rs
index 41640d0bf2d..cd8f09b011b 100644
--- a/rust/lance-encoding/src/version.rs
+++ b/rust/lance-encoding/src/version.rs
@@ -5,6 +5,7 @@ use std::str::FromStr;
 
 use lance_arrow::DataTypeExt;
 use lance_core::datatypes::Field;
+use lance_core::deepsize::{Context, DeepSizeOf};
 use lance_core::{Error, Result};
 
 pub const LEGACY_FORMAT_VERSION: &str = "0.1";
@@ -38,6 +39,12 @@ pub enum LanceFileVersion {
     V2_3,
 }
 
+impl DeepSizeOf for LanceFileVersion {
+    fn deep_size_of_children(&self, _context: &mut Context) -> usize {
+        0
+    }
+}
+
 impl LanceFileVersion {
     /// Convert Stable or Next to the actual version
     pub fn resolve(&self) -> Self {
diff --git a/rust/lance-file/Cargo.toml b/rust/lance-file/Cargo.toml
index c79ffcdb57c..56536e840e9 100644
--- a/rust/lance-file/Cargo.toml
+++ b/rust/lance-file/Cargo.toml
@@ -27,7 +27,6 @@ async-trait.workspace = true
 byteorder.workspace = true
 bytes.workspace = true
 datafusion-common.workspace = true
-deepsize.workspace = true
 futures.workspace = true
 log.workspace = true
 num-traits.workspace = true
@@ -62,5 +61,9 @@ features = ["protoc"]
 name = "reader"
 harness = false
 
+[[bench]]
+name = "schema"
+harness = false
+
 [lints]
 workspace = true
diff --git a/rust/lance-file/benches/schema.rs b/rust/lance-file/benches/schema.rs
new file mode 100644
index 00000000000..b8da23d9777
--- /dev/null
+++ b/rust/lance-file/benches/schema.rs
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::hint::black_box;
+
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use lance_core::datatypes::Schema;
+use lance_file::{datatypes::Fields, format::pb};
+
+fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field {
+    pb::Field {
+        id,
+        parent_id,
+        name,
+        logical_type: logical_type.to_owned(),
+        ..Default::default()
+    }
+}
+
+/// Builds a pre-order flat schema with `num_physical_columns` physical leaves.
+///
+/// Each struct contributes one parent and two `int32` leaves. Root fields use
+/// `-1` as `parent_id`, and each struct consumes a three-ID block.
+fn wide_two_leaf_structs(num_physical_columns: usize) -> Fields {
+    assert_eq!(num_physical_columns % 2, 0);
+    let num_structs = num_physical_columns / 2;
+    let mut fields = Vec::with_capacity(num_structs + num_physical_columns);
+
+    for struct_index in 0..num_structs {
+        let parent_id = (struct_index * 3) as i32;
+        fields.push(proto_field(
+            parent_id,
+            -1,
+            format!("struct_{struct_index}"),
+            "struct",
+        ));
+        fields.push(proto_field(
+            parent_id + 1,
+            parent_id,
+            format!("left_{struct_index}"),
+            "int32",
+        ));
+        fields.push(proto_field(
+            parent_id + 2,
+            parent_id,
+            format!("right_{struct_index}"),
+            "int32",
+        ));
+    }
+
+    Fields(fields)
+}
+
+fn bench_schema_reconstruction(c: &mut Criterion) {
+    let mut group = c.benchmark_group("schema_from_flat_fields");
+
+    for num_physical_columns in [1024, 4096, 16_384, 65_536] {
+        let fields = wide_two_leaf_structs(num_physical_columns);
+        group.throughput(Throughput::Elements(fields.0.len() as u64));
+        group.bench_with_input(
+            BenchmarkId::new("physical_columns", num_physical_columns),
+            &fields,
+            |bencher, fields| {
+                bencher.iter(|| Schema::try_from(black_box(fields)).unwrap());
+            },
+        );
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_schema_reconstruction);
+criterion_main!(benches);
diff --git a/rust/lance-file/src/datatypes.rs b/rust/lance-file/src/datatypes.rs
index ac6a8d7b293..c31cb5c97e7 100644
--- a/rust/lance-file/src/datatypes.rs
+++ b/rust/lance-file/src/datatypes.rs
@@ -99,6 +99,32 @@ impl From<&Field> for pb::Field {
 
 pub struct Fields(pub Vec<pb::Field>);
 
+struct FieldNode {
+    field: Field,
+    child_indices: Vec<usize>,
+}
+
+/// Searches in pre-order depth-first order and returns the first matching node,
+/// preserving the legacy parent tie-break for duplicate field IDs.
+fn first_field_index_by_id(
+    nodes: &[FieldNode],
+    root_indices: &[usize],
+    field_id: i32,
+) -> Option<usize> {
+    let mut to_visit = Vec::with_capacity(nodes.len());
+    to_visit.extend(root_indices.iter().rev().copied());
+
+    while let Some(node_index) = to_visit.pop() {
+        let node = &nodes[node_index];
+        if node.field.id == field_id {
+            return Some(node_index);
+        }
+        to_visit.extend(node.child_indices.iter().rev().copied());
+    }
+
+    None
+}
+
 impl From<&Field> for Fields {
     fn from(field: &Field) -> Self {
         let mut protos = vec![pb::Field::from(field)];
@@ -107,24 +133,119 @@ impl From<&Field> for Fields {
     }
 }
 
-/// Convert list of protobuf `Field` to a Schema.
-impl From<&Fields> for Schema {
-    fn from(fields: &Fields) -> Self {
-        let mut schema = Self {
-            fields: vec![],
-            metadata: HashMap::default(),
-        };
-
-        fields.0.iter().for_each(|f| {
-            if f.parent_id == -1 {
-                schema.fields.push(Field::from(f));
+/// Reconstruct a schema from a flat, pre-order protobuf field list.
+///
+/// Parent fields must appear before their children. Historical manifests may
+/// contain duplicate field IDs, so an ID may not identify a unique parent. For
+/// those references, reconstruction preserves the legacy
+/// [`Schema::mut_field_by_id`] tie-break by selecting the first matching field
+/// in pre-order depth-first traversal.
+///
+/// # Examples
+///
+/// ```
+/// use lance_core::datatypes::Schema;
+/// use lance_file::{datatypes::Fields, format::pb};
+///
+/// let field = pb::Field {
+///     id: 0,
+///     parent_id: -1,
+///     name: "value".to_owned(),
+///     logical_type: "int32".to_owned(),
+///     ..Default::default()
+/// };
+/// let fields = Fields(vec![field]);
+/// let schema = Schema::try_from(&fields)?;
+/// assert_eq!(schema.fields[0].name, "value");
+/// # Ok::<(), lance_core::Error>(())
+/// ```
+impl TryFrom<&Fields> for Schema {
+    type Error = Error;
+
+    fn try_from(fields: &Fields) -> Result<Self> {
+        let mut nodes: Vec<FieldNode> = Vec::with_capacity(fields.0.len());
+        let mut root_indices = Vec::with_capacity(fields.0.len());
+        let mut field_indices: HashMap<i32, Option<usize>> = HashMap::with_capacity(fields.0.len());
+
+        for proto_field in &fields.0 {
+            let parent_index = if proto_field.parent_id == -1 {
+                None
             } else {
-                let parent = schema.mut_field_by_id(f.parent_id).unwrap();
-                parent.children.push(Field::from(f));
+                let parent_index = match field_indices.get(&proto_field.parent_id) {
+                    Some(Some(parent_index)) => *parent_index,
+                    Some(None) => {
+                        // Duplicate IDs are invalid but occur in historical
+                        // manifests. Match the legacy tree traversal only for
+                        // these ambiguous parent references so valid schemas
+                        // retain the linear fast path.
+                        first_field_index_by_id(&nodes, &root_indices, proto_field.parent_id)
+                            .ok_or_else(|| {
+                                Error::internal(format!(
+                                    "Duplicate field id {} has no existing arena node",
+                                    proto_field.parent_id
+                                ))
+                            })?
+                    }
+                    None => {
+                        return Err(Error::schema(format!(
+                            "Field '{}' (id={}) references parent id {}, which must appear earlier in the protobuf field list",
+                            proto_field.name, proto_field.id, proto_field.parent_id
+                        )));
+                    }
+                };
+                Some(parent_index)
+            };
+
+            let node_index = nodes.len();
+            if let Some(parent_index) = parent_index {
+                nodes[parent_index].child_indices.push(node_index);
+            } else {
+                root_indices.push(node_index);
             }
-        });
+            nodes.push(FieldNode {
+                field: Field::from(proto_field),
+                child_indices: Vec::new(),
+            });
+
+            field_indices
+                .entry(proto_field.id)
+                .and_modify(|field_index| *field_index = None)
+                .or_insert(Some(node_index));
+        }
 
-        schema
+        let mut fields_by_node = Vec::with_capacity(nodes.len());
+        fields_by_node.resize_with(nodes.len(), || None);
+        for (node_index, mut node) in nodes.into_iter().enumerate().rev() {
+            node.field.children.reserve(node.child_indices.len());
+            for child_index in node.child_indices {
+                let child = fields_by_node
+                    .get_mut(child_index)
+                    .and_then(Option::take)
+                    .ok_or_else(|| {
+                        Error::internal(format!(
+                            "Schema field arena node {child_index} was not materialized before its parent"
+                        ))
+                    })?;
+                node.field.children.push(child);
+            }
+            fields_by_node[node_index] = Some(node.field);
+        }
+
+        let fields = root_indices
+            .into_iter()
+            .map(|root_index| {
+                fields_by_node[root_index].take().ok_or_else(|| {
+                    Error::internal(format!(
+                        "Schema field arena root node {root_index} was not materialized"
+                    ))
+                })
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        Ok(Self {
+            fields,
+            metadata: HashMap::default(),
+        })
     }
 }
 
@@ -133,9 +254,28 @@ pub struct FieldsWithMeta {
     pub metadata: HashMap<String, Vec<u8>>,
 }
 
-/// Convert list of protobuf `Field` and Metadata to a Schema.
-impl From<FieldsWithMeta> for Schema {
-    fn from(fields_with_meta: FieldsWithMeta) -> Self {
+/// Reconstruct a schema from flat protobuf fields and schema metadata.
+///
+/// # Examples
+///
+/// ```
+/// use std::collections::HashMap;
+///
+/// use lance_core::datatypes::Schema;
+/// use lance_file::datatypes::{Fields, FieldsWithMeta};
+///
+/// let fields = FieldsWithMeta {
+///     fields: Fields(Vec::new()),
+///     metadata: HashMap::from([("owner".to_owned(), b"lance".to_vec())]),
+/// };
+/// let schema = Schema::try_from(fields)?;
+/// assert_eq!(schema.metadata["owner"], "lance");
+/// # Ok::<(), lance_core::Error>(())
+/// ```
+impl TryFrom<FieldsWithMeta> for Schema {
+    type Error = Error;
+
+    fn try_from(fields_with_meta: FieldsWithMeta) -> Result<Self> {
         let lance_metadata = fields_with_meta
             .metadata
             .into_iter()
@@ -145,11 +285,11 @@ impl From<FieldsWithMeta> for Schema {
             })
             .collect();
 
-        let schema_with_fields = Self::from(&fields_with_meta.fields);
-        Self {
+        let schema_with_fields = Self::try_from(&fields_with_meta.fields)?;
+        Ok(Self {
             fields: schema_with_fields.fields,
             metadata: lance_metadata,
-        }
+        })
     }
 }
 
@@ -270,14 +410,27 @@ pub async fn populate_schema_dictionary(schema: &mut Schema, reader: &dyn Reader
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use arrow_schema::DataType;
     use arrow_schema::Field as ArrowField;
     use arrow_schema::Fields as ArrowFields;
     use arrow_schema::Schema as ArrowSchema;
+    use lance_core::Error;
     use lance_core::datatypes::Schema;
-    use std::collections::HashMap;
 
     use super::{Fields, FieldsWithMeta};
+    use crate::format::pb;
+
+    fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field {
+        pb::Field {
+            id,
+            parent_id,
+            name,
+            logical_type: logical_type.to_owned(),
+            ..Default::default()
+        }
+    }
 
     #[test]
     fn test_schema_set_ids() {
@@ -317,10 +470,120 @@ mod tests {
         let expected_schema = Schema::try_from(&arrow_schema).unwrap();
         let fields_with_meta: FieldsWithMeta = (&expected_schema).into();
 
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta).unwrap();
         assert_eq!(expected_schema, schema);
     }
 
+    #[test]
+    fn test_reconstruct_wide_nested_schema() {
+        const NUM_STRUCTS: usize = 4096;
+
+        let mut proto_fields = Vec::with_capacity(NUM_STRUCTS * 3);
+        for struct_index in 0..NUM_STRUCTS {
+            let parent_id = (struct_index * 3) as i32;
+            proto_fields.push(proto_field(
+                parent_id,
+                -1,
+                format!("struct_{struct_index}"),
+                "struct",
+            ));
+            proto_fields.push(proto_field(
+                parent_id + 1,
+                parent_id,
+                format!("left_{struct_index}"),
+                "int32",
+            ));
+            proto_fields.push(proto_field(
+                parent_id + 2,
+                parent_id,
+                format!("right_{struct_index}"),
+                "int32",
+            ));
+        }
+
+        let fields = Fields(proto_fields);
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), NUM_STRUCTS);
+        for (struct_index, field) in schema.fields.iter().enumerate() {
+            let parent_id = (struct_index * 3) as i32;
+            assert_eq!(field.id, parent_id);
+            assert_eq!(field.name, format!("struct_{struct_index}"));
+            assert_eq!(field.children.len(), 2);
+            assert_eq!(field.children[0].id, parent_id + 1);
+            assert_eq!(field.children[0].name, format!("left_{struct_index}"));
+            assert_eq!(field.children[1].id, parent_id + 2);
+            assert_eq!(field.children[1].name, format!("right_{struct_index}"));
+        }
+    }
+
+    #[test]
+    fn test_reconstruct_deep_nested_schema() {
+        const DEPTH: usize = 1024;
+
+        let proto_fields = (0..DEPTH)
+            .map(|depth| {
+                proto_field(
+                    depth as i32,
+                    if depth == 0 { -1 } else { depth as i32 - 1 },
+                    format!("level_{depth}"),
+                    if depth + 1 == DEPTH {
+                        "int32"
+                    } else {
+                        "struct"
+                    },
+                )
+            })
+            .collect();
+
+        let fields = Fields(proto_fields);
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), 1);
+        let mut field = &schema.fields[0];
+        for depth in 0..DEPTH {
+            assert_eq!(field.id, depth as i32);
+            assert_eq!(field.name, format!("level_{depth}"));
+            if depth + 1 == DEPTH {
+                assert!(field.children.is_empty());
+            } else {
+                assert_eq!(field.children.len(), 1);
+                field = &field.children[0];
+            }
+        }
+    }
+
+    #[test]
+    fn test_reconstruct_schema_reports_missing_parent() {
+        let fields = Fields(vec![proto_field(7, 42, "child".to_owned(), "int32")]);
+
+        let error = Schema::try_from(&fields).unwrap_err();
+        assert!(matches!(&error, Error::Schema { .. }));
+        assert!(
+            error.to_string().contains(
+                "Field 'child' (id=7) references parent id 42, which must appear earlier"
+            )
+        );
+    }
+
+    #[test]
+    fn test_reconstruct_schema_preserves_legacy_duplicate_id_match() {
+        let fields = Fields(vec![
+            proto_field(1, -1, "root_a".to_owned(), "struct"),
+            proto_field(2, -1, "root_b".to_owned(), "struct"),
+            proto_field(2, 1, "nested_duplicate".to_owned(), "struct"),
+            proto_field(3, 2, "child".to_owned(), "int32"),
+        ]);
+
+        let schema = Schema::try_from(&fields).unwrap();
+        assert_eq!(schema.fields.len(), 2);
+        assert_eq!(schema.fields[0].name, "root_a");
+        assert_eq!(schema.fields[0].children.len(), 1);
+        assert_eq!(schema.fields[0].children[0].name, "nested_duplicate");
+        assert_eq!(schema.fields[0].children[0].children.len(), 1);
+        assert_eq!(schema.fields[0].children[0].children[0].name, "child");
+        assert_eq!(schema.fields[1].name, "root_b");
+        assert!(schema.fields[1].children.is_empty());
+    }
+
     #[test]
     fn test_clustering_key_roundtrip() {
         let arrow_schema = ArrowSchema::new(vec![
@@ -351,7 +614,7 @@ mod tests {
 
         // Round-trip through protobuf
         let fields_with_meta: FieldsWithMeta = (&schema).into();
-        let restored = Schema::from(fields_with_meta);
+        let restored = Schema::try_from(fields_with_meta).unwrap();
 
         let ck2 = restored.unenforced_clustering_key();
         assert_eq!(ck2.len(), 2);
diff --git a/rust/lance-file/src/io.rs b/rust/lance-file/src/io.rs
index c09e9d8d372..1a8edf92b08 100644
--- a/rust/lance-file/src/io.rs
+++ b/rust/lance-file/src/io.rs
@@ -38,6 +38,16 @@ impl EncodingsIo for LanceEncodingsIo {
         }))
     }
 
+    fn with_io_stats(
+        &self,
+        stats: Arc<dyn lance_core::utils::io_stats::IoStatsRecorder>,
+    ) -> Option<Arc<dyn EncodingsIo>> {
+        Some(Arc::new(Self {
+            scheduler: self.scheduler.with_io_stats(stats),
+            read_chunk_size: self.read_chunk_size,
+        }))
+    }
+
     fn submit_request(
         &self,
         ranges: Vec<std::ops::Range<u64>>,
diff --git a/rust/lance-file/src/previous/format/metadata.rs b/rust/lance-file/src/previous/format/metadata.rs
index 7e4046be893..209d0733cec 100644
--- a/rust/lance-file/src/previous/format/metadata.rs
+++ b/rust/lance-file/src/previous/format/metadata.rs
@@ -6,8 +6,8 @@ use std::ops::Range;
 
 use crate::datatypes::{Fields, FieldsWithMeta};
 use crate::format::pb;
-use deepsize::DeepSizeOf;
 use lance_core::datatypes::Schema;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use lance_io::traits::ProtoStruct;
 
@@ -62,10 +62,10 @@ impl TryFrom<pb::Metadata> for Metadata {
             manifest_position: Some(m.manifest_position as usize),
             stats_metadata: if let Some(stats_meta) = m.statistics {
                 Some(StatisticsMetadata {
-                    schema: Schema::from(FieldsWithMeta {
+                    schema: Schema::try_from(FieldsWithMeta {
                         fields: Fields(stats_meta.schema),
                         metadata: Default::default(),
-                    }),
+                    })?,
                     leaf_field_ids: stats_meta.fields,
                     page_table_position: stats_meta.page_table_position as usize,
                 })
diff --git a/rust/lance-file/src/previous/page_table.rs b/rust/lance-file/src/previous/page_table.rs
index 9a3c0d71437..cc246caa585 100644
--- a/rust/lance-file/src/previous/page_table.rs
+++ b/rust/lance-file/src/previous/page_table.rs
@@ -4,7 +4,7 @@
 use arrow_array::builder::Int64Builder;
 use arrow_array::{Array, Int64Array};
 use arrow_schema::DataType;
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use lance_io::encodings::Decoder;
 use lance_io::encodings::plain::PlainDecoder;
 use std::collections::BTreeMap;
diff --git a/rust/lance-file/src/previous/reader.rs b/rust/lance-file/src/previous/reader.rs
index cf30d30a547..1ab861985e1 100644
--- a/rust/lance-file/src/previous/reader.rs
+++ b/rust/lance-file/src/previous/reader.rs
@@ -19,11 +19,11 @@ use arrow_buffer::ArrowNativeType;
 use arrow_schema::{DataType, FieldRef, Schema as ArrowSchema};
 use arrow_select::concat::{self, concat_batches};
 use async_recursion::async_recursion;
-use deepsize::DeepSizeOf;
 use futures::{Future, FutureExt, StreamExt, TryStreamExt, stream};
 use lance_arrow::*;
 use lance_core::cache::{CacheKey, LanceCache};
 use lance_core::datatypes::{Field, Schema};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use lance_io::encodings::AsyncIndex;
 use lance_io::encodings::dictionary::DictionaryDecoder;
diff --git a/rust/lance-file/src/previous/writer/mod.rs b/rust/lance-file/src/previous/writer/mod.rs
index 4b04f722925..ab13e782367 100644
--- a/rust/lance-file/src/previous/writer/mod.rs
+++ b/rust/lance-file/src/previous/writer/mod.rs
@@ -29,6 +29,7 @@ use tokio::io::AsyncWriteExt;
 use crate::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION};
 use crate::previous::format::metadata::{Metadata, StatisticsMetadata};
 use crate::previous::page_table::{PageInfo, PageTable};
+use crate::writer::FileWriteSummary;
 
 /// The file format currently includes a "manifest" where it stores the schema for
 /// self-describing files.  Historically this has been a table format manifest that
@@ -243,23 +244,29 @@ impl<M: ManifestProvider + Send + Sync> FileWriter<M> {
     pub async fn finish_with_metadata(
         &mut self,
         metadata: &HashMap<String, String>,
-    ) -> Result<usize> {
+    ) -> Result<FileWriteSummary> {
         self.schema
             .metadata
             .extend(metadata.iter().map(|(k, y)| (k.clone(), y.clone())));
         self.finish().await
     }
 
-    pub async fn finish(&mut self) -> Result<usize> {
+    pub async fn finish(&mut self) -> Result<FileWriteSummary> {
         self.write_footer().await?;
-        Writer::shutdown(self.object_writer.as_mut()).await?;
+        // `shutdown` flushes the footer and reports the authoritative on-disk
+        // byte count, so the size is sourced here instead of via a later
+        // `tell()` that would rely on the cursor surviving shutdown.
+        let write_result = Writer::shutdown(self.object_writer.as_mut()).await?;
         let num_rows = self
             .metadata
             .batch_offsets
             .last()
             .cloned()
             .unwrap_or_default();
-        Ok(num_rows as usize)
+        Ok(FileWriteSummary {
+            num_rows: num_rows as u64,
+            size_bytes: write_result.size as u64,
+        })
     }
 
     /// Total records written in this file.
diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs
index 3d1d8e7c361..60debd4067e 100644
--- a/rust/lance-file/src/reader.rs
+++ b/rust/lance-file/src/reader.rs
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use std::{
+    borrow::Cow,
     collections::{BTreeMap, BTreeSet},
     io::Cursor,
     ops::Range,
@@ -13,8 +14,8 @@ use arrow_array::RecordBatchReader;
 use arrow_schema::Schema as ArrowSchema;
 use byteorder::{ByteOrder, LittleEndian, ReadBytesExt};
 use bytes::{Bytes, BytesMut};
-use deepsize::{Context, DeepSizeOf};
 use futures::{Stream, StreamExt, stream::BoxStream};
+use lance_core::deepsize::{Context, DeepSizeOf};
 use lance_encoding::{
     EncodingsIo,
     decoder::{
@@ -31,7 +32,7 @@ use prost::{Message, Name};
 
 use lance_core::{
     Error, Result,
-    cache::LanceCache,
+    cache::{CacheKey, LanceCache},
     datatypes::{Field, Schema},
 };
 use lance_encoding::format::pb as pbenc;
@@ -127,6 +128,14 @@ impl CachedFileMetadata {
     }
 }
 
+fn column_metadata_deep_size(column_metadatas: &[pbfile::ColumnMetadata]) -> usize {
+    column_metadatas
+        .iter()
+        .map(|cm| cm.encoded_len() * 4)
+        .sum::<usize>()
+        + std::mem::size_of_val(column_metadatas)
+}
+
 impl DeepSizeOf for CachedFileMetadata {
     fn deep_size_of_children(&self, context: &mut Context) -> usize {
         let schema_size = self.file_schema.deep_size_of_children(context);
@@ -142,49 +151,15 @@ impl DeepSizeOf for CachedFileMetadata {
         // as a proxy for in-memory size. The decoded representation is typically
         // several times larger than the wire format due to heap-allocated
         // repeated/string/bytes fields, so we apply a 4x multiplier.
-        let column_metadatas_size: usize = self
-            .column_metadatas
-            .iter()
-            .map(|cm| cm.encoded_len() * 4)
-            .sum::<usize>()
-            + std::mem::size_of_val(self.column_metadatas.as_slice());
+        let column_metadatas_size = column_metadata_deep_size(self.column_metadatas.as_slice());
 
         // column_infos is Vec<Arc<ColumnInfo>>. Each ColumnInfo contains
         // page_infos (with protobuf PageEncoding), buffer offsets, and a
         // column-level ColumnEncoding protobuf.
-        let column_infos_size: usize = self
-            .column_infos
-            .iter()
-            .map(|ci| {
-                let pages_size: usize = ci
-                    .page_infos
-                    .iter()
-                    .map(|pi| {
-                        let enc_size = match &pi.encoding {
-                            lance_encoding::decoder::PageEncoding::Legacy(e) => e.encoded_len() * 4,
-                            lance_encoding::decoder::PageEncoding::Structural(e) => {
-                                e.encoded_len() * 4
-                            }
-                        };
-                        enc_size
-                            + std::mem::size_of_val(pi.buffer_offsets_and_sizes.as_ref())
-                            + std::mem::size_of::<u64>() * 2 // num_rows + priority
-                    })
-                    .sum();
-                pages_size
-                    + std::mem::size_of_val(ci.buffer_offsets_and_sizes.as_ref())
-                    + ci.encoding.encoded_len() * 4
-                    + std::mem::size_of::<u32>() // index
-                    + std::mem::size_of::<usize>() * 2 // Arc overhead
-            })
-            .sum();
+        let column_infos_size = self.column_infos.deep_size_of_children(context);
 
         // Global buffer bytes retained for zero-IO reads (copied out of the tail).
-        let retained_buffers_size: usize = self
-            .retained_global_buffers
-            .values()
-            .map(|buf| buf.len())
-            .sum();
+        let retained_buffers_size = self.retained_global_buffers.deep_size_of_children(context);
 
         schema_size
             + buffers_size
@@ -194,6 +169,65 @@ impl DeepSizeOf for CachedFileMetadata {
     }
 }
 
+/// Lightweight file metadata used to locate per-column metadata on demand.
+///
+/// This contains the file-level schema, row count, global buffer descriptors,
+/// and column metadata offset table. Unlike [`CachedFileMetadata`], it does not
+/// hold decoded metadata for every column.
+#[derive(Debug, DeepSizeOf)]
+pub struct FileMetadataIndex {
+    file_schema: Arc<Schema>,
+    num_rows: u64,
+    file_buffers: Vec<BufferDescriptor>,
+    column_metadata_offsets: Arc<[(u64, u64)]>,
+    num_columns: u32,
+    version: LanceFileVersion,
+    file_size_bytes: u64,
+    retained_global_buffers: BTreeMap<u32, Bytes>,
+}
+
+impl FileMetadataIndex {
+    /// Returns the total size of the file in bytes.
+    pub fn file_size(&self) -> u64 {
+        self.file_size_bytes
+    }
+
+    /// Returns the number of physical columns in the file.
+    pub fn num_columns(&self) -> u32 {
+        self.num_columns
+    }
+}
+
+#[derive(Debug)]
+struct CachedColumnMetadata {
+    column_metadata: pbfile::ColumnMetadata,
+    column_info: Arc<ColumnInfo>,
+}
+
+impl DeepSizeOf for CachedColumnMetadata {
+    fn deep_size_of_children(&self, context: &mut Context) -> usize {
+        column_metadata_deep_size(std::slice::from_ref(&self.column_metadata))
+            + self.column_info.deep_size_of_children(context)
+    }
+}
+
+#[derive(Debug, Clone)]
+struct ColumnMetadataCacheKey {
+    column_index: u32,
+}
+
+impl CacheKey for ColumnMetadataCacheKey {
+    type ValueType = CachedColumnMetadata;
+
+    fn key(&self) -> Cow<'_, str> {
+        Cow::Owned(format!("column_metadata/{}", self.column_index))
+    }
+
+    fn type_name() -> &'static str {
+        "ColumnMetadata"
+    }
+}
+
 impl CachedFileMetadata {
     pub fn version(&self) -> LanceFileVersion {
         match (self.major_version, self.minor_version) {
@@ -287,18 +321,15 @@ impl ReaderProjection {
     ) -> Result<()> {
         for field in fields {
             let is_structural = file_version >= LanceFileVersion::V2_1;
+            let (contributes, recurse) = field_column_shape(field, is_structural);
             // In the 2.0 system we needed ids for intermediate fields.  In 2.1+
             // we only need ids for leaf fields.
-            if (!is_structural
-                || field.children.is_empty()
-                || field.is_blob()
-                || field.is_packed_struct())
+            if contributes
                 && let Some(column_idx) = field_id_to_column_index.get(&(field.id as u32)).copied()
             {
                 column_indices.push(column_idx);
             }
-            // Don't recurse into children if the field is a blob or packed struct in 2.1
-            if !is_structural || (!field.is_blob() && !field.is_packed_struct()) {
+            if recurse {
                 Self::from_field_ids_helper(
                     file_version,
                     field.children.iter(),
@@ -430,16 +461,45 @@ impl Default for FileReaderOptions {
 }
 
 #[derive(Debug, Clone)]
-pub struct FileReader {
+struct PreparedProjection {
+    column_infos: Vec<Arc<ColumnInfo>>,
+    decoder_projection: ReaderProjection,
+}
+
+#[derive(Debug, Clone)]
+enum FileMetadataProvider {
+    Full(Arc<CachedFileMetadata>),
+    Indexed(Arc<FileMetadataIndex>),
+}
+
+#[derive(Debug, Clone)]
+struct FileReadCore {
     scheduler: Arc<dyn EncodingsIo>,
-    // The default projection to be applied to all reads
     base_projection: ReaderProjection,
-    num_rows: u64,
-    metadata: Arc<CachedFileMetadata>,
+    metadata_provider: FileMetadataProvider,
     decoder_plugins: Arc<DecoderPlugins>,
     cache: Arc<LanceCache>,
     options: FileReaderOptions,
 }
+
+/// A projection-scoped reader for Lance files.
+///
+/// This reader fixes a base projection at construction time. All later reads
+/// must stay within that projection, which lets the reader load only the column
+/// metadata needed by the base projection when opening from a [`FileMetadataIndex`].
+/// It intentionally does not expose APIs that require synchronous access to full
+/// file metadata.
+#[derive(Debug, Clone)]
+pub struct ProjectedFileReader {
+    core: FileReadCore,
+}
+
+/// A Lance file reader backed by fully decoded file metadata.
+#[derive(Debug, Clone)]
+pub struct FileReader {
+    core: FileReadCore,
+    metadata: Arc<CachedFileMetadata>,
+}
 #[derive(Debug)]
 struct Footer {
     #[allow(dead_code)]
@@ -457,30 +517,206 @@ struct Footer {
 
 const FOOTER_LEN: usize = 40;
 
+// How a field maps onto physical columns, shared by the projection-building and
+// projection-validation walks so they stay in lockstep. In the 2.0 layout every
+// ordinary field (including structs and lists) has its own column; in 2.1 only
+// leaves do. Blob/packed-struct fields are opaque in all versions: they are a
+// single column with no descent, including unloaded blob descriptor schemas.
+// Returns `(contributes, recurse)`: whether the field has its own column and
+// whether to walk into its children. The DFS order is the field's own column (if
+// any) followed by its children, so a field's root (first) column is always the
+// first entry of its sub-slice.
+fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) {
+    if field.is_blob() || field.is_packed_struct() {
+        return (true, false);
+    }
+
+    let contributes = !is_structural || field.children.is_empty();
+    let recurse = !field.children.is_empty();
+    (contributes, recurse)
+}
+
+// Count the V2.1 physical columns required to reconstruct a projected field.
+// This is the same DFS shape consumed by `ColumnInfoIter`: ordinary structural
+// nodes are transparent and leaves contribute columns. Indexed metadata loading
+// can therefore compact any ordinary structural projection into 0..N while
+// preserving this order.
+//
+// Blob and packed-struct fields remain unsupported by indexed projection. Their
+// opaque decode semantics are handled by the existing full-metadata reader.
+fn indexed_projection_column_count(field: &Field) -> Option<usize> {
+    if field.is_blob() || field.is_packed_struct() {
+        return None;
+    }
+
+    let (contributes, recurse) = field_column_shape(field, true);
+    let initial = usize::from(contributes);
+    if !recurse {
+        return Some(initial);
+    }
+
+    field.children.iter().try_fold(initial, |count, child| {
+        count.checked_add(indexed_projection_column_count(child)?)
+    })
+}
+
+// Whether a field's children each cover the same rows as the field itself. Struct
+// children do (one value per parent row), so they must share its length. List,
+// map, and fixed-size-list items have an independent cardinality (item count, not
+// row count) and are validated only against themselves.
+fn children_share_parent_length(field: &Field) -> bool {
+    field.logical_type.is_struct()
+}
+
+// Validate one field's slice of a projection's flat `column_indices`, returning
+// the field's top-level row count (the page-row sum of its root column). Walks the
+// same DFS order as `from_field_ids_helper`, advancing `cursor` past every column
+// the field contributes.
+//
+// `comparable` tracks whether the field's row count shares the read's top-level
+// cardinality. A struct's children must all match that count -- the decoders
+// combine them assuming equal lengths and would otherwise panic or read past a
+// shorter child -- so the equality check runs only while `comparable` holds. Once
+// the walk descends through a list/map/fixed-size-list its items have an
+// independent cardinality (item count, not row count), so `comparable` turns off
+// for that whole subtree and a nested struct's children are no longer compared.
+fn validate_field_length<F: Fn(usize) -> Result<u64>>(
+    field: &Field,
+    is_structural: bool,
+    comparable: bool,
+    column_indices: &[u32],
+    cursor: &mut usize,
+    column_len: &F,
+) -> Result<u64> {
+    let (contributes, recurse) = field_column_shape(field, is_structural);
+    let mut field_rows: Option<u64> = None;
+    if contributes {
+        let column = *column_indices.get(*cursor).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "projection supplied fewer column indices than its fields require \
+                 (ran out at field '{}')",
+                field.name
+            ))
+        })?;
+        *cursor += 1;
+        field_rows = Some(column_len(column as usize)?);
+    }
+    if recurse {
+        // Only enforce equal-length children for a struct whose own count is still
+        // at the top-level cardinality; below a list/map/fixed-size-list the items
+        // have an independent cardinality, so neither this field nor its
+        // descendants are comparable.
+        let enforce_children = comparable && children_share_parent_length(field);
+        for child in &field.children {
+            let child_rows = validate_field_length(
+                child,
+                is_structural,
+                enforce_children,
+                column_indices,
+                cursor,
+                column_len,
+            )?;
+            // A struct that contributes no column of its own (the 2.1 layout)
+            // takes its row count from its first child.
+            let expected = *field_rows.get_or_insert(child_rows);
+            if enforce_children && child_rows != expected {
+                return Err(Error::invalid_input(format!(
+                    "cannot read field '{}': its children have differing lengths \
+                     (child '{}' has {} rows, but the field has {}); a struct's \
+                     children must all have the same length",
+                    field.name, child.name, child_rows, expected
+                )));
+            }
+        }
+    }
+    field_rows.ok_or_else(|| {
+        Error::invalid_input(format!(
+            "projected field '{}' maps to no columns",
+            field.name
+        ))
+    })
+}
+
+// The reader combines a projection's columns into rectangular batches, so they
+// must all have the same length.  Returns that common length, or a descriptive
+// error (naming each column's length) when they differ. Ordinary files always
+// pass; only files written with `FileWriter::write_column` whose columns ended up
+// unequal can fail, and those must be read separately.
+fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result<u64> {
+    let first = field_lengths.first().map_or(0, |&(_, len)| len);
+    if field_lengths.iter().all(|&(_, len)| len == first) {
+        return Ok(first);
+    }
+    let columns = field_lengths
+        .iter()
+        .map(|(name, len)| format!("{name}={len}"))
+        .collect::<Vec<_>>()
+        .join(", ");
+    Err(Error::invalid_input(format!(
+        "cannot read columns of differing lengths together ({columns}); \
+         read each column (or equal-length group) separately"
+    )))
+}
+
 impl FileReader {
     pub fn with_scheduler(&self, scheduler: Arc<dyn EncodingsIo>) -> Self {
         Self {
-            scheduler,
-            base_projection: self.base_projection.clone(),
-            cache: self.cache.clone(),
-            decoder_plugins: self.decoder_plugins.clone(),
+            core: self.core.with_scheduler(scheduler),
             metadata: self.metadata.clone(),
-            options: self.options.clone(),
-            num_rows: self.num_rows,
+        }
+    }
+
+    /// Returns a clone of this reader whose I/O is additionally recorded into
+    /// `stats`, on top of the scheduler's global accounting.
+    ///
+    /// All cached metadata is shared with `self`, so no file is re-opened and
+    /// only a few `Arc` clones are performed.  If the underlying I/O service
+    /// does not support per-scope statistics (e.g. an in-memory scheduler), the
+    /// returned reader is an ordinary, uninstrumented clone.
+    pub fn with_io_stats(
+        &self,
+        stats: Arc<dyn lance_core::utils::io_stats::IoStatsRecorder>,
+    ) -> Self {
+        match self.core.scheduler.with_io_stats(stats) {
+            Some(scheduler) => self.with_scheduler(scheduler),
+            None => self.clone(),
         }
     }
 
     pub fn num_rows(&self) -> u64 {
-        self.num_rows
+        self.core.num_rows()
+    }
+
+    /// The number of rows stored in a single physical column.
+    ///
+    /// For ordinary (rectangular) files every column has the same length, equal
+    /// to [`num_rows`](Self::num_rows). Files written with
+    /// [`FileWriter::write_column`](crate::writer::FileWriter::write_column)
+    /// may have columns of differing lengths; this returns the length of one
+    /// such column, derived by summing its pages' row counts. Errors if
+    /// `column_index` is out of bounds.
+    pub fn column_num_rows(&self, column_index: usize) -> Result<u64> {
+        let column = self
+            .metadata
+            .column_metadatas
+            .get(column_index)
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "column index {} is out of bounds (file has {} columns)",
+                    column_index,
+                    self.metadata.column_metadatas.len()
+                ))
+            })?;
+        Ok(column.pages.iter().map(|page| page.length).sum())
     }
 
     pub fn metadata(&self) -> &Arc<CachedFileMetadata> {
         &self.metadata
     }
 
-    pub fn file_statistics(&self) -> FileStatistics {
-        let column_metadatas = &self.metadata().column_metadatas;
-
+    fn statistics_from_column_metadata(
+        column_metadatas: &[pbfile::ColumnMetadata],
+    ) -> FileStatistics {
         let column_stats = column_metadatas
             .iter()
             .map(|col_metadata| {
@@ -502,22 +738,12 @@ impl FileReader {
         }
     }
 
-    pub async fn read_global_buffer(&self, index: u32) -> Result<Bytes> {
-        let buffer_desc = self.metadata.file_buffers.get(index as usize).ok_or_else(||Error::invalid_input(format!("request for global buffer at index {} but there were only {} global buffers in the file", index, self.metadata.file_buffers.len())))?;
-
-        // If the buffer's bytes were captured by the tail read at open, serve them
-        // from memory with no additional I/O. Larger buffers (outside the window)
-        // are not retained and fall back to a dedicated read.
-        if let Some(bytes) = self.metadata.retained_global_buffers.get(&index) {
-            return Ok(bytes.clone());
-        }
+    pub fn file_statistics(&self) -> FileStatistics {
+        Self::statistics_from_column_metadata(&self.metadata().column_metadatas)
+    }
 
-        self.scheduler
-            .submit_single(
-                buffer_desc.position..buffer_desc.position + buffer_desc.size,
-                0,
-            )
-            .await
+    pub async fn read_global_buffer(&self, index: u32) -> Result<Bytes> {
+        self.core.read_global_buffer(index).await
     }
 
     async fn read_tail(scheduler: &FileScheduler) -> Result<(Bytes, u64)> {
@@ -531,6 +757,47 @@ impl FileReader {
         Ok((tail_bytes, file_size))
     }
 
+    async fn read_range_from_tail_or_scheduler(
+        tail_bytes: &Bytes,
+        tail_offset: u64,
+        scheduler: &FileScheduler,
+        range: Range<u64>,
+    ) -> Result<Bytes> {
+        let tail_end = tail_offset + tail_bytes.len() as u64;
+        if range.start >= tail_offset && range.end <= tail_end {
+            let rel_start = (range.start - tail_offset) as usize;
+            let rel_end = (range.end - tail_offset) as usize;
+            Ok(tail_bytes.slice(rel_start..rel_end))
+        } else {
+            scheduler.submit_single(range, 0).await
+        }
+    }
+
+    fn retained_global_buffers_from_tail(
+        gbo_table: &[BufferDescriptor],
+        tail_bytes: &Bytes,
+        tail_offset: u64,
+    ) -> BTreeMap<u32, Bytes> {
+        let tail_end = tail_offset + tail_bytes.len() as u64;
+        gbo_table
+            .iter()
+            .enumerate()
+            .skip(1)
+            .filter_map(|(index, buffer)| {
+                let start = buffer.position;
+                let end = buffer.position + buffer.size;
+                if start >= tail_offset && end <= tail_end {
+                    let rel_start = (start - tail_offset) as usize;
+                    let rel_end = (end - tail_offset) as usize;
+                    let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]);
+                    Some((index as u32, bytes))
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
+
     // Checks to make sure the footer is written correctly and returns the
     // position of the file descriptor (which comes from the footer)
     fn decode_footer(footer_bytes: &Bytes) -> Result<Footer> {
@@ -585,15 +852,22 @@ impl FileReader {
         let column_metadata_start = footer.column_meta_start;
         // cmo == column_metadata_offsets
         let cmo_table_size = 16 * footer.num_columns as usize;
+        if column_metadata_bytes.len() < cmo_table_size {
+            return Err(Error::invalid_input(format!(
+                "column metadata region has {} bytes but CMO table needs {} bytes for {} columns",
+                column_metadata_bytes.len(),
+                cmo_table_size,
+                footer.num_columns
+            )));
+        }
         let cmo_table = column_metadata_bytes.slice(column_metadata_bytes.len() - cmo_table_size..);
+        let column_metadata_offsets = Self::decode_cmo_table(cmo_table, footer)?;
 
-        (0..footer.num_columns)
-            .map(|col_idx| {
-                let offset = (col_idx * 16) as usize;
-                let position = LittleEndian::read_u64(&cmo_table[offset..offset + 8]);
-                let length = LittleEndian::read_u64(&cmo_table[offset + 8..offset + 16]);
-                let normalized_position = (position - column_metadata_start) as usize;
-                let normalized_end = normalized_position + (length as usize);
+        column_metadata_offsets
+            .iter()
+            .map(|(position, length)| {
+                let normalized_position = (*position - column_metadata_start) as usize;
+                let normalized_end = normalized_position + (*length as usize);
                 Ok(pbfile::ColumnMetadata::decode(
                     &column_metadata_bytes[normalized_position..normalized_end],
                 )?)
@@ -601,6 +875,44 @@ impl FileReader {
             .collect::<Result<Vec<_>>>()
     }
 
+    fn decode_cmo_table(cmo_table: Bytes, footer: &Footer) -> Result<Arc<[(u64, u64)]>> {
+        let expected_size = 16 * footer.num_columns as usize;
+        if cmo_table.len() != expected_size {
+            return Err(Error::invalid_input(format!(
+                "column metadata offset table has {} bytes but expected {} bytes for {} columns",
+                cmo_table.len(),
+                expected_size,
+                footer.num_columns
+            )));
+        }
+
+        let mut offsets = Vec::with_capacity(footer.num_columns as usize);
+        for col_idx in 0..footer.num_columns {
+            let offset = (col_idx * 16) as usize;
+            let position = LittleEndian::read_u64(&cmo_table[offset..offset + 8]);
+            let length = LittleEndian::read_u64(&cmo_table[offset + 8..offset + 16]);
+            let end = position.checked_add(length).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "column metadata range overflows for column index {}, position={}, length={}",
+                    col_idx, position, length
+                ))
+            })?;
+            if position < footer.column_meta_start || end > footer.column_meta_offsets_start {
+                return Err(Error::invalid_input(format!(
+                    "column metadata range for column index {} is outside metadata region: position={}, length={}, metadata_start={}, cmo_start={}",
+                    col_idx,
+                    position,
+                    length,
+                    footer.column_meta_start,
+                    footer.column_meta_offsets_start
+                )));
+            }
+            offsets.push((position, length));
+        }
+
+        Ok(Arc::from(offsets))
+    }
+
     async fn optimistic_tail_read(
         data: &Bytes,
         start_pos: u64,
@@ -673,7 +985,7 @@ impl FileReader {
             fields: Fields(pb_schema.fields),
             metadata: pb_schema.metadata,
         };
-        let schema = lance_core::datatypes::Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)?;
         Ok((num_rows, schema))
     }
 
@@ -740,24 +1052,8 @@ impl FileReader {
         // is decoded above and never fetched via read_global_buffer) out of the tail
         // so read_global_buffer can serve them without I/O. We copy rather than slice
         // so the much larger tail allocation can be released once decoding is done.
-        let tail_end = tail_offset + tail_bytes.len() as u64;
-        let retained_global_buffers = gbo_table
-            .iter()
-            .enumerate()
-            .skip(1)
-            .filter_map(|(index, buffer)| {
-                let start = buffer.position;
-                let end = buffer.position + buffer.size;
-                if start >= tail_offset && end <= tail_end {
-                    let rel_start = (start - tail_offset) as usize;
-                    let rel_end = (end - tail_offset) as usize;
-                    let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]);
-                    Some((index as u32, bytes))
-                } else {
-                    None
-                }
-            })
-            .collect();
+        let retained_global_buffers =
+            Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset);
 
         Ok(CachedFileMetadata {
             file_schema: Arc::new(schema),
@@ -776,6 +1072,87 @@ impl FileReader {
         })
     }
 
+    async fn read_metadata_index_with_known_schema(
+        scheduler: &FileScheduler,
+        known_schema: Option<(Arc<Schema>, u64)>,
+    ) -> Result<FileMetadataIndex> {
+        let (tail_bytes, file_len) = Self::read_tail(scheduler).await?;
+        let tail_offset = file_len - tail_bytes.len() as u64;
+        let footer = Self::decode_footer(&tail_bytes)?;
+
+        let file_version = LanceFileVersion::try_from_major_minor(
+            footer.major_version as u32,
+            footer.minor_version as u32,
+        )?;
+
+        let gbo_table =
+            Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, file_version).await?;
+        if gbo_table.is_empty() {
+            return Err(Error::internal(
+                "File did not contain any global buffers, schema expected".to_string(),
+            ));
+        }
+        let (file_schema, num_rows) = match known_schema {
+            Some((file_schema, num_rows)) => (file_schema, num_rows),
+            None => {
+                let schema_buffer = &gbo_table[0];
+                let schema_bytes = Self::read_range_from_tail_or_scheduler(
+                    &tail_bytes,
+                    tail_offset,
+                    scheduler,
+                    schema_buffer.position..schema_buffer.position + schema_buffer.size,
+                )
+                .await?;
+                let (num_rows, schema) = Self::decode_schema(schema_bytes)?;
+                (Arc::new(schema), num_rows)
+            }
+        };
+
+        let cmo_table = Self::read_range_from_tail_or_scheduler(
+            &tail_bytes,
+            tail_offset,
+            scheduler,
+            footer.column_meta_offsets_start..footer.global_buff_offsets_start,
+        )
+        .await?;
+        let column_metadata_offsets = Self::decode_cmo_table(cmo_table, &footer)?;
+
+        let retained_global_buffers =
+            Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset);
+
+        Ok(FileMetadataIndex {
+            file_schema,
+            num_rows,
+            file_buffers: gbo_table,
+            column_metadata_offsets,
+            num_columns: footer.num_columns,
+            version: file_version,
+            file_size_bytes: file_len,
+            retained_global_buffers,
+        })
+    }
+
+    /// Reads the lightweight metadata index from a file.
+    ///
+    /// This reads the file schema from the schema global buffer. Use
+    /// [`Self::read_metadata_index_with_schema`] when the caller already has
+    /// the schema and row count from a higher-level metadata source.
+    pub async fn read_metadata_index(scheduler: &FileScheduler) -> Result<FileMetadataIndex> {
+        Self::read_metadata_index_with_known_schema(scheduler, None).await
+    }
+
+    /// Reads the metadata index without fetching the schema global buffer.
+    ///
+    /// Use this when the caller already has the file schema and physical row
+    /// count from an enclosing metadata layer, such as a dataset manifest.
+    pub async fn read_metadata_index_with_schema(
+        scheduler: &FileScheduler,
+        file_schema: Arc<Schema>,
+        num_rows: u64,
+    ) -> Result<FileMetadataIndex> {
+        Self::read_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))).await
+    }
+
     fn fetch_encoding<M: Default + Name + Sized>(encoding: &pbfile::Encoding) -> M {
         match &encoding.location {
             Some(pbfile::encoding::Location::Indirect(_)) => todo!(),
@@ -797,61 +1174,67 @@ impl FileReader {
             .iter()
             .enumerate()
             .map(|(col_idx, col_meta)| {
-                let page_infos = col_meta
-                    .pages
-                    .iter()
-                    .map(|page| {
-                        let num_rows = page.length;
-                        let encoding = match file_version {
-                            LanceFileVersion::V2_0 => {
-                                PageEncoding::Legacy(Self::fetch_encoding::<pbenc::ArrayEncoding>(
-                                    page.encoding.as_ref().unwrap(),
-                                ))
-                            }
-                            _ => PageEncoding::Structural(Self::fetch_encoding::<
-                                pbenc21::PageLayout,
-                            >(
-                                page.encoding.as_ref().unwrap()
-                            )),
-                        };
-                        let buffer_offsets_and_sizes = Arc::from(
-                            page.buffer_offsets
-                                .iter()
-                                .zip(page.buffer_sizes.iter())
-                                .map(|(offset, size)| {
-                                    // Starting with version 2.1 we can assert that page buffers are aligned
-                                    assert!(
-                                        file_version < LanceFileVersion::V2_1
-                                            || offset % PAGE_BUFFER_ALIGNMENT as u64 == 0
-                                    );
-                                    (*offset, *size)
-                                })
-                                .collect::<Vec<_>>(),
-                        );
-                        PageInfo {
-                            buffer_offsets_and_sizes,
-                            encoding,
-                            num_rows,
-                            priority: page.priority,
-                        }
-                    })
-                    .collect::<Vec<_>>();
+                Self::meta_to_col_info(col_idx as u32, col_meta, file_version)
+            })
+            .collect::<Vec<_>>()
+    }
+
+    fn meta_to_col_info(
+        col_idx: u32,
+        col_meta: &pbfile::ColumnMetadata,
+        file_version: LanceFileVersion,
+    ) -> Arc<ColumnInfo> {
+        let page_infos = col_meta
+            .pages
+            .iter()
+            .map(|page| {
+                let num_rows = page.length;
+                let encoding = match file_version {
+                    LanceFileVersion::V2_0 => {
+                        PageEncoding::Legacy(Self::fetch_encoding::<pbenc::ArrayEncoding>(
+                            page.encoding.as_ref().unwrap(),
+                        ))
+                    }
+                    _ => PageEncoding::Structural(Self::fetch_encoding::<pbenc21::PageLayout>(
+                        page.encoding.as_ref().unwrap(),
+                    )),
+                };
                 let buffer_offsets_and_sizes = Arc::from(
-                    col_meta
-                        .buffer_offsets
+                    page.buffer_offsets
                         .iter()
-                        .zip(col_meta.buffer_sizes.iter())
-                        .map(|(offset, size)| (*offset, *size))
+                        .zip(page.buffer_sizes.iter())
+                        .map(|(offset, size)| {
+                            // Starting with version 2.1 we can assert that page buffers are aligned
+                            assert!(
+                                file_version < LanceFileVersion::V2_1
+                                    || offset % PAGE_BUFFER_ALIGNMENT as u64 == 0
+                            );
+                            (*offset, *size)
+                        })
                         .collect::<Vec<_>>(),
                 );
-                Arc::new(ColumnInfo {
-                    index: col_idx as u32,
-                    page_infos: Arc::from(page_infos),
+                PageInfo {
                     buffer_offsets_and_sizes,
-                    encoding: Self::fetch_encoding(col_meta.encoding.as_ref().unwrap()),
-                })
+                    encoding,
+                    num_rows,
+                    priority: page.priority,
+                }
             })
-            .collect::<Vec<_>>()
+            .collect::<Vec<_>>();
+        let buffer_offsets_and_sizes = Arc::from(
+            col_meta
+                .buffer_offsets
+                .iter()
+                .zip(col_meta.buffer_sizes.iter())
+                .map(|(offset, size)| (*offset, *size))
+                .collect::<Vec<_>>(),
+        );
+        Arc::new(ColumnInfo {
+            index: col_idx,
+            page_infos: Arc::from(page_infos),
+            buffer_offsets_and_sizes,
+            encoding: Self::fetch_encoding(col_meta.encoding.as_ref().unwrap()),
+        })
     }
 
     fn validate_projection(
@@ -920,6 +1303,7 @@ impl FileReader {
     /// This method also can accept any kind of `EncodingsIo` implementation allowing
     /// for custom strategies to be used for I/O scheduling (e.g. for takes on fast
     /// disks it may be better to avoid asynchronous overhead).
+    /// Opens a data reader backed by fully decoded file metadata.
     pub async fn try_open_with_file_metadata(
         scheduler: Arc<dyn EncodingsIo>,
         path: Path,
@@ -930,22 +1314,17 @@ impl FileReader {
         options: FileReaderOptions,
     ) -> Result<Self> {
         let cache = Arc::new(cache.with_key_prefix(path.as_ref()));
-
-        if let Some(base_projection) = base_projection.as_ref() {
-            Self::validate_projection(base_projection, &file_metadata)?;
-        }
-        let num_rows = file_metadata.num_rows;
-        Ok(Self {
+        let core = FileReadCore::try_new(
             scheduler,
-            base_projection: base_projection.unwrap_or(ReaderProjection::from_whole_schema(
-                file_metadata.file_schema.as_ref(),
-                file_metadata.version(),
-            )),
-            num_rows,
-            metadata: file_metadata,
+            base_projection,
             decoder_plugins,
+            FileMetadataProvider::Full(file_metadata.clone()),
             cache,
             options,
+        )?;
+        Ok(Self {
+            core,
+            metadata: file_metadata,
         })
     }
 
@@ -1014,42 +1393,18 @@ impl FileReader {
         .await
     }
 
-    async fn read_range(
-        &self,
-        range: Range<u64>,
+    #[allow(clippy::too_many_arguments)]
+    async fn do_take_rows(
+        column_infos: Vec<Arc<ColumnInfo>>,
+        io: Arc<dyn EncodingsIo>,
+        cache: Arc<LanceCache>,
+        decoder_plugins: Arc<DecoderPlugins>,
+        indices: Vec<u64>,
         batch_size: u32,
         projection: ReaderProjection,
         filter: FilterExpression,
-    ) -> Result<BoxStream<'static, ReadBatchTask>> {
-        // Create and initialize the stream
-        Self::do_read_range(
-            self.collect_columns_from_projection(&projection)?,
-            self.scheduler.clone(),
-            self.cache.clone(),
-            self.num_rows,
-            self.decoder_plugins.clone(),
-            range,
-            batch_size,
-            projection,
-            filter,
-            self.options.decoder_config.clone(),
-            self.options.batch_size_bytes,
-        )
-        .await
-    }
-
-    #[allow(clippy::too_many_arguments)]
-    async fn do_take_rows(
-        column_infos: Vec<Arc<ColumnInfo>>,
-        io: Arc<dyn EncodingsIo>,
-        cache: Arc<LanceCache>,
-        decoder_plugins: Arc<DecoderPlugins>,
-        indices: Vec<u64>,
-        batch_size: u32,
-        projection: ReaderProjection,
-        filter: FilterExpression,
-        decoder_config: DecoderConfig,
-        batch_size_bytes: Option<u64>,
+        decoder_config: DecoderConfig,
+        batch_size_bytes: Option<u64>,
     ) -> Result<BoxStream<'static, ReadBatchTask>> {
         debug!(
             "Taking {} rows spread across range {}..{} with batch_size {} from columns {:?}",
@@ -1082,28 +1437,6 @@ impl FileReader {
         .await
     }
 
-    async fn take_rows(
-        &self,
-        indices: Vec<u64>,
-        batch_size: u32,
-        projection: ReaderProjection,
-    ) -> Result<BoxStream<'static, ReadBatchTask>> {
-        // Create and initialize the stream
-        Self::do_take_rows(
-            self.collect_columns_from_projection(&projection)?,
-            self.scheduler.clone(),
-            self.cache.clone(),
-            self.decoder_plugins.clone(),
-            indices,
-            batch_size,
-            projection,
-            FilterExpression::no_filter(),
-            self.options.decoder_config.clone(),
-            self.options.batch_size_bytes,
-        )
-        .await
-    }
-
     #[allow(clippy::too_many_arguments)]
     async fn do_read_ranges(
         column_infos: Vec<Arc<ColumnInfo>>,
@@ -1150,28 +1483,6 @@ impl FileReader {
         .await
     }
 
-    async fn read_ranges(
-        &self,
-        ranges: Vec<Range<u64>>,
-        batch_size: u32,
-        projection: ReaderProjection,
-        filter: FilterExpression,
-    ) -> Result<BoxStream<'static, ReadBatchTask>> {
-        Self::do_read_ranges(
-            self.collect_columns_from_projection(&projection)?,
-            self.scheduler.clone(),
-            self.cache.clone(),
-            self.decoder_plugins.clone(),
-            ranges,
-            batch_size,
-            projection,
-            filter,
-            self.options.decoder_config.clone(),
-            self.options.batch_size_bytes,
-        )
-        .await
-    }
-
     /// Creates a stream of "read tasks" to read the data from the file
     ///
     /// The arguments are similar to [`Self::read_stream_projected`] but instead of returning a stream
@@ -1202,72 +1513,9 @@ impl FileReader {
         projection: Option<ReaderProjection>,
         filter: FilterExpression,
     ) -> Result<Pin<Box<dyn Stream<Item = ReadBatchTask> + Send>>> {
-        let projection = projection.unwrap_or_else(|| self.base_projection.clone());
-        Self::validate_projection(&projection, &self.metadata)?;
-        let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| {
-            if bound > self.num_rows || bound == self.num_rows && inclusive {
-                Err(Error::invalid_input(format!(
-                    "cannot read {:?} from file with {} rows",
-                    params, self.num_rows
-                )))
-            } else {
-                Ok(())
-            }
-        };
-        match &params {
-            ReadBatchParams::Indices(indices) => {
-                for idx in indices {
-                    match idx {
-                        None => {
-                            return Err(Error::invalid_input("Null value in indices array"));
-                        }
-                        Some(idx) => {
-                            verify_bound(&params, idx as u64, true)?;
-                        }
-                    }
-                }
-                let indices = indices.iter().map(|idx| idx.unwrap() as u64).collect();
-                self.take_rows(indices, batch_size, projection).await
-            }
-            ReadBatchParams::Range(range) => {
-                verify_bound(&params, range.end as u64, false)?;
-                self.read_range(
-                    range.start as u64..range.end as u64,
-                    batch_size,
-                    projection,
-                    filter,
-                )
-                .await
-            }
-            ReadBatchParams::Ranges(ranges) => {
-                let mut ranges_u64 = Vec::with_capacity(ranges.len());
-                for range in ranges.as_ref() {
-                    verify_bound(&params, range.end, false)?;
-                    ranges_u64.push(range.start..range.end);
-                }
-                self.read_ranges(ranges_u64, batch_size, projection, filter)
-                    .await
-            }
-            ReadBatchParams::RangeFrom(range) => {
-                verify_bound(&params, range.start as u64, true)?;
-                self.read_range(
-                    range.start as u64..self.num_rows,
-                    batch_size,
-                    projection,
-                    filter,
-                )
-                .await
-            }
-            ReadBatchParams::RangeTo(range) => {
-                verify_bound(&params, range.end as u64, false)?;
-                self.read_range(0..range.end as u64, batch_size, projection, filter)
-                    .await
-            }
-            ReadBatchParams::RangeFull => {
-                self.read_range(0..self.num_rows, batch_size, projection, filter)
-                    .await
-            }
-        }
+        self.core
+            .read_tasks(params, batch_size, projection, filter)
+            .await
     }
 
     /// Reads data from the file as a stream of record batches
@@ -1340,11 +1588,11 @@ impl FileReader {
 
         let config = SchedulerDecoderConfig {
             batch_size,
-            cache: self.cache.clone(),
-            decoder_plugins: self.decoder_plugins.clone(),
-            io: self.scheduler.clone(),
-            decoder_config: self.options.decoder_config.clone(),
-            batch_size_bytes: self.options.batch_size_bytes,
+            cache: self.core.cache.clone(),
+            decoder_plugins: self.core.decoder_plugins.clone(),
+            io: self.core.scheduler.clone(),
+            decoder_config: self.core.options.decoder_config.clone(),
+            batch_size_bytes: self.core.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Indices(indices);
@@ -1380,11 +1628,11 @@ impl FileReader {
 
         let config = SchedulerDecoderConfig {
             batch_size,
-            cache: self.cache.clone(),
-            decoder_plugins: self.decoder_plugins.clone(),
-            io: self.scheduler.clone(),
-            decoder_config: self.options.decoder_config.clone(),
-            batch_size_bytes: self.options.batch_size_bytes,
+            cache: self.core.cache.clone(),
+            decoder_plugins: self.core.decoder_plugins.clone(),
+            io: self.core.scheduler.clone(),
+            decoder_config: self.core.options.decoder_config.clone(),
+            batch_size_bytes: self.core.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(ranges);
@@ -1407,7 +1655,7 @@ impl FileReader {
         filter: FilterExpression,
     ) -> Result<Box<dyn RecordBatchReader + Send + 'static>> {
         let column_infos = self.collect_columns_from_projection(&projection)?;
-        let num_rows = self.num_rows;
+        let num_rows = self.core.num_rows();
 
         debug!(
             "Reading range {:?} with batch_size {} from file with {} rows and {} columns into schema with {} columns",
@@ -1420,11 +1668,11 @@ impl FileReader {
 
         let config = SchedulerDecoderConfig {
             batch_size,
-            cache: self.cache.clone(),
-            decoder_plugins: self.decoder_plugins.clone(),
-            io: self.scheduler.clone(),
-            decoder_config: self.options.decoder_config.clone(),
-            batch_size_bytes: self.options.batch_size_bytes,
+            cache: self.core.cache.clone(),
+            decoder_plugins: self.core.decoder_plugins.clone(),
+            io: self.core.scheduler.clone(),
+            decoder_config: self.core.options.decoder_config.clone(),
+            batch_size_bytes: self.core.options.batch_size_bytes,
         };
 
         let requested_rows = RequestedRows::Ranges(vec![range]);
@@ -1457,13 +1705,22 @@ impl FileReader {
         projection: Option<ReaderProjection>,
         filter: FilterExpression,
     ) -> Result<Box<dyn RecordBatchReader + Send + 'static>> {
-        let projection = projection.unwrap_or_else(|| self.base_projection.clone());
+        let projection = projection.unwrap_or_else(|| self.core.base_projection.clone());
         Self::validate_projection(&projection, &self.metadata)?;
+        // Apply the same projection-length validation as the async path.  This
+        // reader is always backed by full metadata, so we can build the prepared
+        // projection synchronously (no column-metadata I/O) and reuse the shared
+        // check.  `read_len` is the projection's common column length, which
+        // `RangeFull`/`RangeFrom` resolve against rather than `num_rows`.
+        let prepared = PreparedProjection {
+            column_infos: self.metadata.column_infos.clone(),
+            decoder_projection: projection.clone(),
+        };
+        let read_len = self.core.prepared_read_length(&prepared)?;
         let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| {
-            if bound > self.num_rows || bound == self.num_rows && inclusive {
+            if bound > read_len || (bound == read_len && inclusive) {
                 Err(Error::invalid_input(format!(
-                    "cannot read {:?} from file with {} rows",
-                    params, self.num_rows
+                    "cannot read {params:?} from columns with {read_len} rows"
                 )))
             } else {
                 Ok(())
@@ -1504,7 +1761,7 @@ impl FileReader {
             ReadBatchParams::RangeFrom(range) => {
                 verify_bound(&params, range.start as u64, true)?;
                 self.read_range_blocking(
-                    range.start as u64..self.num_rows,
+                    range.start as u64..read_len,
                     batch_size,
                     projection,
                     filter,
@@ -1515,7 +1772,7 @@ impl FileReader {
                 self.read_range_blocking(0..range.end as u64, batch_size, projection, filter)
             }
             ReadBatchParams::RangeFull => {
-                self.read_range_blocking(0..self.num_rows, batch_size, projection, filter)
+                self.read_range_blocking(0..read_len, batch_size, projection, filter)
             }
         }
     }
@@ -1542,144 +1799,789 @@ impl FileReader {
             params,
             batch_size,
             batch_readahead,
-            self.base_projection.clone(),
+            self.core.base_projection.clone(),
             filter,
         )
         .await
     }
 
     pub fn schema(&self) -> &Arc<Schema> {
-        &self.metadata.file_schema
+        self.core.schema()
     }
 }
 
-/// Inspects a page and returns a String describing the page's encoding
-pub fn describe_encoding(page: &pbfile::column_metadata::Page) -> String {
-    if let Some(encoding) = &page.encoding {
-        if let Some(style) = &encoding.location {
-            match style {
-                pbfile::encoding::Location::Indirect(indirect) => {
-                    format!(
-                        "IndirectEncoding(pos={},size={})",
-                        indirect.buffer_location, indirect.buffer_length
-                    )
-                }
-                pbfile::encoding::Location::Direct(direct) => {
-                    let encoding_any =
-                        prost_types::Any::decode(Bytes::from(direct.encoding.clone()))
-                            .expect("failed to deserialize encoding as protobuf");
-                    if encoding_any.type_url == "/lance.encodings.ArrayEncoding" {
-                        let encoding = encoding_any.to_msg::<pbenc::ArrayEncoding>();
-                        match encoding {
-                            Ok(encoding) => {
-                                format!("{:#?}", encoding)
-                            }
-                            Err(err) => {
-                                format!("Unsupported(decode_err={})", err)
-                            }
-                        }
-                    } else if encoding_any.type_url == "/lance.encodings21.PageLayout" {
-                        let encoding = encoding_any.to_msg::<pbenc21::PageLayout>();
-                        match encoding {
-                            Ok(encoding) => {
-                                format!("{:#?}", encoding)
-                            }
-                            Err(err) => {
-                                format!("Unsupported(decode_err={})", err)
-                            }
-                        }
-                    } else {
-                        format!("Unrecognized(type_url={})", encoding_any.type_url)
-                    }
-                }
-                pbfile::encoding::Location::None(_) => "NoEncodingDescription".to_string(),
-            }
-        } else {
-            "MISSING STYLE".to_string()
+impl FileMetadataProvider {
+    fn version(&self) -> LanceFileVersion {
+        match self {
+            Self::Full(metadata) => metadata.version(),
+            Self::Indexed(metadata_index) => metadata_index.version,
         }
-    } else {
-        "MISSING".to_string()
     }
-}
-
-pub trait EncodedBatchReaderExt {
-    fn try_from_mini_lance(
-        bytes: Bytes,
-        schema: &Schema,
-        version: LanceFileVersion,
-    ) -> Result<Self>
-    where
-        Self: Sized;
-    fn try_from_self_described_lance(bytes: Bytes) -> Result<Self>
-    where
-        Self: Sized;
-}
-
-impl EncodedBatchReaderExt for EncodedBatch {
-    fn try_from_mini_lance(
-        bytes: Bytes,
-        schema: &Schema,
-        file_version: LanceFileVersion,
-    ) -> Result<Self>
-    where
-        Self: Sized,
-    {
-        let projection = ReaderProjection::from_whole_schema(schema, file_version);
-        let footer = FileReader::decode_footer(&bytes)?;
 
-        // Next, read the metadata for the columns
-        // This is both the column metadata and the CMO table
-        let column_metadata_start = footer.column_meta_start as usize;
-        let column_metadata_end = footer.global_buff_offsets_start as usize;
-        let column_metadata_bytes = bytes.slice(column_metadata_start..column_metadata_end);
-        let column_metadatas =
-            FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?;
+    fn num_rows(&self) -> u64 {
+        match self {
+            Self::Full(metadata) => metadata.num_rows,
+            Self::Indexed(metadata_index) => metadata_index.num_rows,
+        }
+    }
 
-        let file_version = LanceFileVersion::try_from_major_minor(
-            footer.major_version as u32,
-            footer.minor_version as u32,
-        )?;
+    fn schema(&self) -> &Arc<Schema> {
+        match self {
+            Self::Full(metadata) => &metadata.file_schema,
+            Self::Indexed(metadata_index) => &metadata_index.file_schema,
+        }
+    }
 
-        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version);
+    fn file_buffers(&self) -> &Vec<BufferDescriptor> {
+        match self {
+            Self::Full(metadata) => &metadata.file_buffers,
+            Self::Indexed(metadata_index) => &metadata_index.file_buffers,
+        }
+    }
 
-        Ok(Self {
-            data: bytes,
-            num_rows: page_table
-                .first()
-                .map(|col| col.page_infos.iter().map(|page| page.num_rows).sum::<u64>())
-                .unwrap_or(0),
-            page_table,
-            top_level_columns: projection.column_indices,
-            schema: Arc::new(schema.clone()),
-        })
+    fn retained_global_buffers(&self) -> &BTreeMap<u32, Bytes> {
+        match self {
+            Self::Full(metadata) => &metadata.retained_global_buffers,
+            Self::Indexed(metadata_index) => &metadata_index.retained_global_buffers,
+        }
     }
 
-    fn try_from_self_described_lance(bytes: Bytes) -> Result<Self>
-    where
-        Self: Sized,
-    {
-        let footer = FileReader::decode_footer(&bytes)?;
-        let file_version = LanceFileVersion::try_from_major_minor(
-            footer.major_version as u32,
-            footer.minor_version as u32,
-        )?;
+    fn file_statistics(&self) -> Option<FileStatistics> {
+        let metadata = match self {
+            Self::Full(metadata) => metadata,
+            Self::Indexed(_) => return None,
+        };
+        Some(FileReader::statistics_from_column_metadata(
+            &metadata.column_metadatas,
+        ))
+    }
 
-        let gbo_table = FileReader::do_decode_gbo_table(
-            &bytes.slice(footer.global_buff_offsets_start as usize..),
-            &footer,
-            file_version,
-        )?;
-        if gbo_table.is_empty() {
-            return Err(Error::internal(
-                "File did not contain any global buffers, schema expected".to_string(),
-            ));
+    fn supports_indexed_projection(
+        projection: &ReaderProjection,
+        version: LanceFileVersion,
+    ) -> bool {
+        if version < LanceFileVersion::V2_1 || projection.schema.fields.is_empty() {
+            return false;
         }
-        let schema_start = gbo_table[0].position as usize;
-        let schema_size = gbo_table[0].size as usize;
-
-        let schema_bytes = bytes.slice(schema_start..(schema_start + schema_size));
-        let (_, schema) = FileReader::decode_schema(schema_bytes)?;
-        let projection = ReaderProjection::from_whole_schema(&schema, file_version);
+
+        projection
+            .schema
+            .fields
+            .iter()
+            .try_fold(0usize, |count, field| {
+                count.checked_add(indexed_projection_column_count(field)?)
+            })
+            == Some(projection.column_indices.len())
+    }
+
+    fn validate_indexed_projection(
+        projection: &ReaderProjection,
+        metadata_index: &FileMetadataIndex,
+    ) -> Result<()> {
+        if projection.schema.fields.is_empty() {
+            return Err(Error::invalid_input(
+                "Attempt to read zero columns from the file, at least one column must be specified"
+                    .to_string(),
+            ));
+        }
+        let mut column_indices_seen = BTreeSet::new();
+        for column_index in &projection.column_indices {
+            if !column_indices_seen.insert(*column_index) {
+                return Err(Error::invalid_input(format!(
+                    "The projection specified the column index {} more than once",
+                    column_index
+                )));
+            }
+            if *column_index >= metadata_index.num_columns {
+                return Err(Error::invalid_input(format!(
+                    "The projection specified the column index {} but there are only {} columns in the file",
+                    column_index, metadata_index.num_columns
+                )));
+            }
+        }
+        if !Self::supports_indexed_projection(projection, metadata_index.version) {
+            return Err(Error::not_supported(format!(
+                "lazy column metadata loading requires a V2.1+ ordinary structural projection without blob or packed-struct fields whose physical-column count matches the projection; got file version {:?}, {} schema fields, and {} column indices",
+                metadata_index.version,
+                projection.schema.fields.len(),
+                projection.column_indices.len()
+            )));
+        }
+        Ok(())
+    }
+
+    fn validate_projection(&self, projection: &ReaderProjection) -> Result<()> {
+        match self {
+            Self::Full(metadata) => FileReader::validate_projection(projection, metadata),
+            Self::Indexed(metadata_index) => {
+                Self::validate_indexed_projection(projection, metadata_index)
+            }
+        }
+    }
+
+    fn column_metadata_range(
+        metadata_index: &FileMetadataIndex,
+        column_index: u32,
+    ) -> Result<Range<u64>> {
+        let (position, length) = metadata_index
+            .column_metadata_offsets
+            .get(column_index as usize)
+            .copied()
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "The projection specified the column index {} but there are only {} columns in the file",
+                    column_index, metadata_index.num_columns
+                ))
+            })?;
+        let end = position.checked_add(length).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "column metadata range overflows for column index {}, position={}, length={}",
+                column_index, position, length
+            ))
+        })?;
+        Ok(position..end)
+    }
+
+    async fn load_indexed_column_infos(
+        metadata_index: &FileMetadataIndex,
+        io: &Arc<dyn EncodingsIo>,
+        cache: &Arc<LanceCache>,
+        column_indices: &[u32],
+    ) -> Result<Vec<Arc<ColumnInfo>>> {
+        let mut column_infos = vec![None; column_indices.len()];
+        let mut missing_columns = Vec::new();
+
+        for (result_index, column_index) in column_indices.iter().copied().enumerate() {
+            let cache_key = ColumnMetadataCacheKey { column_index };
+            if let Some(cached) = cache.get_with_key(&cache_key).await {
+                column_infos[result_index] = Some(cached.column_info.clone());
+            } else {
+                let range = Self::column_metadata_range(metadata_index, column_index)?;
+                missing_columns.push((result_index, column_index, range));
+            }
+        }
+
+        missing_columns.sort_by_key(|(_, _, range)| range.start);
+        if !missing_columns.is_empty() {
+            let ranges = missing_columns
+                .iter()
+                .map(|(_, _, range)| range.clone())
+                .collect::<Vec<_>>();
+            let metadata_bytes = io.submit_request(ranges, 0).await?;
+            for ((result_index, column_index, _), bytes) in
+                missing_columns.into_iter().zip(metadata_bytes)
+            {
+                let column_metadata = pbfile::ColumnMetadata::decode(bytes)?;
+                let column_info = FileReader::meta_to_col_info(
+                    column_index,
+                    &column_metadata,
+                    metadata_index.version,
+                );
+                let cached = Arc::new(CachedColumnMetadata {
+                    column_metadata,
+                    column_info: column_info.clone(),
+                });
+                let cache_key = ColumnMetadataCacheKey { column_index };
+                cache.insert_with_key(&cache_key, cached).await;
+                column_infos[result_index] = Some(column_info);
+            }
+        }
+
+        column_infos
+            .into_iter()
+            .enumerate()
+            .map(|(idx, column_info)| {
+                column_info.ok_or_else(|| {
+                    Error::internal(format!(
+                        "lazy metadata loader did not load requested projection column at position {}",
+                        idx
+                    ))
+                })
+            })
+            .collect()
+    }
+
+    async fn prepare_projection(
+        &self,
+        projection: &ReaderProjection,
+        io: &Arc<dyn EncodingsIo>,
+        cache: &Arc<LanceCache>,
+    ) -> Result<PreparedProjection> {
+        self.validate_projection(projection)?;
+        match self {
+            Self::Full(metadata) => Ok(PreparedProjection {
+                column_infos: metadata.column_infos.clone(),
+                decoder_projection: projection.clone(),
+            }),
+            Self::Indexed(metadata_index) => {
+                let column_infos = Self::load_indexed_column_infos(
+                    metadata_index,
+                    io,
+                    cache,
+                    &projection.column_indices,
+                )
+                .await?;
+                let decoder_projection = ReaderProjection {
+                    schema: projection.schema.clone(),
+                    column_indices: (0..projection.column_indices.len())
+                        .map(|idx| idx as u32)
+                        .collect(),
+                };
+                Ok(PreparedProjection {
+                    column_infos,
+                    decoder_projection,
+                })
+            }
+        }
+    }
+}
+
+impl FileReadCore {
+    fn try_new(
+        scheduler: Arc<dyn EncodingsIo>,
+        base_projection: Option<ReaderProjection>,
+        decoder_plugins: Arc<DecoderPlugins>,
+        metadata_provider: FileMetadataProvider,
+        cache: Arc<LanceCache>,
+        options: FileReaderOptions,
+    ) -> Result<Self> {
+        if let Some(base_projection) = base_projection.as_ref() {
+            metadata_provider.validate_projection(base_projection)?;
+        }
+        let base_projection = base_projection.unwrap_or(ReaderProjection::from_whole_schema(
+            metadata_provider.schema().as_ref(),
+            metadata_provider.version(),
+        ));
+        Ok(Self {
+            scheduler,
+            base_projection,
+            metadata_provider,
+            decoder_plugins,
+            cache,
+            options,
+        })
+    }
+
+    fn with_scheduler(&self, scheduler: Arc<dyn EncodingsIo>) -> Self {
+        Self {
+            scheduler,
+            base_projection: self.base_projection.clone(),
+            metadata_provider: self.metadata_provider.clone(),
+            decoder_plugins: self.decoder_plugins.clone(),
+            cache: self.cache.clone(),
+            options: self.options.clone(),
+        }
+    }
+
+    fn version(&self) -> LanceFileVersion {
+        self.metadata_provider.version()
+    }
+
+    fn num_rows(&self) -> u64 {
+        self.metadata_provider.num_rows()
+    }
+
+    fn schema(&self) -> &Arc<Schema> {
+        self.metadata_provider.schema()
+    }
+
+    async fn read_global_buffer(&self, index: u32) -> Result<Bytes> {
+        let file_buffers = self.metadata_provider.file_buffers();
+        let buffer_desc = file_buffers.get(index as usize).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "request for global buffer at index {} but there were only {} global buffers in the file",
+                index,
+                file_buffers.len()
+            ))
+        })?;
+
+        if let Some(bytes) = self.metadata_provider.retained_global_buffers().get(&index) {
+            return Ok(bytes.clone());
+        }
+
+        let bytes = self
+            .scheduler
+            .submit_request(
+                vec![buffer_desc.position..buffer_desc.position + buffer_desc.size],
+                0,
+            )
+            .await?;
+        bytes.into_iter().next().ok_or_else(|| {
+            Error::internal(format!(
+                "global buffer read for index {} returned no bytes",
+                index
+            ))
+        })
+    }
+
+    // The common length to read across a prepared projection, after validating
+    // its columns can be combined into rectangular batches. Each top-level field
+    // is checked for internal consistency (see `validate_field_length`); the
+    // top-level fields must then share a length, since one read combines them.
+    // Ordinary files always pass (every column has `num_rows` rows); files
+    // written with `FileWriter::write_column` whose columns ended up unequal are
+    // rejected here and must be read separately.
+    //
+    // `column_infos` and `decoder_projection.column_indices` line up for both
+    // metadata providers: the full provider keeps absolute indices into the whole
+    // file, while the indexed (lazy) provider loads only the projected columns and
+    // renumbers them 0..N -- in either case `column_infos[column_index]` is the
+    // requested column.
+    fn prepared_read_length(&self, prepared: &PreparedProjection) -> Result<u64> {
+        let is_structural = self.version() >= LanceFileVersion::V2_1;
+        let column_infos = &prepared.column_infos;
+        let column_len = |column: usize| -> Result<u64> {
+            let info = column_infos.get(column).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "projection references column index {} but only {} columns are available",
+                    column,
+                    column_infos.len()
+                ))
+            })?;
+            Ok(info.page_infos.iter().map(|page| page.num_rows).sum())
+        };
+        let column_indices = &prepared.decoder_projection.column_indices;
+        let fields = &prepared.decoder_projection.schema.fields;
+        let mut cursor = 0usize;
+        let mut field_lengths = Vec::with_capacity(fields.len());
+        for field in fields {
+            let rows = validate_field_length(
+                field,
+                is_structural,
+                true,
+                column_indices,
+                &mut cursor,
+                &column_len,
+            )?;
+            field_lengths.push((field.name.as_str(), rows));
+        }
+        if cursor != column_indices.len() {
+            return Err(Error::invalid_input(format!(
+                "projection supplied {} column indices but its fields require {}",
+                column_indices.len(),
+                cursor
+            )));
+        }
+        verify_uniform_lengths(&field_lengths)
+    }
+
+    async fn read_range(
+        &self,
+        range: Range<u64>,
+        batch_size: u32,
+        prepared: PreparedProjection,
+        filter: FilterExpression,
+    ) -> Result<BoxStream<'static, ReadBatchTask>> {
+        FileReader::do_read_range(
+            prepared.column_infos,
+            self.scheduler.clone(),
+            self.cache.clone(),
+            self.num_rows(),
+            self.decoder_plugins.clone(),
+            range,
+            batch_size,
+            prepared.decoder_projection,
+            filter,
+            self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
+        )
+        .await
+    }
+
+    async fn take_rows(
+        &self,
+        indices: Vec<u64>,
+        batch_size: u32,
+        prepared: PreparedProjection,
+    ) -> Result<BoxStream<'static, ReadBatchTask>> {
+        FileReader::do_take_rows(
+            prepared.column_infos,
+            self.scheduler.clone(),
+            self.cache.clone(),
+            self.decoder_plugins.clone(),
+            indices,
+            batch_size,
+            prepared.decoder_projection,
+            FilterExpression::no_filter(),
+            self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
+        )
+        .await
+    }
+
+    async fn read_ranges(
+        &self,
+        ranges: Vec<Range<u64>>,
+        batch_size: u32,
+        prepared: PreparedProjection,
+        filter: FilterExpression,
+    ) -> Result<BoxStream<'static, ReadBatchTask>> {
+        FileReader::do_read_ranges(
+            prepared.column_infos,
+            self.scheduler.clone(),
+            self.cache.clone(),
+            self.decoder_plugins.clone(),
+            ranges,
+            batch_size,
+            prepared.decoder_projection,
+            filter,
+            self.options.decoder_config.clone(),
+            self.options.batch_size_bytes,
+        )
+        .await
+    }
+
+    async fn read_tasks(
+        &self,
+        params: ReadBatchParams,
+        batch_size: u32,
+        projection: Option<ReaderProjection>,
+        filter: FilterExpression,
+    ) -> Result<Pin<Box<dyn Stream<Item = ReadBatchTask> + Send>>> {
+        let projection = projection.unwrap_or_else(|| self.base_projection.clone());
+        let prepared = self
+            .metadata_provider
+            .prepare_projection(&projection, &self.scheduler, &self.cache)
+            .await?;
+        // All projected columns must share a length: the reader combines them
+        // into rectangular batches.  Ordinary files satisfy this (every column
+        // has `num_rows` rows); files written with `FileWriter::write_column`
+        // may not, and such columns must be read separately.  `read_len` is that
+        // common length, which `RangeFull`/`RangeFrom` resolve against (rather
+        // than `num_rows`, the file's longest column).
+        let read_len = self.prepared_read_length(&prepared)?;
+        let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| {
+            if bound > read_len || (bound == read_len && inclusive) {
+                Err(Error::invalid_input(format!(
+                    "cannot read {params:?} from columns with {read_len} rows"
+                )))
+            } else {
+                Ok(())
+            }
+        };
+        match &params {
+            ReadBatchParams::Indices(indices) => {
+                for idx in indices {
+                    match idx {
+                        None => {
+                            return Err(Error::invalid_input("Null value in indices array"));
+                        }
+                        Some(idx) => {
+                            verify_bound(&params, idx as u64, true)?;
+                        }
+                    }
+                }
+                let indices = indices.iter().map(|idx| idx.unwrap() as u64).collect();
+                self.take_rows(indices, batch_size, prepared).await
+            }
+            ReadBatchParams::Range(range) => {
+                verify_bound(&params, range.end as u64, false)?;
+                self.read_range(
+                    range.start as u64..range.end as u64,
+                    batch_size,
+                    prepared,
+                    filter,
+                )
+                .await
+            }
+            ReadBatchParams::Ranges(ranges) => {
+                let mut ranges_u64 = Vec::with_capacity(ranges.len());
+                for range in ranges.as_ref() {
+                    verify_bound(&params, range.end, false)?;
+                    ranges_u64.push(range.start..range.end);
+                }
+                self.read_ranges(ranges_u64, batch_size, prepared, filter)
+                    .await
+            }
+            ReadBatchParams::RangeFrom(range) => {
+                verify_bound(&params, range.start as u64, true)?;
+                self.read_range(range.start as u64..read_len, batch_size, prepared, filter)
+                    .await
+            }
+            ReadBatchParams::RangeTo(range) => {
+                verify_bound(&params, range.end as u64, false)?;
+                self.read_range(0..range.end as u64, batch_size, prepared, filter)
+                    .await
+            }
+            ReadBatchParams::RangeFull => {
+                self.read_range(0..read_len, batch_size, prepared, filter)
+                    .await
+            }
+        }
+    }
+}
+
+impl ProjectedFileReader {
+    /// Opens a data reader backed by indexed column metadata.
+    ///
+    /// `base_projection` must be a supported indexed projection. Reads that do
+    /// not pass an explicit projection use this base projection.
+    pub async fn try_open(
+        scheduler: FileScheduler,
+        base_projection: Option<ReaderProjection>,
+        decoder_plugins: Arc<DecoderPlugins>,
+        cache: &LanceCache,
+        options: FileReaderOptions,
+    ) -> Result<Self> {
+        let base_projection = Self::require_indexed_base_projection(base_projection)?;
+        let metadata_index = Arc::new(FileReader::read_metadata_index(&scheduler).await?);
+        let path = scheduler.reader().path().clone();
+        let encodings_io =
+            LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size);
+        Self::try_open_with_metadata_index(
+            Arc::new(encodings_io),
+            path,
+            Some(base_projection),
+            decoder_plugins,
+            metadata_index,
+            cache,
+            options,
+        )
+        .await
+    }
+
+    /// Opens a data reader from a previously loaded metadata index.
+    ///
+    /// `base_projection` must be a supported indexed projection. Use
+    /// [`Self::try_open_with_file_metadata`] when the default read should cover
+    /// the whole file schema.
+    pub async fn try_open_with_metadata_index(
+        scheduler: Arc<dyn EncodingsIo>,
+        path: Path,
+        base_projection: Option<ReaderProjection>,
+        decoder_plugins: Arc<DecoderPlugins>,
+        metadata_index: Arc<FileMetadataIndex>,
+        cache: &LanceCache,
+        options: FileReaderOptions,
+    ) -> Result<Self> {
+        let base_projection = Self::require_indexed_base_projection(base_projection)?;
+        let cache = Arc::new(cache.with_key_prefix(path.as_ref()));
+        let core = FileReadCore::try_new(
+            scheduler,
+            Some(base_projection),
+            decoder_plugins,
+            FileMetadataProvider::Indexed(metadata_index),
+            cache,
+            options,
+        )?;
+        Ok(Self { core })
+    }
+
+    fn require_indexed_base_projection(
+        base_projection: Option<ReaderProjection>,
+    ) -> Result<ReaderProjection> {
+        base_projection.ok_or_else(|| {
+            Error::invalid_input("ProjectedFileReader requires an explicit base projection")
+        })
+    }
+
+    pub async fn try_open_with_file_metadata(
+        scheduler: Arc<dyn EncodingsIo>,
+        path: Path,
+        base_projection: Option<ReaderProjection>,
+        decoder_plugins: Arc<DecoderPlugins>,
+        file_metadata: Arc<CachedFileMetadata>,
+        cache: &LanceCache,
+        options: FileReaderOptions,
+    ) -> Result<Self> {
+        let cache = Arc::new(cache.with_key_prefix(path.as_ref()));
+        let core = FileReadCore::try_new(
+            scheduler,
+            base_projection,
+            decoder_plugins,
+            FileMetadataProvider::Full(file_metadata),
+            cache,
+            options,
+        )?;
+        Ok(Self { core })
+    }
+
+    /// Returns whether a projection can be served by indexed column metadata.
+    pub fn supports_projection(projection: &ReaderProjection, version: LanceFileVersion) -> bool {
+        FileMetadataProvider::supports_indexed_projection(projection, version)
+    }
+
+    /// Returns a clone of this reader using a different scheduler.
+    pub fn with_scheduler(&self, scheduler: Arc<dyn EncodingsIo>) -> Self {
+        Self {
+            core: self.core.with_scheduler(scheduler),
+        }
+    }
+
+    /// Returns the Lance file version.
+    pub fn version(&self) -> LanceFileVersion {
+        self.core.version()
+    }
+
+    /// Returns the number of rows in the file.
+    pub fn num_rows(&self) -> u64 {
+        self.core.num_rows()
+    }
+
+    /// Returns the file schema visible to this reader.
+    pub fn schema(&self) -> &Arc<Schema> {
+        self.core.schema()
+    }
+
+    /// Returns file statistics when this reader has full file metadata.
+    pub fn file_statistics(&self) -> Option<FileStatistics> {
+        self.core.metadata_provider.file_statistics()
+    }
+
+    #[cfg(test)]
+    fn metadata_index(&self) -> Option<&Arc<FileMetadataIndex>> {
+        match &self.core.metadata_provider {
+            FileMetadataProvider::Indexed(metadata_index) => Some(metadata_index),
+            FileMetadataProvider::Full(_) => None,
+        }
+    }
+
+    /// Reads a global buffer by index.
+    pub async fn read_global_buffer(&self, index: u32) -> Result<Bytes> {
+        self.core.read_global_buffer(index).await
+    }
+
+    /// Creates a stream of read tasks for the requested rows and projection.
+    pub async fn read_tasks(
+        &self,
+        params: ReadBatchParams,
+        batch_size: u32,
+        projection: Option<ReaderProjection>,
+        filter: FilterExpression,
+    ) -> Result<Pin<Box<dyn Stream<Item = ReadBatchTask> + Send>>> {
+        self.core
+            .read_tasks(params, batch_size, projection, filter)
+            .await
+    }
+}
+
+/// Inspects a page and returns a String describing the page's encoding
+pub fn describe_encoding(page: &pbfile::column_metadata::Page) -> String {
+    if let Some(encoding) = &page.encoding {
+        if let Some(style) = &encoding.location {
+            match style {
+                pbfile::encoding::Location::Indirect(indirect) => {
+                    format!(
+                        "IndirectEncoding(pos={},size={})",
+                        indirect.buffer_location, indirect.buffer_length
+                    )
+                }
+                pbfile::encoding::Location::Direct(direct) => {
+                    let encoding_any =
+                        prost_types::Any::decode(Bytes::from(direct.encoding.clone()))
+                            .expect("failed to deserialize encoding as protobuf");
+                    if encoding_any.type_url == "/lance.encodings.ArrayEncoding" {
+                        let encoding = encoding_any.to_msg::<pbenc::ArrayEncoding>();
+                        match encoding {
+                            Ok(encoding) => {
+                                format!("{:#?}", encoding)
+                            }
+                            Err(err) => {
+                                format!("Unsupported(decode_err={})", err)
+                            }
+                        }
+                    } else if encoding_any.type_url == "/lance.encodings21.PageLayout" {
+                        let encoding = encoding_any.to_msg::<pbenc21::PageLayout>();
+                        match encoding {
+                            Ok(encoding) => {
+                                format!("{:#?}", encoding)
+                            }
+                            Err(err) => {
+                                format!("Unsupported(decode_err={})", err)
+                            }
+                        }
+                    } else {
+                        format!("Unrecognized(type_url={})", encoding_any.type_url)
+                    }
+                }
+                pbfile::encoding::Location::None(_) => "NoEncodingDescription".to_string(),
+            }
+        } else {
+            "MISSING STYLE".to_string()
+        }
+    } else {
+        "MISSING".to_string()
+    }
+}
+
+pub trait EncodedBatchReaderExt {
+    fn try_from_mini_lance(
+        bytes: Bytes,
+        schema: &Schema,
+        version: LanceFileVersion,
+    ) -> Result<Self>
+    where
+        Self: Sized;
+    fn try_from_self_described_lance(bytes: Bytes) -> Result<Self>
+    where
+        Self: Sized;
+}
+
+impl EncodedBatchReaderExt for EncodedBatch {
+    fn try_from_mini_lance(
+        bytes: Bytes,
+        schema: &Schema,
+        file_version: LanceFileVersion,
+    ) -> Result<Self>
+    where
+        Self: Sized,
+    {
+        let projection = ReaderProjection::from_whole_schema(schema, file_version);
+        let footer = FileReader::decode_footer(&bytes)?;
+
+        // Next, read the metadata for the columns
+        // This is both the column metadata and the CMO table
+        let column_metadata_start = footer.column_meta_start as usize;
+        let column_metadata_end = footer.global_buff_offsets_start as usize;
+        let column_metadata_bytes = bytes.slice(column_metadata_start..column_metadata_end);
+        let column_metadatas =
+            FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?;
+
+        let file_version = LanceFileVersion::try_from_major_minor(
+            footer.major_version as u32,
+            footer.minor_version as u32,
+        )?;
+
+        let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version);
+
+        Ok(Self {
+            data: bytes,
+            num_rows: page_table
+                .first()
+                .map(|col| col.page_infos.iter().map(|page| page.num_rows).sum::<u64>())
+                .unwrap_or(0),
+            page_table,
+            top_level_columns: projection.column_indices,
+            schema: Arc::new(schema.clone()),
+        })
+    }
+
+    fn try_from_self_described_lance(bytes: Bytes) -> Result<Self>
+    where
+        Self: Sized,
+    {
+        let footer = FileReader::decode_footer(&bytes)?;
+        let file_version = LanceFileVersion::try_from_major_minor(
+            footer.major_version as u32,
+            footer.minor_version as u32,
+        )?;
+
+        let gbo_table = FileReader::do_decode_gbo_table(
+            &bytes.slice(footer.global_buff_offsets_start as usize..),
+            &footer,
+            file_version,
+        )?;
+        if gbo_table.is_empty() {
+            return Err(Error::internal(
+                "File did not contain any global buffers, schema expected".to_string(),
+            ));
+        }
+        let schema_start = gbo_table[0].position as usize;
+        let schema_size = gbo_table[0].size as usize;
+
+        let schema_bytes = bytes.slice(schema_start..(schema_start + schema_size));
+        let (_, schema) = FileReader::decode_schema(schema_bytes)?;
+        let projection = ReaderProjection::from_whole_schema(&schema, file_version);
 
         // Next, read the metadata for the columns
         // This is both the column metadata and the CMO table
@@ -1706,7 +2608,11 @@ impl EncodedBatchReaderExt for EncodedBatch {
 
 #[cfg(test)]
 mod tests {
-    use std::{collections::BTreeMap, pin::Pin, sync::Arc};
+    use std::{
+        collections::{BTreeMap, HashMap},
+        pin::Pin,
+        sync::Arc,
+    };
 
     use arrow_array::{
         RecordBatch, UInt32Array,
@@ -1715,11 +2621,13 @@ mod tests {
     use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema};
     use bytes::Bytes;
     use futures::{StreamExt, prelude::stream::TryStreamExt};
-    use lance_arrow::RecordBatchExt;
+    use lance_arrow::{BLOB_META_KEY, RecordBatchExt};
     use lance_core::{ArrowResult, datatypes::Schema};
-    use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch};
+    use lance_datagen::{ArrayGeneratorExt, BatchCount, ByteCount, RowCount, array, gen_batch};
     use lance_encoding::{
-        decoder::{DecodeBatchScheduler, DecoderPlugins, FilterExpression, decode_batch},
+        decoder::{
+            DecodeBatchScheduler, DecoderPlugins, FilterExpression, ReadBatchTask, decode_batch,
+        },
         encoder::{EncodedBatch, EncodingOptions, default_encoding_strategy, encode_batch},
         version::LanceFileVersion,
     };
@@ -1728,7 +2636,10 @@ mod tests {
     use rstest::rstest;
     use tokio::sync::mpsc;
 
-    use crate::reader::{EncodedBatchReaderExt, FileReader, FileReaderOptions, ReaderProjection};
+    use crate::reader::{
+        EncodedBatchReaderExt, FileReader, FileReaderOptions, ProjectedFileReader,
+        ReaderProjection, validate_field_length, verify_uniform_lengths,
+    };
     use crate::testing::{FsFixture, WrittenFile, test_cache, write_lance_file};
     use crate::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions};
     use lance_encoding::decoder::DecoderConfig;
@@ -1748,13 +2659,85 @@ mod tests {
         if version <= LanceFileVersion::V2_0 {
             reader = reader.col("large_bin", array::rand_type(&DataType::LargeBinary));
         }
-        let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100));
+        let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100));
+
+        write_lance_file(
+            reader,
+            fs,
+            FileWriterOptions {
+                format_version: Some(version),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    async fn create_wide_direct_file(fs: &FsFixture, num_columns: usize) -> WrittenFile {
+        let mut reader = gen_batch();
+        for column_idx in 0..num_columns {
+            reader = reader.col(format!("c{column_idx}"), array::step::<Int32Type>());
+        }
+        let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100));
+
+        write_lance_file(
+            reader,
+            fs,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    async fn create_wide_fixed_size_list_file(fs: &FsFixture, num_columns: usize) -> WrittenFile {
+        let data_type =
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4);
+        let mut reader = gen_batch();
+        for column_idx in 0..num_columns {
+            reader = reader.col(
+                format!("c{column_idx}"),
+                array::rand_type(&data_type).with_random_nulls(0.1),
+            );
+        }
+        let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4));
+
+        write_lance_file(
+            reader,
+            fs,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            },
+        )
+        .await
+    }
+
+    async fn create_wide_structural_file(fs: &FsFixture, num_groups: usize) -> WrittenFile {
+        let struct_type = DataType::Struct(Fields::from(vec![
+            Field::new("x", DataType::Int32, true),
+            Field::new("y", DataType::Int32, true),
+        ]));
+        let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+        let mut reader = gen_batch();
+        for group_idx in 0..num_groups {
+            reader = reader
+                .col(
+                    format!("s{group_idx}"),
+                    array::rand_type(&struct_type).with_random_nulls(0.5),
+                )
+                .col(
+                    format!("l{group_idx}"),
+                    array::rand_type(&list_type).with_random_nulls(0.5),
+                );
+        }
+        let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4));
 
         write_lance_file(
             reader,
             fs,
             FileWriterOptions {
-                format_version: Some(version),
+                format_version: Some(LanceFileVersion::V2_1),
                 ..Default::default()
             },
         )
@@ -1810,287 +2793,814 @@ mod tests {
                 }
             }
         }
-        assert_eq!(remaining, 0);
-    }
-
-    #[tokio::test]
-    async fn test_round_trip() {
-        let fs = FsFixture::default();
-
-        let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await;
+        assert_eq!(remaining, 0);
+    }
+
+    async fn collect_read_tasks(
+        tasks: Pin<Box<dyn futures::Stream<Item = ReadBatchTask> + Send>>,
+        readahead: usize,
+    ) -> Vec<RecordBatch> {
+        tasks
+            .map(|task| task.task)
+            .buffered(readahead)
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_round_trip() {
+        let fs = FsFixture::default();
+
+        let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await;
+
+        for read_size in [32, 1024, 1024 * 1024] {
+            let file_scheduler = fs
+                .scheduler
+                .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+                .await
+                .unwrap();
+            let file_reader = FileReader::try_open(
+                file_scheduler,
+                None,
+                Arc::<DecoderPlugins>::default(),
+                &test_cache(),
+                FileReaderOptions::default(),
+            )
+            .await
+            .unwrap();
+
+            let schema = file_reader.schema();
+            assert_eq!(schema.metadata.get("foo").unwrap(), "bar");
+
+            let batch_stream = file_reader
+                .read_stream(
+                    lance_io::ReadBatchParams::RangeFull,
+                    read_size,
+                    16,
+                    FilterExpression::no_filter(),
+                )
+                .await
+                .unwrap();
+
+            verify_expected(&data, batch_stream, read_size, None).await;
+        }
+    }
+
+    #[rstest]
+    #[test_log::test(tokio::test)]
+    async fn test_encoded_batch_round_trip(
+        // TODO: Add V2_1 (currently fails)
+        #[values(LanceFileVersion::V2_0)] version: LanceFileVersion,
+    ) {
+        let data = gen_batch()
+            .col("x", array::rand::<Int32Type>())
+            .col("y", array::rand_utf8(ByteCount::from(16), false))
+            .into_batch_rows(RowCount::from(10000))
+            .unwrap();
+
+        let lance_schema = Arc::new(Schema::try_from(data.schema().as_ref()).unwrap());
+
+        let encoding_options = EncodingOptions {
+            cache_bytes_per_column: 4096,
+            max_page_bytes: 32 * 1024 * 1024,
+            keep_original_array: true,
+            buffer_alignment: 64,
+            version,
+        };
+
+        let encoding_strategy = default_encoding_strategy(version);
+
+        let encoded_batch = encode_batch(
+            &data,
+            lance_schema.clone(),
+            encoding_strategy.as_ref(),
+            &encoding_options,
+        )
+        .await
+        .unwrap();
+
+        // Test self described
+        let bytes = encoded_batch.try_to_self_described_lance(version).unwrap();
+
+        let decoded_batch = EncodedBatch::try_from_self_described_lance(bytes).unwrap();
+
+        let decoded = decode_batch(
+            &decoded_batch,
+            &FilterExpression::no_filter(),
+            Arc::<DecoderPlugins>::default(),
+            false,
+            version,
+            None,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(data, decoded);
+
+        // Test mini
+        let bytes = encoded_batch.try_to_mini_lance(version).unwrap();
+        let decoded_batch =
+            EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref(), LanceFileVersion::V2_0)
+                .unwrap();
+        let decoded = decode_batch(
+            &decoded_batch,
+            &FilterExpression::no_filter(),
+            Arc::<DecoderPlugins>::default(),
+            false,
+            version,
+            None,
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(data, decoded);
+    }
+
+    #[rstest]
+    #[test_log::test(tokio::test)]
+    async fn test_projection(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)]
+        version: LanceFileVersion,
+    ) {
+        let fs = FsFixture::default();
+
+        let written_file = create_some_file(&fs, version).await;
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+
+        let field_id_mapping = written_file
+            .field_id_mapping
+            .iter()
+            .copied()
+            .collect::<BTreeMap<_, _>>();
+
+        let empty_projection = ReaderProjection {
+            column_indices: Vec::default(),
+            schema: Arc::new(Schema::default()),
+        };
+
+        for columns in [
+            vec!["score"],
+            vec!["location"],
+            vec!["categories"],
+            vec!["score.x"],
+            vec!["score", "categories"],
+            vec!["score", "location"],
+            vec!["location", "categories"],
+            vec!["score.y", "location", "categories"],
+        ] {
+            debug!("Testing round trip with projection {:?}", columns);
+            for use_field_ids in [true, false] {
+                // We can specify the projection as part of the read operation via read_stream_projected
+                let file_reader = FileReader::try_open(
+                    file_scheduler.clone(),
+                    None,
+                    Arc::<DecoderPlugins>::default(),
+                    &test_cache(),
+                    FileReaderOptions::default(),
+                )
+                .await
+                .unwrap();
+
+                let projected_schema = written_file.schema.project(&columns).unwrap();
+                let projection = if use_field_ids {
+                    ReaderProjection::from_field_ids(
+                        file_reader.metadata.version(),
+                        &projected_schema,
+                        &field_id_mapping,
+                    )
+                    .unwrap()
+                } else {
+                    ReaderProjection::from_column_names(
+                        file_reader.metadata.version(),
+                        &written_file.schema,
+                        &columns,
+                    )
+                    .unwrap()
+                };
+
+                let batch_stream = file_reader
+                    .read_stream_projected(
+                        lance_io::ReadBatchParams::RangeFull,
+                        1024,
+                        16,
+                        projection.clone(),
+                        FilterExpression::no_filter(),
+                    )
+                    .await
+                    .unwrap();
+
+                let projection_arrow = ArrowSchema::from(projection.schema.as_ref());
+                verify_expected(
+                    &written_file.data,
+                    batch_stream,
+                    1024,
+                    Some(Box::new(move |batch: &RecordBatch| {
+                        batch.project_by_schema(&projection_arrow).unwrap()
+                    })),
+                )
+                .await;
+
+                // We can also specify the projection as a base projection when we open the file
+                let file_reader = FileReader::try_open(
+                    file_scheduler.clone(),
+                    Some(projection.clone()),
+                    Arc::<DecoderPlugins>::default(),
+                    &test_cache(),
+                    FileReaderOptions::default(),
+                )
+                .await
+                .unwrap();
+
+                let batch_stream = file_reader
+                    .read_stream(
+                        lance_io::ReadBatchParams::RangeFull,
+                        1024,
+                        16,
+                        FilterExpression::no_filter(),
+                    )
+                    .await
+                    .unwrap();
+
+                let projection_arrow = ArrowSchema::from(projection.schema.as_ref());
+                verify_expected(
+                    &written_file.data,
+                    batch_stream,
+                    1024,
+                    Some(Box::new(move |batch: &RecordBatch| {
+                        batch.project_by_schema(&projection_arrow).unwrap()
+                    })),
+                )
+                .await;
+
+                assert!(
+                    file_reader
+                        .read_stream_projected(
+                            lance_io::ReadBatchParams::RangeFull,
+                            1024,
+                            16,
+                            empty_projection.clone(),
+                            FilterExpression::no_filter(),
+                        )
+                        .await
+                        .is_err()
+                );
+            }
+        }
 
-        for read_size in [32, 1024, 1024 * 1024] {
-            let file_scheduler = fs
-                .scheduler
-                .open_file(&fs.tmp_path, &CachedFileSize::unknown())
-                .await
-                .unwrap();
-            let file_reader = FileReader::try_open(
-                file_scheduler,
-                None,
+        assert!(
+            FileReader::try_open(
+                file_scheduler.clone(),
+                Some(empty_projection),
                 Arc::<DecoderPlugins>::default(),
                 &test_cache(),
                 FileReaderOptions::default(),
             )
             .await
-            .unwrap();
+            .is_err()
+        );
 
-            let schema = file_reader.schema();
-            assert_eq!(schema.metadata.get("foo").unwrap(), "bar");
+        let arrow_schema = ArrowSchema::new(vec![
+            Field::new("x", DataType::Int32, true),
+            Field::new("y", DataType::Int32, true),
+        ]);
+        let schema = Schema::try_from(&arrow_schema).unwrap();
 
-            let batch_stream = file_reader
-                .read_stream(
-                    lance_io::ReadBatchParams::RangeFull,
-                    read_size,
-                    16,
-                    FilterExpression::no_filter(),
-                )
-                .await
-                .unwrap();
+        let projection_with_dupes = ReaderProjection {
+            column_indices: vec![0, 0],
+            schema: Arc::new(schema),
+        };
 
-            verify_expected(&data, batch_stream, read_size, None).await;
-        }
+        assert!(
+            FileReader::try_open(
+                file_scheduler.clone(),
+                Some(projection_with_dupes),
+                Arc::<DecoderPlugins>::default(),
+                &test_cache(),
+                FileReaderOptions::default(),
+            )
+            .await
+            .is_err()
+        );
     }
 
-    #[rstest]
-    #[test_log::test(tokio::test)]
-    async fn test_encoded_batch_round_trip(
-        // TODO: Add V2_1 (currently fails)
-        #[values(LanceFileVersion::V2_0)] version: LanceFileVersion,
-    ) {
-        let data = gen_batch()
-            .col("x", array::rand::<Int32Type>())
-            .col("y", array::rand_utf8(ByteCount::from(16), false))
-            .into_batch_rows(RowCount::from(10000))
+    #[tokio::test]
+    async fn test_lazy_reader_direct_projection_matches_eager_reader() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_direct_file(&fs, 16).await;
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
             .unwrap();
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["c10"],
+        )
+        .unwrap();
 
-        let lance_schema = Arc::new(Schema::try_from(data.schema().as_ref()).unwrap());
+        let eager_reader = FileReader::try_open(
+            file_scheduler.clone(),
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let expected = eager_reader
+            .read_stream_projected(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                16,
+                projection.clone(),
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
 
-        let encoding_options = EncodingOptions {
-            cache_bytes_per_column: 4096,
-            max_page_bytes: 32 * 1024 * 1024,
-            keep_original_array: true,
-            buffer_alignment: 64,
-            version,
-        };
+        let cache = test_cache();
+        let lazy_reader = ProjectedFileReader::try_open(
+            file_scheduler,
+            Some(projection.clone()),
+            Arc::<DecoderPlugins>::default(),
+            &cache,
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let actual = collect_read_tasks(tasks, 16).await;
 
-        let encoding_strategy = default_encoding_strategy(version);
+        assert_eq!(expected, actual);
+    }
 
-        let encoded_batch = encode_batch(
-            &data,
-            lance_schema.clone(),
-            encoding_strategy.as_ref(),
-            &encoding_options,
+    #[tokio::test]
+    async fn test_lazy_reader_loads_only_requested_column_metadata() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_direct_file(&fs, 512).await;
+
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["c0"],
+        )
+        .unwrap();
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let lazy_reader = ProjectedFileReader::try_open(
+            file_scheduler,
+            Some(projection.clone()),
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
         )
         .await
         .unwrap();
+        let selected_column = projection.column_indices[0] as usize;
+        let requested_metadata_bytes = lazy_reader
+            .metadata_index()
+            .unwrap()
+            .column_metadata_offsets[selected_column]
+            .1;
+        let total_metadata_bytes = lazy_reader
+            .metadata_index()
+            .unwrap()
+            .column_metadata_offsets
+            .iter()
+            .map(|(_, length)| *length)
+            .sum::<u64>();
+        assert!(
+            total_metadata_bytes > 8 * fs.object_store.block_size() as u64,
+            "test file metadata is too small to prove lazy loading: {total_metadata_bytes} bytes"
+        );
 
-        // Test self described
-        let bytes = encoded_batch.try_to_self_described_lance(version).unwrap();
+        fs.object_store.io_stats_incremental();
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::Range(0..0),
+                1024,
+                Some(projection.clone()),
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let batches = collect_read_tasks(tasks, 1).await;
+        assert!(batches.is_empty());
 
-        let decoded_batch = EncodedBatch::try_from_self_described_lance(bytes).unwrap();
+        let stats = fs.object_store.io_stats_incremental();
+        assert!(
+            stats.read_bytes < total_metadata_bytes / 2,
+            "lazy read fetched too much metadata: read {} bytes, requested column metadata is {} bytes, total column metadata is {} bytes",
+            stats.read_bytes,
+            requested_metadata_bytes,
+            total_metadata_bytes
+        );
 
-        let decoded = decode_batch(
-            &decoded_batch,
-            &FilterExpression::no_filter(),
-            Arc::<DecoderPlugins>::default(),
-            false,
-            version,
+        fs.object_store.io_stats_incremental();
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::Range(0..0),
+                1024,
+                Some(projection),
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let batches = collect_read_tasks(tasks, 1).await;
+        assert!(batches.is_empty());
+
+        let stats = fs.object_store.io_stats_incremental();
+        assert_eq!(
+            stats.read_iops, 0,
+            "cached column metadata should avoid repeat metadata I/O"
+        );
+        assert_eq!(
+            stats.read_bytes, 0,
+            "cached column metadata should avoid repeat metadata reads"
+        );
+    }
+
+    async fn assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+        fs: &FsFixture,
+        projection: ReaderProjection,
+        shape: &str,
+    ) -> Vec<RecordBatch> {
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let eager_reader = FileReader::try_open(
+            file_scheduler.clone(),
             None,
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
         )
         .await
         .unwrap();
+        let expected = eager_reader
+            .read_stream_projected(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                16,
+                projection.clone(),
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
 
-        assert_eq!(data, decoded);
-
-        // Test mini
-        let bytes = encoded_batch.try_to_mini_lance(version).unwrap();
-        let decoded_batch =
-            EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref(), LanceFileVersion::V2_0)
-                .unwrap();
-        let decoded = decode_batch(
-            &decoded_batch,
-            &FilterExpression::no_filter(),
+        let cache = test_cache();
+        let lazy_reader = ProjectedFileReader::try_open(
+            file_scheduler,
+            Some(projection.clone()),
             Arc::<DecoderPlugins>::default(),
-            false,
-            version,
-            None,
+            &cache,
+            FileReaderOptions::default(),
         )
         .await
         .unwrap();
+        let metadata_index = lazy_reader.metadata_index().unwrap();
+        let requested_metadata_bytes = projection
+            .column_indices
+            .iter()
+            .map(|column_index| metadata_index.column_metadata_offsets[*column_index as usize].1)
+            .sum::<u64>();
+        let total_metadata_bytes = metadata_index
+            .column_metadata_offsets
+            .iter()
+            .map(|(_, length)| *length)
+            .sum::<u64>();
+        assert!(total_metadata_bytes > requested_metadata_bytes * 8);
 
-        assert_eq!(data, decoded);
+        fs.object_store.io_stats_incremental();
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::Range(0..0),
+                127,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        assert!(collect_read_tasks(tasks, 1).await.is_empty());
+        let metadata_stats = fs.object_store.io_stats_incremental();
+        assert!(
+            metadata_stats.read_bytes < total_metadata_bytes / 2,
+            "lazy {shape} read fetched too much metadata: read {} bytes, requested column metadata is {} bytes, total column metadata is {} bytes",
+            metadata_stats.read_bytes,
+            requested_metadata_bytes,
+            total_metadata_bytes
+        );
+
+        let tasks = lazy_reader
+            .read_tasks(
+                lance_io::ReadBatchParams::RangeFull,
+                127,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let actual = collect_read_tasks(tasks, 16).await;
+        assert_eq!(expected, actual);
+        actual
+    }
+
+    #[tokio::test]
+    async fn test_lazy_reader_fixed_size_list_projection_matches_eager_reader() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_fixed_size_list_file(&fs, 512).await;
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["c17", "c509"],
+        )
+        .unwrap();
+        assert!(ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
+        assert!(!ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_0
+        ));
+        assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+            &fs,
+            projection,
+            "fixed-size-list",
+        )
+        .await;
+    }
+
+    #[tokio::test]
+    async fn test_lazy_reader_nested_projection_compacts_physical_columns() {
+        let fs = FsFixture::default();
+        let written_file = create_wide_structural_file(&fs, 128).await;
+        let projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["s97.y", "l4", "s3"],
+        )
+        .unwrap();
+
+        assert_eq!(
+            projection
+                .schema
+                .fields
+                .iter()
+                .map(|field| field.name.as_str())
+                .collect::<Vec<_>>(),
+            vec!["s97", "l4", "s3"]
+        );
+        assert_eq!(projection.schema.fields[0].children.len(), 1);
+        assert_eq!(projection.schema.fields[0].children[0].name, "y");
+        assert_eq!(projection.schema.fields[2].children.len(), 2);
+        assert_eq!(projection.column_indices.len(), 4);
+        assert!(
+            projection
+                .column_indices
+                .windows(2)
+                .any(|indices| indices[0] > indices[1]),
+            "the projection must reorder physical columns to exercise compact remapping"
+        );
+        assert!(ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
+        let actual = assert_lazy_projection_matches_eager_and_reads_metadata_subset(
+            &fs, projection, "nested",
+        )
+        .await;
+        assert!(
+            actual
+                .iter()
+                .flat_map(|batch| batch.columns())
+                .any(|column| column.null_count() > 0),
+            "the structural projection must exercise nullable arrays"
+        );
     }
 
     #[rstest]
-    #[test_log::test(tokio::test)]
-    async fn test_projection(
-        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)]
-        version: LanceFileVersion,
+    #[case::before_metadata_region(90, 5)]
+    #[case::after_metadata_region(190, 20)]
+    fn test_decode_cmo_table_rejects_out_of_range_offsets(
+        #[case] position: u64,
+        #[case] length: u64,
     ) {
+        let mut cmo_table = [0; 16];
+        cmo_table[0..8].copy_from_slice(&position.to_le_bytes());
+        cmo_table[8..16].copy_from_slice(&length.to_le_bytes());
+        let footer = super::Footer {
+            column_meta_start: 100,
+            column_meta_offsets_start: 200,
+            global_buff_offsets_start: 200,
+            num_global_buffers: 0,
+            num_columns: 1,
+            major_version: 2,
+            minor_version: 1,
+        };
+
+        let err = FileReader::decode_cmo_table(Bytes::copy_from_slice(&cmo_table), &footer)
+            .expect_err("out-of-range CMO entries must be rejected");
+        assert!(
+            matches!(err, lance_core::Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+    }
+
+    #[rstest]
+    #[case::blob(BLOB_META_KEY)]
+    #[case::packed_struct("lance-encoding:packed")]
+    #[tokio::test]
+    async fn test_lazy_reader_rejects_opaque_projection(#[case] metadata_key: &str) {
         let fs = FsFixture::default();
+        let written_file = create_some_file(&fs, LanceFileVersion::V2_1).await;
+
+        let ordinary_projection = ReaderProjection::from_column_names(
+            LanceFileVersion::V2_1,
+            &written_file.schema,
+            &["location.x"],
+        )
+        .unwrap();
+        assert_eq!(ordinary_projection.schema.fields[0].children.len(), 1);
+        assert!(ProjectedFileReader::supports_projection(
+            &ordinary_projection,
+            LanceFileVersion::V2_1
+        ));
 
-        let written_file = create_some_file(&fs, version).await;
         let file_scheduler = fs
             .scheduler
             .open_file(&fs.tmp_path, &CachedFileSize::unknown())
             .await
             .unwrap();
+        let err = ProjectedFileReader::try_open(
+            file_scheduler.clone(),
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            matches!(err, lance_core::Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
 
-        let field_id_mapping = written_file
-            .field_id_mapping
-            .iter()
-            .copied()
-            .collect::<BTreeMap<_, _>>();
-
-        let empty_projection = ReaderProjection {
-            column_indices: Vec::default(),
-            schema: Arc::new(Schema::default()),
-        };
-
-        for columns in [
-            vec!["score"],
-            vec!["location"],
-            vec!["categories"],
-            vec!["score.x"],
-            vec!["score", "categories"],
-            vec!["score", "location"],
-            vec!["location", "categories"],
-            vec!["score.y", "location", "categories"],
-        ] {
-            debug!("Testing round trip with projection {:?}", columns);
-            for use_field_ids in [true, false] {
-                // We can specify the projection as part of the read operation via read_stream_projected
-                let file_reader = FileReader::try_open(
-                    file_scheduler.clone(),
-                    None,
-                    Arc::<DecoderPlugins>::default(),
-                    &test_cache(),
-                    FileReaderOptions::default(),
-                )
-                .await
-                .unwrap();
+        let mut projection = ordinary_projection;
+        Arc::make_mut(&mut projection.schema).fields[0]
+            .metadata
+            .insert(metadata_key.to_string(), "true".to_string());
+        assert!(!ProjectedFileReader::supports_projection(
+            &projection,
+            LanceFileVersion::V2_1
+        ));
 
-                let projected_schema = written_file.schema.project(&columns).unwrap();
-                let projection = if use_field_ids {
-                    ReaderProjection::from_field_ids(
-                        file_reader.metadata.version(),
-                        &projected_schema,
-                        &field_id_mapping,
-                    )
-                    .unwrap()
-                } else {
-                    ReaderProjection::from_column_names(
-                        file_reader.metadata.version(),
-                        &written_file.schema,
-                        &columns,
-                    )
-                    .unwrap()
-                };
+        let err = ProjectedFileReader::try_open(
+            file_scheduler,
+            Some(projection),
+            Arc::<DecoderPlugins>::default(),
+            &test_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            matches!(err, lance_core::Error::NotSupported { .. }),
+            "expected NotSupported for {metadata_key}, got {err:?}"
+        );
+    }
 
-                let batch_stream = file_reader
-                    .read_stream_projected(
-                        lance_io::ReadBatchParams::RangeFull,
-                        1024,
-                        16,
-                        projection.clone(),
-                        FilterExpression::no_filter(),
-                    )
-                    .await
-                    .unwrap();
+    // The projection-length validation lives in `FileReadCore`, shared by the
+    // eager and the lazy (indexed) metadata providers. The indexed provider loads
+    // only the projected columns and renumbers them 0..N, so this checks that the
+    // renumbered `column_infos`/`column_indices` still line up for the length
+    // check: a mismatched-length projection is rejected through
+    // `ProjectedFileReader`, and a single short column resolves to its own length.
+    #[tokio::test]
+    async fn test_lazy_reader_validates_unequal_length_projection() {
+        use arrow_array::Int32Array;
+        use lance_io::ReadBatchParams;
 
-                let projection_arrow = ArrowSchema::from(projection.schema.as_ref());
-                verify_expected(
-                    &written_file.data,
-                    batch_stream,
-                    1024,
-                    Some(Box::new(move |batch: &RecordBatch| {
-                        batch.project_by_schema(&projection_arrow).unwrap()
-                    })),
-                )
-                .await;
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("a", DataType::Int32, true),
+            Field::new("c", DataType::Int32, true),
+        ]));
+        let lance_schema = Schema::try_from(arrow_schema.as_ref()).unwrap();
 
-                // We can also specify the projection as a base projection when we open the file
-                let file_reader = FileReader::try_open(
-                    file_scheduler.clone(),
-                    Some(projection.clone()),
-                    Arc::<DecoderPlugins>::default(),
-                    &test_cache(),
-                    FileReaderOptions::default(),
-                )
-                .await
-                .unwrap();
+        let fs = FsFixture::default();
+        let options = FileWriterOptions {
+            format_version: Some(LanceFileVersion::V2_1),
+            ..Default::default()
+        };
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            options,
+        )
+        .unwrap();
+        // "a" has 5 rows, "c" has 1 -- an unequal-length file.
+        writer
+            .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])))
+            .await
+            .unwrap();
+        writer
+            .write_column(1, Arc::new(Int32Array::from(vec![100])))
+            .await
+            .unwrap();
+        writer.finish().await.unwrap();
 
-                let batch_stream = file_reader
-                    .read_stream(
-                        lance_io::ReadBatchParams::RangeFull,
-                        1024,
-                        16,
-                        FilterExpression::no_filter(),
-                    )
-                    .await
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let cache = test_cache();
+        let open_indexed = |names: &[&str]| {
+            let projection =
+                ReaderProjection::from_column_names(LanceFileVersion::V2_1, &lance_schema, names)
                     .unwrap();
-
-                let projection_arrow = ArrowSchema::from(projection.schema.as_ref());
-                verify_expected(
-                    &written_file.data,
-                    batch_stream,
-                    1024,
-                    Some(Box::new(move |batch: &RecordBatch| {
-                        batch.project_by_schema(&projection_arrow).unwrap()
-                    })),
-                )
-                .await;
-
-                assert!(
-                    file_reader
-                        .read_stream_projected(
-                            lance_io::ReadBatchParams::RangeFull,
-                            1024,
-                            16,
-                            empty_projection.clone(),
-                            FilterExpression::no_filter(),
-                        )
-                        .await
-                        .is_err()
-                );
-            }
-        }
-
-        assert!(
-            FileReader::try_open(
+            ProjectedFileReader::try_open(
                 file_scheduler.clone(),
-                Some(empty_projection),
+                Some(projection),
                 Arc::<DecoderPlugins>::default(),
-                &test_cache(),
+                &cache,
                 FileReaderOptions::default(),
             )
-            .await
-            .is_err()
-        );
-
-        let arrow_schema = ArrowSchema::new(vec![
-            Field::new("x", DataType::Int32, true),
-            Field::new("y", DataType::Int32, true),
-        ]);
-        let schema = Schema::try_from(&arrow_schema).unwrap();
-
-        let projection_with_dupes = ReaderProjection {
-            column_indices: vec![0, 0],
-            schema: Arc::new(schema),
         };
 
-        assert!(
-            FileReader::try_open(
-                file_scheduler.clone(),
-                Some(projection_with_dupes),
-                Arc::<DecoderPlugins>::default(),
-                &test_cache(),
-                FileReaderOptions::default(),
+        // A mismatched-length projection [a, c] (5 vs 1) is rejected at read time,
+        // through the indexed provider's renumbered column infos.
+        let lazy = open_indexed(&["a", "c"]).await.unwrap();
+        // (`read_tasks` yields a stream, which is not `Debug`, so match rather
+        // than `unwrap_err`.)
+        let err = match lazy
+            .read_tasks(
+                ReadBatchParams::RangeFull,
+                1024,
+                None,
+                FilterExpression::no_filter(),
             )
             .await
-            .is_err()
+        {
+            Ok(_) => panic!("expected the mismatched-length projection to be rejected"),
+            Err(e) => e.to_string(),
+        };
+        assert!(
+            err.contains("a=5") && err.contains("c=1"),
+            "error should name each column's length, got: {err}"
         );
+
+        // A single short column resolves to its own length (1), not the file's
+        // longest column.
+        let lazy = open_indexed(&["c"]).await.unwrap();
+        let tasks = lazy
+            .read_tasks(
+                ReadBatchParams::RangeFull,
+                1024,
+                None,
+                FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        let batches = collect_read_tasks(tasks, 16).await;
+        let values: Vec<Option<i32>> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>()
+            })
+            .collect();
+        assert_eq!(values, vec![Some(100)]);
     }
 
     #[test_log::test(tokio::test)]
@@ -2317,7 +3827,7 @@ mod tests {
             &vec![],
             total_rows as u64,
             Arc::<DecoderPlugins>::default(),
-            file_reader.scheduler.clone(),
+            file_reader.core.scheduler.clone(),
             test_cache(),
             &FilterExpression::no_filter(),
             &DecoderConfig::default(),
@@ -2337,7 +3847,7 @@ mod tests {
             range,
             &FilterExpression::no_filter(),
             tx,
-            file_reader.scheduler.clone(),
+            file_reader.core.scheduler.clone(),
         )
     }
 
@@ -2511,7 +4021,7 @@ mod tests {
         // column_metadatas and column_infos, otherwise the moka cache weigher
         // dramatically underestimates entry sizes and never evicts, causing
         // unbounded memory growth on random-access workloads.
-        use deepsize::DeepSizeOf;
+        use lance_core::deepsize::DeepSizeOf;
 
         let fs = FsFixture::default();
         let _written = create_some_file(&fs, version).await;
@@ -2591,4 +4101,158 @@ mod tests {
         let msg = err.to_string();
         assert!(msg.contains('2'), "error should mention the index: {msg}");
     }
+
+    // Exercises the projection length-validation walk in isolation, feeding
+    // synthetic per-column lengths so we can reach cases no current writer can
+    // actually produce -- in particular a struct whose children diverge in
+    // length, which the decoders would otherwise panic on or misread.
+    #[rstest]
+    fn test_validate_struct_child_lengths(#[values(false, true)] is_structural: bool) {
+        let run = |dt: DataType, indices: &[u32], lengths: Vec<u64>| -> lance_core::Result<u64> {
+            let arrow = ArrowSchema::new(vec![Field::new("s", dt, true)]);
+            let schema = Schema::try_from(&arrow).unwrap();
+            let column_len = |c: usize| Ok(lengths[c]);
+            let mut cursor = 0usize;
+            let mut field_lengths = Vec::new();
+            for field in &schema.fields {
+                let rows = validate_field_length(
+                    field,
+                    is_structural,
+                    true,
+                    indices,
+                    &mut cursor,
+                    &column_len,
+                )?;
+                field_lengths.push((field.name.as_str(), rows));
+            }
+            verify_uniform_lengths(&field_lengths)
+        };
+
+        let struct_ty = || {
+            DataType::Struct(Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Int32, true),
+            ]))
+        };
+
+        // In 2.1 a struct contributes no column of its own (just its two leaves);
+        // in 2.0 it also has its own column first.
+        let (indices, equal, unequal): (&[u32], Vec<u64>, Vec<u64>) = if is_structural {
+            (&[0, 1], vec![5, 5], vec![5, 3])
+        } else {
+            (&[0, 1, 2], vec![5, 5, 5], vec![5, 5, 3])
+        };
+
+        assert_eq!(run(struct_ty(), indices, equal).unwrap(), 5);
+
+        let err = run(struct_ty(), indices, unequal).unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("differing lengths") && msg.contains('b'),
+            "expected a child-length error naming 'b', got: {msg}"
+        );
+    }
+
+    #[test]
+    fn test_validate_v2_0_unloaded_blob_projection_is_opaque() {
+        let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let arrow = ArrowSchema::new(vec![
+            Field::new("blob", DataType::LargeBinary, true).with_metadata(metadata),
+        ]);
+        let mut schema = Schema::try_from(&arrow).unwrap();
+        schema.fields[0].unloaded_mut();
+        let projection = ReaderProjection {
+            schema: Arc::new(schema),
+            column_indices: vec![0],
+        };
+        let column_len = |column: usize| {
+            assert_eq!(column, 0);
+            Ok(3)
+        };
+        let mut cursor = 0usize;
+
+        let rows = validate_field_length(
+            &projection.schema.fields[0],
+            false,
+            true,
+            &projection.column_indices,
+            &mut cursor,
+            &column_len,
+        )
+        .unwrap();
+
+        assert_eq!(rows, 3);
+        assert_eq!(cursor, 1);
+    }
+
+    #[test]
+    fn test_validate_length_list_and_empty_struct() {
+        let validate = |dt: DataType,
+                        is_structural: bool,
+                        indices: &[u32],
+                        lengths: Vec<u64>|
+         -> lance_core::Result<u64> {
+            let arrow = ArrowSchema::new(vec![Field::new("f", dt, true)]);
+            let schema = Schema::try_from(&arrow).unwrap();
+            let column_len = |c: usize| Ok(lengths[c]);
+            let mut cursor = 0usize;
+            validate_field_length(
+                &schema.fields[0],
+                is_structural,
+                true,
+                indices,
+                &mut cursor,
+                &column_len,
+            )
+        };
+
+        // A list's items have a different cardinality than its rows; that gap
+        // must not be flagged as a mismatch. In 2.0 the list is an offsets column
+        // (rows) plus an items column (item count); in 2.1 it is a single column
+        // whose page rows are the top-level row count.
+        let list_ty = DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
+        assert_eq!(
+            validate(list_ty.clone(), false, &[0, 1], vec![5, 17]).unwrap(),
+            5
+        );
+        assert_eq!(validate(list_ty, true, &[0], vec![5]).unwrap(), 5);
+
+        // A list of structs: the struct's children sit below the list boundary, so
+        // their (item-count) lengths must NOT be compared against the list's row
+        // count. In 2.0 each field has a column [list, struct, a, b] = [6, 6, 29,
+        // 29]; the list resolves to its own row count (6) and the struct's longer
+        // children are not flagged. (Regression: this previously errored because
+        // the nested struct's children were checked against the struct's count.)
+        let list_of_struct = DataType::List(Arc::new(Field::new(
+            "item",
+            DataType::Struct(Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Int32, true),
+            ])),
+            true,
+        )));
+        assert_eq!(
+            validate(
+                list_of_struct.clone(),
+                false,
+                &[0, 1, 2, 3],
+                vec![6, 6, 29, 29]
+            )
+            .unwrap(),
+            6
+        );
+        // In 2.1 only the two leaves carry columns; still no false mismatch.
+        assert_eq!(
+            validate(list_of_struct, true, &[0, 1], vec![29, 29]).unwrap(),
+            29
+        );
+
+        // An empty struct contributes a single column and validates to its length.
+        let empty_struct = DataType::Struct(Fields::empty());
+        assert_eq!(
+            validate(empty_struct.clone(), false, &[0], vec![9]).unwrap(),
+            9
+        );
+        assert_eq!(validate(empty_struct, true, &[0], vec![9]).unwrap(), 9);
+    }
 }
diff --git a/rust/lance-file/src/writer.rs b/rust/lance-file/src/writer.rs
index 902c462c820..f7042b6b99c 100644
--- a/rust/lance-file/src/writer.rs
+++ b/rust/lance-file/src/writer.rs
@@ -6,7 +6,7 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use std::sync::atomic::AtomicBool;
 
-use arrow_array::RecordBatch;
+use arrow_array::{ArrayRef, RecordBatch};
 
 use arrow_data::ArrayData;
 use bytes::{Buf, BufMut, Bytes, BytesMut};
@@ -49,6 +49,15 @@ const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT];
 const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024;
 const ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES: &str = "LANCE_FILE_WRITER_MAX_PAGE_BYTES";
 
+/// Summary of a completed Lance file write.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct FileWriteSummary {
+    /// The number of rows written to the file.
+    pub num_rows: u64,
+    /// The final size of the file in bytes.
+    pub size_bytes: u64,
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct FileWriterOptions {
     /// How many bytes to use for buffering column data
@@ -212,6 +221,11 @@ pub struct FileWriter {
     field_id_to_column_indices: Vec<(u32, u32)>,
     num_columns: u32,
     rows_written: u64,
+    // The number of rows written for each top-level field (i.e. each entry in
+    // `column_writers`). With `write_batch` every field advances together and
+    // these are all equal, but `write_column` advances one field at a time, so
+    // a single file may end up with columns of differing item counts.
+    field_rows_written: Vec<u64>,
     global_buffers: Vec<(u64, u64)>,
     schema_metadata: HashMap<String, String>,
     options: FileWriterOptions,
@@ -268,6 +282,7 @@ impl FileWriter {
             column_metadata: Vec::new(),
             num_columns: 0,
             rows_written: 0,
+            field_rows_written: Vec::new(),
             field_id_to_column_indices: Vec::new(),
             global_buffers: Vec::new(),
             schema_metadata: HashMap::new(),
@@ -303,7 +318,7 @@ impl FileWriter {
         for batch in batches {
             writer.write_batch(&batch).await?;
         }
-        Ok(writer.finish().await? as usize)
+        Ok(writer.finish().await?.num_rows as usize)
     }
 
     async fn do_write_buffer(writer: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> Result<()> {
@@ -458,6 +473,7 @@ impl FileWriter {
             BatchEncoder::try_new(&schema, encoding_strategy.as_ref(), &encoding_options)?;
         self.num_columns = encoder.num_columns();
 
+        self.field_rows_written = vec![0; encoder.field_encoders.len()];
         self.column_writers = encoder.field_encoders;
         self.column_metadata = vec![initial_column_metadata(); self.num_columns as usize];
         self.field_id_to_column_indices = encoder.field_id_to_column_index;
@@ -481,13 +497,14 @@ impl FileWriter {
         batch: &RecordBatch,
         external_buffers: &mut OutOfLineBuffers,
     ) -> Result<Vec<Vec<EncodeTask>>> {
-        self.schema
+        let field_arrays = self
+            .schema
             .as_ref()
             .unwrap()
             .fields
             .iter()
-            .zip(self.column_writers.iter_mut())
-            .map(|(field, column_writer)| {
+            .enumerate()
+            .map(|(field_idx, field)| {
                 let array =
                     batch
                         .column_by_name(&field.name)
@@ -498,19 +515,57 @@ impl FileWriter {
                             )
                             .into(),
                         ))?;
+                Ok((field_idx, array.clone()))
+            })
+            .collect::<Result<Vec<_>>>()?;
+        self.encode_columns(&field_arrays, external_buffers)
+    }
+
+    // Encode a set of `(field index, array)` pairs, each advancing only its own
+    // column. Each task captures its field's current row offset at encode time,
+    // so `advance_columns` must run after this call (never before); the order of
+    // the returned tasks relative to `write_pages` does not matter.
+    fn encode_columns(
+        &mut self,
+        field_arrays: &[(usize, ArrayRef)],
+        external_buffers: &mut OutOfLineBuffers,
+    ) -> Result<Vec<Vec<EncodeTask>>> {
+        // Snapshot the starting row number of each field before borrowing the
+        // column writers mutably below.
+        let row_numbers = field_arrays
+            .iter()
+            .map(|(field_idx, _)| self.field_rows_written[*field_idx])
+            .collect::<Vec<_>>();
+        field_arrays
+            .iter()
+            .zip(row_numbers)
+            .map(|((field_idx, array), row_number)| {
                 let repdef = RepDefBuilder::default();
                 let num_rows = array.len() as u64;
-                column_writer.maybe_encode(
+                self.column_writers[*field_idx].maybe_encode(
                     array.clone(),
                     external_buffers,
                     repdef,
-                    self.rows_written,
+                    row_number,
                     num_rows,
                 )
             })
             .collect::<Result<Vec<_>>>()
     }
 
+    // Advance the per-field row counters after a set of columns has been
+    // written, keeping `rows_written` (the file's logical length) in sync as the
+    // longest column. Only the written fields move, so their new totals fold into
+    // `rows_written` directly without rescanning every field. (`write_batch`
+    // advances every field uniformly and tracks this inline instead.)
+    fn advance_columns(&mut self, field_arrays: &[(usize, ArrayRef)]) {
+        for (field_idx, array) in field_arrays {
+            let new_total = self.field_rows_written[*field_idx] + array.len() as u64;
+            self.field_rows_written[*field_idx] = new_total;
+            self.rows_written = self.rows_written.max(new_total);
+        }
+    }
+
     /// Schedule a batch of data to be written to the file
     ///
     /// Note: the future returned by this method may complete before the data has been fully
@@ -548,18 +603,97 @@ impl FileWriter {
             .flatten()
             .collect::<FuturesOrdered<_>>();
 
-        self.rows_written = match self.rows_written.checked_add(batch.num_rows() as u64) {
-            Some(rows_written) => rows_written,
-            None => {
-                return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into()));
-            }
-        };
+        // `write_batch` advances every field by the same amount, so the longest
+        // column simply grows by `num_rows`. Guard against overflowing the row
+        // counter.
+        if self.rows_written.checked_add(num_rows).is_none() {
+            return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into()));
+        }
+        for field_rows in self.field_rows_written.iter_mut() {
+            *field_rows += num_rows;
+        }
+        self.rows_written += num_rows;
 
         self.write_pages(encoding_tasks).await?;
 
         Ok(())
     }
 
+    /// Write a single column, advancing only that column's row counter.
+    ///
+    /// Unlike [`write_batch`](Self::write_batch), which advances every column
+    /// from a single shared row counter, this method advances one column
+    /// independently. Used across calls it produces a single file whose columns
+    /// may have different item counts.
+    ///
+    /// `column_index` refers to a top-level field in the writer's schema (the
+    /// same order as the schema's fields); a nested child cannot be targeted on
+    /// its own. Because each call writes the whole field from a single array, the
+    /// children of a struct field always advance together and stay equal-length;
+    /// only different top-level fields can diverge in length. A column may be
+    /// written across multiple calls; its values are appended. A field that is
+    /// never written ends up as a zero-length column. The writer must have been
+    /// created with an explicit schema (via [`try_new`](Self::try_new)); a lazy
+    /// schema cannot be inferred here because individual calls need not cover
+    /// every field.
+    ///
+    /// ```
+    /// # use arrow_array::{ArrayRef, Int32Array};
+    /// # use std::sync::Arc;
+    /// # use lance_file::writer::FileWriter;
+    /// # async fn example(writer: &mut FileWriter) -> lance_core::Result<()> {
+    /// // Field 0 gets three values, field 1 gets one — a non-rectangular file.
+    /// writer.write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))).await?;
+    /// writer.write_column(1, Arc::new(Int32Array::from(vec![10]))).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> {
+        let schema = self.schema.as_ref().ok_or_else(|| {
+            Error::invalid_input_source(
+                "write_column requires the writer to be created with an explicit schema".into(),
+            )
+        })?;
+        let field = schema.fields.get(column_index).ok_or_else(|| {
+            Error::invalid_input_source(
+                format!(
+                    "write_column: field index {} is out of bounds (schema has {} fields)",
+                    column_index,
+                    schema.fields.len()
+                )
+                .into(),
+            )
+        })?;
+        if array.len() as u64 > u32::MAX as u64 {
+            return Err(Error::invalid_input_source(
+                "cannot write Lance files with more than 2^32 rows".into(),
+            ));
+        }
+        Self::verify_field_nullability(&array.to_data(), field)?;
+
+        // A never-advanced field simply remains a zero-length column, which the
+        // encoders handle at `finish` time.
+        if array.is_empty() {
+            return Ok(());
+        }
+
+        let columns = [(column_index, array)];
+        let mut external_buffers =
+            OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64);
+        let encoding_tasks = self.encode_columns(&columns, &mut external_buffers)?;
+        for external_buffer in external_buffers.take_buffers() {
+            Self::do_write_buffer(&mut self.writer, &external_buffer).await?;
+        }
+        let encoding_tasks = encoding_tasks
+            .into_iter()
+            .flatten()
+            .collect::<FuturesOrdered<_>>();
+
+        self.advance_columns(&columns);
+        self.write_pages(encoding_tasks).await?;
+        Ok(())
+    }
+
     async fn write_column_metadata(
         &mut self,
         metadata: pbfile::ColumnMetadata,
@@ -624,14 +758,11 @@ impl FileWriter {
     async fn write_global_buffers(&mut self) -> Result<Vec<(u64, u64)>> {
         let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided.  Schema is unknown and file cannot be created"))?;
         schema.metadata = std::mem::take(&mut self.schema_metadata);
-        // Use descriptor layout for blob v2 in the footer to avoid exposing logical child fields.
-        //
-        // TODO(xuanwo): this doesn't work on nested struct, need better solution like fields_per_order_mut?
-        schema.fields.iter_mut().for_each(|f| {
-            if f.is_blob_v2() {
-                f.unloaded_mut();
-            }
-        });
+        // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields.
+        schema
+            .fields
+            .iter_mut()
+            .for_each(|f| f.unload_blobs_recursive());
 
         let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?;
         let file_descriptor_bytes = file_descriptor.encode_to_vec();
@@ -755,8 +886,8 @@ impl FileWriter {
     /// will write the file metadata and the footer.  It will not return until all
     /// data has been flushed and the file has been closed.
     ///
-    /// Returns the total number of rows written
-    pub async fn finish(&mut self) -> Result<u64> {
+    /// Returns a summary of the completed file write.
+    pub async fn finish(&mut self) -> Result<FileWriteSummary> {
         // 1. flush any remaining data and write out those pages
         let mut external_buffers =
             OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64);
@@ -812,9 +943,12 @@ impl FileWriter {
         self.writer.write_all(MAGIC).await?;
 
         // 7. close the writer
-        Writer::shutdown(self.writer.as_mut()).await?;
+        let write_result = Writer::shutdown(self.writer.as_mut()).await?;
 
-        Ok(self.rows_written)
+        Ok(FileWriteSummary {
+            num_rows: self.rows_written,
+            size_bytes: write_result.size as u64,
+        })
     }
 
     pub async fn abort(&mut self) {
@@ -965,11 +1099,11 @@ mod tests {
     use std::collections::HashMap;
     use std::sync::Arc;
 
-    use crate::reader::{FileReader, FileReaderOptions, describe_encoding};
+    use crate::reader::{FileReader, FileReaderOptions, ReaderProjection, describe_encoding};
     use crate::testing::FsFixture;
     use crate::writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriter, FileWriterOptions};
     use arrow_array::builder::{Float32Builder, Int32Builder};
-    use arrow_array::{Int32Array, RecordBatch, UInt64Array};
+    use arrow_array::{ArrayRef, Int32Array, RecordBatch, UInt64Array};
     use arrow_array::{RecordBatchReader, StringArray, types::Float64Type};
     use arrow_schema::{DataType, Field, Field as ArrowField, Schema, Schema as ArrowSchema};
     use lance_core::cache::LanceCache;
@@ -981,6 +1115,7 @@ mod tests {
     use lance_encoding::version::LanceFileVersion;
     use lance_io::object_store::ObjectStore;
     use lance_io::utils::CachedFileSize;
+    use rstest::rstest;
 
     #[tokio::test]
     async fn test_basic_write() {
@@ -1031,6 +1166,611 @@ mod tests {
         file_writer.finish().await.unwrap();
     }
 
+    // Read a single column back at an explicit range/index set, returning its
+    // `Int32` values. Reading one column (or an equal-length group) at a time is
+    // how unequal-length files are consumed: a full scan across columns of
+    // differing lengths cannot form a single rectangular batch.
+    async fn read_int32_column(
+        reader: &FileReader,
+        schema: &LanceSchema,
+        version: LanceFileVersion,
+        name: &str,
+        params: lance_io::ReadBatchParams,
+    ) -> Vec<Option<i32>> {
+        use futures::TryStreamExt;
+        use lance_encoding::decoder::FilterExpression;
+
+        let projection = ReaderProjection::from_column_names(version, schema, &[name]).unwrap();
+        let batches: Vec<RecordBatch> = reader
+            .read_stream_projected(params, 1024, 16, projection, FilterExpression::no_filter())
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        batches
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>()
+            })
+            .collect()
+    }
+
+    /// A single file may hold columns of differing item counts, written by
+    /// advancing each column's row counter independently (no shared global
+    /// counter).
+    #[rstest]
+    #[tokio::test]
+    async fn test_write_columns_unequal_lengths(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+    ) {
+        use lance_io::ReadBatchParams;
+
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+            ArrowField::new("c", DataType::Int32, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let fs = FsFixture::default();
+        let options = FileWriterOptions {
+            format_version: Some(version),
+            ..Default::default()
+        };
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            options,
+        )
+        .unwrap();
+
+        // Field "a" gets 5 values across two calls (appending), field "b" gets a
+        // single value, and field "c" is never written (a zero-length column).
+        let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        let b: ArrayRef = Arc::new(Int32Array::from(vec![10]));
+        writer.write_column(0, a1).await.unwrap();
+        writer.write_column(1, b).await.unwrap();
+        let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5]));
+        writer.write_column(0, a2).await.unwrap();
+        // An empty array is a no-op whether or not the field already has rows:
+        // field "a" keeps its 5 rows, field "c" stays a zero-length column.
+        let empty: ArrayRef = Arc::new(Int32Array::from(Vec::<i32>::new()));
+        writer.write_column(0, empty.clone()).await.unwrap();
+        writer.write_column(2, empty).await.unwrap();
+
+        let summary = writer.finish().await.unwrap();
+        // The file's logical length is the longest column.
+        assert_eq!(summary.num_rows, 5);
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        // Per-column row counts are recorded in / derivable from file metadata.
+        assert_eq!(reader.num_rows(), 5);
+        assert_eq!(reader.column_num_rows(0).unwrap(), 5);
+        assert_eq!(reader.column_num_rows(1).unwrap(), 1);
+        assert_eq!(reader.column_num_rows(2).unwrap(), 0);
+        assert!(reader.column_num_rows(3).is_err());
+
+        // Each column reads back independently at its own length.
+        assert_eq!(
+            read_int32_column(
+                &reader,
+                &lance_schema,
+                version,
+                "a",
+                ReadBatchParams::Range(0..5)
+            )
+            .await,
+            vec![Some(1), Some(2), Some(3), Some(4), Some(5)],
+        );
+        assert_eq!(
+            read_int32_column(
+                &reader,
+                &lance_schema,
+                version,
+                "b",
+                ReadBatchParams::Range(0..1)
+            )
+            .await,
+            vec![Some(10)],
+        );
+
+        // Random access by position within the longer column returns the right
+        // value even though other columns are shorter. (The take path requires
+        // strictly increasing indices.)
+        assert_eq!(
+            read_int32_column(
+                &reader,
+                &lance_schema,
+                version,
+                "a",
+                ReadBatchParams::Indices(arrow_array::UInt32Array::from(vec![0, 2, 4])),
+            )
+            .await,
+            vec![Some(1), Some(3), Some(5)],
+        );
+    }
+
+    /// Reading an unequal-length file:
+    /// - a projection whose columns are equal length full-scans normally;
+    /// - a full scan across columns of differing length is rejected up front,
+    ///   before any batch is produced (even though a prefix would be rectangular);
+    /// - a bounded read is valid as long as every projected column covers it;
+    /// - a single-column `RangeFull` resolves to that column's own length, not
+    ///   the file's (maximum) length.
+    #[rstest]
+    #[tokio::test]
+    async fn test_read_unequal_length_projection(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+    ) {
+        use futures::TryStreamExt;
+        use lance_encoding::decoder::FilterExpression;
+        use lance_io::ReadBatchParams;
+
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+            ArrowField::new("c", DataType::Int32, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+        let fs = FsFixture::default();
+        let options = FileWriterOptions {
+            format_version: Some(version),
+            ..Default::default()
+        };
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            options,
+        )
+        .unwrap();
+        // "a" and "b" are equal length (5); "c" is shorter (1).
+        writer
+            .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])))
+            .await
+            .unwrap();
+        writer
+            .write_column(1, Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10])))
+            .await
+            .unwrap();
+        writer
+            .write_column(2, Arc::new(Int32Array::from(vec![100])))
+            .await
+            .unwrap();
+        writer.finish().await.unwrap();
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        let read = |names: &'static [&'static str], params: ReadBatchParams| {
+            let projection =
+                ReaderProjection::from_column_names(version, &lance_schema, names).unwrap();
+            async {
+                match reader
+                    .read_stream_projected(
+                        params,
+                        1024,
+                        16,
+                        projection,
+                        FilterExpression::no_filter(),
+                    )
+                    .await
+                {
+                    Ok(stream) => stream.try_collect::<Vec<RecordBatch>>().await,
+                    Err(e) => Err(e),
+                }
+            }
+        };
+        let col_values = |batches: &[RecordBatch], idx: usize| -> Vec<Option<i32>> {
+            batches
+                .iter()
+                .flat_map(|b| {
+                    b.column(idx)
+                        .as_any()
+                        .downcast_ref::<Int32Array>()
+                        .unwrap()
+                        .iter()
+                        .collect::<Vec<_>>()
+                })
+                .collect()
+        };
+
+        // Equal-length projection [a, b] full-scans into rectangular batches.
+        let batches = read(&["a", "b"], ReadBatchParams::RangeFull).await.unwrap();
+        assert_eq!(
+            col_values(&batches, 0),
+            vec![Some(1), Some(2), Some(3), Some(4), Some(5)]
+        );
+        assert_eq!(
+            col_values(&batches, 1),
+            vec![Some(6), Some(7), Some(8), Some(9), Some(10)]
+        );
+
+        // A mismatched-length projection [a, c] (5 vs 1) is rejected before any
+        // batch is yielded, regardless of the read params — its columns cannot
+        // be combined into rectangular batches. The error names each column's
+        // length so the caller can see which column is the odd one out.
+        let err = read(&["a", "c"], ReadBatchParams::RangeFull)
+            .await
+            .unwrap_err()
+            .to_string();
+        assert!(
+            err.contains("a=5") && err.contains("c=1"),
+            "error should name each column's length, got: {err}"
+        );
+        assert!(
+            read(&["a", "c"], ReadBatchParams::Range(0..1))
+                .await
+                .is_err(),
+            "even a common-prefix read of unequal-length columns must error"
+        );
+
+        // A single-column RangeFull resolves to that column's own length.
+        let batches = read(&["c"], ReadBatchParams::RangeFull).await.unwrap();
+        assert_eq!(col_values(&batches, 0), vec![Some(100)]);
+        let batches = read(&["a"], ReadBatchParams::RangeFull).await.unwrap();
+        assert_eq!(
+            col_values(&batches, 0),
+            vec![Some(1), Some(2), Some(3), Some(4), Some(5)]
+        );
+
+        // RangeFrom/RangeTo likewise resolve against the projected column's own
+        // length rather than the file's longest column.
+        let batches = read(&["a"], ReadBatchParams::RangeFrom(2..)).await.unwrap();
+        assert_eq!(col_values(&batches, 0), vec![Some(3), Some(4), Some(5)]);
+        // RangeFrom on the short column "c" resolves to length 1, not 5.
+        let batches = read(&["c"], ReadBatchParams::RangeFrom(0..)).await.unwrap();
+        assert_eq!(col_values(&batches, 0), vec![Some(100)]);
+        let batches = read(&["a"], ReadBatchParams::RangeTo(..3)).await.unwrap();
+        assert_eq!(col_values(&batches, 0), vec![Some(1), Some(2), Some(3)]);
+        // A bound past the projected column's length errors.
+        assert!(
+            read(&["a"], ReadBatchParams::RangeTo(..6)).await.is_err(),
+            "RangeTo past the column length must error"
+        );
+        assert!(
+            read(&["c"], ReadBatchParams::RangeFrom(2..)).await.is_err(),
+            "RangeFrom past the column length must error"
+        );
+    }
+
+    /// A struct and a list column each map to multiple physical columns, and a
+    /// list's item column is longer than its top-level row count. The
+    /// projection-length check must partition `column_indices` by top-level
+    /// field and use each field's root column, so an ordinary (rectangular) file
+    /// with nested columns still reads under the new validation path.
+    #[rstest]
+    #[tokio::test]
+    async fn test_read_nested_columns_under_validation(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+    ) {
+        use arrow_array::types::Int32Type;
+        use arrow_array::{ListArray, StructArray};
+        use futures::TryStreamExt;
+        use lance_encoding::decoder::FilterExpression;
+        use lance_io::ReadBatchParams;
+
+        let struct_type = DataType::Struct(
+            vec![
+                ArrowField::new("x", DataType::Int32, true),
+                ArrowField::new("y", DataType::Int32, true),
+            ]
+            .into(),
+        );
+        let list_type = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true)));
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("s", struct_type, true),
+            ArrowField::new("lst", list_type, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3]));
+        let s: ArrayRef = Arc::new(StructArray::from(vec![
+            (
+                Arc::new(ArrowField::new("x", DataType::Int32, true)),
+                Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef,
+            ),
+            (
+                Arc::new(ArrowField::new("y", DataType::Int32, true)),
+                Arc::new(Int32Array::from(vec![11, 21, 31])) as ArrayRef,
+            ),
+        ]));
+        // 3 lists, 6 items: the item column is longer than the top-level rows.
+        let lst: ArrayRef = Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+            Some(vec![Some(1), Some(2)]),
+            Some(vec![Some(3)]),
+            Some(vec![Some(4), Some(5), Some(6)]),
+        ]));
+        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![a, s, lst]).unwrap();
+
+        let fs = FsFixture::default();
+        let options = FileWriterOptions {
+            format_version: Some(version),
+            ..Default::default()
+        };
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            options,
+        )
+        .unwrap();
+        writer.write_batch(&batch).await.unwrap();
+        writer.finish().await.unwrap();
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+
+        // If `validate_field_length` mispartitioned the physical columns, the
+        // length check would read the wrong root column (e.g. the list's item
+        // column, length 6) and spuriously reject this rectangular file.
+        for names in [&["a", "s", "lst"][..], &["a", "lst"][..], &["a", "s"][..]] {
+            let projection =
+                ReaderProjection::from_column_names(version, &lance_schema, names).unwrap();
+            let batches: Vec<RecordBatch> = reader
+                .read_stream_projected(
+                    ReadBatchParams::RangeFull,
+                    1024,
+                    16,
+                    projection,
+                    FilterExpression::no_filter(),
+                )
+                .await
+                .unwrap()
+                .try_collect()
+                .await
+                .unwrap();
+            let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+            assert_eq!(
+                total_rows, 3,
+                "projection {names:?} should read 3 top-level rows"
+            );
+        }
+    }
+
+    /// `write_column` rejects invalid inputs at the API boundary with
+    /// descriptive errors: a writer without an explicit schema, an
+    /// out-of-bounds field index, and a null written into a non-nullable field.
+    #[tokio::test]
+    async fn test_write_column_validation_errors() {
+        // A lazy-schema writer cannot infer the schema from a single column.
+        let fs = FsFixture::default();
+        let mut lazy_writer = FileWriter::new_lazy(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            FileWriterOptions::default(),
+        );
+        let err = lazy_writer
+            .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3])))
+            .await
+            .unwrap_err()
+            .to_string();
+        assert!(
+            err.contains("explicit schema"),
+            "expected explicit-schema error, got: {err}"
+        );
+
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, false),
+            ArrowField::new("b", DataType::Int32, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        // An out-of-bounds field index is rejected, naming the index and count.
+        let fs = FsFixture::default();
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            FileWriterOptions::default(),
+        )
+        .unwrap();
+        let err = writer
+            .write_column(5, Arc::new(Int32Array::from(vec![1])))
+            .await
+            .unwrap_err()
+            .to_string();
+        assert!(
+            err.contains('5') && err.contains('2'),
+            "expected out-of-bounds error naming index 5 and 2 fields, got: {err}"
+        );
+
+        // A null in a non-nullable field ("a") is rejected.
+        let err = writer
+            .write_column(0, Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])))
+            .await
+            .unwrap_err()
+            .to_string();
+        assert!(
+            err.contains("non-null"),
+            "expected nullability error, got: {err}"
+        );
+    }
+
+    /// The blocking read path applies the same projection-length validation as
+    /// the async path: a short single column resolves to its own length, and a
+    /// mismatched-length projection errors up front.
+    #[rstest]
+    #[tokio::test]
+    async fn test_blocking_read_unequal_length(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+    ) {
+        use lance_encoding::decoder::FilterExpression;
+        use lance_io::ReadBatchParams;
+
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("c", DataType::Int32, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+        let fs = FsFixture::default();
+        let options = FileWriterOptions {
+            format_version: Some(version),
+            ..Default::default()
+        };
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema.clone(),
+            options,
+        )
+        .unwrap();
+        writer
+            .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])))
+            .await
+            .unwrap();
+        writer
+            .write_column(1, Arc::new(Int32Array::from(vec![100])))
+            .await
+            .unwrap();
+        writer.finish().await.unwrap();
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = Arc::new(
+            FileReader::try_open(
+                file_scheduler,
+                None,
+                Arc::<DecoderPlugins>::default(),
+                &LanceCache::no_cache(),
+                FileReaderOptions::default(),
+            )
+            .await
+            .unwrap(),
+        );
+
+        // Single short column: RangeFull resolves to its own length (1).
+        let proj_c = ReaderProjection::from_column_names(version, &lance_schema, &["c"]).unwrap();
+        let reader_c = reader.clone();
+        let batches = tokio::task::spawn_blocking(move || {
+            reader_c
+                .read_stream_projected_blocking(
+                    ReadBatchParams::RangeFull,
+                    1024,
+                    Some(proj_c),
+                    FilterExpression::no_filter(),
+                )
+                .unwrap()
+                .collect::<std::result::Result<Vec<RecordBatch>, _>>()
+                .unwrap()
+        })
+        .await
+        .unwrap();
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total_rows, 1);
+
+        // A mismatched projection [a, c] errors on the blocking path too.
+        let proj_ac =
+            ReaderProjection::from_column_names(version, &lance_schema, &["a", "c"]).unwrap();
+        let reader_ac = reader.clone();
+        let is_err = tokio::task::spawn_blocking(move || {
+            reader_ac
+                .read_stream_projected_blocking(
+                    ReadBatchParams::RangeFull,
+                    1024,
+                    Some(proj_ac),
+                    FilterExpression::no_filter(),
+                )
+                .is_err()
+        })
+        .await
+        .unwrap();
+        assert!(
+            is_err,
+            "blocking full scan across unequal-length columns must error"
+        );
+    }
+
+    /// Files written the ordinary (rectangular) way keep equal column lengths,
+    /// so the unequal-length support is backwards compatible.
+    #[tokio::test]
+    async fn test_write_batch_keeps_equal_lengths() {
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+        ]));
+        let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let fs = FsFixture::default();
+        let mut writer = FileWriter::try_new(
+            fs.object_store.create(&fs.tmp_path).await.unwrap(),
+            lance_schema,
+            FileWriterOptions::default(),
+        )
+        .unwrap();
+        let batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(Int32Array::from(vec![4, 5, 6])),
+            ],
+        )
+        .unwrap();
+        writer.write_batch(&batch).await.unwrap();
+        let summary = writer.finish().await.unwrap();
+        assert_eq!(summary.num_rows, 3);
+
+        let file_scheduler = fs
+            .scheduler
+            .open_file(&fs.tmp_path, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = FileReader::try_open(
+            file_scheduler,
+            None,
+            Arc::<DecoderPlugins>::default(),
+            &LanceCache::no_cache(),
+            FileReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        assert_eq!(reader.column_num_rows(0).unwrap(), 3);
+        assert_eq!(reader.column_num_rows(1).unwrap(), 3);
+    }
+
     #[tokio::test]
     async fn test_max_page_bytes_enforced() {
         let arrow_field = Field::new("data", DataType::UInt64, false);
@@ -1581,8 +2321,12 @@ mod tests {
         .unwrap();
 
         writer.write_batch(&batch).await.unwrap();
-        let num_rows = writer.finish().await.unwrap();
-        assert_eq!(num_rows, 2);
+        let write_summary = writer.finish().await.unwrap();
+        assert_eq!(write_summary.num_rows, 2);
+        assert_eq!(
+            write_summary.size_bytes,
+            fs.object_store.size(&path).await.unwrap()
+        );
 
         // Read back with split configuration
         let file_scheduler = fs
diff --git a/rust/lance-index-core/Cargo.toml b/rust/lance-index-core/Cargo.toml
new file mode 100644
index 00000000000..5024f2eee0b
--- /dev/null
+++ b/rust/lance-index-core/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "lance-index-core"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+readme = "README.md"
+description = "Core traits and types for Lance index plugins"
+keywords.workspace = true
+categories.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+async-trait.workspace = true
+arrow-array.workspace = true
+arrow-schema.workspace = true
+arrow-select.workspace = true
+bytes.workspace = true
+datafusion-common.workspace = true
+datafusion-expr.workspace = true
+datafusion.workspace = true
+futures.workspace = true
+lance-core.workspace = true
+lance-io.workspace = true
+lance-select.workspace = true
+prost-types.workspace = true
+roaring.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+
+[lints]
+workspace = true
diff --git a/rust/lance-index-core/src/lib.rs b/rust/lance-index-core/src/lib.rs
new file mode 100644
index 00000000000..393064014e4
--- /dev/null
+++ b/rust/lance-index-core/src/lib.rs
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{any::Any, sync::Arc};
+
+use async_trait::async_trait;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result};
+use roaring::RoaringBitmap;
+use serde::{Deserialize, Serialize};
+use std::convert::TryFrom;
+
+pub mod metrics;
+pub mod scalar;
+
+/// Generic methods common across all types of secondary indices
+///
+#[async_trait]
+pub trait Index: Send + Sync + DeepSizeOf {
+    /// Cast to [Any].
+    fn as_any(&self) -> &dyn Any;
+
+    /// Cast to [Index]
+    fn as_index(self: Arc<Self>) -> Arc<dyn Index>;
+
+    /// Retrieve index statistics as a JSON Value
+    fn statistics(&self) -> Result<serde_json::Value>;
+
+    /// Prewarm the index.
+    ///
+    /// This will load the index into memory and cache it.
+    async fn prewarm(&self) -> Result<()>;
+
+    /// Get the type of the index
+    fn index_type(&self) -> IndexType;
+
+    /// Read through the index and determine which fragment ids are covered by the index
+    ///
+    /// This is a kind of slow operation.  It's better to use the fragment_bitmap.  This
+    /// only exists for cases where the fragment_bitmap has become corrupted or missing.
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap>;
+}
+
+/// Index Type
+#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf, Serialize, Deserialize)]
+pub enum IndexType {
+    // Preserve 0-100 for simple indices.
+    Scalar = 0, // Legacy scalar index, alias to BTree
+
+    BTree = 1, // BTree
+
+    Bitmap = 2, // Bitmap
+
+    LabelList = 3, // LabelList
+
+    Inverted = 4, // Inverted
+
+    NGram = 5, // NGram
+
+    FragmentReuse = 6,
+
+    MemWal = 7,
+
+    ZoneMap = 8, // ZoneMap
+
+    BloomFilter = 9, // Bloom filter
+
+    RTree = 10, // RTree
+
+    Fm = 11, // FM-Index
+
+    // 100+ and up for vector index.
+    /// Flat vector index.
+    Vector = 100, // Legacy vector index, alias to IvfPq
+    IvfFlat = 101,
+    IvfSq = 102,
+    IvfPq = 103,
+    IvfHnswSq = 104,
+    IvfHnswPq = 105,
+    IvfHnswFlat = 106,
+    IvfRq = 107,
+}
+
+impl std::fmt::Display for IndexType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::Scalar | Self::BTree => write!(f, "BTree"),
+            Self::Bitmap => write!(f, "Bitmap"),
+            Self::LabelList => write!(f, "LabelList"),
+            Self::Inverted => write!(f, "Inverted"),
+            Self::NGram => write!(f, "NGram"),
+            Self::FragmentReuse => write!(f, "FragmentReuse"),
+            Self::MemWal => write!(f, "MemWal"),
+            Self::ZoneMap => write!(f, "ZoneMap"),
+            Self::BloomFilter => write!(f, "BloomFilter"),
+            Self::RTree => write!(f, "RTree"),
+            Self::Fm => write!(f, "Fm"),
+            Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"),
+            Self::IvfFlat => write!(f, "IVF_FLAT"),
+            Self::IvfSq => write!(f, "IVF_SQ"),
+            Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
+            Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
+            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
+            Self::IvfRq => write!(f, "IVF_RQ"),
+        }
+    }
+}
+
+impl TryFrom<i32> for IndexType {
+    type Error = Error;
+
+    fn try_from(value: i32) -> Result<Self> {
+        match value {
+            v if v == Self::Scalar as i32 => Ok(Self::Scalar),
+            v if v == Self::BTree as i32 => Ok(Self::BTree),
+            v if v == Self::Bitmap as i32 => Ok(Self::Bitmap),
+            v if v == Self::LabelList as i32 => Ok(Self::LabelList),
+            v if v == Self::NGram as i32 => Ok(Self::NGram),
+            v if v == Self::Inverted as i32 => Ok(Self::Inverted),
+            v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse),
+            v if v == Self::MemWal as i32 => Ok(Self::MemWal),
+            v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap),
+            v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter),
+            v if v == Self::RTree as i32 => Ok(Self::RTree),
+            v if v == Self::Fm as i32 => Ok(Self::Fm),
+            v if v == Self::Vector as i32 => Ok(Self::Vector),
+            v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat),
+            v if v == Self::IvfSq as i32 => Ok(Self::IvfSq),
+            v if v == Self::IvfPq as i32 => Ok(Self::IvfPq),
+            v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq),
+            v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq),
+            v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat),
+            v if v == Self::IvfRq as i32 => Ok(Self::IvfRq),
+            _ => Err(Error::invalid_input_source(
+                format!("the input value {} is not a valid IndexType", value).into(),
+            )),
+        }
+    }
+}
+
+impl TryFrom<&str> for IndexType {
+    type Error = Error;
+
+    fn try_from(value: &str) -> Result<Self> {
+        match value {
+            "BTree" | "BTREE" => Ok(Self::BTree),
+            "Bitmap" | "BITMAP" => Ok(Self::Bitmap),
+            "LabelList" | "LABELLIST" => Ok(Self::LabelList),
+            "Inverted" | "INVERTED" => Ok(Self::Inverted),
+            "NGram" | "NGRAM" => Ok(Self::NGram),
+            "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap),
+            "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter),
+            "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree),
+            "Fm" | "FM" => Ok(Self::Fm),
+            "Vector" | "VECTOR" => Ok(Self::Vector),
+            "IVF_FLAT" => Ok(Self::IvfFlat),
+            "IVF_SQ" => Ok(Self::IvfSq),
+            "IVF_PQ" => Ok(Self::IvfPq),
+            "IVF_RQ" => Ok(Self::IvfRq),
+            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
+            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
+            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
+            "FragmentReuse" => Ok(Self::FragmentReuse),
+            "MemWal" => Ok(Self::MemWal),
+            _ => Err(Error::invalid_input(format!(
+                "invalid index type: {}",
+                value
+            ))),
+        }
+    }
+}
+
+impl IndexType {
+    pub fn is_scalar(&self) -> bool {
+        matches!(
+            self,
+            Self::Scalar
+                | Self::BTree
+                | Self::Bitmap
+                | Self::LabelList
+                | Self::Inverted
+                | Self::NGram
+                | Self::ZoneMap
+                | Self::BloomFilter
+                | Self::RTree
+                | Self::Fm,
+        )
+    }
+
+    pub fn is_vector(&self) -> bool {
+        matches!(
+            self,
+            Self::Vector
+                | Self::IvfPq
+                | Self::IvfHnswSq
+                | Self::IvfHnswPq
+                | Self::IvfHnswFlat
+                | Self::IvfFlat
+                | Self::IvfSq
+                | Self::IvfRq
+        )
+    }
+
+    pub fn is_system(&self) -> bool {
+        matches!(self, Self::FragmentReuse | Self::MemWal)
+    }
+
+    /// Returns the current format version of the index type,
+    /// bump this when the index format changes.
+    /// Indices which higher version than these will be ignored for compatibility,
+    /// This would happen when creating index in a newer version of Lance,
+    /// but then opening the index in older version of Lance
+    pub fn version(&self) -> i32 {
+        match self {
+            Self::Scalar => 0,
+            Self::BTree => 0,
+            Self::Bitmap => 0,
+            Self::LabelList => 0,
+            Self::Inverted => 0,
+            Self::NGram => 0,
+            Self::FragmentReuse => 0,
+            Self::MemWal => 0,
+            Self::ZoneMap => 0,
+            Self::BloomFilter => 0,
+            Self::RTree => 0,
+            Self::Fm => 0,
+
+            // IMPORTANT: if any vector index subtype needs a format bump that is
+            // not backward compatible, its new version must be set to
+            // (current max vector index version + 1), even if only one subtype
+            // changed. Compatibility filtering currently cannot distinguish vector
+            // subtypes from details-only metadata, so vector versions effectively
+            // share one global monotonic compatibility level.
+            Self::Vector
+            | Self::IvfFlat
+            | Self::IvfSq
+            | Self::IvfPq
+            | Self::IvfHnswSq
+            | Self::IvfHnswPq
+            | Self::IvfHnswFlat => 1,
+            Self::IvfRq => 2,
+        }
+    }
+
+    /// Returns the target partition size for the index type.
+    ///
+    /// This is used to compute the number of partitions for the index.
+    /// The partition size is optimized for the best performance of the index.
+    ///
+    /// This is for vector indices only.
+    pub fn target_partition_size(&self) -> usize {
+        match self {
+            Self::Vector => 8192,
+            Self::IvfFlat => 4096,
+            Self::IvfSq => 8192,
+            Self::IvfPq => 8192,
+            Self::IvfRq => 4096,
+            Self::IvfHnswFlat => 1 << 20,
+            Self::IvfHnswSq => 1 << 20,
+            Self::IvfHnswPq => 1 << 20,
+            _ => 8192,
+        }
+    }
+
+    /// Returns the highest supported vector index version in this Lance build.
+    pub fn max_vector_version() -> u32 {
+        [
+            Self::Vector,
+            Self::IvfFlat,
+            Self::IvfSq,
+            Self::IvfPq,
+            Self::IvfHnswSq,
+            Self::IvfHnswPq,
+            Self::IvfHnswFlat,
+            Self::IvfRq,
+        ]
+        .into_iter()
+        .map(|index_type| index_type.version() as u32)
+        .max()
+        .unwrap_or(1)
+    }
+}
+
+pub trait IndexParams: Send + Sync {
+    fn as_any(&self) -> &dyn Any;
+
+    fn index_name(&self) -> &str;
+}
diff --git a/rust/lance-index-core/src/metrics.rs b/rust/lance-index-core/src/metrics.rs
new file mode 100644
index 00000000000..8c0c119a3c3
--- /dev/null
+++ b/rust/lance-index-core/src/metrics.rs
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen";
+pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return";
+pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores";
+pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected";
+
+/// A trait used by the index to report metrics
+///
+/// Callers can implement this trait to collect metrics
+pub trait MetricsCollector: Send + Sync {
+    /// Record partition loads
+    ///
+    /// Many indices consist of partitions that may need to be loaded
+    /// into cache.  For example, an inverted index or ngram index has a
+    /// posting list for each token.
+    ///
+    /// In the ideal case, these shards are in the cache and will not need
+    /// to be loaded from disk.  This method should not be called if the
+    /// shard is in the cache.
+    fn record_parts_loaded(&self, num_parts: usize);
+
+    /// Record a shard load
+    fn record_part_load(&self) {
+        self.record_parts_loaded(1);
+    }
+
+    /// Record an index load
+    ///
+    /// This should be called when a scalar index is loaded from storage.
+    /// It should not be called if the index is already in memory.
+    fn record_index_loads(&self, num_indexes: usize);
+
+    /// Record an index load
+    fn record_index_load(&self) {
+        self.record_index_loads(1);
+    }
+
+    /// Record the number of "comparisons" made by the index
+    ///
+    /// What exactly constitutes a comparison depends on the index type.
+    /// For example, a B-tree index may make comparisons while searching for a value.
+    /// On the other hand, a bitmap index makes comparisons when computing the intersection
+    /// of two bitmaps.
+    ///
+    /// The goal is to provide some visibility into the compute cost of the search
+    fn record_comparisons(&self, num_comparisons: usize);
+
+    /// Record AND candidates returned from WAND alignment to the scoring loop.
+    ///
+    /// This excludes candidates pruned before `next()` returns. Use this with
+    /// `record_and_candidates_pruned_before_return` to recover total aligned
+    /// AND candidates.
+    fn record_and_candidates_seen(&self, _num_candidates: usize) {}
+
+    /// Record AND candidates pruned during WAND alignment before `next()` returns.
+    fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {}
+
+    fn record_and_full_scores(&self, _num_scores: usize) {}
+
+    fn record_freqs_collected(&self, _num_collections: usize) {}
+
+    /// Returns an optional sink for recording exact I/O statistics (bytes read,
+    /// IOPS, and requests) performed on behalf of this collector.
+    ///
+    /// Index implementations that read from a
+    /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to
+    /// their file readers so the I/O performed for a single query is measured
+    /// and attributed here.  The default returns `None`, meaning the caller does
+    /// not want I/O measured (and index implementations should then take their
+    /// normal, uninstrumented read path).
+    fn io_stats(&self) -> Option<lance_io::scheduler::IoStats> {
+        None
+    }
+}
+
+/// A no-op metrics collector that does nothing
+pub struct NoOpMetricsCollector;
+
+impl MetricsCollector for NoOpMetricsCollector {
+    fn record_parts_loaded(&self, _num_parts: usize) {}
+    fn record_index_loads(&self, _num_indexes: usize) {}
+    fn record_comparisons(&self, _num_comparisons: usize) {}
+}
+
+#[derive(Default)]
+pub struct LocalMetricsCollector {
+    pub parts_loaded: AtomicUsize,
+    pub index_loads: AtomicUsize,
+    pub comparisons: AtomicUsize,
+}
+
+impl LocalMetricsCollector {
+    pub fn dump_into(self, other: &dyn MetricsCollector) {
+        other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed));
+        other.record_index_loads(self.index_loads.load(Ordering::Relaxed));
+        other.record_comparisons(self.comparisons.load(Ordering::Relaxed));
+    }
+}
+
+impl MetricsCollector for LocalMetricsCollector {
+    fn record_parts_loaded(&self, num_parts: usize) {
+        self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed);
+    }
+
+    fn record_index_loads(&self, num_indexes: usize) {
+        self.index_loads.fetch_add(num_indexes, Ordering::Relaxed);
+    }
+
+    fn record_comparisons(&self, num_comparisons: usize) {
+        self.comparisons
+            .fetch_add(num_comparisons, Ordering::Relaxed);
+    }
+}
diff --git a/rust/lance-index-core/src/scalar.rs b/rust/lance-index-core/src/scalar.rs
new file mode 100644
index 00000000000..e209a32cdb8
--- /dev/null
+++ b/rust/lance-index-core/src/scalar.rs
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Abstract scalar index traits and types for Lance index plugins
+
+use arrow_array::{BooleanArray, RecordBatch, UInt64Array};
+use arrow_schema::Schema;
+use async_trait::async_trait;
+use bytes::Bytes;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion_common::scalar::ScalarValue;
+use datafusion_expr::Expr;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use lance_core::{Error, Result};
+use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter};
+use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
+use roaring::{RoaringBitmap, RoaringTreemap};
+use serde::Serialize;
+use std::collections::HashMap;
+use std::fmt::Debug;
+use std::pin::Pin;
+use std::{any::Any, sync::Arc};
+
+use crate::metrics::MetricsCollector;
+use crate::{Index, IndexParams, IndexType};
+
+/// Metadata about a single file within an index segment.
+#[derive(Debug, Clone, PartialEq, DeepSizeOf)]
+pub struct IndexFile {
+    /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx")
+    pub path: String,
+    /// Size of the file in bytes
+    pub size_bytes: u64,
+}
+
+pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index";
+
+/// Builtin index types supported by the Lance library
+///
+/// This is primarily for convenience to avoid a bunch of string
+/// constants and provide some auto-complete.  This type should not
+/// be used in the manifest as plugins cannot add new entries.
+#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
+pub enum BuiltinIndexType {
+    BTree,
+    Bitmap,
+    LabelList,
+    NGram,
+    ZoneMap,
+    BloomFilter,
+    RTree,
+    Inverted,
+    Fm,
+}
+
+impl BuiltinIndexType {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Self::BTree => "btree",
+            Self::Bitmap => "bitmap",
+            Self::LabelList => "labellist",
+            Self::NGram => "ngram",
+            Self::ZoneMap => "zonemap",
+            Self::Inverted => "inverted",
+            Self::BloomFilter => "bloomfilter",
+            Self::RTree => "rtree",
+            Self::Fm => "fm",
+        }
+    }
+}
+
+impl TryFrom<IndexType> for BuiltinIndexType {
+    type Error = Error;
+
+    fn try_from(value: IndexType) -> Result<Self> {
+        match value {
+            IndexType::BTree => Ok(Self::BTree),
+            IndexType::Bitmap => Ok(Self::Bitmap),
+            IndexType::LabelList => Ok(Self::LabelList),
+            IndexType::NGram => Ok(Self::NGram),
+            IndexType::ZoneMap => Ok(Self::ZoneMap),
+            IndexType::Inverted => Ok(Self::Inverted),
+            IndexType::BloomFilter => Ok(Self::BloomFilter),
+            IndexType::RTree => Ok(Self::RTree),
+            IndexType::Fm => Ok(Self::Fm),
+            _ => Err(Error::index("Invalid index type".to_string())),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct ScalarIndexParams {
+    /// The type of index to create
+    ///
+    /// Plugins may add additional index types.  Index type lookup is case-insensitive.
+    pub index_type: String,
+    /// The parameters to train the index
+    ///
+    /// This should be a JSON string.  The contents of the JSON string will be specific to the
+    /// index type.  If not set, then default parameters will be used for the index type.
+    pub params: Option<String>,
+}
+
+impl Default for ScalarIndexParams {
+    fn default() -> Self {
+        Self {
+            index_type: BuiltinIndexType::BTree.as_str().to_string(),
+            params: None,
+        }
+    }
+}
+
+impl ScalarIndexParams {
+    /// Creates a new ScalarIndexParams from one of the builtin index types
+    pub fn for_builtin(index_type: BuiltinIndexType) -> Self {
+        Self {
+            index_type: index_type.as_str().to_string(),
+            params: None,
+        }
+    }
+
+    /// Create a new ScalarIndexParams with the given index type
+    pub fn new(index_type: String) -> Self {
+        Self {
+            index_type,
+            params: None,
+        }
+    }
+
+    /// Set the parameters for the index
+    pub fn with_params<ParamsType: Serialize>(mut self, params: &ParamsType) -> Self {
+        self.params = Some(serde_json::to_string(params).unwrap());
+        self
+    }
+}
+
+impl IndexParams for ScalarIndexParams {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn index_name(&self) -> &str {
+        LANCE_SCALAR_INDEX
+    }
+}
+
+/// Trait for storing an index (or parts of an index) into storage
+#[async_trait]
+pub trait IndexWriter: Send {
+    /// Writes a record batch into the file, returning the 0-based index of the batch in the file
+    ///
+    /// E.g. if this is the third time this is called this method will return 2
+    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64>;
+    /// Adds a global buffer and returns its index.
+    async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index writer",
+        ))
+    }
+    /// Finishes writing the file and closes the file
+    async fn finish(&mut self) -> Result<IndexFile>;
+    /// Finishes writing the file and closes the file with additional metadata
+    async fn finish_with_metadata(
+        &mut self,
+        metadata: HashMap<String, String>,
+    ) -> Result<IndexFile>;
+}
+
+/// Trait for reading an index (or parts of an index) from storage
+#[async_trait]
+pub trait IndexReader: Send + Sync {
+    /// Read the n-th record batch from the file
+    async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch>;
+    /// Reads a global buffer by index.
+    async fn read_global_buffer(&self, _index: u32) -> Result<Bytes> {
+        Err(Error::not_supported(
+            "global buffers are not supported by this index reader",
+        ))
+    }
+    /// Read the range of rows from the file.
+    /// If projection is Some, only return the columns in the projection,
+    /// nested columns like Some(&["x.y"]) are not supported.
+    /// If projection is None, return all columns.
+    async fn read_range(
+        &self,
+        range: std::ops::Range<usize>,
+        projection: Option<&[&str]>,
+    ) -> Result<RecordBatch>;
+    /// Read multiple ranges and concatenate into a single batch.
+    /// Default impl runs `read_range`s in parallel via `try_join_all`.
+    async fn read_ranges(
+        &self,
+        ranges: &[std::ops::Range<usize>],
+        projection: Option<&[&str]>,
+    ) -> Result<RecordBatch> {
+        if ranges.is_empty() {
+            return self.read_range(0..0, projection).await;
+        }
+        let futures = ranges
+            .iter()
+            .map(|r| self.read_range(r.clone(), projection));
+        let batches = futures::future::try_join_all(futures).await?;
+        let schema = batches[0].schema();
+        Ok(arrow_select::concat::concat_batches(&schema, &batches)?)
+    }
+    /// Read a range of rows as a stream of record batches.
+    ///
+    /// This allows the caller to process rows incrementally without loading the
+    /// entire range into memory at once.
+    ///
+    /// The default implementation falls back to [`Self::read_range`] and wraps
+    /// the result in a single-item stream.
+    async fn read_range_stream(
+        &self,
+        range: std::ops::Range<usize>,
+        projection: Option<&[&str]>,
+    ) -> Result<Pin<Box<dyn RecordBatchStream>>> {
+        let batch = self.read_range(range, projection).await?;
+        let schema = batch.schema();
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move { Ok(batch) }),
+        )))
+    }
+    /// Return the number of batches in the file
+    async fn num_batches(&self, batch_size: u64) -> u32;
+    /// Return the number of rows in the file
+    fn num_rows(&self) -> usize;
+    /// Return the metadata of the file
+    fn schema(&self) -> &lance_core::datatypes::Schema;
+    /// Best-effort on-disk byte size of the file when the reader already knows it
+    /// without extra I/O, else `None`. Used to size prewarm chunks.
+    fn file_size_bytes(&self) -> Option<u64> {
+        None
+    }
+}
+
+/// Trait abstracting I/O away from index logic
+///
+/// Scalar indices are currently serialized as indexable arrow record batches stored in
+/// named "files".  The index store is responsible for serializing and deserializing
+/// these batches into file data (e.g. as .lance files or .parquet files, etc.)
+#[async_trait]
+pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
+    fn as_any(&self) -> &dyn Any;
+    fn clone_arc(&self) -> Arc<dyn IndexStore>;
+
+    /// Suggested I/O parallelism for the store
+    fn io_parallelism(&self) -> usize;
+
+    /// Create a new file and return a writer to store data in the file
+    async fn new_index_file(&self, name: &str, schema: Arc<Schema>)
+    -> Result<Box<dyn IndexWriter>>;
+
+    /// Open an existing file for retrieval
+    async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>>;
+
+    /// Return a store that submits its I/O at the given base priority.
+    fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore>;
+
+    /// Copy a range of batches from an index file from this store to another
+    ///
+    /// This is often useful when remapping or updating
+    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<IndexFile>;
+
+    /// Copy an index file from this store to a new name in another store, leaving the source intact
+    async fn copy_index_file_to(
+        &self,
+        name: &str,
+        new_name: &str,
+        dest_store: &dyn IndexStore,
+    ) -> Result<IndexFile> {
+        if name == new_name {
+            self.copy_index_file(name, dest_store).await
+        } else {
+            Err(Error::not_supported(format!(
+                "copying index file {name} to {new_name} is not supported by this index store"
+            )))
+        }
+    }
+
+    /// Rename an index file
+    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile>;
+
+    /// Delete an index file (used in the tmp spill store to keep tmp size down)
+    async fn delete_index_file(&self, name: &str) -> Result<()>;
+
+    /// List all files in the index directory with their sizes.
+    ///
+    /// Returns a list of (relative_path, size_bytes) tuples.
+    /// Used to capture file metadata after index creation/modification.
+    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>>;
+}
+
+/// Different scalar indices may support different kinds of queries
+///
+/// For example, a btree index can support a wide range of queries (e.g. x > 7)
+/// while an index based on FTS only supports queries like "x LIKE 'foo'"
+///
+/// This trait is used when we need an object that can represent any kind of query
+///
+/// Note: if you are implementing this trait for a query type then you probably also
+/// need to implement the scalar query parser trait to create instances of your query at parse time.
+pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync {
+    /// Cast the query as Any to allow for downcasting
+    fn as_any(&self) -> &dyn Any;
+    /// Format the query as a string for display purposes
+    fn format(&self, col: &str) -> String;
+    /// Convert the query to a datafusion expression
+    fn to_expr(&self, col: String) -> Expr;
+    /// Compare this query to another query
+    fn dyn_eq(&self, other: &dyn AnyQuery) -> bool;
+}
+
+impl PartialEq for dyn AnyQuery {
+    fn eq(&self, other: &Self) -> bool {
+        self.dyn_eq(other)
+    }
+}
+
+/// The result of a search operation against a scalar index
+#[derive(Debug, PartialEq)]
+pub enum SearchResult {
+    /// The exact row ids that satisfy the query
+    Exact(NullableRowAddrSet),
+    /// Any row id satisfying the query will be in this set but not every
+    /// row id in this set will satisfy the query, a further recheck step
+    /// is needed
+    AtMost(NullableRowAddrSet),
+    /// All of the given row ids satisfy the query but there may be more
+    ///
+    /// No scalar index actually returns this today but it can arise from
+    /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x)))
+    AtLeast(NullableRowAddrSet),
+}
+
+impl SearchResult {
+    pub fn exact(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn at_most(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn at_least(row_ids: impl Into<RowAddrTreeMap>) -> Self {
+        Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default()))
+    }
+
+    pub fn with_nulls(self, nulls: impl Into<RowAddrTreeMap>) -> Self {
+        match self {
+            Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())),
+            Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())),
+            Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())),
+        }
+    }
+
+    pub fn row_addrs(&self) -> &NullableRowAddrSet {
+        match self {
+            Self::Exact(row_addrs) => row_addrs,
+            Self::AtMost(row_addrs) => row_addrs,
+            Self::AtLeast(row_addrs) => row_addrs,
+        }
+    }
+
+    pub fn is_exact(&self) -> bool {
+        matches!(self, Self::Exact(_))
+    }
+}
+
+/// Brief information about an index that was created
+pub struct CreatedIndex {
+    /// The details of the index that was created
+    ///
+    /// These should be stored somewhere as they will be needed to
+    /// load the index later.
+    pub index_details: prost_types::Any,
+    /// The version of the index that was created
+    ///
+    /// This can be used to determine if a reader is able to load the index.
+    pub index_version: u32,
+    /// List of files and their sizes for this index
+    ///
+    /// This enables skipping HEAD calls when opening indices and provides
+    /// visibility into index storage size via describe_indices().
+    pub files: Vec<IndexFile>,
+}
+
+/// The ordering that training data must satisfy
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrainingOrdering {
+    /// The input will arrive sorted by the value column in ascending order
+    Values,
+    /// The input will arrive sorted by the address column in ascending order
+    Addresses,
+    /// The input will arrive in an arbitrary order
+    None,
+}
+
+#[derive(Debug, Clone)]
+pub struct TrainingCriteria {
+    pub ordering: TrainingOrdering,
+    pub needs_row_ids: bool,
+    pub needs_row_addrs: bool,
+}
+
+impl TrainingCriteria {
+    pub fn new(ordering: TrainingOrdering) -> Self {
+        Self {
+            ordering,
+            needs_row_ids: false,
+            needs_row_addrs: false,
+        }
+    }
+
+    pub fn with_row_id(mut self) -> Self {
+        self.needs_row_ids = true;
+        self
+    }
+
+    pub fn with_row_addr(mut self) -> Self {
+        self.needs_row_addrs = true;
+        self
+    }
+}
+
+/// The criteria that specifies how to update an index
+pub struct UpdateCriteria {
+    /// If true, then we need to read the old data to update the index
+    ///
+    /// This should be avoided if possible but is left in for some legacy paths
+    pub requires_old_data: bool,
+    /// The criteria required for data (both old and new)
+    pub data_criteria: TrainingCriteria,
+}
+
+/// Filter used when merging existing scalar-index rows during update.
+///
+/// The caller must pick a filter mode that matches the row-id semantics of the
+/// dataset:
+/// - address-style row IDs: fragment filtering is valid
+/// - stable row IDs: use exact row-id membership instead
+#[derive(Debug, Clone)]
+pub enum OldIndexDataFilter {
+    /// Keeps track of which fragments are still valid and which are no longer valid.
+    ///
+    /// This is valid for address-style row IDs.
+    Fragments {
+        to_keep: RoaringBitmap,
+        to_remove: RoaringBitmap,
+    },
+    /// Keep old rows whose row IDs are in this exact allow-list.
+    ///
+    /// This is required for stable row IDs, where row IDs are opaque and
+    /// should not be interpreted as encoded row addresses.
+    RowIds(RowAddrTreeMap),
+}
+
+impl OldIndexDataFilter {
+    /// Build a boolean mask that keeps only row IDs selected by this filter.
+    pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray {
+        match self {
+            Self::Fragments { to_keep, .. } => row_ids
+                .iter()
+                .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32)))
+                .collect(),
+            Self::RowIds(valid_row_ids) => row_ids
+                .iter()
+                .map(|id| id.map(|id| valid_row_ids.contains(id)))
+                .collect(),
+        }
+    }
+
+    /// Apply this filter in place to a set of existing (old) row ids/addresses,
+    /// retaining only the rows the filter selects to keep. Used by index types
+    /// that merge old postings directly (e.g. bitmap) instead of re-scanning a
+    /// row-id array through [`Self::filter_row_ids`].
+    pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) {
+        match self {
+            Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()),
+            Self::RowIds(valid_row_ids) => *rows &= valid_row_ids,
+        }
+    }
+}
+
+impl UpdateCriteria {
+    pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self {
+        Self {
+            requires_old_data: true,
+            data_criteria,
+        }
+    }
+
+    pub fn only_new_data(data_criteria: TrainingCriteria) -> Self {
+        Self {
+            requires_old_data: false,
+            data_criteria,
+        }
+    }
+}
+
+/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries
+#[async_trait]
+pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
+    /// Search the scalar index
+    ///
+    /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered
+    async fn search(
+        &self,
+        query: &dyn AnyQuery,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<SearchResult>;
+
+    /// Returns true if this index reports matches as physical row addresses
+    /// (`fragment_id << 32 | offset`) rather than row ids
+    ///
+    /// Address-domain indices (e.g. zone map, bloom filter) are built over the
+    /// `_rowaddr` column. On a dataset with stable row ids the address and
+    /// row-id domains diverge, so these results must be translated back to row
+    /// ids (via the per-fragment row-id sequences, known only at the dataset
+    /// layer) before they are combined with row-id results or handed to the
+    /// scan. The default (row-id domain) needs no translation.
+    fn results_are_row_addresses(&self) -> bool {
+        false
+    }
+
+    /// Returns true if the remap operation is supported
+    fn can_remap(&self) -> bool;
+
+    /// Remap the row ids, creating a new remapped version of this index in `dest_store`
+    async fn remap(
+        &self,
+        mapping: &RowAddrRemap,
+        dest_store: &dyn IndexStore,
+    ) -> Result<CreatedIndex>;
+
+    /// Add the new data into the index, creating an updated version of the index in `dest_store`
+    ///
+    /// If `old_data_filter` is provided, old index data will be filtered before
+    /// merge according to the chosen filter mode.
+    async fn update(
+        &self,
+        new_data: SendableRecordBatchStream,
+        dest_store: &dyn IndexStore,
+        old_data_filter: Option<OldIndexDataFilter>,
+    ) -> Result<CreatedIndex>;
+
+    /// Returns the criteria that will be used to update the index
+    fn update_criteria(&self) -> UpdateCriteria;
+
+    /// Derive the index parameters from the current index
+    ///
+    /// This returns a ScalarIndexParams that can be used to recreate an index
+    /// with the same configuration on another dataset.
+    fn derive_index_params(&self) -> Result<ScalarIndexParams>;
+
+    /// Global `[min, max]` of the indexed column from index metadata, without a
+    /// scan, or `None` if this index type cannot supply a sound bound. When
+    /// `Some`, the range is a superset of live values (conservative under
+    /// deletes): safe to prune with, not guaranteed tight.
+    fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> {
+        None
+    }
+}
+
+/// Abstraction over any type that can remap row IDs during index loading.
+///
+/// This decouples scalar index plugins from the table-level frag reuse index type.
+/// The frag reuse index implements this trait, but callers may also supply custom
+/// implementations for testing or other remapping strategies.
+pub trait RowIdRemapper: Send + Sync + std::fmt::Debug {
+    /// Remap a single row id.  Returns `None` if the row was deleted.
+    fn remap_row_id(&self, row_id: u64) -> Option<u64>;
+    /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows.
+    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap;
+    /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows.
+    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap;
+    /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows.
+    fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch>;
+}
diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml
index 5ff94574a70..652bf73cc26 100644
--- a/rust/lance-index/Cargo.toml
+++ b/rust/lance-index/Cargo.toml
@@ -26,7 +26,6 @@ datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datafusion-physical-expr.workspace = true
 datafusion.workspace = true
-deepsize.workspace = true
 dirs.workspace = true
 fst.workspace = true
 futures.workspace = true
@@ -40,16 +39,18 @@ jsonb.workspace = true
 lance-arrow.workspace = true
 lance-arrow-stats.workspace = true
 lance-core.workspace = true
+lance-index-core.workspace = true
 lance-datafusion.workspace = true
 lance-encoding.workspace = true
 lance-file.workspace = true
 lance-geo = { workspace = true, optional = true }
 lance-io.workspace = true
+lance-bitpacking.workspace = true
+libsais-rs = "0.2"
 lance-linalg.workspace = true
 lance-select.workspace = true
 lance-tokenizer.workspace = true
 lance-table.workspace = true
-libm.workspace = true
 log.workspace = true
 ndarray.workspace = true
 num-traits.workspace = true
@@ -57,6 +58,7 @@ object_store.workspace = true
 prost.workspace = true
 prost-types.workspace = true
 rand.workspace = true
+regex-syntax.workspace = true
 roaring.workspace = true
 rayon.workspace = true
 serde_json.workspace = true
@@ -69,9 +71,7 @@ crossbeam-queue.workspace = true
 bytes.workspace = true
 chrono.workspace = true
 uuid.workspace = true
-twox-hash = "2.0"
 async-channel = "2.3.1"
-bitpacking = { version = "0.9.2", features = ["bitpacker4x"] }
 rand_distr.workspace = true
 lance-datagen.workspace = true
 rangemap.workspace = true
diff --git a/rust/lance-index/benches/geo.rs b/rust/lance-index/benches/geo.rs
index a9c403cc6bf..078aaefc6fb 100644
--- a/rust/lance-index/benches/geo.rs
+++ b/rust/lance-index/benches/geo.rs
@@ -15,7 +15,7 @@ use geoarrow_schema::Dimension;
 use lance_core::cache::LanceCache;
 use lance_core::{Error, ROW_ID};
 use lance_index::scalar::lance_format::LanceIndexStore;
-use lance_index::scalar::registry::ScalarIndexPlugin;
+use lance_index::scalar::registry::BasicTrainer;
 use lance_index::scalar::rtree::{BoundingBox, RTreeIndex, RTreeIndexPlugin, RTreeTrainingRequest};
 use lance_index::scalar::{GeoQuery, RelationQuery, ScalarIndex};
 use lance_io::object_store::ObjectStore;
diff --git a/rust/lance-index/benches/hnsw.rs b/rust/lance-index/benches/hnsw.rs
index 1aafd30188c..0a9b10bf42c 100644
--- a/rust/lance-index/benches/hnsw.rs
+++ b/rust/lance-index/benches/hnsw.rs
@@ -95,6 +95,67 @@ fn bench_hnsw(c: &mut Criterion) {
     });
 }
 
+fn bench_hnsw_load(c: &mut Criterion) {
+    const DIMENSION: usize = 128;
+    const TOTAL: usize = 100_000;
+    const SEED: [u8; 32] = [42; 32];
+    const K: usize = 100;
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    let data = generate_random_array_with_seed::<Float32Type>(TOTAL * DIMENSION, SEED);
+    let fsl = FixedSizeListArray::try_new_from_values(data, DIMENSION as i32).unwrap();
+    let vectors = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+
+    let search_build_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+    let hnsw = search_build_pool
+        .install(|| HNSW::index_vectors(vectors.as_ref(), HnswBuildParams::default()))
+        .unwrap();
+    let batch = hnsw.to_batch().unwrap();
+
+    // Load cost -- the path #6746 targets. `RecordBatch::clone` is an Arrow
+    // refcount bump (what production does anyway: each partition-cache IPC
+    // read yields a fresh batch), so it does not mask the load work measured.
+    c.bench_function(format!("load_hnsw({TOTAL}x{DIMENSION})").as_str(), |b| {
+        b.iter(|| {
+            let loaded = HNSW::load(batch.clone()).unwrap();
+            assert_eq!(loaded.len(), TOTAL);
+        })
+    });
+
+    // Search on the Arrow-backed loaded graph -- same TOTAL/DIMENSION/K/ef as
+    // the `search_hnsw` bench, so the two are directly comparable and confirm
+    // the new backend keeps search latency unchanged (issue #6746).
+    let loaded = HNSW::load(batch).unwrap();
+    let query = fsl.value(0);
+    c.bench_function(
+        format!("search_hnsw_loaded{TOTAL}x{DIMENSION}").as_str(),
+        |b| {
+            b.to_async(&rt).iter(|| async {
+                let uids: HashSet<u32> = loaded
+                    .search_basic(
+                        query.clone(),
+                        K,
+                        &HnswQueryParams {
+                            ef: 300,
+                            lower_bound: None,
+                            upper_bound: None,
+                            dist_q_c: 0.0,
+                        },
+                        None,
+                        vectors.as_ref(),
+                    )
+                    .unwrap()
+                    .iter()
+                    .map(|node| node.id)
+                    .collect();
+
+                assert_eq!(uids.len(), K);
+            })
+        },
+    );
+}
+
 fn bench_hnsw_sq(c: &mut Criterion) {
     const DIMENSION: usize = 128;
     const TOTAL: usize = 100_000;
@@ -291,7 +352,7 @@ criterion_group!(
         .measurement_time(Duration::from_secs(10))
         .sample_size(10)
         .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
-    targets = bench_hnsw, bench_hnsw_sq, bench_hnsw_pq);
+    targets = bench_hnsw, bench_hnsw_load, bench_hnsw_sq, bench_hnsw_pq);
 
 // Non-linux version does not support pprof.
 #[cfg(not(target_os = "linux"))]
@@ -300,6 +361,6 @@ criterion_group!(
     config = Criterion::default()
         .measurement_time(Duration::from_secs(10))
         .sample_size(10);
-    targets = bench_hnsw, bench_hnsw_sq, bench_hnsw_pq);
+    targets = bench_hnsw, bench_hnsw_load, bench_hnsw_sq, bench_hnsw_pq);
 
 criterion_main!(benches);
diff --git a/rust/lance-index/benches/rq.rs b/rust/lance-index/benches/rq.rs
index 4a7364d1313..72e0c49820d 100644
--- a/rust/lance-index/benches/rq.rs
+++ b/rust/lance-index/benches/rq.rs
@@ -17,11 +17,16 @@ use lance_datagen::array::rand_type;
 use lance_datagen::{BatchGeneratorBuilder, RowCount};
 use lance_index::vector::bq::RQRotationType;
 use lance_index::vector::bq::builder::RabitQuantizer;
+use lance_index::vector::bq::ex_dot::{
+    blocked_ex_code_bytes, ex_dot_kernel, pack_blocked_row, packed_ex_code_value,
+};
 use lance_index::vector::bq::storage::*;
 use lance_index::vector::bq::transform::{ADD_FACTORS_COLUMN, SCALE_FACTORS_COLUMN};
 use lance_index::vector::quantizer::{Quantization, QuantizerStorage};
 use lance_index::vector::storage::{DistCalculator, VectorStore};
 use lance_linalg::distance::DistanceType;
+use rand::rngs::SmallRng;
+use rand::{Rng, SeedableRng};
 
 const DIM: usize = 128;
 const TOTAL: usize = 16 * 1000;
@@ -119,16 +124,526 @@ fn compute_distances(c: &mut Criterion) {
     }
 }
 
-#[cfg(target_os = "linux")]
-criterion_group!(
-    name=benches;
-    config = Criterion::default().measurement_time(Duration::from_secs(10));
-    targets = construct_dist_table, compute_distances);
+/// The table-gather ex distance used before the dedicated ex-dot kernels,
+/// kept here as the baseline: per dim, extract the packed code and gather
+/// `query[d] * code` from a `dim * 2^ex_bits` table.
+fn gather_ex_distance(row_codes: &[u8], dim: usize, ex_bits: u8, ex_dist_table: &[f32]) -> f32 {
+    let entries_per_dim = 1usize << ex_bits;
+    (0..dim)
+        .map(|dim_idx| {
+            let code = packed_ex_code_value(row_codes, dim_idx, ex_bits) as usize;
+            ex_dist_table[dim_idx * entries_per_dim + code]
+        })
+        .sum()
+}
+
+fn ex_dot_kernels(c: &mut Criterion) {
+    for ex_dim in [1536usize, 2048] {
+        ex_dot_kernels_for_dim(c, ex_dim);
+    }
+}
+
+fn ex_dot_kernels_for_dim(c: &mut Criterion, ex_dim: usize) {
+    const NUM_ROWS: usize = 1024;
+
+    let mut rng = SmallRng::seed_from_u64(42);
+    let query = (0..ex_dim)
+        .map(|_| rng.random_range(-1.0f32..1.0))
+        .collect::<Vec<_>>();
+
+    for ex_bits in 1..=8u8 {
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        let values = (0..NUM_ROWS * ex_dim)
+            .map(|_| rng.random_range(0..=max_code))
+            .collect::<Vec<_>>();
+
+        // The gather baseline reads the legacy sequential layout it shipped
+        // with; the kernel reads the blocked layout.
+        let seq_code_len = (ex_dim * ex_bits as usize).div_ceil(8);
+        let mut seq_codes = vec![0u8; NUM_ROWS * seq_code_len];
+        for (row, row_values) in seq_codes
+            .chunks_exact_mut(seq_code_len)
+            .zip(values.chunks_exact(ex_dim))
+        {
+            for (dim, &value) in row_values.iter().enumerate() {
+                let bit_offset = dim * ex_bits as usize;
+                let bits = (value as u16) << (bit_offset % 8);
+                row[bit_offset / 8] |= bits as u8;
+                if bits >> 8 != 0 {
+                    row[bit_offset / 8 + 1] |= (bits >> 8) as u8;
+                }
+            }
+        }
+
+        let kernel_code_len = blocked_ex_code_bytes(ex_dim, ex_bits);
+        let mut kernel_codes = vec![0u8; NUM_ROWS * kernel_code_len];
+        for (row, row_values) in kernel_codes
+            .chunks_exact_mut(kernel_code_len)
+            .zip(values.chunks_exact(ex_dim))
+        {
+            pack_blocked_row(row_values, ex_bits, row);
+        }
+
+        // ex_dim is block-aligned here, so the kernels read the query as-is.
+        let ex_query = &query;
+        let kernel = ex_dot_kernel(ex_bits);
+        c.bench_function(
+            format!("RQ ex_dot kernel: ex_bits={ex_bits}, DIM={ex_dim}, rows={NUM_ROWS}").as_str(),
+            |b| {
+                b.iter(|| {
+                    let mut sum = 0.0f32;
+                    for row in kernel_codes.chunks_exact(kernel_code_len) {
+                        sum += kernel(ex_query, row);
+                    }
+                    black_box(sum)
+                })
+            },
+        );
+
+        let entries_per_dim = 1usize << ex_bits;
+        let mut ex_dist_table = vec![0.0f32; ex_dim * entries_per_dim];
+        for (dim, table) in ex_dist_table.chunks_exact_mut(entries_per_dim).enumerate() {
+            for (code, value) in table.iter_mut().enumerate() {
+                *value = query[dim] * code as f32;
+            }
+        }
+        c.bench_function(
+            format!("RQ ex_dot table-gather: ex_bits={ex_bits}, DIM={ex_dim}, rows={NUM_ROWS}")
+                .as_str(),
+            |b| {
+                b.iter(|| {
+                    let mut sum = 0.0f32;
+                    for row in seq_codes.chunks_exact(seq_code_len) {
+                        sum += gather_ex_distance(row, ex_dim, ex_bits, &ex_dist_table);
+                    }
+                    black_box(sum)
+                })
+            },
+        );
+    }
+}
+
+/// Storage load cost per format: blocked-format ex codes are aliased as-is,
+/// legacy sequential ex codes are repacked row by row.
+fn ex_code_storage_load(c: &mut Criterion) {
+    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, UInt8Array, UInt64Array};
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::vector::bq::ex_dot::repack_sequential_row;
+    use lance_index::vector::bq::rabit_ex_code_bytes;
+    use lance_index::vector::bq::transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN};
+    use std::sync::Arc;
+
+    const LOAD_DIM: usize = 1536;
+    const LOAD_ROWS: usize = 8192;
+    const NUM_BITS: u8 = 4; // ex_bits=3, a bit-plane width
+
+    let ex_bits = NUM_BITS - 1;
+    let mut rng = SmallRng::seed_from_u64(7);
+    let metadata = RabitQuantizationMetadata {
+        rotate_mat: None,
+        rotate_mat_position: None,
+        fast_rotation_signs: None,
+        rotation_type: RQRotationType::Fast,
+        code_dim: LOAD_DIM as u32,
+        num_bits: NUM_BITS,
+        packed: true,
+        query_estimator: RabitQueryEstimator::RawQuery,
+    };
+    let code_len = LOAD_DIM / 8;
+    let binary_codes = (0..LOAD_ROWS * code_len)
+        .map(|_| rng.random_range(0..=u8::MAX))
+        .collect::<Vec<_>>();
+    let seq_code_len = rabit_ex_code_bytes(LOAD_DIM, ex_bits).unwrap();
+    let seq_codes = (0..LOAD_ROWS * seq_code_len)
+        .map(|_| rng.random_range(0..=u8::MAX))
+        .collect::<Vec<_>>();
+    let blocked_code_len = blocked_ex_code_bytes(LOAD_DIM, ex_bits);
+    let mut blocked_codes = vec![0u8; LOAD_ROWS * blocked_code_len];
+    for (seq_row, blocked_row) in seq_codes
+        .chunks_exact(seq_code_len)
+        .zip(blocked_codes.chunks_exact_mut(blocked_code_len))
+    {
+        repack_sequential_row(seq_row, LOAD_DIM, ex_bits, blocked_row);
+    }
+
+    let make_batch = |ex_column: &str, ex_values: Vec<u8>, ex_code_len: usize| {
+        arrow_array::RecordBatch::try_from_iter(vec![
+            (
+                ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..LOAD_ROWS as u64)) as ArrayRef,
+            ),
+            (
+                RABIT_CODE_COLUMN,
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(binary_codes.clone()),
+                        code_len as i32,
+                    )
+                    .unwrap(),
+                ) as ArrayRef,
+            ),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; LOAD_ROWS])) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; LOAD_ROWS])) as ArrayRef,
+            ),
+            (
+                ex_column,
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(ex_values),
+                        ex_code_len as i32,
+                    )
+                    .unwrap(),
+                ) as ArrayRef,
+            ),
+            (
+                EX_ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; LOAD_ROWS])) as ArrayRef,
+            ),
+            (
+                EX_SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; LOAD_ROWS])) as ArrayRef,
+            ),
+        ])
+        .unwrap()
+    };
+
+    let blocked_batch = make_batch(
+        RABIT_BLOCKED_EX_CODE_COLUMN,
+        blocked_codes,
+        blocked_code_len,
+    );
+    c.bench_function(
+        format!("RQ storage load (blocked ex codes): num_bits={NUM_BITS}, DIM={LOAD_DIM}, rows={LOAD_ROWS}")
+            .as_str(),
+        |b| {
+            b.iter(|| {
+                black_box(
+                    RabitQuantizationStorage::try_from_batch(
+                        blocked_batch.clone(),
+                        &metadata,
+                        DistanceType::L2,
+                        None,
+                    )
+                    .unwrap(),
+                )
+            })
+        },
+    );
+
+    let legacy_batch = make_batch(RABIT_EX_CODE_COLUMN, seq_codes, seq_code_len);
+    c.bench_function(
+        format!("RQ storage load (legacy ex codes): num_bits={NUM_BITS}, DIM={LOAD_DIM}, rows={LOAD_ROWS}")
+            .as_str(),
+        |b| {
+            b.iter(|| {
+                black_box(
+                    RabitQuantizationStorage::try_from_batch(
+                        legacy_batch.clone(),
+                        &metadata,
+                        DistanceType::L2,
+                        None,
+                    )
+                    .unwrap(),
+                )
+            })
+        },
+    );
+}
+
+/// Bulk-scoring cost of the ex stage: the quantized ex-FastScan LUT path
+/// (inside `distance_all`) vs the exact per-row ex-dot kernel. The
+/// binary-only run isolates the shared binary stage so the ex cost is the
+/// difference from the full run.
+fn ex_bulk_paths(c: &mut Criterion) {
+    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, UInt8Array, UInt64Array};
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::vector::ApproxMode;
+    use lance_index::vector::bq::ex_dot::pad_query_into;
+    use lance_index::vector::bq::transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN};
+    use lance_index::vector::storage::DistanceCalculatorOptions;
+    use std::sync::Arc;
+
+    const BULK_DIM: usize = 1536;
+    const BULK_ROWS: usize = 16384;
+
+    let mut rng = SmallRng::seed_from_u64(13);
+    for num_bits in [3u8, 5, 9] {
+        let ex_bits = num_bits - 1;
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+
+        let rq = RabitQuantizer::new_with_rotation::<Float32Type>(
+            num_bits,
+            BULK_DIM as i32,
+            RQRotationType::Fast,
+        );
+        let metadata = rq.metadata(None);
+
+        let code_len = BULK_DIM / 8;
+        let binary_codes = (0..BULK_ROWS * code_len)
+            .map(|_| rng.random_range(0..=u8::MAX))
+            .collect::<Vec<_>>();
+        let ex_code_len = blocked_ex_code_bytes(BULK_DIM, ex_bits);
+        let mut ex_codes = vec![0u8; BULK_ROWS * ex_code_len];
+        let values = (0..BULK_DIM)
+            .map(|_| rng.random_range(0..=max_code))
+            .collect::<Vec<_>>();
+        for row in ex_codes.chunks_exact_mut(ex_code_len) {
+            pack_blocked_row(&values, ex_bits, row);
+        }
+
+        // No error factors: `distance_all` takes the FastScan ex bulk branch.
+        let batch = arrow_array::RecordBatch::try_from_iter(vec![
+            (
+                ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..BULK_ROWS as u64)) as ArrayRef,
+            ),
+            (
+                RABIT_CODE_COLUMN,
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(binary_codes),
+                        code_len as i32,
+                    )
+                    .unwrap(),
+                ) as ArrayRef,
+            ),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; BULK_ROWS])) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; BULK_ROWS])) as ArrayRef,
+            ),
+            (
+                RABIT_BLOCKED_EX_CODE_COLUMN,
+                Arc::new(
+                    FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(ex_codes.clone()),
+                        ex_code_len as i32,
+                    )
+                    .unwrap(),
+                ) as ArrayRef,
+            ),
+            (
+                EX_ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0f32; BULK_ROWS])) as ArrayRef,
+            ),
+            (
+                EX_SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![1.0f32; BULK_ROWS])) as ArrayRef,
+            ),
+        ])
+        .unwrap();
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+
+        let query: ArrayRef = Arc::new(Float32Array::from(
+            (0..BULK_DIM)
+                .map(|_| rng.random_range(-1.0f32..1.0))
+                .collect::<Vec<_>>(),
+        ));
+
+        for (label, approx_mode) in [
+            ("full distance_all (binary + ex LUT)", ApproxMode::Normal),
+            ("binary-only distance_all (fast mode)", ApproxMode::Fast),
+        ] {
+            let mut f32_scratch = Vec::new();
+            let calc = storage.dist_calculator_with_scratch(
+                query.clone(),
+                0.0,
+                None,
+                &mut f32_scratch,
+                DistanceCalculatorOptions { approx_mode },
+            );
+            let mut dists = Vec::new();
+            let mut u16_scratch = Vec::new();
+            let mut u8_scratch = Vec::new();
+            let mut u32_scratch = Vec::new();
+            c.bench_function(
+                format!("RQ bulk {label}: num_bits={num_bits}, DIM={BULK_DIM}, rows={BULK_ROWS}")
+                    .as_str(),
+                |b| {
+                    b.iter(|| {
+                        calc.distance_all_with_scratch(
+                            0,
+                            &mut dists,
+                            &mut u16_scratch,
+                            &mut u8_scratch,
+                            &mut u32_scratch,
+                        );
+                        black_box(dists.len())
+                    })
+                },
+            );
+        }
+
+        let kernel = ex_dot_kernel(ex_bits);
+        let mut ex_query = vec![0.0f32; BULK_DIM];
+        pad_query_into(
+            query
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap()
+                .values(),
+            &mut ex_query,
+        );
+        c.bench_function(
+            format!(
+                "RQ bulk ex kernel loop: num_bits={num_bits}, DIM={BULK_DIM}, rows={BULK_ROWS}"
+            )
+            .as_str(),
+            |b| {
+                b.iter(|| {
+                    let mut sum = 0.0f32;
+                    for row in ex_codes.chunks_exact(ex_code_len) {
+                        sum += kernel(&ex_query, row);
+                    }
+                    black_box(sum)
+                })
+            },
+        );
+    }
+}
+
+/// Top-k accumulation through the gated raw-query multi-bit path: binary
+/// FastScan, the per-row lower-bound pruning scan, and the exact rerank of
+/// the surviving rows. Error factors are present so the gating is enabled.
+fn heap_topk(c: &mut Criterion) {
+    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, UInt8Array, UInt64Array};
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::vector::ApproxMode;
+    use lance_index::vector::bq::transform::{
+        ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN,
+    };
+    use lance_index::vector::storage::DistanceCalculatorOptions;
+    use std::collections::BinaryHeap;
+    use std::sync::Arc;
+
+    const TOPK_DIM: usize = 1536;
+    const TOPK_ROWS: usize = 4096;
+    const TOPK_K: usize = 10;
+    const NUM_BITS: u8 = 5;
+    let ex_bits = NUM_BITS - 1;
+
+    let mut rng = SmallRng::seed_from_u64(99);
+    let rq = RabitQuantizer::new_with_rotation::<Float32Type>(
+        NUM_BITS,
+        TOPK_DIM as i32,
+        RQRotationType::Fast,
+    );
+    let metadata = rq.metadata(None);
+
+    let code_len = TOPK_DIM / 8;
+    let binary_codes = (0..TOPK_ROWS * code_len)
+        .map(|_| rng.random())
+        .collect::<Vec<u8>>();
+    let ex_code_len = blocked_ex_code_bytes(TOPK_DIM, ex_bits);
+    let ex_codes = (0..TOPK_ROWS * ex_code_len)
+        .map(|_| rng.random())
+        .collect::<Vec<u8>>();
+    // Factor magnitudes chosen so the lower bounds spread mostly with the add
+    // factors; once the heap is full the threshold prunes the vast majority
+    // of rows, like a production multi-partition scan.
+    let mut rand_factors = |low: f32, high: f32| {
+        Arc::new(Float32Array::from(
+            (0..TOPK_ROWS)
+                .map(|_| rng.random_range(low..high))
+                .collect::<Vec<_>>(),
+        )) as ArrayRef
+    };
+    let batch = arrow_array::RecordBatch::try_from_iter(vec![
+        (
+            ROW_ID,
+            Arc::new(UInt64Array::from_iter_values(0..TOPK_ROWS as u64)) as ArrayRef,
+        ),
+        (
+            RABIT_CODE_COLUMN,
+            Arc::new(
+                FixedSizeListArray::try_new_from_values(
+                    UInt8Array::from(binary_codes),
+                    code_len as i32,
+                )
+                .unwrap(),
+            ) as ArrayRef,
+        ),
+        (ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)),
+        (SCALE_FACTORS_COLUMN, rand_factors(0.0005, 0.0015)),
+        (ERROR_FACTORS_COLUMN, rand_factors(0.0, 0.01)),
+        (
+            RABIT_BLOCKED_EX_CODE_COLUMN,
+            Arc::new(
+                FixedSizeListArray::try_new_from_values(
+                    UInt8Array::from(ex_codes),
+                    ex_code_len as i32,
+                )
+                .unwrap(),
+            ) as ArrayRef,
+        ),
+        (EX_ADD_FACTORS_COLUMN, rand_factors(0.0, 1.0)),
+        (EX_SCALE_FACTORS_COLUMN, rand_factors(0.00003, 0.0001)),
+    ])
+    .unwrap();
+    let storage =
+        RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None).unwrap();
+    let query: ArrayRef = Arc::new(Float32Array::from(
+        (0..TOPK_DIM)
+            .map(|_| rng.random_range(-1.0f32..1.0))
+            .collect::<Vec<_>>(),
+    ));
+
+    for (label, approx_mode) in [
+        ("normal", ApproxMode::Normal),
+        ("accurate", ApproxMode::Accurate),
+    ] {
+        let mut f32_scratch = Vec::new();
+        let calc = storage.dist_calculator_with_scratch(
+            query.clone(),
+            1.0,
+            None,
+            &mut f32_scratch,
+            DistanceCalculatorOptions { approx_mode },
+        );
+        let mut heap = BinaryHeap::with_capacity(TOPK_K + 1);
+        let mut dists = Vec::new();
+        let mut u16_scratch = Vec::new();
+        let mut u8_scratch = Vec::new();
+        let mut u32_scratch = Vec::new();
+        c.bench_function(
+            format!(
+                "RQ heap topk ({label}): num_bits={NUM_BITS}, DIM={TOPK_DIM}, rows={TOPK_ROWS}, k={TOPK_K}"
+            )
+            .as_str(),
+            |b| {
+                b.iter(|| {
+                    heap.clear();
+                    calc.accumulate_topk_with_scratch(
+                        TOPK_K,
+                        None,
+                        None,
+                        |id| id as u64,
+                        &mut heap,
+                        &mut dists,
+                        &mut u16_scratch,
+                        &mut u8_scratch,
+                        &mut u32_scratch,
+                    );
+                    black_box(heap.len())
+                })
+            },
+        );
+    }
+}
 
-#[cfg(not(target_os = "linux"))]
 criterion_group!(
     name=benches;
     config = Criterion::default().measurement_time(Duration::from_secs(10));
-    targets = construct_dist_table, compute_distances);
+    targets = construct_dist_table, compute_distances, ex_dot_kernels, ex_code_storage_load, ex_bulk_paths, heap_topk);
 
 criterion_main!(benches);
diff --git a/rust/lance-index/build.rs b/rust/lance-index/build.rs
index 0617de8c806..b47744f7b5a 100644
--- a/rust/lance-index/build.rs
+++ b/rust/lance-index/build.rs
@@ -6,6 +6,9 @@ use std::io::Result;
 
 fn main() -> Result<()> {
     println!("cargo:rerun-if-changed=protos");
+    // Cache-entry protos are library-internal serialization, not part of the
+    // on-disk format spec, so they live here rather than in the shared `protos/`.
+    println!("cargo:rerun-if-changed=protos-cache");
 
     #[cfg(feature = "protoc")]
     // Use vendored protobuf compiler if requested.
@@ -17,8 +20,12 @@ fn main() -> Result<()> {
     prost_build.protoc_arg("--experimental_allow_proto3_optional");
     prost_build.enable_type_names();
     prost_build.compile_protos(
-        &["./protos/index.proto", "./protos/index_old.proto"],
-        &["./protos"],
+        &[
+            "./protos/index.proto",
+            "./protos/index_old.proto",
+            "./protos-cache/cache.proto",
+        ],
+        &["./protos", "./protos-cache"],
     )?;
 
     let rust_toolchain = env::var("RUSTUP_TOOLCHAIN")
diff --git a/rust/lance-index/protos-cache/cache.proto b/rust/lance-index/protos-cache/cache.proto
new file mode 100644
index 00000000000..a861e2ef652
--- /dev/null
+++ b/rust/lance-index/protos-cache/cache.proto
@@ -0,0 +1,199 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+// Protobuf headers for serialized index cache entries.
+//
+// These messages describe the *cache* serialization format, not the on-disk
+// Lance format spec, so they live with the library (lance-index) rather than in
+// the top-level `protos/` spec folder.
+//
+// Field numbers and enum values are append-only across all messages here: never
+// renumber or reuse them. A change the proto cannot express transparently
+// (adding/removing/reordering the IPC/raw sections that follow a header) must
+// bump the relevant codec's `CURRENT_VERSION` instead.
+
+syntax = "proto3";
+
+package lance.index.cache;
+
+// ---------------------------------------------------------------------------
+// Full-text search (FTS) posting lists
+// ---------------------------------------------------------------------------
+
+// Header for a serialized `CompressedPostingList` cache entry.
+message CompressedPostingHeader {
+  float max_score = 1;
+  uint32 length = 2;
+  PostingTailCodec posting_tail_codec = 3;
+  PositionStorage position_storage = 4;
+  // Only meaningful when position_storage == POSITION_STORAGE_SHARED.
+  PositionStreamCodec position_stream_codec = 5;
+  // Number of documents in each compressed posting block. Older cache entries
+  // omit this field and decode as the legacy 128-doc block size.
+  uint32 block_size = 6;
+  // Whether an impact IPC section follows the posting/position sections.
+  bool has_impacts = 7;
+}
+
+// Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow
+// IPC section of (row_ids: UInt64, frequencies: Float32), then — when
+// position_storage == POSITION_STORAGE_LEGACY — an IPC section of the per-doc
+// position list. Plain postings never carry a shared position stream.
+message PlainPostingHeader {
+  // Absent when the posting has no precomputed block-max score (the in-memory
+  // `max_score` is `None`); present otherwise.
+  optional float max_score = 1;
+  // POSITION_STORAGE_NONE or POSITION_STORAGE_LEGACY only.
+  PositionStorage position_storage = 2;
+}
+
+// Header for a serialized standalone `Positions` cache entry. Followed by the
+// position sections framed per `position_storage`, which is never
+// POSITION_STORAGE_NONE for a standalone entry.
+message PositionsHeader {
+  PositionStorage position_storage = 1;
+  // Only meaningful when position_storage == POSITION_STORAGE_SHARED.
+  PositionStreamCodec position_stream_codec = 2;
+}
+
+// Header for a serialized `PostingListGroup`: a member count followed by that
+// many `PostingList` bodies written inline. Each member body is
+// self-delimiting, so members need no length prefixes, and writing them inline
+// keeps their Arrow IPC sections 64-byte aligned within the group entry.
+message PostingListGroupHeader {
+  uint32 count = 1;
+}
+
+// Tail-block encoding of a compressed posting list.
+enum PostingTailCodec {
+  POSTING_TAIL_CODEC_FIXED32 = 0;
+  POSTING_TAIL_CODEC_VARINT_DELTA = 1;
+}
+
+// Encoding of a shared position stream's byte buffer.
+enum PositionStreamCodec {
+  POSITION_STREAM_CODEC_VARINT_DOC_DELTA = 0;
+  POSITION_STREAM_CODEC_PACKED_DELTA = 1;
+}
+
+// Which (if any) positions accompany the posting list, and how they are framed
+// in the sections after the header.
+enum PositionStorage {
+  POSITION_STORAGE_NONE = 0;
+  // Legacy per-doc positions as a single Arrow IPC section.
+  POSITION_STORAGE_LEGACY = 1;
+  // Shared stream: an Arrow IPC section of block offsets, then a raw blob of
+  // the (codec-encoded) position bytes.
+  POSITION_STORAGE_SHARED = 2;
+}
+
+// ---------------------------------------------------------------------------
+// Scalar indices
+// ---------------------------------------------------------------------------
+
+// Header for a serialized `BTreeIndexState` cache entry, followed by a single
+// Arrow IPC section holding the page-lookup batch.
+message BTreeIndexHeader {
+  uint64 batch_size = 1;
+  // Whether an explicit page-range -> file mapping is present. Distinguishes a
+  // non-range-partitioned index (false) from a range-partitioned one whose map
+  // happens to be empty (true with no entries).
+  bool has_ranges_to_files = 2;
+  repeated RangeToFile ranges_to_files = 3;
+}
+
+// One entry of a `BTreeIndexState` page-range -> file mapping. The range is
+// inclusive on both ends (a `RangeInclusive<u32>`).
+message RangeToFile {
+  uint32 start = 1;
+  uint32 end = 2;
+  uint32 page_offset = 3;
+  string path = 4;
+}
+
+// ---------------------------------------------------------------------------
+// Vector indices (IVF partitions)
+// ---------------------------------------------------------------------------
+
+// Headers for serialized IVF partition cache entries (`PartitionEntry<S, Q>`).
+//
+// Each header is followed by 64-byte-aligned Arrow IPC sections in a fixed,
+// version-keyed order (sub-index, then any quantizer-specific arrays, then the
+// quantizer storage batches).
+
+// Distance metric a quantizer's storage was built for.
+enum DistanceType {
+  DISTANCE_TYPE_L2 = 0;
+  DISTANCE_TYPE_COSINE = 1;
+  DISTANCE_TYPE_DOT = 2;
+  DISTANCE_TYPE_HAMMING = 3;
+}
+
+// Rotation applied by a RabitQ quantizer.
+enum RotationType {
+  ROTATION_TYPE_MATRIX = 0;
+  ROTATION_TYPE_FAST = 1;
+}
+
+// Estimator a RabitQ quantizer uses at query time.
+enum RabitQueryEstimator {
+  RABIT_QUERY_ESTIMATOR_RESIDUAL_QUERY = 0;
+  RABIT_QUERY_ESTIMATOR_RAW_QUERY = 1;
+}
+
+// Product quantizer. Sections: sub-index IPC, codebook IPC, storage IPC.
+message PqPartitionHeader {
+  DistanceType distance_type = 1;
+  uint32 nbits = 2;
+  uint64 num_sub_vectors = 3;
+  uint64 dimension = 4;
+  bool transposed = 5;
+}
+
+// Flat (float) and flat-binary quantizers. Sections: sub-index IPC, storage IPC.
+message FlatPartitionHeader {
+  DistanceType distance_type = 1;
+  uint64 dim = 2;
+}
+
+// Scalar quantizer. Sections: sub-index IPC, storage IPC (possibly multi-batch).
+message SqPartitionHeader {
+  DistanceType distance_type = 1;
+  uint32 num_bits = 2;
+  uint64 dim = 3;
+  double bounds_start = 4;
+  double bounds_end = 5;
+}
+
+// Header for a serialized IVF index state (`IvfIndexState<Q>`), followed by
+// three raw blobs: the IVF model protobuf, the quantizer's extra-metadata
+// buffer (may be empty), and the auxiliary IVF model protobuf.
+message IvfStateHeader {
+  string index_file_path = 1;
+  string uuid = 2;
+  string distance_type = 3;
+  repeated string sub_index_metadata = 4;
+  string sub_index_type = 5;
+  string quantization_type = 6;
+  // Per-quantizer `Q::Metadata` as JSON. Kept as a string because the metadata
+  // type is generic over the quantizer; the proto envelope still provides
+  // additive evolution for the surrounding fields.
+  string quantizer_metadata_json = 7;
+  string cache_key_prefix = 8;
+  uint64 index_file_size = 9;
+  uint64 aux_file_size = 10;
+}
+
+// RabitQ quantizer. Sections: sub-index IPC, rotate-matrix IPC (Matrix rotation
+// only), storage IPC.
+message RabitPartitionHeader {
+  DistanceType distance_type = 1;
+  uint32 num_bits = 2;
+  uint32 code_dim = 3;
+  RotationType rotation_type = 4;
+  // Fast-rotation sign vector; present only when rotation_type ==
+  // ROTATION_TYPE_FAST (the Matrix case stores its rotation as an IPC section).
+  optional bytes fast_rotation_signs = 5;
+  // Estimator the RabitQ storage uses at query time (residual vs raw query).
+  RabitQueryEstimator query_estimator = 6;
+}
diff --git a/rust/lance-index/src/frag_reuse.rs b/rust/lance-index/src/frag_reuse.rs
index d145108d3c0..7072cdbe446 100644
--- a/rust/lance-index/src/frag_reuse.rs
+++ b/rust/lance-index/src/frag_reuse.rs
@@ -1,355 +1,45 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use crate::{Index, IndexType};
-use arrow_array::cast::AsArray;
-use arrow_array::types::UInt64Type;
-use arrow_array::{Array, ArrayRef, PrimitiveArray, RecordBatch, UInt64Array};
+//! `Index`-trait adapter for the fragment-reuse system index.
+//!
+//! The data structures and table-format logic live in
+//! [`lance_table::system_index::frag_reuse`]; this module re-exports them and
+//! provides newtype wrappers that implement the [`Index`] and [`RowIdRemapper`]
+//! traits.
+
+use std::any::Any;
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
 use async_trait::async_trait;
-use deepsize::{Context, DeepSizeOf};
-use itertools::Itertools;
-use lance_core::{Error, Result};
+use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use lance_select::RowAddrTreeMap;
-use lance_table::format::pb::fragment_reuse_index_details::InlineContent;
-use lance_table::format::{ExternalFile, Fragment, pb};
 use roaring::{RoaringBitmap, RoaringTreemap};
-use serde::{Deserialize, Serialize};
-use std::{any::Any, collections::HashMap, sync::Arc};
-use uuid::Uuid;
-
-pub const FRAG_REUSE_INDEX_NAME: &str = "__lance_frag_reuse";
-pub const FRAG_REUSE_DETAILS_FILE_NAME: &str = "details.binpb";
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FragDigest {
-    pub id: u64,
-    pub physical_rows: usize,
-    pub num_deleted_rows: usize,
-}
-
-impl From<&FragDigest> for pb::fragment_reuse_index_details::FragmentDigest {
-    fn from(digest: &FragDigest) -> Self {
-        Self {
-            id: digest.id,
-            physical_rows: digest.physical_rows as u64,
-            num_deleted_rows: digest.num_deleted_rows as u64,
-        }
-    }
-}
-
-impl From<&Fragment> for FragDigest {
-    fn from(fragment: &Fragment) -> Self {
-        Self {
-            id: fragment.id,
-            physical_rows: fragment
-                .physical_rows
-                .expect("Fragment doesn't have physical rows recorded"),
-            num_deleted_rows: fragment
-                .deletion_file
-                .as_ref()
-                .and_then(|d| d.num_deleted_rows)
-                .unwrap_or(0),
-        }
-    }
-}
-
-impl TryFrom<pb::fragment_reuse_index_details::FragmentDigest> for FragDigest {
-    type Error = Error;
-
-    fn try_from(digest: pb::fragment_reuse_index_details::FragmentDigest) -> Result<Self> {
-        Ok(Self {
-            id: digest.id,
-            physical_rows: digest.physical_rows as usize,
-            num_deleted_rows: digest.num_deleted_rows as usize,
-        })
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FragReuseGroup {
-    pub changed_row_addrs: Vec<u8>,
-    pub old_frags: Vec<FragDigest>,
-    pub new_frags: Vec<FragDigest>,
-}
-
-impl From<&FragReuseGroup> for pb::fragment_reuse_index_details::Group {
-    fn from(group: &FragReuseGroup) -> Self {
-        Self {
-            changed_row_addrs: group.changed_row_addrs.clone(),
-            old_fragments: group.old_frags.iter().map(|f| f.into()).collect(),
-            new_fragments: group.new_frags.iter().map(|f| f.into()).collect(),
-        }
-    }
-}
-
-impl TryFrom<pb::fragment_reuse_index_details::Group> for FragReuseGroup {
-    type Error = Error;
-
-    fn try_from(group: pb::fragment_reuse_index_details::Group) -> Result<Self> {
-        Ok(Self {
-            changed_row_addrs: group.changed_row_addrs,
-            old_frags: group
-                .old_fragments
-                .into_iter()
-                .map(FragDigest::try_from)
-                .collect::<Result<_>>()?,
-            new_frags: group
-                .new_fragments
-                .into_iter()
-                .map(FragDigest::try_from)
-                .collect::<Result<_>>()?,
-        })
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FragReuseVersion {
-    pub dataset_version: u64,
-    pub groups: Vec<FragReuseGroup>,
-}
-
-impl From<&FragReuseVersion> for pb::fragment_reuse_index_details::Version {
-    fn from(version: &FragReuseVersion) -> Self {
-        Self {
-            dataset_version: version.dataset_version,
-            groups: version.groups.iter().map(|g| g.into()).collect(),
-        }
-    }
-}
-
-impl TryFrom<pb::fragment_reuse_index_details::Version> for FragReuseVersion {
-    type Error = Error;
-
-    fn try_from(version: pb::fragment_reuse_index_details::Version) -> Result<Self> {
-        Ok(Self {
-            dataset_version: version.dataset_version,
-            groups: version
-                .groups
-                .into_iter()
-                .map(FragReuseGroup::try_from)
-                .collect::<Result<_>>()?,
-        })
-    }
-}
-
-impl FragReuseVersion {
-    pub fn old_frag_ids(&self) -> Vec<u64> {
-        self.groups
-            .iter()
-            .flat_map(|g| g.old_frags.iter().map(|f| f.id))
-            .collect::<Vec<_>>()
-    }
-
-    pub fn new_frag_ids(&self) -> Vec<u64> {
-        self.groups
-            .iter()
-            .flat_map(|g| g.new_frags.iter().map(|f| f.id))
-            .collect::<Vec<_>>()
-    }
-
-    pub fn new_frag_bitmap(&self) -> RoaringBitmap {
-        RoaringBitmap::from_iter(self.new_frag_ids().iter().map(|&id| id as u32))
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub enum FragReuseIndexDetailsContentType {
-    Inline(FragReuseIndexDetails),
-    External(ExternalFile),
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FragReuseIndexDetails {
-    pub versions: Vec<FragReuseVersion>,
-}
-
-impl From<&FragReuseIndexDetails> for InlineContent {
-    fn from(details: &FragReuseIndexDetails) -> Self {
-        Self {
-            versions: details
-                .versions
-                .iter()
-                .map(|m| m.into())
-                // sort from oldest to latest version
-                .sorted_by_key(|v: &pb::fragment_reuse_index_details::Version| v.dataset_version)
-                .collect(),
-        }
-    }
-}
+use serde::Serialize;
 
-impl TryFrom<InlineContent> for FragReuseIndexDetails {
-    type Error = Error;
+pub use lance_table::system_index::frag_reuse::*;
 
-    fn try_from(content: InlineContent) -> Result<Self> {
-        Ok(Self {
-            versions: content
-                .versions
-                .into_iter()
-                .map(|m| m.try_into())
-                .collect::<Result<Vec<_>>>()?,
-        })
-    }
-}
-
-impl FragReuseIndexDetails {
-    pub fn new_frag_bitmap(&self) -> RoaringBitmap {
-        RoaringBitmap::from_iter(
-            self.versions
-                .iter()
-                .flat_map(|v| v.new_frag_ids().into_iter().map(|id| id as u32)),
-        )
-    }
-}
+use crate::scalar::RowIdRemapper;
+use crate::{Index, IndexType};
 
-/// An index that stores row ID maps.
-/// A row ID map describes the mapping from old row address to new address after compactions.
-/// Each version contains the mapping for one round of compaction.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct FragReuseIndex {
-    pub uuid: Uuid,
-    pub row_id_maps: Vec<HashMap<u64, Option<u64>>>,
-    pub details: FragReuseIndexDetails,
-}
+/// Newtype wrapping [`FragReuseIndex`] so that `lance-index` can implement
+/// the `Index` and `RowIdRemapper` traits (orphan rules prevent implementing
+/// them directly in `lance-table`).
+pub struct FragReuseIndexHandle(pub Arc<FragReuseIndex>);
 
-impl DeepSizeOf for FragReuseIndex {
-    fn deep_size_of_children(&self, cx: &mut Context) -> usize {
-        self.row_id_maps.deep_size_of_children(cx) + self.details.deep_size_of_children(cx)
+impl std::fmt::Debug for FragReuseIndexHandle {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("FragReuseIndexHandle")
+            .field(&self.0)
+            .finish()
     }
 }
 
-impl FragReuseIndex {
-    pub fn new(
-        uuid: Uuid,
-        row_id_maps: Vec<HashMap<u64, Option<u64>>>,
-        details: FragReuseIndexDetails,
-    ) -> Self {
-        Self {
-            uuid,
-            row_id_maps,
-            details,
-        }
-    }
-
-    pub fn remap_row_id(&self, row_id: u64) -> Option<u64> {
-        let mut mapped_value = Some(row_id);
-        for row_id_map in self.row_id_maps.iter() {
-            if mapped_value.is_some() {
-                mapped_value = row_id_map
-                    .get(&mapped_value.unwrap())
-                    .copied()
-                    .unwrap_or(mapped_value);
-            }
-        }
-
-        mapped_value
-    }
-
-    pub fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
-        RowAddrTreeMap::from_iter(row_addrs.row_addrs().unwrap().filter_map(|addr| {
-            let addr_as_u64 = u64::from(addr);
-            self.remap_row_id(addr_as_u64)
-        }))
-    }
-
-    pub fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
-        RoaringTreemap::from_iter(row_ids.iter().filter_map(|addr| self.remap_row_id(addr)))
-    }
-
-    /// Remap a record batch that contains a row_id column at index `row_id_idx`
-    /// Currently this assumes there are only 2 columns in the schema,
-    /// which is the case for all indexes.
-    /// For example, for btree, the schema is (value, row_id).
-    /// For vector index storage, the schema is (row_id, vector).
-    pub fn remap_row_ids_record_batch(
-        &self,
-        batch: RecordBatch,
-        row_id_idx: usize,
-    ) -> Result<RecordBatch> {
-        assert_eq!(batch.schema().fields().len(), 2);
-        let other_column_idx = 1 - row_id_idx;
-        let row_ids = batch.column(row_id_idx).as_primitive::<UInt64Type>();
-        let (val_indices, new_row_ids): (Vec<u64>, Vec<u64>) = row_ids
-            .values()
-            .iter()
-            .enumerate()
-            .filter_map(|(idx, old_id)| {
-                self.remap_row_id(*old_id)
-                    .map(|new_id| (idx as u64, new_id))
-            })
-            .unzip();
-        let new_val_indices = UInt64Array::from_iter_values(val_indices);
-        let new_vals =
-            arrow_select::take::take(batch.column(other_column_idx), &new_val_indices, None)?;
-
-        let mut batch_data: Vec<(usize, ArrayRef)> = vec![
-            (
-                row_id_idx,
-                Arc::new(UInt64Array::from_iter_values(new_row_ids)) as ArrayRef,
-            ),
-            (other_column_idx, Arc::new(new_vals)),
-        ];
-        batch_data.sort_by_key(|(i, _)| *i);
-        Ok(RecordBatch::try_new(
-            batch.schema(),
-            batch_data.into_iter().map(|(_, item)| item).collect(),
-        )?)
-    }
-
-    pub fn remap_row_ids_array(&self, array: ArrayRef) -> PrimitiveArray<UInt64Type> {
-        let primitive_array = array
-            .as_any()
-            .downcast_ref::<PrimitiveArray<UInt64Type>>()
-            .expect("expected row IDs to be uint64 array");
-        (0..primitive_array.len())
-            .map(|i| {
-                if primitive_array.is_null(i) {
-                    None
-                } else {
-                    self.remap_row_id(primitive_array.value(i))
-                }
-            })
-            .collect()
-    }
-
-    pub fn remap_fragment_bitmap(&self, fragment_bitmap: &mut RoaringBitmap) -> Result<()> {
-        for version in self.details.versions.iter() {
-            for group in version.groups.iter() {
-                let mut removed = 0;
-                for old_frag in group.old_frags.iter() {
-                    if fragment_bitmap.remove(old_frag.id as u32) {
-                        removed += 1;
-                    }
-                }
-
-                if removed > 0 {
-                    if removed != group.old_frags.len() {
-                        // Straddle: the index covered only part of this rewrite
-                        // group. Caused by the bug fixed in
-                        // <https://github.com/lance-format/lance/pull/6610>.
-                        // We've already removed the indexed old_frags from the
-                        // bitmap above; deliberately do NOT insert new_frags,
-                        // since the merged fragment also contains rows that
-                        // were never indexed. Affected rows fall through to
-                        // flat scan until the next optimize_indices. The fix
-                        // is persisted on the next write via build_manifest.
-                        tracing::warn!(
-                            "Healing straddling fragment-reuse rewrite group in index bitmap: \
-                             group {:?} was only partially indexed ({} of {} old fragments). \
-                             Affected rows will use flat scan until the next optimize_indices.",
-                            group.old_frags,
-                            removed,
-                            group.old_frags.len(),
-                        );
-                        continue;
-                    }
-
-                    for new_frag in group.new_frags.iter() {
-                        fragment_bitmap.insert(new_frag.id as u32);
-                    }
-                }
-            }
-        }
-        Ok(())
+impl DeepSizeOf for FragReuseIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
     }
 }
 
@@ -359,7 +49,7 @@ struct FragReuseStatistics {
 }
 
 #[async_trait]
-impl Index for FragReuseIndex {
+impl Index for FragReuseIndexHandle {
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -368,18 +58,12 @@ impl Index for FragReuseIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::not_supported_source(
-            "FragReuseIndex is not a vector index".into(),
-        ))
-    }
-
     fn statistics(&self) -> Result<serde_json::Value> {
         let stats = FragReuseStatistics {
-            num_versions: self.details.versions.len(),
+            num_versions: self.0.details.versions.len(),
         };
         serde_json::to_value(stats).map_err(|e| {
-            Error::internal(format!(
+            lance_core::Error::internal(format!(
                 "failed to serialize fragment reuse index statistics: {}",
                 e
             ))
@@ -399,133 +83,24 @@ impl Index for FragReuseIndex {
     }
 }
 
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    #[tokio::test]
-    async fn test_serialize_deserialize_index_details() {
-        // Create sample FragReuseVersions with different dataset versions
-        let version1 = FragReuseVersion {
-            dataset_version: 2,
-            groups: vec![FragReuseGroup {
-                changed_row_addrs: vec![1, 2, 3],
-                old_frags: vec![FragDigest {
-                    id: 1,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                }],
-                new_frags: vec![
-                    FragDigest {
-                        id: 2,
-                        physical_rows: 1,
-                        num_deleted_rows: 0,
-                    },
-                    FragDigest {
-                        id: 3,
-                        physical_rows: 1,
-                        num_deleted_rows: 0,
-                    },
-                ],
-            }],
-        };
-
-        let version2 = FragReuseVersion {
-            dataset_version: 1,
-            groups: vec![FragReuseGroup {
-                changed_row_addrs: vec![4, 5, 6],
-                old_frags: vec![FragDigest {
-                    id: 2,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                }],
-                new_frags: vec![
-                    FragDigest {
-                        id: 4,
-                        physical_rows: 1,
-                        num_deleted_rows: 0,
-                    },
-                    FragDigest {
-                        id: 5,
-                        physical_rows: 1,
-                        num_deleted_rows: 0,
-                    },
-                ],
-            }],
-        };
-
-        // Create FragReuseIndexDetails with versions in reverse order
-        let details = FragReuseIndexDetails {
-            versions: vec![version1, version2],
-        };
-
-        // Convert to protobuf format
-        let inline_content: InlineContent = (&details).into();
-
-        // Convert back to FragReuseIndexDetails
-        let roundtrip_details = FragReuseIndexDetails::try_from(inline_content).unwrap();
+impl RowIdRemapper for FragReuseIndexHandle {
+    fn remap_row_id(&self, row_id: u64) -> Option<u64> {
+        self.0.remap_row_id(row_id)
+    }
 
-        // Verify the roundtrip
-        assert_eq!(roundtrip_details.versions.len(), 2);
+    fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
+        self.0.remap_row_addrs_tree_map(row_addrs)
+    }
 
-        // Verify versions are sorted by dataset_version (oldest to latest)
-        assert_eq!(roundtrip_details.versions[0].dataset_version, 1);
-        assert_eq!(
-            roundtrip_details.versions[0].groups[0].changed_row_addrs,
-            vec![4, 5, 6]
-        );
-        assert_eq!(
-            roundtrip_details.versions[0].groups[0].new_frags,
-            vec![
-                FragDigest {
-                    id: 4,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                },
-                FragDigest {
-                    id: 5,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                }
-            ]
-        );
-        assert_eq!(
-            roundtrip_details.versions[0].groups[0].old_frags,
-            vec![FragDigest {
-                id: 2,
-                physical_rows: 1,
-                num_deleted_rows: 0,
-            }]
-        );
+    fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
+        self.0.remap_row_ids_roaring_tree_map(row_ids)
+    }
 
-        assert_eq!(roundtrip_details.versions[1].dataset_version, 2);
-        assert_eq!(
-            roundtrip_details.versions[1].groups[0].changed_row_addrs,
-            vec![1, 2, 3]
-        );
-        assert_eq!(
-            roundtrip_details.versions[1].groups[0].new_frags,
-            vec![
-                FragDigest {
-                    id: 2,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                },
-                FragDigest {
-                    id: 3,
-                    physical_rows: 1,
-                    num_deleted_rows: 0,
-                }
-            ]
-        );
-        assert_eq!(
-            roundtrip_details.versions[1].groups[0].old_frags,
-            vec![FragDigest {
-                id: 1,
-                physical_rows: 1,
-                num_deleted_rows: 0,
-            }]
-        );
+    fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch> {
+        self.0.remap_row_ids_record_batch(batch, row_id_idx)
     }
 }
diff --git a/rust/lance-index/src/lib.rs b/rust/lance-index/src/lib.rs
index 0ed6ddd4e2d..cf60ece2fcf 100644
--- a/rust/lance-index/src/lib.rs
+++ b/rust/lance-index/src/lib.rs
@@ -9,16 +9,9 @@
 //! API stability is not guaranteed.
 //! </section>
 
-use std::{any::Any, sync::Arc};
-
 use crate::frag_reuse::FRAG_REUSE_INDEX_NAME;
 use crate::mem_wal::MEM_WAL_INDEX_NAME;
-use async_trait::async_trait;
-use deepsize::DeepSizeOf;
-use lance_core::{Error, Result};
-use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
-use std::convert::TryFrom;
 
 pub mod frag_reuse;
 pub mod mem_wal;
@@ -33,6 +26,9 @@ pub mod vector;
 
 pub use crate::traits::*;
 
+// Re-export core traits from lance-index-core
+pub use lance_index_core::{Index, IndexParams, IndexType};
+
 pub const INDEX_FILE_NAME: &str = "index.idx";
 /// The name of the auxiliary index file.
 ///
@@ -68,273 +64,11 @@ pub mod pbold {
     include!(concat!(env!("OUT_DIR"), "/lance.table.rs"));
 }
 
-/// Generic methods common across all types of secondary indices
-///
-#[async_trait]
-pub trait Index: Send + Sync + DeepSizeOf {
-    /// Cast to [Any].
-    fn as_any(&self) -> &dyn Any;
-
-    /// Cast to [Index]
-    fn as_index(self: Arc<Self>) -> Arc<dyn Index>;
-
-    /// Cast to [vector::VectorIndex]
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn vector::VectorIndex>>;
-
-    /// Retrieve index statistics as a JSON Value
-    fn statistics(&self) -> Result<serde_json::Value>;
-
-    /// Prewarm the index.
-    ///
-    /// This will load the index into memory and cache it.
-    async fn prewarm(&self) -> Result<()>;
-
-    /// Get the type of the index
-    fn index_type(&self) -> IndexType;
-
-    /// Read through the index and determine which fragment ids are covered by the index
-    ///
-    /// This is a kind of slow operation.  It's better to use the fragment_bitmap.  This
-    /// only exists for cases where the fragment_bitmap has become corrupted or missing.
-    async fn calculate_included_frags(&self) -> Result<RoaringBitmap>;
-}
-
-/// Index Type
-#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf)]
-pub enum IndexType {
-    // Preserve 0-100 for simple indices.
-    Scalar = 0, // Legacy scalar index, alias to BTree
-
-    BTree = 1, // BTree
-
-    Bitmap = 2, // Bitmap
-
-    LabelList = 3, // LabelList
-
-    Inverted = 4, // Inverted
-
-    NGram = 5, // NGram
-
-    FragmentReuse = 6,
-
-    MemWal = 7,
-
-    ZoneMap = 8, // ZoneMap
-
-    BloomFilter = 9, // Bloom filter
-
-    RTree = 10, // RTree
-
-    // 100+ and up for vector index.
-    /// Flat vector index.
-    Vector = 100, // Legacy vector index, alias to IvfPq
-    IvfFlat = 101,
-    IvfSq = 102,
-    IvfPq = 103,
-    IvfHnswSq = 104,
-    IvfHnswPq = 105,
-    IvfHnswFlat = 106,
-    IvfRq = 107,
-}
-
-impl std::fmt::Display for IndexType {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        match self {
-            Self::Scalar | Self::BTree => write!(f, "BTree"),
-            Self::Bitmap => write!(f, "Bitmap"),
-            Self::LabelList => write!(f, "LabelList"),
-            Self::Inverted => write!(f, "Inverted"),
-            Self::NGram => write!(f, "NGram"),
-            Self::FragmentReuse => write!(f, "FragmentReuse"),
-            Self::MemWal => write!(f, "MemWal"),
-            Self::ZoneMap => write!(f, "ZoneMap"),
-            Self::BloomFilter => write!(f, "BloomFilter"),
-            Self::RTree => write!(f, "RTree"),
-            Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"),
-            Self::IvfFlat => write!(f, "IVF_FLAT"),
-            Self::IvfSq => write!(f, "IVF_SQ"),
-            Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"),
-            Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"),
-            Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"),
-            Self::IvfRq => write!(f, "IVF_RQ"),
-        }
-    }
-}
-
-impl TryFrom<i32> for IndexType {
-    type Error = Error;
-
-    fn try_from(value: i32) -> Result<Self> {
-        match value {
-            v if v == Self::Scalar as i32 => Ok(Self::Scalar),
-            v if v == Self::BTree as i32 => Ok(Self::BTree),
-            v if v == Self::Bitmap as i32 => Ok(Self::Bitmap),
-            v if v == Self::LabelList as i32 => Ok(Self::LabelList),
-            v if v == Self::NGram as i32 => Ok(Self::NGram),
-            v if v == Self::Inverted as i32 => Ok(Self::Inverted),
-            v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse),
-            v if v == Self::MemWal as i32 => Ok(Self::MemWal),
-            v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap),
-            v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter),
-            v if v == Self::RTree as i32 => Ok(Self::RTree),
-            v if v == Self::Vector as i32 => Ok(Self::Vector),
-            v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat),
-            v if v == Self::IvfSq as i32 => Ok(Self::IvfSq),
-            v if v == Self::IvfPq as i32 => Ok(Self::IvfPq),
-            v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq),
-            v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq),
-            v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat),
-            v if v == Self::IvfRq as i32 => Ok(Self::IvfRq),
-            _ => Err(Error::invalid_input_source(
-                format!("the input value {} is not a valid IndexType", value).into(),
-            )),
-        }
-    }
-}
-
-impl TryFrom<&str> for IndexType {
-    type Error = Error;
-
-    fn try_from(value: &str) -> Result<Self> {
-        match value {
-            "BTree" | "BTREE" => Ok(Self::BTree),
-            "Bitmap" | "BITMAP" => Ok(Self::Bitmap),
-            "LabelList" | "LABELLIST" => Ok(Self::LabelList),
-            "Inverted" | "INVERTED" => Ok(Self::Inverted),
-            "NGram" | "NGRAM" => Ok(Self::NGram),
-            "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap),
-            "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter),
-            "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree),
-            "Vector" | "VECTOR" => Ok(Self::Vector),
-            "IVF_FLAT" => Ok(Self::IvfFlat),
-            "IVF_SQ" => Ok(Self::IvfSq),
-            "IVF_PQ" => Ok(Self::IvfPq),
-            "IVF_RQ" => Ok(Self::IvfRq),
-            "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat),
-            "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq),
-            "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq),
-            "FragmentReuse" => Ok(Self::FragmentReuse),
-            "MemWal" => Ok(Self::MemWal),
-            _ => Err(Error::invalid_input(format!(
-                "invalid index type: {}",
-                value
-            ))),
-        }
-    }
-}
-
-impl IndexType {
-    pub fn is_scalar(&self) -> bool {
-        matches!(
-            self,
-            Self::Scalar
-                | Self::BTree
-                | Self::Bitmap
-                | Self::LabelList
-                | Self::Inverted
-                | Self::NGram
-                | Self::ZoneMap
-                | Self::BloomFilter
-                | Self::RTree,
-        )
-    }
-
-    pub fn is_vector(&self) -> bool {
-        matches!(
-            self,
-            Self::Vector
-                | Self::IvfPq
-                | Self::IvfHnswSq
-                | Self::IvfHnswPq
-                | Self::IvfHnswFlat
-                | Self::IvfFlat
-                | Self::IvfSq
-                | Self::IvfRq
-        )
-    }
-
-    pub fn is_system(&self) -> bool {
-        matches!(self, Self::FragmentReuse | Self::MemWal)
-    }
-
-    /// Returns the current format version of the index type,
-    /// bump this when the index format changes.
-    /// Indices which higher version than these will be ignored for compatibility,
-    /// This would happen when creating index in a newer version of Lance,
-    /// but then opening the index in older version of Lance
-    pub fn version(&self) -> i32 {
-        match self {
-            Self::Scalar => 0,
-            Self::BTree => 0,
-            Self::Bitmap => 0,
-            Self::LabelList => 0,
-            Self::Inverted => 0,
-            Self::NGram => 0,
-            Self::FragmentReuse => 0,
-            Self::MemWal => 0,
-            Self::ZoneMap => 0,
-            Self::BloomFilter => 0,
-            Self::RTree => 0,
-
-            // IMPORTANT: if any vector index subtype needs a format bump that is
-            // not backward compatible, its new version must be set to
-            // (current max vector index version + 1), even if only one subtype
-            // changed. Compatibility filtering currently cannot distinguish vector
-            // subtypes from details-only metadata, so vector versions effectively
-            // share one global monotonic compatibility level.
-            Self::Vector
-            | Self::IvfFlat
-            | Self::IvfSq
-            | Self::IvfPq
-            | Self::IvfHnswSq
-            | Self::IvfHnswPq
-            | Self::IvfHnswFlat => VECTOR_INDEX_VERSION as i32,
-            Self::IvfRq => IVF_RQ_INDEX_VERSION as i32,
-        }
-    }
-
-    /// Returns the target partition size for the index type.
-    ///
-    /// This is used to compute the number of partitions for the index.
-    /// The partition size is optimized for the best performance of the index.
-    ///
-    /// This is for vector indices only.
-    pub fn target_partition_size(&self) -> usize {
-        match self {
-            Self::Vector => 8192,
-            Self::IvfFlat => 4096,
-            Self::IvfSq => 8192,
-            Self::IvfPq => 8192,
-            Self::IvfHnswFlat => 1 << 20,
-            Self::IvfHnswSq => 1 << 20,
-            Self::IvfHnswPq => 1 << 20,
-            _ => 8192,
-        }
-    }
-
-    /// Returns the highest supported vector index version in this Lance build.
-    pub fn max_vector_version() -> u32 {
-        [
-            Self::Vector,
-            Self::IvfFlat,
-            Self::IvfSq,
-            Self::IvfPq,
-            Self::IvfHnswSq,
-            Self::IvfHnswPq,
-            Self::IvfHnswFlat,
-            Self::IvfRq,
-        ]
-        .into_iter()
-        .map(|index_type| index_type.version() as u32)
-        .max()
-        .unwrap_or(VECTOR_INDEX_VERSION)
-    }
-}
-
-pub trait IndexParams: Send + Sync {
-    fn as_any(&self) -> &dyn Any;
-
-    fn index_name(&self) -> &str;
+/// Protobuf headers for serialized index cache entries (FTS posting lists,
+/// scalar indices, and IVF vector partitions).
+pub mod cache_pb {
+    #![allow(clippy::use_self)]
+    include!(concat!(env!("OUT_DIR"), "/lance.index.cache.rs"));
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -375,6 +109,11 @@ mod tests {
         assert_eq!(IndexType::max_vector_version(), IVF_RQ_INDEX_VERSION);
     }
 
+    #[test]
+    fn test_ivf_rq_target_partition_size() {
+        assert_eq!(IndexType::IvfRq.target_partition_size(), 4096);
+    }
+
     #[test]
     fn test_index_type_try_from_i32_covers_all_variants() {
         let all = [
@@ -389,6 +128,7 @@ mod tests {
             IndexType::ZoneMap,
             IndexType::BloomFilter,
             IndexType::RTree,
+            IndexType::Fm,
             IndexType::Vector,
             IndexType::IvfFlat,
             IndexType::IvfSq,
@@ -430,6 +170,8 @@ mod tests {
             ("RTree", IndexType::RTree),
             ("RTREE", IndexType::RTree),
             ("R_TREE", IndexType::RTree),
+            ("Fm", IndexType::Fm),
+            ("FM", IndexType::Fm),
             ("Vector", IndexType::Vector),
             ("VECTOR", IndexType::Vector),
             ("IVF_FLAT", IndexType::IvfFlat),
diff --git a/rust/lance-index/src/mem_wal.rs b/rust/lance-index/src/mem_wal.rs
index 4310db88908..b169d8c6994 100644
--- a/rust/lance-index/src/mem_wal.rs
+++ b/rust/lance-index/src/mem_wal.rs
@@ -1,406 +1,33 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+//! `Index`-trait adapter for the MemWAL system index.
+//!
+//! The data structures and table-format logic live in
+//! [`lance_table::system_index::mem_wal`]; this module re-exports them and
+//! provides a newtype wrapper that implements the [`Index`] trait.
+
 use std::any::Any;
-use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use deepsize::DeepSizeOf;
-use lance_core::Error;
-use lance_table::format::pb;
+use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use roaring::RoaringBitmap;
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use crate::{Index, IndexType};
-
-pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal";
-
-/// Type alias for shard identifier (UUID v4).
-pub type ShardId = Uuid;
-
-/// A flushed MemTable generation and its storage location.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct FlushedGeneration {
-    pub generation: u64,
-    pub path: String,
-}
-
-impl From<&FlushedGeneration> for pb::FlushedGeneration {
-    fn from(fg: &FlushedGeneration) -> Self {
-        Self {
-            generation: fg.generation,
-            path: fg.path.clone(),
-        }
-    }
-}
-
-impl From<pb::FlushedGeneration> for FlushedGeneration {
-    fn from(fg: pb::FlushedGeneration) -> Self {
-        Self {
-            generation: fg.generation,
-            path: fg.path,
-        }
-    }
-}
-
-/// A shard's merged generation, used in MemWalIndexDetails.
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
-pub struct MergedGeneration {
-    pub shard_id: Uuid,
-    pub generation: u64,
-}
-
-impl DeepSizeOf for MergedGeneration {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
-        0 // UUID is 16 bytes fixed size, no heap allocations
-    }
-}
-
-impl MergedGeneration {
-    pub fn new(shard_id: Uuid, generation: u64) -> Self {
-        Self {
-            shard_id,
-            generation,
-        }
-    }
-}
-
-impl From<&MergedGeneration> for pb::MergedGeneration {
-    fn from(mg: &MergedGeneration) -> Self {
-        Self {
-            shard_id: Some((&mg.shard_id).into()),
-            generation: mg.generation,
-        }
-    }
-}
-
-impl TryFrom<pb::MergedGeneration> for MergedGeneration {
-    type Error = Error;
-
-    fn try_from(mg: pb::MergedGeneration) -> lance_core::Result<Self> {
-        let shard_id = mg
-            .shard_id
-            .as_ref()
-            .map(Uuid::try_from)
-            .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??;
-        Ok(Self {
-            shard_id,
-            generation: mg.generation,
-        })
-    }
-}
-
-/// Tracks which merged generation a base table index has been rebuilt to cover.
-/// Used to determine whether to read from flushed MemTable indexes or base table.
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct IndexCatchupProgress {
-    pub index_name: String,
-    pub caught_up_generations: Vec<MergedGeneration>,
-}
-
-impl IndexCatchupProgress {
-    pub fn new(index_name: String, caught_up_generations: Vec<MergedGeneration>) -> Self {
-        Self {
-            index_name,
-            caught_up_generations,
-        }
-    }
-
-    /// Get the caught up generation for a specific shard.
-    /// Returns None if the shard is not present (assumed fully caught up).
-    pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
-        self.caught_up_generations
-            .iter()
-            .find(|mg| &mg.shard_id == shard_id)
-            .map(|mg| mg.generation)
-    }
-}
-
-impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress {
-    fn from(icp: &IndexCatchupProgress) -> Self {
-        Self {
-            index_name: icp.index_name.clone(),
-            caught_up_generations: icp
-                .caught_up_generations
-                .iter()
-                .map(|mg| mg.into())
-                .collect(),
-        }
-    }
-}
-
-impl TryFrom<pb::IndexCatchupProgress> for IndexCatchupProgress {
-    type Error = Error;
-
-    fn try_from(icp: pb::IndexCatchupProgress) -> lance_core::Result<Self> {
-        Ok(Self {
-            index_name: icp.index_name,
-            caught_up_generations: icp
-                .caught_up_generations
-                .into_iter()
-                .map(MergedGeneration::try_from)
-                .collect::<lance_core::Result<_>>()?,
-        })
-    }
-}
-
-/// Shard manifest containing epoch-based fencing and WAL state.
-/// Each shard has exactly one active writer at any time.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub struct ShardManifest {
-    pub shard_id: Uuid,
-    pub version: u64,
-    pub shard_spec_id: u32,
-    /// Computed shard field values as raw Arrow scalar bytes, keyed by field id.
-    /// The byte encoding follows Arrow's little-endian convention: int32 is 4 LE
-    /// bytes, utf8 is raw UTF-8 bytes, etc. The result_type in the corresponding
-    /// ShardingField from the ShardingSpec determines how to interpret each value.
-    pub shard_field_values: HashMap<String, Vec<u8>>,
-    pub writer_epoch: u64,
-    /// The most recent WAL entry position flushed to a MemTable.
-    /// Recovery replays from `replay_after_wal_entry_position + 1`. The
-    /// default value 0 means "no flush has ever stamped this shard" — WAL
-    /// positions themselves are 1-based, so 0 is never a valid covered
-    /// position.
-    pub replay_after_wal_entry_position: u64,
-    /// The most recent WAL entry position observed at manifest write time.
-    /// Default 0 means "no entry has been written yet"; WAL positions are
-    /// 1-based.
-    pub wal_entry_position_last_seen: u64,
-    pub current_generation: u64,
-    pub flushed_generations: Vec<FlushedGeneration>,
-}
+use serde::Serialize;
 
-impl DeepSizeOf for ShardManifest {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.shard_field_values.deep_size_of_children(context)
-            + self.flushed_generations.deep_size_of_children(context)
-    }
-}
-
-impl From<&ShardManifest> for pb::ShardManifest {
-    fn from(rm: &ShardManifest) -> Self {
-        Self {
-            shard_id: Some((&rm.shard_id).into()),
-            version: rm.version,
-            shard_spec_id: rm.shard_spec_id,
-            shard_field_entries: rm
-                .shard_field_values
-                .iter()
-                .map(|(k, v)| pb::ShardFieldEntry {
-                    field_id: k.clone(),
-                    value: v.clone(),
-                })
-                .collect(),
-            writer_epoch: rm.writer_epoch,
-            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
-            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
-            current_generation: rm.current_generation,
-            flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(),
-        }
-    }
-}
-
-impl TryFrom<pb::ShardManifest> for ShardManifest {
-    type Error = Error;
-
-    fn try_from(rm: pb::ShardManifest) -> lance_core::Result<Self> {
-        let shard_id = rm
-            .shard_id
-            .as_ref()
-            .map(Uuid::try_from)
-            .ok_or_else(|| Error::invalid_input("Missing shard_id in ShardManifest"))??;
-        let shard_field_values = rm
-            .shard_field_entries
-            .into_iter()
-            .map(|e| (e.field_id, e.value))
-            .collect();
-        Ok(Self {
-            shard_id,
-            version: rm.version,
-            shard_spec_id: rm.shard_spec_id,
-            shard_field_values,
-            writer_epoch: rm.writer_epoch,
-            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
-            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
-            current_generation: rm.current_generation,
-            flushed_generations: rm
-                .flushed_generations
-                .into_iter()
-                .map(FlushedGeneration::from)
-                .collect(),
-        })
-    }
-}
-
-/// Sharding field definition.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct ShardingField {
-    pub field_id: String,
-    pub source_ids: Vec<i32>,
-    pub transform: Option<String>,
-    pub expression: Option<String>,
-    pub result_type: String,
-    pub parameters: HashMap<String, String>,
-}
-
-impl From<&ShardingField> for pb::ShardingField {
-    fn from(rf: &ShardingField) -> Self {
-        Self {
-            field_id: rf.field_id.clone(),
-            source_ids: rf.source_ids.clone(),
-            transform: rf.transform.clone(),
-            expression: rf.expression.clone(),
-            result_type: rf.result_type.clone(),
-            parameters: rf.parameters.clone(),
-        }
-    }
-}
-
-impl From<pb::ShardingField> for ShardingField {
-    fn from(rf: pb::ShardingField) -> Self {
-        Self {
-            field_id: rf.field_id,
-            source_ids: rf.source_ids,
-            transform: rf.transform,
-            expression: rf.expression,
-            result_type: rf.result_type,
-            parameters: rf.parameters,
-        }
-    }
-}
-
-/// Sharding spec definition.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct ShardingSpec {
-    pub spec_id: u32,
-    pub fields: Vec<ShardingField>,
-}
-
-impl From<&ShardingSpec> for pb::ShardingSpec {
-    fn from(rs: &ShardingSpec) -> Self {
-        Self {
-            spec_id: rs.spec_id,
-            fields: rs.fields.iter().map(|f| f.into()).collect(),
-        }
-    }
-}
+pub use lance_table::system_index::mem_wal::*;
 
-impl From<pb::ShardingSpec> for ShardingSpec {
-    fn from(rs: pb::ShardingSpec) -> Self {
-        Self {
-            spec_id: rs.spec_id,
-            fields: rs.fields.into_iter().map(ShardingField::from).collect(),
-        }
-    }
-}
-
-/// Index details for MemWAL Index, stored in IndexMetadata.index_details.
-#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
-pub struct MemWalIndexDetails {
-    pub snapshot_ts_millis: i64,
-    pub num_shards: u32,
-    pub inline_snapshots: Option<Vec<u8>>,
-    pub sharding_specs: Vec<ShardingSpec>,
-    pub maintained_indexes: Vec<String>,
-    pub merged_generations: Vec<MergedGeneration>,
-    pub index_catchup: Vec<IndexCatchupProgress>,
-    /// Default `ShardWriter` configuration values for this MemWAL index.
-    ///
-    /// Persisted so every writer — across processes and restarts — starts
-    /// from the same default writer configuration. These are defaults only;
-    /// an individual writer may still override any value at runtime in its
-    /// own (non-persisted) `ShardWriterConfig`.
-    pub writer_config_defaults: HashMap<String, String>,
-}
-
-impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails {
-    fn from(details: &MemWalIndexDetails) -> Self {
-        Self {
-            snapshot_ts_millis: details.snapshot_ts_millis,
-            num_shards: details.num_shards,
-            inline_snapshots: details.inline_snapshots.clone(),
-            sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(),
-            maintained_indexes: details.maintained_indexes.clone(),
-            merged_generations: details
-                .merged_generations
-                .iter()
-                .map(|mg| mg.into())
-                .collect(),
-            index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(),
-            writer_config_defaults: details.writer_config_defaults.clone(),
-        }
-    }
-}
-
-impl TryFrom<pb::MemWalIndexDetails> for MemWalIndexDetails {
-    type Error = Error;
-
-    fn try_from(details: pb::MemWalIndexDetails) -> lance_core::Result<Self> {
-        Ok(Self {
-            snapshot_ts_millis: details.snapshot_ts_millis,
-            num_shards: details.num_shards,
-            inline_snapshots: details.inline_snapshots,
-            sharding_specs: details
-                .sharding_specs
-                .into_iter()
-                .map(ShardingSpec::from)
-                .collect(),
-            maintained_indexes: details.maintained_indexes,
-            merged_generations: details
-                .merged_generations
-                .into_iter()
-                .map(MergedGeneration::try_from)
-                .collect::<lance_core::Result<_>>()?,
-            index_catchup: details
-                .index_catchup
-                .into_iter()
-                .map(IndexCatchupProgress::try_from)
-                .collect::<lance_core::Result<_>>()?,
-            writer_config_defaults: details.writer_config_defaults,
-        })
-    }
-}
-
-/// MemWAL Index provides access to MemWAL configuration and state.
-#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
-pub struct MemWalIndex {
-    pub details: MemWalIndexDetails,
-}
-
-impl MemWalIndex {
-    pub fn new(details: MemWalIndexDetails) -> Self {
-        Self { details }
-    }
-
-    pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
-        self.details
-            .merged_generations
-            .iter()
-            .find(|mg| &mg.shard_id == shard_id)
-            .map(|mg| mg.generation)
-    }
-
-    /// Get the caught up generation for a specific index and shard.
-    /// Returns None if the index is not tracked (assumed fully caught up).
-    pub fn index_caught_up_generation(&self, index_name: &str, shard_id: &Uuid) -> Option<u64> {
-        self.details
-            .index_catchup
-            .iter()
-            .find(|icp| icp.index_name == index_name)
-            .and_then(|icp| icp.caught_up_generation_for_shard(shard_id))
-    }
+use crate::{Index, IndexType};
 
-    /// Check if an index is fully caught up for a shard.
-    /// Returns true if the index covers all merged data for the shard.
-    pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool {
-        let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0);
-        let caught_up_gen = self.index_caught_up_generation(index_name, shard_id);
+/// Newtype wrapping [`MemWalIndex`] so that `lance-index` can implement
+/// the `Index` trait (orphan rules prevent implementing it directly in
+/// `lance-table`).
+pub struct MemWalIndexHandle(pub Arc<MemWalIndex>);
 
-        // If not tracked in index_catchup, assumed fully caught up
-        caught_up_gen.is_none_or(|generation| generation >= merged_gen)
+impl DeepSizeOf for MemWalIndexHandle {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
     }
 }
 
@@ -414,7 +41,7 @@ struct MemWalStatistics {
 }
 
 #[async_trait]
-impl Index for MemWalIndex {
+impl Index for MemWalIndexHandle {
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -423,29 +50,23 @@ impl Index for MemWalIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> lance_core::Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::not_supported_source(
-            "MemWalIndex is not a vector index".into(),
-        ))
-    }
-
-    fn statistics(&self) -> lance_core::Result<serde_json::Value> {
+    fn statistics(&self) -> Result<serde_json::Value> {
         let stats = MemWalStatistics {
-            num_shards: self.details.num_shards,
-            num_merged_generations: self.details.merged_generations.len(),
-            num_shard_specs: self.details.sharding_specs.len(),
-            num_maintained_indexes: self.details.maintained_indexes.len(),
-            num_index_catchup_entries: self.details.index_catchup.len(),
+            num_shards: self.0.details.num_shards,
+            num_merged_generations: self.0.details.merged_generations.len(),
+            num_shard_specs: self.0.details.sharding_specs.len(),
+            num_maintained_indexes: self.0.details.maintained_indexes.len(),
+            num_index_catchup_entries: self.0.details.index_catchup.len(),
         };
         serde_json::to_value(stats).map_err(|e| {
-            Error::internal(format!(
+            lance_core::Error::internal(format!(
                 "failed to serialize MemWAL index statistics: {}",
                 e
             ))
         })
     }
 
-    async fn prewarm(&self) -> lance_core::Result<()> {
+    async fn prewarm(&self) -> Result<()> {
         Ok(())
     }
 
@@ -453,7 +74,7 @@ impl Index for MemWalIndex {
         IndexType::MemWal
     }
 
-    async fn calculate_included_frags(&self) -> lance_core::Result<RoaringBitmap> {
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap> {
         Ok(RoaringBitmap::new())
     }
 }
diff --git a/rust/lance-index/src/metrics.rs b/rust/lance-index/src/metrics.rs
index 9e2161ae8f9..7d1ea2964c6 100644
--- a/rust/lance-index/src/metrics.rs
+++ b/rust/lance-index/src/metrics.rs
@@ -1,85 +1,4 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-/// A trait used by the index to report metrics
-///
-/// Callers can implement this trait to collect metrics
-pub trait MetricsCollector: Send + Sync {
-    /// Record partition loads
-    ///
-    /// Many indices consist of partitions that may need to be loaded
-    /// into cache.  For example, an inverted index or ngram index has a
-    /// posting list for each token.
-    ///
-    /// In the ideal case, these shards are in the cache and will not need
-    /// to be loaded from disk.  This method should not be called if the
-    /// shard is in the cache.
-    fn record_parts_loaded(&self, num_parts: usize);
-
-    /// Record a shard load
-    fn record_part_load(&self) {
-        self.record_parts_loaded(1);
-    }
-
-    /// Record an index load
-    ///
-    /// This should be called when a scalar index is loaded from storage.
-    /// It should not be called if the index is already in memory.
-    fn record_index_loads(&self, num_indexes: usize);
-
-    /// Record an index load
-    fn record_index_load(&self) {
-        self.record_index_loads(1);
-    }
-
-    /// Record the number of "comparisons" made by the index
-    ///
-    /// What exactly constitutes a comparison depends on the index type.
-    /// For example, a B-tree index may make comparisons while searching for a value.
-    /// On the other hand, a bitmap index makes comparisons when computing the intersection
-    /// of two bitmaps.
-    ///
-    /// The goal is to provide some visibility into the compute cost of the search
-    fn record_comparisons(&self, num_comparisons: usize);
-}
-
-/// A no-op metrics collector that does nothing
-pub struct NoOpMetricsCollector;
-
-impl MetricsCollector for NoOpMetricsCollector {
-    fn record_parts_loaded(&self, _num_parts: usize) {}
-    fn record_index_loads(&self, _num_indexes: usize) {}
-    fn record_comparisons(&self, _num_comparisons: usize) {}
-}
-
-#[derive(Default)]
-pub struct LocalMetricsCollector {
-    pub parts_loaded: AtomicUsize,
-    pub index_loads: AtomicUsize,
-    pub comparisons: AtomicUsize,
-}
-
-impl LocalMetricsCollector {
-    pub fn dump_into(self, other: &dyn MetricsCollector) {
-        other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed));
-        other.record_index_loads(self.index_loads.load(Ordering::Relaxed));
-        other.record_comparisons(self.comparisons.load(Ordering::Relaxed));
-    }
-}
-
-impl MetricsCollector for LocalMetricsCollector {
-    fn record_parts_loaded(&self, num_parts: usize) {
-        self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed);
-    }
-
-    fn record_index_loads(&self, num_indexes: usize) {
-        self.index_loads.fetch_add(num_indexes, Ordering::Relaxed);
-    }
-
-    fn record_comparisons(&self, num_comparisons: usize) {
-        self.comparisons
-            .fetch_add(num_comparisons, Ordering::Relaxed);
-    }
-}
+pub use lance_index_core::metrics::*;
diff --git a/rust/lance-index/src/registry.rs b/rust/lance-index/src/registry.rs
index 8ab65d38896..bd7448240fe 100644
--- a/rust/lance-index/src/registry.rs
+++ b/rust/lance-index/src/registry.rs
@@ -10,11 +10,27 @@ use crate::{
     pb, pbold,
     scalar::{
         bitmap::BitmapIndexPlugin, bloomfilter::BloomFilterIndexPlugin, btree::BTreeIndexPlugin,
-        inverted::InvertedIndexPlugin, json::JsonIndexPlugin, label_list::LabelListIndexPlugin,
-        ngram::NGramIndexPlugin, registry::ScalarIndexPlugin, zonemap::ZoneMapIndexPlugin,
+        fmindex::FMIndexPlugin, inverted::InvertedIndexPlugin, json::JsonIndexPlugin,
+        label_list::LabelListIndexPlugin, ngram::NGramIndexPlugin, registry::ScalarIndexPlugin,
+        zonemap::ZoneMapIndexPlugin,
     },
 };
 
+/// Derive a human-readable index type name from a details type URL.
+///
+/// The display name is the final `.`-separated segment of the type URL with any
+/// trailing `IndexDetails` removed. For example, `/lance.index.pb.VectorIndexDetails`
+/// yields `Vector`. Used as a best-effort fallback when no plugin is registered
+/// for the type URL, so the index type is never reported as opaque "Unknown"
+/// while valid index details exist.
+pub fn display_type_from_url(type_url: &str) -> &str {
+    let segment = type_url.rsplit('.').next().unwrap_or(type_url);
+    segment
+        .strip_suffix("IndexDetails")
+        .filter(|stripped| !stripped.is_empty())
+        .unwrap_or(segment)
+}
+
 /// A registry of index plugins
 pub struct IndexPluginRegistry {
     plugins: HashMap<String, Box<dyn ScalarIndexPlugin>>,
@@ -66,6 +82,7 @@ impl IndexPluginRegistry {
         registry.add_plugin::<pb::BloomFilterIndexDetails, BloomFilterIndexPlugin>();
         registry.add_plugin::<pbold::InvertedIndexDetails, InvertedIndexPlugin>();
         registry.add_plugin::<pb::JsonIndexDetails, JsonIndexPlugin>();
+        registry.add_plugin::<pb::FmIndexDetails, FMIndexPlugin>();
         #[cfg(feature = "geo")]
         registry.add_plugin::<pb::RTreeIndexDetails, RTreeIndexPlugin>();
 
@@ -110,6 +127,24 @@ impl IndexPluginRegistry {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_display_type_from_url() {
+        assert_eq!(
+            display_type_from_url("/lance.index.pb.VectorIndexDetails"),
+            "Vector"
+        );
+        assert_eq!(display_type_from_url("BTreeIndexDetails"), "BTree");
+        // Segment without the IndexDetails suffix is returned verbatim.
+        assert_eq!(
+            display_type_from_url("/lance.pb.SomethingElse"),
+            "SomethingElse"
+        );
+        // A bare "IndexDetails" segment has nothing left after stripping, so it
+        // is returned as-is rather than an empty string.
+        assert_eq!(display_type_from_url("IndexDetails"), "IndexDetails");
+        assert_eq!(display_type_from_url(""), "");
+    }
+
     #[test]
     fn test_get_plugin_by_name_accepts_case_insensitive_builtin_names() {
         let registry = IndexPluginRegistry::with_default_plugins();
@@ -121,6 +156,7 @@ mod tests {
             ("NGRAM", "NGram"),
             ("ZONEMAP", "ZoneMap"),
             ("BLOOMFILTER", "BloomFilter"),
+            ("FM", "Fm"),
             ("JSON", "Json"),
         ] {
             let plugin = registry.get_plugin_by_name(requested_name).unwrap();
diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs
index 5ab138ff481..33e4082e114 100644
--- a/rust/lance-index/src/scalar.rs
+++ b/rust/lance-index/src/scalar.rs
@@ -4,37 +4,37 @@
 //! Scalar indices for metadata search & filtering
 
 use arrow::buffer::{OffsetBuffer, ScalarBuffer};
-use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array};
-use arrow_schema::{Field, Schema};
-use async_trait::async_trait;
-use bytes::Bytes;
+use arrow_array::ListArray;
+use arrow_schema::Field;
+use datafusion::functions::regex::regexplike::RegexpLikeFunc;
 use datafusion::functions::string::contains::ContainsFunc;
 use datafusion::functions_nested::array_has;
-use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion_common::{Column, scalar::ScalarValue};
-use std::collections::{HashMap, HashSet};
-use std::fmt::Debug;
-use std::pin::Pin;
+use std::collections::HashSet;
 use std::{any::Any, ops::Bound, sync::Arc};
 
-use datafusion_expr::{Expr, expr::ScalarFunction};
-use deepsize::DeepSizeOf;
+use datafusion_expr::{
+    Expr,
+    expr::{Like, ScalarFunction},
+};
 use inverted::query::{FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, fill_fts_query_column};
-use lance_core::{Error, Result};
-use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter};
-use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
-use roaring::RoaringBitmap;
-use serde::Serialize;
+use lance_core::Result;
 
-use crate::metrics::MetricsCollector;
-use crate::scalar::registry::TrainingCriteria;
-use crate::{Index, IndexParams, IndexType};
-pub use lance_table::format::IndexFile;
+use lance_datafusion::udf::CONTAINS_TOKENS_UDF;
+
+use crate::IndexParams;
+pub use crate::metrics::MetricsCollector;
+pub use lance_index_core::scalar::{
+    AnyQuery, BuiltinIndexType, CreatedIndex, IndexFile, IndexReader, IndexStore, IndexWriter,
+    LANCE_SCALAR_INDEX, OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams,
+    SearchResult, TrainingCriteria, TrainingOrdering, UpdateCriteria,
+};
 
 pub mod bitmap;
 pub mod bloomfilter;
 pub mod btree;
 pub mod expression;
+pub mod fmindex;
 pub mod inverted;
 pub mod json;
 pub mod label_list;
@@ -46,116 +46,40 @@ pub mod rtree;
 pub mod zoned;
 pub mod zonemap;
 
-use crate::frag_reuse::FragReuseIndex;
 pub use inverted::tokenizer::InvertedIndexParams;
-use lance_datafusion::udf::CONTAINS_TOKENS_UDF;
 
-pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index";
-
-/// Builtin index types supported by the Lance library
+/// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a
+/// `Vec<`[`lance_table::format::IndexFile`]`>`.
 ///
-/// This is primarily for convenience to avoid a bunch of string
-/// constants and provide some auto-complete.  This type should not
-/// be used in the manifest as plugins cannot add new entries.
-#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
-pub enum BuiltinIndexType {
-    BTree,
-    Bitmap,
-    LabelList,
-    NGram,
-    ZoneMap,
-    BloomFilter,
-    RTree,
-    Inverted,
-}
-
-impl BuiltinIndexType {
-    pub fn as_str(&self) -> &str {
-        match self {
-            Self::BTree => "btree",
-            Self::Bitmap => "bitmap",
-            Self::LabelList => "labellist",
-            Self::NGram => "ngram",
-            Self::ZoneMap => "zonemap",
-            Self::Inverted => "inverted",
-            Self::BloomFilter => "bloomfilter",
-            Self::RTree => "rtree",
-        }
-    }
-}
-
-impl TryFrom<IndexType> for BuiltinIndexType {
-    type Error = Error;
-
-    fn try_from(value: IndexType) -> Result<Self> {
-        match value {
-            IndexType::BTree => Ok(Self::BTree),
-            IndexType::Bitmap => Ok(Self::Bitmap),
-            IndexType::LabelList => Ok(Self::LabelList),
-            IndexType::NGram => Ok(Self::NGram),
-            IndexType::ZoneMap => Ok(Self::ZoneMap),
-            IndexType::Inverted => Ok(Self::Inverted),
-            IndexType::BloomFilter => Ok(Self::BloomFilter),
-            IndexType::RTree => Ok(Self::RTree),
-            _ => Err(Error::index("Invalid index type".to_string())),
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct ScalarIndexParams {
-    /// The type of index to create
-    ///
-    /// Plugins may add additional index types.  Index type lookup is case-insensitive.
-    pub index_type: String,
-    /// The parameters to train the index
-    ///
-    /// This should be a JSON string.  The contents of the JSON string will be specific to the
-    /// index type.  If not set, then default parameters will be used for the index type.
-    pub params: Option<String>,
-}
-
-impl Default for ScalarIndexParams {
-    fn default() -> Self {
-        Self {
-            index_type: BuiltinIndexType::BTree.as_str().to_string(),
-            params: None,
-        }
-    }
-}
-
-impl ScalarIndexParams {
-    /// Creates a new ScalarIndexParams from one of the builtin index types
-    pub fn for_builtin(index_type: BuiltinIndexType) -> Self {
-        Self {
-            index_type: index_type.as_str().to_string(),
-            params: None,
-        }
-    }
-
-    /// Create a new ScalarIndexParams with the given index type
-    pub fn new(index_type: String) -> Self {
-        Self {
-            index_type,
-            params: None,
-        }
-    }
-
-    /// Set the parameters for the index
-    pub fn with_params<ParamsType: Serialize>(mut self, params: &ParamsType) -> Self {
-        self.params = Some(serde_json::to_string(params).unwrap());
-        self
-    }
-}
-
-impl IndexParams for ScalarIndexParams {
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
-    fn index_name(&self) -> &str {
-        LANCE_SCALAR_INDEX
-    }
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn index_files_to_table(
+    files: Vec<lance_index_core::scalar::IndexFile>,
+) -> Vec<lance_table::format::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_table::format::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
+}
+
+/// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a
+/// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`.
+///
+/// These two structs have identical fields; this helper bridges the crate
+/// boundary without relying on orphan-rule–violating `From` impls.
+pub fn table_files_to_index(
+    files: Vec<lance_table::format::IndexFile>,
+) -> Vec<lance_index_core::scalar::IndexFile> {
+    files
+        .into_iter()
+        .map(|f| lance_index_core::scalar::IndexFile {
+            path: f.path,
+            size_bytes: f.size_bytes,
+        })
+        .collect()
 }
 
 impl IndexParams for InvertedIndexParams {
@@ -168,136 +92,6 @@ impl IndexParams for InvertedIndexParams {
     }
 }
 
-/// Trait for storing an index (or parts of an index) into storage
-#[async_trait]
-pub trait IndexWriter: Send {
-    /// Writes a record batch into the file, returning the 0-based index of the batch in the file
-    ///
-    /// E.g. if this is the third time this is called this method will return 2
-    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64>;
-    /// Adds a global buffer and returns its index.
-    async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
-        Err(Error::not_supported(
-            "global buffers are not supported by this index writer",
-        ))
-    }
-    /// Finishes writing the file and closes the file
-    async fn finish(&mut self) -> Result<()>;
-    /// Finishes writing the file and closes the file with additional metadata
-    async fn finish_with_metadata(&mut self, metadata: HashMap<String, String>) -> Result<()>;
-}
-
-/// Trait for reading an index (or parts of an index) from storage
-#[async_trait]
-pub trait IndexReader: Send + Sync {
-    /// Read the n-th record batch from the file
-    async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch>;
-    /// Reads a global buffer by index.
-    async fn read_global_buffer(&self, _index: u32) -> Result<Bytes> {
-        Err(Error::not_supported(
-            "global buffers are not supported by this index reader",
-        ))
-    }
-    /// Read the range of rows from the file.
-    /// If projection is Some, only return the columns in the projection,
-    /// nested columns like Some(&["x.y"]) are not supported.
-    /// If projection is None, return all columns.
-    async fn read_range(
-        &self,
-        range: std::ops::Range<usize>,
-        projection: Option<&[&str]>,
-    ) -> Result<RecordBatch>;
-    /// Read a range of rows as a stream of record batches.
-    ///
-    /// This allows the caller to process rows incrementally without loading the
-    /// entire range into memory at once.
-    ///
-    /// The default implementation falls back to [`Self::read_range`] and wraps
-    /// the result in a single-item stream.
-    async fn read_range_stream(
-        &self,
-        range: std::ops::Range<usize>,
-        projection: Option<&[&str]>,
-    ) -> Result<Pin<Box<dyn RecordBatchStream>>> {
-        let batch = self.read_range(range, projection).await?;
-        let schema = batch.schema();
-        Ok(Box::pin(RecordBatchStreamAdapter::new(
-            schema,
-            futures::stream::once(async move { Ok(batch) }),
-        )))
-    }
-    /// Return the number of batches in the file
-    async fn num_batches(&self, batch_size: u64) -> u32;
-    /// Return the number of rows in the file
-    fn num_rows(&self) -> usize;
-    /// Return the metadata of the file
-    fn schema(&self) -> &lance_core::datatypes::Schema;
-}
-
-/// Trait abstracting I/O away from index logic
-///
-/// Scalar indices are currently serialized as indexable arrow record batches stored in
-/// named "files".  The index store is responsible for serializing and deserializing
-/// these batches into file data (e.g. as .lance files or .parquet files, etc.)
-#[async_trait]
-pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf {
-    fn as_any(&self) -> &dyn Any;
-    fn clone_arc(&self) -> Arc<dyn IndexStore>;
-
-    /// Suggested I/O parallelism for the store
-    fn io_parallelism(&self) -> usize;
-
-    /// Create a new file and return a writer to store data in the file
-    async fn new_index_file(&self, name: &str, schema: Arc<Schema>)
-    -> Result<Box<dyn IndexWriter>>;
-
-    /// Open an existing file for retrieval
-    async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>>;
-
-    /// Copy a range of batches from an index file from this store to another
-    ///
-    /// This is often useful when remapping or updating
-    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<()>;
-
-    /// Rename an index file
-    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<()>;
-
-    /// Delete an index file (used in the tmp spill store to keep tmp size down)
-    async fn delete_index_file(&self, name: &str) -> Result<()>;
-
-    /// List all files in the index directory with their sizes.
-    ///
-    /// Returns a list of (relative_path, size_bytes) tuples.
-    /// Used to capture file metadata after index creation/modification.
-    async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>>;
-}
-
-/// Different scalar indices may support different kinds of queries
-///
-/// For example, a btree index can support a wide range of queries (e.g. x > 7)
-/// while an index based on FTS only supports queries like "x LIKE 'foo'"
-///
-/// This trait is used when we need an object that can represent any kind of query
-///
-/// Note: if you are implementing this trait for a query type then you probably also
-/// need to implement the [crate::scalar::expression::ScalarQueryParser] trait to
-/// create instances of your query at parse time.
-pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync {
-    /// Cast the query as Any to allow for downcasting
-    fn as_any(&self) -> &dyn Any;
-    /// Format the query as a string for display purposes
-    fn format(&self, col: &str) -> String;
-    /// Convert the query to a datafusion expression
-    fn to_expr(&self, col: String) -> Expr;
-    /// Compare this query to another query
-    fn dyn_eq(&self, other: &dyn AnyQuery) -> bool;
-}
-
-impl PartialEq for dyn AnyQuery {
-    fn eq(&self, other: &Self) -> bool {
-        self.dyn_eq(other)
-    }
-}
 /// A full text search query
 #[derive(Debug, Clone, PartialEq)]
 pub struct FullTextSearchQuery {
@@ -406,6 +200,19 @@ pub enum SargableQuery {
     LikePrefix(ScalarValue),
 }
 
+/// Escape the LIKE metacharacters (`\`, `%`, `_`) in a literal string so it can be
+/// embedded in a LIKE pattern and matched literally (paired with `ESCAPE '\'`).
+fn escape_like_pattern(s: &str) -> String {
+    let mut out = String::with_capacity(s.len() + 2);
+    for c in s.chars() {
+        if matches!(c, '\\' | '%' | '_') {
+            out.push('\\');
+        }
+        out.push(c);
+    }
+    out
+}
+
 impl AnyQuery for SargableQuery {
     fn as_any(&self) -> &dyn Any {
         self
@@ -507,16 +314,43 @@ impl AnyQuery for SargableQuery {
             )),
             Self::IsNull() => col_expr.is_null(),
             Self::Equals(value) => col_expr.eq(Expr::Literal(value.clone(), None)),
-            Self::LikePrefix(prefix) => {
-                let pattern = match prefix {
-                    ScalarValue::Utf8(Some(s)) => ScalarValue::Utf8(Some(format!("{}%", s))),
-                    ScalarValue::LargeUtf8(Some(s)) => {
-                        ScalarValue::LargeUtf8(Some(format!("{}%", s)))
+            Self::LikePrefix(prefix) => match prefix {
+                ScalarValue::Utf8(Some(s))
+                | ScalarValue::LargeUtf8(Some(s))
+                | ScalarValue::Utf8View(Some(s)) => {
+                    // The prefix is a literal string. If it contains LIKE metacharacters
+                    // (`_`, `%`, `\`) they must be escaped before appending the `%` wildcard;
+                    // otherwise an inexact recheck (e.g. zone maps) would treat them as
+                    // wildcards and over-match rows that do not start with the literal prefix.
+                    // When the prefix has no metacharacters we keep the plain
+                    // `col LIKE 'prefix%'` form (no `ESCAPE`), identical to the prior behavior,
+                    // so DataFusion's optimized prefix matcher still applies.
+                    // A `Utf8View` prefix is handled here too (rather than falling through to
+                    // the catch-all arm, which would rebuild the recheck without the trailing
+                    // `%` and silently turn prefix matching into equality matching) and is
+                    // normalized to a `Utf8` pattern below, mirroring how the parser normalizes
+                    // `Utf8View` to `Utf8` (see `expression.rs`).
+                    let escaped = escape_like_pattern(s);
+                    let needs_escape = escaped.as_str() != s.as_str();
+                    let pattern = format!("{}%", escaped);
+                    let pattern_value = match prefix {
+                        ScalarValue::LargeUtf8(_) => ScalarValue::LargeUtf8(Some(pattern)),
+                        _ => ScalarValue::Utf8(Some(pattern)),
+                    };
+                    if needs_escape {
+                        Expr::Like(Like {
+                            negated: false,
+                            expr: Box::new(col_expr),
+                            pattern: Box::new(Expr::Literal(pattern_value, None)),
+                            escape_char: Some('\\'),
+                            case_insensitive: false,
+                        })
+                    } else {
+                        col_expr.like(Expr::Literal(pattern_value, None))
                     }
-                    other => other.clone(),
-                };
-                col_expr.like(Expr::Literal(pattern, None))
-            }
+                }
+                other => col_expr.like(Expr::Literal(other.clone(), None)),
+            },
         }
     }
 
@@ -604,9 +438,15 @@ impl AnyQuery for LabelListQuery {
 pub enum TextQuery {
     /// Retrieve all row ids where the text contains the given string
     StringContains(String),
-    // TODO: In the future we should be able to do string-insensitive contains
-    // as well as partial matches (e.g. LIKE 'foo%') and potentially even
-    // some regular expressions
+    /// Retrieve all row ids whose text matches the given regular expression.
+    ///
+    /// The pattern is a full regular expression (as accepted by `regexp_like`).
+    /// The index returns a candidate superset that the scan rechecks, so any
+    /// pattern is sound; patterns with no usable trigram structure simply fall
+    /// back to rechecking every row.
+    Regex(String),
+    // TODO: In the future we should be able to do case-insensitive contains
+    // as well as partial matches (e.g. LIKE 'foo%').
 }
 
 impl AnyQuery for TextQuery {
@@ -627,6 +467,17 @@ impl AnyQuery for TextQuery {
                     Expr::Literal(ScalarValue::Utf8(Some(substr.clone())), None),
                 ],
             }),
+            // `regexp_like` returns Boolean directly, so the reconstructed
+            // expression can be used as-is for the recheck filter (no IsNotNull
+            // wrapper, unlike `regexp_match`). It is the semantic equivalent of
+            // the original predicate for the "does it match" question.
+            Self::Regex(pattern) => Expr::ScalarFunction(ScalarFunction {
+                func: Arc::new(RegexpLikeFunc::new().into()),
+                args: vec![
+                    Expr::Column(Column::new_unqualified(col)),
+                    Expr::Literal(ScalarValue::Utf8(Some(pattern.clone())), None),
+                ],
+            }),
         }
     }
 
@@ -783,138 +634,6 @@ impl AnyQuery for GeoQuery {
     }
 }
 
-/// The result of a search operation against a scalar index
-#[derive(Debug, PartialEq)]
-pub enum SearchResult {
-    /// The exact row ids that satisfy the query
-    Exact(NullableRowAddrSet),
-    /// Any row id satisfying the query will be in this set but not every
-    /// row id in this set will satisfy the query, a further recheck step
-    /// is needed
-    AtMost(NullableRowAddrSet),
-    /// All of the given row ids satisfy the query but there may be more
-    ///
-    /// No scalar index actually returns this today but it can arise from
-    /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x)))
-    AtLeast(NullableRowAddrSet),
-}
-
-impl SearchResult {
-    pub fn exact(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn at_most(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn at_least(row_ids: impl Into<RowAddrTreeMap>) -> Self {
-        Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default()))
-    }
-
-    pub fn with_nulls(self, nulls: impl Into<RowAddrTreeMap>) -> Self {
-        match self {
-            Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())),
-            Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())),
-            Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())),
-        }
-    }
-
-    pub fn row_addrs(&self) -> &NullableRowAddrSet {
-        match self {
-            Self::Exact(row_addrs) => row_addrs,
-            Self::AtMost(row_addrs) => row_addrs,
-            Self::AtLeast(row_addrs) => row_addrs,
-        }
-    }
-
-    pub fn is_exact(&self) -> bool {
-        matches!(self, Self::Exact(_))
-    }
-}
-
-/// Brief information about an index that was created
-pub struct CreatedIndex {
-    /// The details of the index that was created
-    ///
-    /// These should be stored somewhere as they will be needed to
-    /// load the index later.
-    pub index_details: prost_types::Any,
-    /// The version of the index that was created
-    ///
-    /// This can be used to determine if a reader is able to load the index.
-    pub index_version: u32,
-    /// List of files and their sizes for this index
-    ///
-    /// This enables skipping HEAD calls when opening indices and provides
-    /// visibility into index storage size via describe_indices().
-    pub files: Option<Vec<IndexFile>>,
-}
-
-/// The criteria that specifies how to update an index
-pub struct UpdateCriteria {
-    /// If true, then we need to read the old data to update the index
-    ///
-    /// This should be avoided if possible but is left in for some legacy paths
-    pub requires_old_data: bool,
-    /// The criteria required for data (both old and new)
-    pub data_criteria: TrainingCriteria,
-}
-
-/// Filter used when merging existing scalar-index rows during update.
-///
-/// The caller must pick a filter mode that matches the row-id semantics of the
-/// dataset:
-/// - address-style row IDs: fragment filtering is valid
-/// - stable row IDs: use exact row-id membership instead
-#[derive(Debug, Clone)]
-pub enum OldIndexDataFilter {
-    /// Keeps track of which fragments are still valid and which are no longer valid.
-    ///
-    /// This is valid for address-style row IDs.
-    Fragments {
-        to_keep: RoaringBitmap,
-        to_remove: RoaringBitmap,
-    },
-    /// Keep old rows whose row IDs are in this exact allow-list.
-    ///
-    /// This is required for stable row IDs, where row IDs are opaque and
-    /// should not be interpreted as encoded row addresses.
-    RowIds(RowAddrTreeMap),
-}
-
-impl OldIndexDataFilter {
-    /// Build a boolean mask that keeps only row IDs selected by this filter.
-    pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray {
-        match self {
-            Self::Fragments { to_keep, .. } => row_ids
-                .iter()
-                .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32)))
-                .collect(),
-            Self::RowIds(valid_row_ids) => row_ids
-                .iter()
-                .map(|id| id.map(|id| valid_row_ids.contains(id)))
-                .collect(),
-        }
-    }
-}
-
-impl UpdateCriteria {
-    pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self {
-        Self {
-            requires_old_data: true,
-            data_criteria,
-        }
-    }
-
-    pub fn only_new_data(data_criteria: TrainingCriteria) -> Self {
-        Self {
-            requires_old_data: false,
-            data_criteria,
-        }
-    }
-}
-
 /// Compute the lexicographically next prefix by incrementing the last character's code point.
 /// Returns None if no valid upper bound exists.
 ///
@@ -971,45 +690,61 @@ fn next_unicode_char(c: char) -> Option<char> {
     char::from_u32(next_cp)
 }
 
-/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries
-#[async_trait]
-pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf {
-    /// Search the scalar index
-    ///
-    /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered
-    async fn search(
-        &self,
-        query: &dyn AnyQuery,
-        metrics: &dyn MetricsCollector,
-    ) -> Result<SearchResult>;
-
-    /// Returns true if the remap operation is supported
-    fn can_remap(&self) -> bool;
-
-    /// Remap the row ids, creating a new remapped version of this index in `dest_store`
-    async fn remap(
-        &self,
-        mapping: &HashMap<u64, Option<u64>>,
-        dest_store: &dyn IndexStore,
-    ) -> Result<CreatedIndex>;
-
-    /// Add the new data into the index, creating an updated version of the index in `dest_store`
-    ///
-    /// If `old_data_filter` is provided, old index data will be filtered before
-    /// merge according to the chosen filter mode.
-    async fn update(
-        &self,
-        new_data: SendableRecordBatchStream,
-        dest_store: &dyn IndexStore,
-        old_data_filter: Option<OldIndexDataFilter>,
-    ) -> Result<CreatedIndex>;
-
-    /// Returns the criteria that will be used to update the index
-    fn update_criteria(&self) -> UpdateCriteria;
-
-    /// Derive the index parameters from the current index
-    ///
-    /// This returns a ScalarIndexParams that can be used to recreate an index
-    /// with the same configuration on another dataset.
-    fn derive_index_params(&self) -> Result<ScalarIndexParams>;
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_like_prefix_to_expr_escapes_metacharacters() {
+        // The stored prefix is a literal string, so LIKE metacharacters in it must be
+        // escaped when the recheck predicate is rebuilt; otherwise `_`/`%` would act as
+        // wildcards and over-match. The reconstructed expression uses `ESCAPE '\'`.
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("a_b%x".to_string())));
+        let Expr::Like(like) = query.to_expr("name".to_string()) else {
+            panic!("expected a LIKE expression");
+        };
+        assert_eq!(like.escape_char, Some('\\'));
+        assert!(!like.negated);
+        assert!(!like.case_insensitive);
+        let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = like.pattern.as_ref() else {
+            panic!("expected a Utf8 literal pattern");
+        };
+        assert_eq!(pattern.as_str(), "a\\_b\\%x%");
+
+        // A prefix without metacharacters only gains the trailing wildcard and keeps the
+        // plain `LIKE 'app%'` form (no `ESCAPE`) so the optimized prefix matcher still applies.
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("app".to_string())));
+        let Expr::Like(like) = query.to_expr("name".to_string()) else {
+            panic!("expected a LIKE expression");
+        };
+        assert_eq!(like.escape_char, None);
+        let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = like.pattern.as_ref() else {
+            panic!("expected a Utf8 literal pattern");
+        };
+        assert_eq!(pattern.as_str(), "app%");
+
+        // A `Utf8View` prefix must get the same treatment instead of falling through to the
+        // catch-all arm: it is normalized to a `Utf8` pattern that keeps the trailing `%`
+        // (and `ESCAPE '\'` when the prefix has metacharacters), so prefix pruning preserves
+        // starts-with semantics rather than degrading to equality.
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8View(Some("a_b%x".to_string())));
+        let Expr::Like(like) = query.to_expr("name".to_string()) else {
+            panic!("expected a LIKE expression");
+        };
+        assert_eq!(like.escape_char, Some('\\'));
+        let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = like.pattern.as_ref() else {
+            panic!("expected a Utf8 literal pattern");
+        };
+        assert_eq!(pattern.as_str(), "a\\_b\\%x%");
+
+        let query = SargableQuery::LikePrefix(ScalarValue::Utf8View(Some("app".to_string())));
+        let Expr::Like(like) = query.to_expr("name".to_string()) else {
+            panic!("expected a LIKE expression");
+        };
+        assert_eq!(like.escape_char, None);
+        let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = like.pattern.as_ref() else {
+            panic!("expected a Utf8 literal pattern");
+        };
+        assert_eq!(pattern.as_str(), "app%");
+    }
 }
diff --git a/rust/lance-index/src/scalar/bitmap.rs b/rust/lance-index/src/scalar/bitmap.rs
index eb0276dcb9f..91595b277f0 100644
--- a/rust/lance-index/src/scalar/bitmap.rs
+++ b/rust/lance-index/src/scalar/bitmap.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
     any::Any,
     cmp::Reverse,
@@ -17,15 +18,14 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion_common::ScalarValue;
-use deepsize::DeepSizeOf;
 use futures::{StreamExt, TryStreamExt, stream};
-use lance_arrow::ipc::{
-    read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream,
-    write_len_prefixed_bytes,
-};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{
     Error, ROW_ID, Result,
-    cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache},
+    cache::{
+        CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache,
+        WeakLanceCache,
+    },
     error::LanceOptionExt,
     utils::tokio::get_num_compute_intensive_cpus,
 };
@@ -36,21 +36,20 @@ use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
 use tracing::{instrument, warn};
 
-use super::{AnyQuery, IndexStore, ScalarIndex};
+use super::{AnyQuery, IndexFile, IndexStore, ScalarIndex};
 use super::{
     BuiltinIndexType, SargableQuery, ScalarIndexParams, SearchResult, btree::OrderableScalarValue,
 };
 use crate::pbold;
 use crate::{Index, IndexType, metrics::MetricsCollector};
 use crate::{
-    frag_reuse::FragReuseIndex,
     progress::IndexBuildProgress,
     scalar::{
-        CreatedIndex, UpdateCriteria,
+        CreatedIndex, RowIdRemapper, UpdateCriteria,
         expression::SargableQueryParser,
         registry::{
-            ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
-            VALUE_COLUMN_NAME,
+            BasicTrainer, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering,
+            TrainingRequest, VALUE_COLUMN_NAME, single_flight_open,
         },
     },
 };
@@ -116,7 +115,7 @@ pub struct BitmapIndex {
     /// Maps each unique value to its bitmap location in the index file
     /// The usize value is the row offset in the bitmap_page_lookup.lance file
     /// for quickly locating the row and reading it out
-    index_map: BTreeMap<OrderableScalarValue, usize>,
+    index_map: Arc<BTreeMap<OrderableScalarValue, usize>>,
 
     null_map: Arc<RowAddrTreeMap>,
 
@@ -126,7 +125,7 @@ pub struct BitmapIndex {
 
     index_cache: WeakLanceCache,
 
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
 
     lazy_reader: LazyIndexReader,
 }
@@ -173,11 +172,17 @@ pub struct BitmapIndexState {
     /// Cached separately from the schema for the empty-index case where the
     /// `lookup_batch` is empty but we still need to remember the column type.
     value_type: DataType,
+    /// Parsed form of `lookup_batch`. Not serialized — populated eagerly in
+    /// both [`BitmapIndexState::from_index`] and [`CacheCodecImpl::deserialize`].
+    /// Stored as `Arc` so cloning into a new [`BitmapIndex`] is O(1).
+    index_map: Arc<BTreeMap<OrderableScalarValue, usize>>,
 }
 
 impl DeepSizeOf for BitmapIndexState {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.lookup_batch.get_array_memory_size() + self.null_map.deep_size_of_children(context)
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.lookup_batch.get_array_memory_size()
+            + self.null_map.deep_size_of_children(context)
+            + self.index_map.deep_size_of_children(context)
     }
 }
 
@@ -187,25 +192,63 @@ impl BitmapIndexState {
             lookup_batch: build_lookup_batch(&index.index_map, &index.value_type)?,
             null_map: index.null_map.clone(),
             value_type: index.value_type.clone(),
+            index_map: index.index_map.clone(),
         })
     }
 
-    pub(crate) fn into_bitmap_index(
-        self,
+    fn from_scalar_index(index: &dyn ScalarIndex) -> Result<Self> {
+        let bitmap = index
+            .as_any()
+            .downcast_ref::<BitmapIndex>()
+            .ok_or_else(|| {
+                Error::internal(
+                    "BitmapIndexState::from_scalar_index called with a non-bitmap index",
+                )
+            })?;
+        Self::from_index(bitmap)
+    }
+
+    pub(crate) fn to_bitmap_index(
+        &self,
         store: Arc<dyn IndexStore>,
         index_cache: &LanceCache,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Arc<BitmapIndex>> {
-        let index_map = parse_lookup_batch(&self.lookup_batch)?;
         Ok(Arc::new(BitmapIndex::new(
-            index_map,
-            self.null_map,
-            self.value_type,
+            self.index_map.clone(),
+            self.null_map.clone(),
+            self.value_type.clone(),
             store,
             WeakLanceCache::from(index_cache),
             frag_reuse_index,
         )))
     }
+
+    /// Build a state directly from its parts, for codec tests in sibling
+    /// modules (e.g. the label-list index, which nests a bitmap state).
+    #[cfg(test)]
+    pub(crate) fn new_for_test(
+        index_map: BTreeMap<OrderableScalarValue, usize>,
+        null_map: RowAddrTreeMap,
+        value_type: DataType,
+    ) -> Result<Self> {
+        Ok(Self {
+            lookup_batch: build_lookup_batch(&index_map, &value_type)?,
+            null_map: Arc::new(null_map),
+            value_type,
+            index_map: Arc::new(index_map),
+        })
+    }
+
+    #[cfg(test)]
+    pub(crate) fn lookup_batch(&self) -> &RecordBatch {
+        &self.lookup_batch
+    }
+
+    #[cfg(test)]
+    pub(crate) fn null_map(&self) -> &RowAddrTreeMap {
+        &self.null_map
+    }
 }
 
 fn build_lookup_batch(
@@ -245,30 +288,34 @@ fn parse_lookup_batch(batch: &RecordBatch) -> Result<BTreeMap<OrderableScalarVal
 }
 
 impl CacheCodecImpl for BitmapIndexState {
+    const TYPE_ID: &'static str = "lance.scalar.BitmapIndexState";
+    const CURRENT_VERSION: u32 = 1;
+
     /// Wire format:
     /// ```text
-    /// [u64 null_map_len][null_map bytes]
-    /// [arrow IPC stream: (keys: <value_type>, offsets: UInt64)]
+    /// RAW_BLOB  : null_map (roaring tree map, portable encoding)
+    /// ARROW_IPC : (keys: <value_type>, offsets: UInt64)
     /// ```
-    /// The value type is recovered from the IPC stream schema.
-    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()> {
+    /// The value type is recovered from the IPC section schema.
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         let mut null_bytes = Vec::with_capacity(self.null_map.serialized_size());
         self.null_map.serialize_into(&mut null_bytes)?;
-        write_len_prefixed_bytes(writer, &null_bytes)?;
-        write_ipc_stream(&self.lookup_batch, writer)?;
+        w.write_raw(&null_bytes)?;
+        w.write_ipc(&self.lookup_batch)?;
         Ok(())
     }
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let null_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let null_bytes = r.read_raw()?;
         let null_map = Arc::new(RowAddrTreeMap::deserialize_from(null_bytes.as_ref())?);
-        let lookup_batch = read_ipc_stream_single_at(data, &mut offset)?;
+        let lookup_batch = r.read_ipc()?;
         let value_type = lookup_batch.schema().field(0).data_type().clone();
+        let index_map = Arc::new(parse_lookup_batch(&lookup_batch)?);
         Ok(Self {
             lookup_batch,
             null_map,
             value_type,
+            index_map,
         })
     }
 }
@@ -295,12 +342,12 @@ impl CacheKey for BitmapIndexStateKey {
 
 impl BitmapIndex {
     fn new(
-        index_map: BTreeMap<OrderableScalarValue, usize>,
+        index_map: Arc<BTreeMap<OrderableScalarValue, usize>>,
         null_map: Arc<RowAddrTreeMap>,
         value_type: DataType,
         store: Arc<dyn IndexStore>,
         index_cache: WeakLanceCache,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Self {
         let lazy_reader = LazyIndexReader::new(store.clone());
         Self {
@@ -316,7 +363,7 @@ impl BitmapIndex {
 
     pub(crate) async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let page_lookup_file = store.open_index_file(BITMAP_LOOKUP_NAME).await?;
@@ -326,7 +373,7 @@ impl BitmapIndex {
             let schema = page_lookup_file.schema();
             let data_type = schema.fields[0].data_type();
             return Ok(Arc::new(Self::new(
-                BTreeMap::new(),
+                Arc::new(BTreeMap::new()),
                 Arc::new(RowAddrTreeMap::default()),
                 data_type,
                 store,
@@ -381,7 +428,7 @@ impl BitmapIndex {
         }
 
         Ok(Arc::new(Self::new(
-            index_map,
+            Arc::new(index_map),
             null_map,
             value_type,
             store,
@@ -465,13 +512,8 @@ impl BitmapIndex {
 }
 
 impl DeepSizeOf for BitmapIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        let mut total_size = 0;
-
-        total_size += self.index_map.deep_size_of_children(context);
-        total_size += self.store.deep_size_of_children(context);
-
-        total_size
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.index_map.deep_size_of_children(context) + self.store.deep_size_of_children(context)
     }
 }
 
@@ -521,12 +563,6 @@ impl Index for BitmapIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::not_supported_source(
-            "BitmapIndex is not a vector index".into(),
-        ))
-    }
-
     async fn prewarm(&self) -> Result<()> {
         let page_lookup_file = self.lazy_reader.get().await?;
         let total_rows = page_lookup_file.num_rows();
@@ -760,18 +796,20 @@ impl ScalarIndex for BitmapIndex {
     /// Remap the row ids, creating a new remapped version of this index in `dest_store`
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         let state = self.load_bitmap_index_state().await?;
         let remapped_state = BitmapIndexPlugin::remap_bitmap_state(state, mapping);
-        BitmapIndexPlugin::write_bitmap_index(remapped_state, dest_store, &self.value_type).await?;
+        let file =
+            BitmapIndexPlugin::write_bitmap_index(remapped_state, dest_store, &self.value_type)
+                .await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -780,13 +818,14 @@ impl ScalarIndex for BitmapIndex {
         &self,
         new_data: SendableRecordBatchStream,
         dest_store: &dyn IndexStore,
-        _old_data_filter: Option<super::OldIndexDataFilter>,
+        old_data_filter: Option<super::OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        BitmapIndexPlugin::streaming_build_and_write(
+        let file = BitmapIndexPlugin::streaming_build_and_write(
             new_data,
             Some(self),
             dest_store,
             BITMAP_LOOKUP_NAME,
+            old_data_filter.as_ref(),
         )
         .await?;
 
@@ -794,7 +833,7 @@ impl ScalarIndex for BitmapIndex {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -852,8 +891,7 @@ impl BitmapBatchWriter {
             return Ok(());
         }
         let keys_array =
-            ScalarValue::iter_to_array(self.keys.drain(..).collect::<Vec<_>>().into_iter())
-                .unwrap();
+            ScalarValue::iter_to_array(self.keys.drain(..).collect::<Vec<_>>()).unwrap();
         let total_size: usize = self.serialized.iter().map(|b| b.len()).sum();
         let mut binary_builder = BinaryBuilder::with_capacity(self.serialized.len(), total_size);
         for b in self.serialized.drain(..) {
@@ -867,7 +905,7 @@ impl BitmapBatchWriter {
     }
 
     /// Flush any remaining data, write index statistics, and finalize the file.
-    async fn finish(mut self) -> Result<()> {
+    async fn finish(mut self) -> Result<IndexFile> {
         self.flush().await?;
         let stats_json = serde_json::to_string(&BitmapStatistics {
             num_bitmaps: self.num_bitmaps,
@@ -875,8 +913,7 @@ impl BitmapBatchWriter {
         .map_err(|e| Error::internal(format!("failed to serialize bitmap statistics: {e}")))?;
         let mut metadata = HashMap::new();
         metadata.insert(INDEX_STATS_METADATA_KEY.to_string(), stats_json);
-        self.file.finish_with_metadata(metadata).await?;
-        Ok(())
+        self.file.finish_with_metadata(metadata).await
     }
 }
 
@@ -1085,10 +1122,7 @@ async fn drain_same_key_bitmaps(
     let merged_key = OrderableScalarValue(key);
     advance_cursor_and_push(cursors, heap, item.shard_idx).await?;
 
-    loop {
-        let Some(Reverse(next_item)) = heap.peek() else {
-            break;
-        };
+    while let Some(Reverse(next_item)) = heap.peek() {
         if next_item.key != merged_key {
             break;
         }
@@ -1166,6 +1200,19 @@ async fn cleanup_bitmap_shard_files(store: &dyn IndexStore, shard_files: &[Strin
 #[derive(Debug, Default)]
 pub struct BitmapIndexPlugin;
 
+/// Drop the rows an old posting should no longer expose -- rows whose fragment
+/// was removed, or (under stable row ids) rows rewritten by an update -- keeping
+/// only those `filter` still considers valid. A no-op when `filter` is `None`.
+fn retain_valid(
+    mut bitmap: RowAddrTreeMap,
+    filter: Option<&super::OldIndexDataFilter>,
+) -> RowAddrTreeMap {
+    if let Some(filter) = filter {
+        filter.retain_old_rows(&mut bitmap);
+    }
+    bitmap
+}
+
 impl BitmapIndexPlugin {
     fn get_batch_from_arrays(
         keys: Arc<dyn Array>,
@@ -1185,7 +1232,7 @@ impl BitmapIndexPlugin {
         state: HashMap<ScalarValue, RowAddrTreeMap>,
         index_store: &dyn IndexStore,
         value_type: &DataType,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         Self::write_bitmap_index_with_extras(
             state,
             index_store,
@@ -1203,7 +1250,7 @@ impl BitmapIndexPlugin {
         value_type: &DataType,
         mut metadata: HashMap<String, String>,
         global_buffers: Vec<(String, Bytes)>,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let num_bitmaps = state.len();
         let schema = Arc::new(Schema::new(vec![
             Field::new("keys", value_type.clone(), true),
@@ -1229,7 +1276,7 @@ impl BitmapIndexPlugin {
             let bitmap_size = bytes.len();
 
             if cur_bytes + bitmap_size > MAX_BITMAP_ARRAY_LENGTH {
-                let keys_array = ScalarValue::iter_to_array(cur_keys.clone().into_iter()).unwrap();
+                let keys_array = ScalarValue::iter_to_array(cur_keys.clone()).unwrap();
                 let mut binary_builder = BinaryBuilder::new();
                 for b in &cur_bitmaps {
                     binary_builder.append_value(b);
@@ -1267,9 +1314,7 @@ impl BitmapIndexPlugin {
             .map_err(|e| Error::internal(format!("failed to serialize bitmap statistics: {e}")))?;
         metadata.insert(INDEX_STATS_METADATA_KEY.to_string(), stats_json);
 
-        bitmap_index_file.finish_with_metadata(metadata).await?;
-
-        Ok(())
+        bitmap_index_file.finish_with_metadata(metadata).await
     }
 
     /// Builds bitmap index state from a `(value, row_id)` stream without writing it.
@@ -1298,8 +1343,8 @@ impl BitmapIndexPlugin {
     pub async fn train_bitmap_index(
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
-    ) -> Result<()> {
-        Self::streaming_build_and_write(data, None, index_store, BITMAP_LOOKUP_NAME).await
+    ) -> Result<IndexFile> {
+        Self::streaming_build_and_write(data, None, index_store, BITMAP_LOOKUP_NAME, None).await
     }
 
     async fn train_bitmap_shard(
@@ -1308,15 +1353,16 @@ impl BitmapIndexPlugin {
         fragment_ids: &[u32],
         shard_id: Option<u32>,
         progress: Arc<dyn crate::progress::IndexBuildProgress>,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let partition_id = bitmap_shard_partition_id(fragment_ids, shard_id)?;
         let file_name = bitmap_shard_file_name(partition_id);
         progress
             .stage_start("build_bitmap_shard", None, "rows")
             .await?;
-        Self::streaming_build_and_write(data, None, index_store, &file_name).await?;
+        let file =
+            Self::streaming_build_and_write(data, None, index_store, &file_name, None).await?;
         progress.stage_complete("build_bitmap_shard").await?;
-        Ok(())
+        Ok(file)
     }
 
     /// Builds and writes a bitmap index in a streaming fashion from value-sorted
@@ -1331,7 +1377,8 @@ impl BitmapIndexPlugin {
         old_index: Option<&BitmapIndex>,
         index_store: &dyn IndexStore,
         output_file_name: &str,
-    ) -> Result<()> {
+        old_data_filter: Option<&super::OldIndexDataFilter>,
+    ) -> Result<IndexFile> {
         let value_type = data_source.schema().field(0).data_type().clone();
 
         let mut writer =
@@ -1377,6 +1424,7 @@ impl BitmapIndexPlugin {
                                 &mut old_pos,
                                 &mut emitted_null,
                                 &mut writer,
+                                old_data_filter,
                             )
                             .await?;
                         }
@@ -1399,6 +1447,7 @@ impl BitmapIndexPlugin {
                 &mut old_pos,
                 &mut emitted_null,
                 &mut writer,
+                old_data_filter,
             )
             .await?;
         }
@@ -1406,7 +1455,13 @@ impl BitmapIndexPlugin {
         // Emit any remaining old-only entries.
         if let Some(idx) = old_index {
             while old_pos < old_keys.len() {
-                let old_bitmap = idx.load_bitmap(&old_keys[old_pos], None).await?;
+                let old_bitmap = retain_valid(
+                    idx.load_bitmap(&old_keys[old_pos], None)
+                        .await?
+                        .as_ref()
+                        .clone(),
+                    old_data_filter,
+                );
                 writer
                     .emit(old_keys[old_pos].0.clone(), &old_bitmap)
                     .await?;
@@ -1421,17 +1476,17 @@ impl BitmapIndexPlugin {
         {
             let null_key = new_null_array(&value_type, 1);
             let null_key = ScalarValue::try_from_array(null_key.as_ref(), 0)?;
-            writer.emit(null_key, &idx.null_map).await?;
+            let null_bitmap = retain_valid((*idx.null_map).clone(), old_data_filter);
+            writer.emit(null_key, &null_bitmap).await?;
         }
 
-        writer.finish().await?;
-
-        Ok(())
+        writer.finish().await
     }
 
     /// Flush a completed value-run from the new data stream, emitting any
     /// old-only entries that sort before it and merging the old bitmap if the
     /// key exists in both old and new.
+    #[allow(clippy::too_many_arguments)]
     async fn finish_run(
         key: ScalarValue,
         bitmap: &mut RowAddrTreeMap,
@@ -1440,13 +1495,14 @@ impl BitmapIndexPlugin {
         old_pos: &mut usize,
         emitted_null: &mut bool,
         writer: &mut BitmapBatchWriter,
+        old_data_filter: Option<&super::OldIndexDataFilter>,
     ) -> Result<()> {
         if key.is_null() {
             // Null values are stored separately in the old index's null_map.
             if let Some(idx) = old_index
                 && !idx.null_map.is_empty()
             {
-                *bitmap |= &*idx.null_map;
+                *bitmap |= &retain_valid((*idx.null_map).clone(), old_data_filter);
             }
             *emitted_null = true;
             writer.emit(key, bitmap).await?;
@@ -1455,7 +1511,13 @@ impl BitmapIndexPlugin {
 
             // Emit old-only entries that sort before this key.
             while *old_pos < old_keys.len() && old_keys[*old_pos] < orderable {
-                let old_bitmap = idx.load_bitmap(&old_keys[*old_pos], None).await?;
+                let old_bitmap = retain_valid(
+                    idx.load_bitmap(&old_keys[*old_pos], None)
+                        .await?
+                        .as_ref()
+                        .clone(),
+                    old_data_filter,
+                );
                 writer
                     .emit(old_keys[*old_pos].0.clone(), &old_bitmap)
                     .await?;
@@ -1464,8 +1526,13 @@ impl BitmapIndexPlugin {
 
             // If the old index also has this key, merge its bitmap.
             if *old_pos < old_keys.len() && old_keys[*old_pos] == orderable {
-                let old_bitmap = idx.load_bitmap(&old_keys[*old_pos], None).await?;
-                *bitmap |= &*old_bitmap;
+                *bitmap |= &retain_valid(
+                    idx.load_bitmap(&old_keys[*old_pos], None)
+                        .await?
+                        .as_ref()
+                        .clone(),
+                    old_data_filter,
+                );
                 *old_pos += 1;
             }
 
@@ -1479,7 +1546,7 @@ impl BitmapIndexPlugin {
     /// Remaps every bitmap in a materialized bitmap-index state using row-id mappings.
     pub(crate) fn remap_bitmap_state(
         state: HashMap<ScalarValue, RowAddrTreeMap>,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
     ) -> HashMap<ScalarValue, RowAddrTreeMap> {
         state
             .into_iter()
@@ -1487,10 +1554,7 @@ impl BitmapIndexPlugin {
                 let remapped_bitmap =
                     RowAddrTreeMap::from_iter(bitmap.row_addrs().unwrap().filter_map(|addr| {
                         let addr_as_u64 = u64::from(addr);
-                        mapping
-                            .get(&addr_as_u64)
-                            .copied()
-                            .unwrap_or(Some(addr_as_u64))
+                        mapping.get(addr_as_u64).unwrap_or(Some(addr_as_u64))
                     }));
                 (key, remapped_bitmap)
             })
@@ -1517,7 +1581,7 @@ impl BitmapIndexPlugin {
         store: &dyn IndexStore,
         shard_files: &[String],
         progress: Arc<dyn IndexBuildProgress>,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         progress
             .stage_start("merge_bitmap_shards", None, "bitmaps")
             .await?;
@@ -1567,10 +1631,10 @@ impl BitmapIndexPlugin {
         progress
             .stage_start("write_bitmap_index", Some(1), "files")
             .await?;
-        writer.finish().await?;
+        let file = writer.finish().await?;
         progress.stage_progress("write_bitmap_index", 1).await?;
         progress.stage_complete("write_bitmap_index").await?;
-        Ok(())
+        Ok(file)
     }
 }
 
@@ -1591,12 +1655,65 @@ pub async fn merge_index_files(
     Ok(())
 }
 
-#[async_trait]
-impl ScalarIndexPlugin for BitmapIndexPlugin {
-    fn name(&self) -> &str {
-        "Bitmap"
+pub async fn merge_bitmap_indices(
+    source_indices: &[Arc<BitmapIndex>],
+    dest_store: &dyn IndexStore,
+    progress: Arc<dyn IndexBuildProgress>,
+) -> Result<CreatedIndex> {
+    if source_indices.is_empty() {
+        return Err(Error::invalid_input(
+            "Bitmap segment merge requires at least one source segment".to_string(),
+        ));
+    }
+
+    let value_type = source_indices[0].value_type().clone();
+    let mut merged_state = HashMap::<ScalarValue, RowAddrTreeMap>::new();
+
+    progress
+        .stage_start(
+            "merge_bitmap_segments",
+            Some(source_indices.len() as u64),
+            "segments",
+        )
+        .await?;
+    for (idx, source_index) in source_indices.iter().enumerate() {
+        if source_index.value_type() != &value_type {
+            return Err(Error::invalid_input(format!(
+                "Bitmap segment has value type {:?}, expected {:?}",
+                source_index.value_type(),
+                value_type
+            )));
+        }
+
+        let state = source_index.load_bitmap_index_state().await?;
+        for (key, bitmap) in state {
+            merged_state
+                .entry(key)
+                .and_modify(|existing| *existing |= &bitmap)
+                .or_insert(bitmap);
+        }
+        progress
+            .stage_progress("merge_bitmap_segments", (idx + 1) as u64)
+            .await?;
     }
+    progress.stage_complete("merge_bitmap_segments").await?;
+
+    progress
+        .stage_start("write_bitmap_index", Some(1), "files")
+        .await?;
+    let file = BitmapIndexPlugin::write_bitmap_index(merged_state, dest_store, &value_type).await?;
+    progress.stage_progress("write_bitmap_index", 1).await?;
+    progress.stage_complete("write_bitmap_index").await?;
+
+    Ok(CreatedIndex {
+        index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default()).unwrap(),
+        index_version: BITMAP_INDEX_VERSION,
+        files: vec![file],
+    })
+}
 
+#[async_trait]
+impl BasicTrainer for BitmapIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -1615,26 +1732,6 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
         Ok(Box::new(BitmapTrainingRequest::new(params)))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        true
-    }
-
-    fn version(&self) -> u32 {
-        BITMAP_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        Some(Box::new(SargableQueryParser::new(
-            index_name,
-            self.name().to_string(),
-            false,
-        )))
-    }
-
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -1652,7 +1749,7 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
                         .to_string(),
                 )
             })?;
-        if let Some(fragment_ids) = fragment_ids.as_ref() {
+        let file = if let Some(fragment_ids) = fragment_ids.as_ref() {
             Self::train_bitmap_shard(
                 data,
                 index_store,
@@ -1660,29 +1757,61 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
                 request.parameters.shard_id,
                 progress,
             )
-            .await?;
+            .await?
         } else if request.parameters.shard_id.is_some() {
             return Err(Error::invalid_input(
                 "Bitmap shard_id requires fragment_ids and is only supported for distributed shard builds"
                     .to_string(),
             ));
         } else {
-            Self::train_bitmap_index(data, index_store).await?;
-        }
+            Self::train_bitmap_index(data, index_store).await?
+        };
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BitmapIndexDetails::default())
                 .unwrap(),
             index_version: BITMAP_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for BitmapIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "Bitmap"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        true
+    }
+
+    fn version(&self) -> u32 {
+        BITMAP_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        // Bitmap indexes cannot answer `LikePrefix` queries (see `search`), so the parser
+        // is configured to skip them and let such predicates fall back to ordinary filtering.
+        Some(Box::new(
+            SargableQueryParser::new(index_name, self.name().to_string(), false)
+                .without_like_prefix(),
+        ))
+    }
 
     /// Load an index from storage
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(BitmapIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
@@ -1691,31 +1820,44 @@ impl ScalarIndexPlugin for BitmapIndexPlugin {
     async fn get_from_cache(
         &self,
         index_store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Option<Arc<dyn ScalarIndex>>> {
         let Some(state) = cache.get_with_key(&BitmapIndexStateKey).await else {
             return Ok(None);
         };
-        let state = (*state).clone();
-        let index = state.into_bitmap_index(index_store, cache, frag_reuse_index)?;
+        let index = state.to_bitmap_index(index_store, cache, frag_reuse_index)?;
         Ok(Some(index as Arc<dyn ScalarIndex>))
     }
 
     async fn put_in_cache(&self, cache: &LanceCache, index: Arc<dyn ScalarIndex>) -> Result<()> {
-        let bitmap = index
-            .as_any()
-            .downcast_ref::<BitmapIndex>()
-            .ok_or_else(|| {
-                Error::internal("BitmapIndexPlugin::put_in_cache called with a non-bitmap index")
-            })?;
-        let state = BitmapIndexState::from_index(bitmap)?;
+        let state = BitmapIndexState::from_scalar_index(index.as_ref())?;
         cache
             .insert_with_key(&BitmapIndexStateKey, Arc::new(state))
             .await;
         Ok(())
     }
 
+    async fn get_or_insert_in_cache(
+        &self,
+        index_store: Arc<dyn IndexStore>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        cache: &LanceCache,
+        load: ScalarIndexLoad<'_>,
+    ) -> Result<Arc<dyn ScalarIndex>> {
+        single_flight_open(
+            cache,
+            BitmapIndexStateKey,
+            load,
+            BitmapIndexState::from_scalar_index,
+            move |state| {
+                Ok(state.to_bitmap_index(index_store, cache, frag_reuse_index)?
+                    as Arc<dyn ScalarIndex>)
+            },
+        )
+        .await
+    }
+
     async fn load_statistics(
         &self,
         index_store: Arc<dyn IndexStore>,
@@ -1761,12 +1903,16 @@ mod tests {
     use lance_core::utils::{address::RowAddress, tempfile::TempObjDir};
     use lance_io::object_store::ObjectStore;
     use lance_select::RowSetOps;
-    use std::collections::HashMap;
+    use rstest::rstest;
 
     fn assert_state_roundtrips(state: &BitmapIndexState) {
         let mut buf = Vec::new();
-        state.serialize(&mut buf).unwrap();
-        let restored = BitmapIndexState::deserialize(&bytes::Bytes::from(buf)).unwrap();
+        state
+            .serialize(&mut CacheEntryWriter::new(&mut buf))
+            .unwrap();
+        let data = bytes::Bytes::from(buf);
+        let mut reader = CacheEntryReader::new(&data, 0, BitmapIndexState::CURRENT_VERSION);
+        let restored = BitmapIndexState::deserialize(&mut reader).unwrap();
         assert_eq!(restored.lookup_batch, state.lookup_batch);
         assert_eq!(&*restored.null_map, &*state.null_map);
         assert_eq!(restored.value_type, state.value_type);
@@ -1786,6 +1932,7 @@ mod tests {
             lookup_batch: build_lookup_batch(&index_map, &DataType::Int32).unwrap(),
             null_map: Arc::new(null_map),
             value_type: DataType::Int32,
+            index_map: Arc::new(index_map),
         };
         assert_state_roundtrips(&state);
 
@@ -1794,10 +1941,58 @@ mod tests {
             lookup_batch: build_lookup_batch(&BTreeMap::new(), &DataType::Utf8).unwrap(),
             null_map: Arc::new(RowAddrTreeMap::new()),
             value_type: DataType::Utf8,
+            index_map: Arc::new(BTreeMap::new()),
         };
         assert_state_roundtrips(&empty_state);
     }
 
+    /// The lookup batch must decode zero-copy through the full envelope-bearing
+    /// [`CacheCodec`] even though the envelope pushes the IPC section to a
+    /// non-aligned starting offset.
+    #[test]
+    fn test_bitmap_index_state_lookup_is_zero_copy() {
+        const ALIGN: usize = 64;
+        let mut index_map = BTreeMap::new();
+        for k in 0..32i32 {
+            index_map.insert(
+                OrderableScalarValue(ScalarValue::Int32(Some(k))),
+                k as usize,
+            );
+        }
+        let state = BitmapIndexState {
+            lookup_batch: build_lookup_batch(&index_map, &DataType::Int32).unwrap(),
+            null_map: Arc::new(RowAddrTreeMap::new()),
+            value_type: DataType::Int32,
+            index_map: Arc::new(index_map),
+        };
+
+        let codec = CacheCodec::from_impl::<BitmapIndexState>();
+        let any: Arc<dyn std::any::Any + Send + Sync> = Arc::new(state);
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+
+        // Model a backend reading into a 64-byte-aligned buffer.
+        let mut v = vec![0u8; buf.len() + ALIGN];
+        let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+        v[pad..pad + buf.len()].copy_from_slice(&buf);
+        let data = bytes::Bytes::from(v).slice(pad..pad + buf.len());
+
+        let restored = codec.deserialize(&data).hit().unwrap();
+        let restored = restored.downcast::<BitmapIndexState>().unwrap();
+
+        let base = data.as_ptr() as usize;
+        let end = base + data.len();
+        for col in restored.lookup_batch.columns() {
+            for buffer in col.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= base && ptr < end,
+                    "lookup batch buffer was realigned out of the input — misaligned IPC section",
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_bitmap_lazy_loading_and_cache() {
         // Create a temporary directory for the index
@@ -1933,6 +2128,85 @@ mod tests {
         }
     }
 
+    // Regression test for the O(N log N) warm-cache rebuild introduced in
+    // commit 4de5ce67d.  BitmapIndexState now caches the parsed Arc<BTreeMap>
+    // so that get_from_cache skips parse_lookup_batch on warm hits.
+    // IS NULL is the worst case: the actual bitmap lookup is O(1) but
+    // reconstruction of the BTreeMap touched every row in the lookup batch.
+    #[tokio::test]
+    async fn test_bitmap_cache_fast_path() {
+        use arrow_array::Int32Array;
+
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // High-cardinality: 1 000 unique integers + 5 null rows.
+        const N: u64 = 1_000;
+        const NULL_COUNT: u64 = 5;
+        // nulls first (sorted batch: nulls precede values)
+        let null_values: Vec<Option<i32>> =
+            std::iter::repeat_n(None, NULL_COUNT as usize).collect();
+        let non_null_values: Vec<Option<i32>> = (0..N as i32).map(Some).collect();
+        let all_values: Vec<Option<i32>> = null_values.into_iter().chain(non_null_values).collect();
+        let all_row_ids: Vec<u64> = (0..N + NULL_COUNT).collect();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("value", DataType::Int32, true),
+            Field::new("_rowid", DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(all_values)),
+                Arc::new(UInt64Array::from(all_row_ids)),
+            ],
+        )
+        .unwrap();
+        let stream = stream::once(async move { Ok(batch) });
+        let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+        BitmapIndexPlugin::train_bitmap_index(stream, store.as_ref())
+            .await
+            .unwrap();
+
+        let cache = LanceCache::with_capacity(16 * 1024 * 1024);
+        let index = BitmapIndex::load(store.clone(), None, &cache)
+            .await
+            .unwrap();
+
+        let plugin = BitmapIndexPlugin;
+        let index_arc: Arc<dyn ScalarIndex> = index.clone() as Arc<dyn ScalarIndex>;
+        plugin.put_in_cache(&cache, index_arc).await.unwrap();
+
+        // get_from_cache must return Some, and the BitmapIndexState's OnceLock
+        // must have been populated by put_in_cache so no parse_lookup_batch occurs.
+        let cached = plugin
+            .get_from_cache(store.clone(), None, &cache)
+            .await
+            .unwrap()
+            .expect("get_from_cache must return Some after put_in_cache");
+
+        // IS NULL: trivial work once the index is in hand.
+        let query = SargableQuery::IsNull();
+        match cached.search(&query, &NoOpMetricsCollector).await.unwrap() {
+            SearchResult::Exact(row_set) => {
+                let mut null_rows: Vec<u64> = row_set
+                    .true_rows()
+                    .row_addrs()
+                    .unwrap()
+                    .map(u64::from)
+                    .collect();
+                null_rows.sort();
+                let expected: Vec<u64> = (0..NULL_COUNT).collect();
+                assert_eq!(null_rows, expected);
+            }
+            _ => panic!("Expected Exact result for IS NULL"),
+        }
+    }
+
     #[tokio::test]
     #[ignore]
     async fn test_big_bitmap_index() {
@@ -2180,8 +2454,44 @@ mod tests {
         assert_eq!(red_rows_2, vec![0, 3, 6, 10, 11]);
     }
 
+    // frags 1 and 2 (3 rows each) are compacted into frag 3: the 6 rows are
+    // rewritten in order to frag 3 offsets 0..6.
+    fn bitmap_remap_compact() -> RowAddrRemap {
+        use lance_core::utils::row_addr_remap::GroupInput;
+        use roaring::RoaringTreemap;
+        RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter(
+                (0..3)
+                    .map(|o| u64::from(RowAddress::new_from_parts(1, o)))
+                    .chain((0..3).map(|o| u64::from(RowAddress::new_from_parts(2, o)))),
+            ),
+            old_frag_ids: vec![1, 2],
+            new_frags: vec![(3, 6)],
+        }])
+        .unwrap()
+    }
+
+    fn bitmap_remap_explicit() -> RowAddrRemap {
+        // The same mapping, listed out explicitly.
+        RowAddrRemap::direct(
+            (0..6u32)
+                .map(|i| {
+                    let (f, o) = if i < 3 { (1, i) } else { (2, i - 3) };
+                    (
+                        u64::from(RowAddress::new_from_parts(f, o)),
+                        Some(u64::from(RowAddress::new_from_parts(3, i))),
+                    )
+                })
+                .collect(),
+        )
+    }
+
+    // remap must behave identically whether the mapping is compact or explicit.
+    #[rstest]
+    #[case(bitmap_remap_compact())]
+    #[case(bitmap_remap_explicit())]
     #[tokio::test]
-    async fn test_remap_bitmap_with_null() {
+    async fn test_remap_bitmap_with_null(#[case] remap: RowAddrRemap) {
         use arrow_array::UInt32Array;
 
         // Create a temporary store.
@@ -2246,38 +2556,8 @@ mod tests {
         assert_eq!(index.index_map.len(), 2); // 2 non-null values (1 and 2)
         assert!(!index.null_map.is_empty()); // Should have null values
 
-        // Create a remap that simulates compaction of frags 1 and 2 into frag 3
-        let mut row_addr_map = HashMap::<u64, Option<u64>>::new();
-        row_addr_map.insert(
-            RowAddress::new_from_parts(1, 0).into(),
-            Some(RowAddress::new_from_parts(3, 0).into()),
-        );
-        row_addr_map.insert(
-            RowAddress::new_from_parts(1, 1).into(),
-            Some(RowAddress::new_from_parts(3, 1).into()),
-        );
-        row_addr_map.insert(
-            RowAddress::new_from_parts(1, 2).into(),
-            Some(RowAddress::new_from_parts(3, 2).into()),
-        );
-        row_addr_map.insert(
-            RowAddress::new_from_parts(2, 0).into(),
-            Some(RowAddress::new_from_parts(3, 3).into()),
-        );
-        row_addr_map.insert(
-            RowAddress::new_from_parts(2, 1).into(),
-            Some(RowAddress::new_from_parts(3, 4).into()),
-        );
-        row_addr_map.insert(
-            RowAddress::new_from_parts(2, 2).into(),
-            Some(RowAddress::new_from_parts(3, 5).into()),
-        );
-
         // Perform remap
-        index
-            .remap(&row_addr_map, test_store.as_ref())
-            .await
-            .unwrap();
+        index.remap(&remap, test_store.as_ref()).await.unwrap();
 
         // Reload and check
         let reloaded_idx = BitmapIndex::load(test_store, None, &LanceCache::no_cache())
diff --git a/rust/lance-index/src/scalar/bloomfilter.rs b/rust/lance-index/src/scalar/bloomfilter.rs
index af37f982d1c..1f1ffea7ebd 100644
--- a/rust/lance-index/src/scalar/bloomfilter.rs
+++ b/rust/lance-index/src/scalar/bloomfilter.rs
@@ -7,43 +7,47 @@
 //! It is a space-efficient data structure that can be used to test whether an element is a member of a set.
 //! It's an inexact filter - they may include false positives that require rechecking.
 
-use crate::scalar::bloomfilter::sbbf::{Sbbf, SbbfBuilder};
+use crate::pb;
 use crate::scalar::expression::{BloomFilterQueryParser, ScalarQueryParser};
 use crate::scalar::registry::{
-    ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
+    BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
 };
 use crate::scalar::{
-    BloomFilterQuery, BuiltinIndexType, CreatedIndex, ScalarIndexParams, UpdateCriteria,
+    BloomFilterQuery, BuiltinIndexType, CreatedIndex, IndexFile, ScalarIndexParams, UpdateCriteria,
 };
-use crate::{Any, pb};
 use arrow_array::{Array, UInt64Array};
-mod as_bytes;
-pub mod sbbf;
 use arrow_schema::{DataType, Field};
 use lance_arrow_stats::StatisticsAccumulator;
+use lance_core::utils::bloomfilter::as_bytes;
+use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder};
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use lance_select::RowAddrTreeMap;
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 
+use std::collections::HashMap;
 use std::sync::LazyLock;
 
 use datafusion::execution::SendableRecordBatchStream;
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
 
-use crate::scalar::FragReuseIndex;
-use crate::scalar::{AnyQuery, IndexStore, MetricsCollector, ScalarIndex, SearchResult};
-use crate::vector::VectorIndex;
+use crate::scalar::{
+    AnyQuery, IndexStore, MetricsCollector, RowIdRemapper, ScalarIndex, SearchResult,
+};
 use crate::{Index, IndexType};
 use arrow_array::{ArrayRef, RecordBatch};
 use async_trait::async_trait;
-use deepsize::DeepSizeOf;
 use lance_core::Error;
 use lance_core::Result;
 use lance_core::cache::LanceCache;
+use lance_core::deepsize::DeepSizeOf;
 use roaring::RoaringBitmap;
 
 use super::zoned::{ZoneBound, ZoneProcessor, ZoneTrainer, rebuild_zones, search_zones};
 
 const BLOOMFILTER_FILENAME: &str = "bloomfilter.lance";
 const BLOOMFILTER_ITEM_META_KEY: &str = "bloomfilter_item";
+const NULL_BITMAP_META_KEY: &str = "null_bitmap";
 const BLOOMFILTER_PROBABILITY_META_KEY: &str = "bloomfilter_probability";
 const BLOOMFILTER_INDEX_VERSION: u32 = 0;
 
@@ -59,7 +63,7 @@ struct BloomFilterStatistics {
 }
 
 impl DeepSizeOf for BloomFilterStatistics {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         // Estimate the size of the bloom filter
         // We could try to get the actual size from the Sbbf if it has a method for that,
         // but for now we'll estimate based on the number of bytes it serializes to
@@ -80,18 +84,20 @@ pub struct BloomFilterIndex {
     number_of_items: u64,
     // Probability of false positives, fraction between 0 and 1
     probability: f64,
+    // Exact set of null row addresses; None for older indices without this bitmap.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl DeepSizeOf for BloomFilterIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.zones.deep_size_of_children(context)
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context)
     }
 }
 
 impl BloomFilterIndex {
     async fn load(
         store: Arc<dyn IndexStore>,
-        _fri: Option<Arc<FragReuseIndex>>,
+        _fri: Option<Arc<dyn RowIdRemapper>>,
         _index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let index_file = store.open_index_file(BLOOMFILTER_FILENAME).await?;
@@ -112,10 +118,21 @@ impl BloomFilterIndex {
             .and_then(|bs| bs.parse().ok())
             .unwrap_or(*DEFAULT_PROBABILITY);
 
+        let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) {
+            let idx = idx_str.parse::<u32>().map_err(|e| {
+                Error::invalid_input(format!("invalid null bitmap buffer index: {e}"))
+            })?;
+            let bytes = index_file.read_global_buffer(idx).await?;
+            Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?)
+        } else {
+            None
+        };
+
         Ok(Arc::new(Self::try_from_serialized(
             bloom_data,
             number_of_items,
             probability,
+            null_rows,
         )?))
     }
 
@@ -123,13 +140,14 @@ impl BloomFilterIndex {
         data: RecordBatch,
         number_of_items: u64,
         probability: f64,
+        null_rows: Option<RowAddrTreeMap>,
     ) -> Result<Self> {
         if data.num_rows() == 0 {
-            // Return empty index for empty data
             return Ok(Self {
                 zones: Vec::new(),
                 number_of_items,
                 probability,
+                null_rows,
             });
         }
 
@@ -210,6 +228,7 @@ impl BloomFilterIndex {
             zones: blocks,
             number_of_items,
             probability,
+            null_rows,
         })
     }
 
@@ -378,12 +397,6 @@ impl Index for BloomFilterIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Err(Error::invalid_input_source(
-            "BloomFilter is not a vector index".into(),
-        ))
-    }
-
     async fn prewarm(&self) -> Result<()> {
         Ok(())
     }
@@ -421,18 +434,28 @@ impl ScalarIndex for BloomFilterIndex {
         metrics: &dyn MetricsCollector,
     ) -> Result<SearchResult> {
         let query = query.as_any().downcast_ref::<BloomFilterQuery>().unwrap();
+        if let BloomFilterQuery::IsNull() = query
+            && let Some(null_rows) = &self.null_rows
+        {
+            return Ok(SearchResult::exact(null_rows.clone()));
+        }
+
         search_zones(&self.zones, metrics, |block| {
             self.evaluate_block_against_query(block, query)
         })
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        true
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
 
     async fn remap(
         &self,
-        _mapping: &HashMap<u64, Option<u64>>,
+        _mapping: &RowAddrRemap,
         _dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         Err(Error::invalid_input_source(
@@ -454,18 +477,29 @@ impl ScalarIndex for BloomFilterIndex {
 
         let processor = BloomFilterProcessor::new(params.clone())?;
         let trainer = ZoneTrainer::new(processor, params.number_of_items)?;
-        let updated_blocks = rebuild_zones(&self.zones, trainer, new_data).await?;
+        let (updated_blocks, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?;
+
+        // Merge existing and new null rows.  If the existing index had no null bitmap
+        // (legacy format — null positions unknown), preserve that None: updating cannot
+        // recover the missing information, and claiming the result has zero nulls would
+        // be a false negative.  Only a full retrain produces a fresh, complete bitmap.
+        let merged_null_rows = self.null_rows.as_ref().map(|existing| {
+            let mut merged = existing.clone();
+            merged |= &new_null_rows;
+            merged
+        });
 
         // Write the combined zones back to storage
         let mut builder = BloomFilterIndexBuilder::try_new(params)?;
         builder.blocks = updated_blocks;
-        builder.write_index(dest_store).await?;
+        builder.null_rows = merged_null_rows;
+        let files = builder.write_index(dest_store).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
                 .unwrap(),
             index_version: BLOOMFILTER_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -548,6 +582,10 @@ impl BloomFilterIndexBuilderParams {
 pub struct BloomFilterIndexBuilder {
     params: BloomFilterIndexBuilderParams,
     blocks: Vec<BloomFilterStatistics>,
+    // None means "legacy index — null positions unknown"; Some means a complete bitmap.
+    // write_index omits the null-bitmap global buffer when this is None, preserving the
+    // legacy format so that downstream searches remain conservative.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl BloomFilterIndexBuilder {
@@ -555,6 +593,7 @@ impl BloomFilterIndexBuilder {
         Ok(Self {
             params,
             blocks: Vec::new(),
+            null_rows: None,
         })
     }
 
@@ -564,7 +603,9 @@ impl BloomFilterIndexBuilder {
     pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> {
         let processor = BloomFilterProcessor::new(self.params.clone())?;
         let trainer = ZoneTrainer::new(processor, self.params.number_of_items)?;
-        self.blocks = trainer.train(batches_source).await?;
+        let (blocks, null_rows) = trainer.train(batches_source).await?;
+        self.blocks = blocks;
+        self.null_rows = Some(null_rows);
         Ok(())
     }
 
@@ -621,7 +662,7 @@ impl BloomFilterIndexBuilder {
         Ok(RecordBatch::try_new(schema, columns)?)
     }
 
-    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<()> {
+    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
         let record_batch = self.bloomfilter_stats_as_batch()?;
 
         let mut file_schema = record_batch.schema().as_ref().clone();
@@ -629,7 +670,6 @@ impl BloomFilterIndexBuilder {
             BLOOMFILTER_ITEM_META_KEY.to_string(),
             self.params.number_of_items.to_string(),
         );
-
         file_schema.metadata.insert(
             BLOOMFILTER_PROBABILITY_META_KEY.to_string(),
             self.params.probability.to_string(),
@@ -639,8 +679,24 @@ impl BloomFilterIndexBuilder {
             .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema))
             .await?;
         index_file.write_record_batch(record_batch).await?;
-        index_file.finish().await?;
-        Ok(())
+
+        let bloomfilter_file = if let Some(null_rows) = self.null_rows {
+            let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size());
+            null_rows.serialize_into(&mut null_bitmap_bytes)?;
+            let null_bitmap_idx = index_file
+                .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes))
+                .await?;
+            index_file
+                .finish_with_metadata(HashMap::from([(
+                    NULL_BITMAP_META_KEY.to_string(),
+                    null_bitmap_idx.to_string(),
+                )]))
+                .await?
+        } else {
+            index_file.finish_with_metadata(HashMap::new()).await?
+        };
+
+        Ok(vec![bloomfilter_file])
     }
 }
 
@@ -987,22 +1043,17 @@ impl BloomFilterIndexPlugin {
         batches_source: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         options: Option<BloomFilterIndexBuilderParams>,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         let mut builder = BloomFilterIndexBuilder::try_new(options.unwrap_or_default())?;
 
         builder.train(batches_source).await?;
 
-        builder.write_index(index_store).await?;
-        Ok(())
+        builder.write_index(index_store).await
     }
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for BloomFilterIndexPlugin {
-    fn name(&self) -> &str {
-        "BloomFilter"
-    }
-
+impl BasicTrainer for BloomFilterIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -1077,14 +1128,21 @@ impl ScalarIndexPlugin for BloomFilterIndexPlugin {
                     "must provide training request created by new_training_request".into(),
                 )
             })?;
-        Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?;
+        let files = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())
                 .unwrap(),
             index_version: BLOOMFILTER_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files,
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for BloomFilterIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
 
     fn provides_exact_answer(&self) -> bool {
         false
@@ -1094,6 +1152,10 @@ impl ScalarIndexPlugin for BloomFilterIndexPlugin {
         BLOOMFILTER_INDEX_VERSION
     }
 
+    fn name(&self) -> &str {
+        "BloomFilter"
+    }
+
     fn new_query_parser(
         &self,
         index_name: String,
@@ -1110,7 +1172,7 @@ impl ScalarIndexPlugin for BloomFilterIndexPlugin {
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(
@@ -1170,7 +1232,7 @@ mod tests {
     use lance_select::RowAddrTreeMap;
 
     use crate::scalar::{
-        BloomFilterQuery, ScalarIndex, SearchResult,
+        BloomFilterQuery, IndexStore, ScalarIndex, SearchResult,
         bloomfilter::{BloomFilterIndex, BloomFilterIndexBuilderParams},
         lance_format::LanceIndexStore,
     };
@@ -2038,10 +2100,10 @@ mod tests {
         expected.insert_range(500..750); // Should match the zone containing 500
         assert_eq!(result, SearchResult::at_most(expected));
 
-        // Test IsNull query
+        // Test IsNull query (no nulls in data, should return exact empty set)
         let query = BloomFilterQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); // No nulls in the data
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
 
         // Test IsIn query
         let query = BloomFilterQuery::IsIn(vec![
@@ -2134,4 +2196,152 @@ mod tests {
             _ => panic!("Expected AtMost search result from bloomfilter"),
         }
     }
+
+    // Writes a bloomfilter file in the legacy format (no null bitmap global buffer),
+    // simulating an index created before the null bitmap feature was added.
+    async fn write_legacy_bloomfilter(store: &dyn IndexStore, has_null: bool) {
+        use crate::scalar::bloomfilter::{
+            BLOOMFILTER_FILENAME, BLOOMFILTER_ITEM_META_KEY, BLOOMFILTER_PROBABILITY_META_KEY,
+        };
+        use arrow_array::BooleanArray;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+            Field::new("has_null", DataType::Boolean, false),
+            Field::new("bloom_filter_data", DataType::Binary, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![3u64])) as _,
+                Arc::new(BooleanArray::from(vec![has_null])) as _,
+                Arc::new(arrow_array::BinaryArray::from_vec(vec![b"".as_ref()])) as _,
+            ],
+        )
+        .unwrap();
+        let mut file_schema = schema.as_ref().clone();
+        file_schema
+            .metadata
+            .insert(BLOOMFILTER_ITEM_META_KEY.to_string(), "1000".to_string());
+        file_schema.metadata.insert(
+            BLOOMFILTER_PROBABILITY_META_KEY.to_string(),
+            "0.01".to_string(),
+        );
+        let mut writer = store
+            .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema))
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    // Updating a legacy (null_rows = None) index must not silently treat None as
+    // "no nulls".  The bug: `self.null_rows.clone().unwrap_or_default()` collapses
+    // None into an empty RowAddrTreeMap; after the merge the updated index has
+    // `null_rows = Some(empty)`, so an IsNull search returns `exact(empty)` — a
+    // false negative even though the legacy zone recorded has_null = true.
+    #[tokio::test]
+    async fn test_update_legacy_none_null_rows_not_treated_as_no_nulls() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Write a legacy-format index (no null bitmap) with has_null=true in its zone.
+        write_legacy_bloomfilter(store.as_ref(), true).await;
+
+        let index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert!(
+            index.null_rows.is_none(),
+            "precondition: legacy null_rows is None"
+        );
+
+        // Update with new data from fragment 1 (no nulls).  The destination is the
+        // same store so we can reload from it afterwards.
+        let new_schema = Arc::new(Schema::new(vec![
+            Field::new(VALUE_COLUMN_NAME, DataType::Int32, true),
+            Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let new_batch = RecordBatch::try_new(
+            new_schema.clone(),
+            vec![
+                Arc::new(arrow_array::Int32Array::from(vec![
+                    Some(10i32),
+                    Some(20),
+                    Some(30),
+                ])) as _,
+                Arc::new(UInt64Array::from_iter_values(
+                    (0u64..3).map(|i| (1u64 << 32) | i),
+                )) as _,
+            ],
+        )
+        .unwrap();
+        let new_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            new_schema,
+            stream::once(std::future::ready(Ok(new_batch))),
+        ));
+
+        index
+            .update(new_stream, store.as_ref(), None)
+            .await
+            .unwrap();
+
+        let updated_index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // The legacy zone had has_null=true, so there ARE nulls at unknown positions.
+        // An IsNull search on the updated index must NOT claim "no nulls" (exact empty).
+        // It must be conservative and return AtMost, falling back to the has_null scan.
+        let result = updated_index
+            .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+
+        // With the bug: null_rows = Some(empty) → returns exact(empty) ← FALSE NEGATIVE
+        // With the fix: null_rows = None        → falls through to has_null scan → AtMost
+        assert!(
+            !result.is_exact(),
+            "IsNull on an updated legacy index must not return exact(empty); \
+             the legacy zone had has_null=true so nulls exist at unknown positions"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_legacy_bloomfilter_no_null_bitmap() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_legacy_bloomfilter(store.as_ref(), true).await;
+
+        let index = BloomFilterIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .expect("failed to load legacy bloomfilter");
+
+        assert!(
+            index.null_rows.is_none(),
+            "legacy index should have no null bitmap"
+        );
+
+        // IS NULL should fall back to the has_null zone scan and return AtMost, not Exact.
+        let result = index
+            .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(
+            !result.is_exact(),
+            "IS NULL on a legacy index should not be exact"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/btree.rs b/rust/lance-index/src/scalar/btree.rs
index ba6d3dd142d..5bf89dfa29f 100644
--- a/rust/lance-index/src/scalar/btree.rs
+++ b/rust/lance-index/src/scalar/btree.rs
@@ -1,53 +1,71 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
     any::Any,
     cmp::Ordering,
-    collections::{BTreeMap, BinaryHeap, HashMap, HashSet},
+    collections::{HashMap, HashSet},
     fmt::{Debug, Display},
     ops::Bound,
     sync::Arc,
 };
 
 use super::{
-    AnyQuery, BuiltinIndexType, IndexReader, IndexStore, IndexWriter, MetricsCollector,
+    AnyQuery, BuiltinIndexType, IndexFile, IndexReader, IndexStore, IndexWriter, MetricsCollector,
     OldIndexDataFilter, SargableQuery, ScalarIndex, ScalarIndexParams, SearchResult,
     compute_next_prefix,
 };
+use crate::cache_pb::{BTreeIndexHeader, RangeToFile};
 use crate::{Index, IndexType};
+use crate::{metrics::NoOpMetricsCollector, scalar::registry::TrainingCriteria};
+use crate::{pbold, scalar::btree::flat::FlatIndex};
 use crate::{
-    frag_reuse::FragReuseIndex,
     progress::{IndexBuildProgress, noop_progress},
     scalar::{
-        CreatedIndex, UpdateCriteria,
+        CreatedIndex, RowIdRemapper, UpdateCriteria,
         expression::{SargableQueryParser, ScalarQueryParser},
-        registry::{ScalarIndexPlugin, TrainingOrdering, TrainingRequest, VALUE_COLUMN_NAME},
+        registry::{
+            BasicTrainer, ScalarIndexLoad, ScalarIndexPlugin, TrainingOrdering, TrainingRequest,
+            VALUE_COLUMN_NAME, single_flight_open,
+        },
     },
 };
-use crate::{metrics::NoOpMetricsCollector, scalar::registry::TrainingCriteria};
-use crate::{pbold, scalar::btree::flat::FlatIndex};
 use arrow_arith::numeric::add;
-use arrow_array::{Array, RecordBatch, UInt32Array, new_empty_array};
-use arrow_schema::{DataType, Field, Schema, SortOptions};
+use arrow_array::{
+    Array, ArrayAccessor, ArrowNativeTypeOp, PrimitiveArray, RecordBatch, UInt32Array,
+    cast::AsArray,
+    new_empty_array,
+    types::{
+        ArrowPrimitiveType, Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type,
+        Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
+    },
+};
+use arrow_ord::ord::make_comparator;
+use arrow_schema::{DataType, Field, IntervalUnit, Schema, SortOptions};
 use async_trait::async_trait;
 use datafusion::physical_plan::{
     ExecutionPlan, SendableRecordBatchStream,
     sorts::sort_preserving_merge::SortPreservingMergeExec, stream::RecordBatchStreamAdapter,
     union::UnionExec,
 };
-use datafusion_common::{DataFusionError, ScalarValue};
-use datafusion_physical_expr::{PhysicalSortExpr, expressions::Column};
-use deepsize::DeepSizeOf;
+use datafusion_common::{DFSchema, DataFusionError, ScalarValue};
+use datafusion_expr::execution_props::ExecutionProps;
+use datafusion_physical_expr::{
+    PhysicalExpr, PhysicalSortExpr, create_physical_expr, expressions::Column,
+};
 use futures::{
     FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt,
     future::BoxFuture,
     stream::{self},
 };
-use lance_arrow::ipc::{read_ipc_stream_single_at, write_ipc_stream};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{
     Error, ROW_ID, Result,
-    cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache},
+    cache::{
+        CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache,
+        WeakLanceCache,
+    },
     error::LanceOptionExt,
     utils::{
         tokio::get_num_compute_intensive_cpus,
@@ -58,7 +76,7 @@ use lance_datafusion::{
     chunker::chunk_concat_stream,
     exec::{LanceExecutionOptions, OneShotExec, execute_plan},
 };
-use lance_select::NullableRowAddrSet;
+use lance_select::{NullableRowAddrSet, RowSetOps};
 use log::{debug, warn};
 use object_store::Error as ObjectStoreError;
 use rangemap::RangeInclusiveMap;
@@ -68,7 +86,7 @@ use tracing::{info, instrument};
 
 mod flat;
 
-const BTREE_LOOKUP_NAME: &str = "page_lookup.lance";
+pub const BTREE_LOOKUP_NAME: &str = "page_lookup.lance";
 const BTREE_PAGES_NAME: &str = "page_data.lance";
 pub const DEFAULT_BTREE_BATCH_SIZE: u64 = 4096;
 const BATCH_SIZE_META_KEY: &str = "batch_size";
@@ -84,7 +102,7 @@ pub(crate) const BTREE_IDS_COLUMN: &str = "ids";
 pub struct OrderableScalarValue(pub ScalarValue);
 
 impl DeepSizeOf for OrderableScalarValue {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         // deepsize and size both factor in the size of the ScalarValue
         self.0.size() - std::mem::size_of::<ScalarValue>()
     }
@@ -379,6 +397,8 @@ impl Ord for OrderableScalarValue {
                 panic!("Attempt to compare List with non-List")
             }
             (LargeList(_), _) => todo!(),
+            (ListView(_), _) => todo!(),
+            (LargeListView(_), _) => todo!(),
             (Map(_), Map(_)) => todo!(),
             (Map(left), Null) => {
                 if left.is_null(0) {
@@ -580,7 +600,7 @@ impl Ord for OrderableScalarValue {
                 }
             }
             (Struct(_arr), _) => panic!("Attempt to compare Struct with non-Struct"),
-            (Dictionary(_k1, _v1), Dictionary(_k2, _v2)) => todo!(),
+            (Dictionary(_k1, v1), Dictionary(_k2, v2)) => Self(*v1.clone()).cmp(&Self(*v2.clone())),
             (Dictionary(_, v1), Null) => Self(*v1.clone()).cmp(&Self(ScalarValue::Null)),
             (Dictionary(_, _), _) => panic!("Attempt to compare Dictionary with non-Dictionary"),
             // What would a btree of unions even look like?  May not be possible.
@@ -594,44 +614,114 @@ impl Ord for OrderableScalarValue {
     }
 }
 
-#[derive(Debug, DeepSizeOf, PartialEq, Eq)]
-struct PageRecord {
-    max: OrderableScalarValue,
-    page_number: u32,
+/// Returns the first index `i` in `[lo, hi)` for which `pred(i)` is `false`.
+///
+/// `pred` must be `true` for a (possibly empty) prefix of the range and `false`
+/// for the rest, i.e. the range is partitioned by `pred`.
+fn partition_point(lo: usize, hi: usize, mut pred: impl FnMut(usize) -> bool) -> usize {
+    let mut lo = lo;
+    let mut hi = hi;
+    while lo < hi {
+        let mid = lo + (hi - lo) / 2;
+        if pred(mid) {
+            lo = mid + 1;
+        } else {
+            hi = mid;
+        }
+    }
+    lo
 }
 
-trait BTreeMapExt<K, V> {
-    fn largest_node_less(&self, key: &K) -> Option<(&K, &V)>;
+/// Builds a comparator over two array accessors of the same `Ord` item type,
+/// matching arrow's NULLs-first ascending order (`null < non-null`, `null == null`).
+///
+/// Unlike [`make_comparator`], the returned closure is generic (not boxed), so the
+/// element comparison inlines into the scan instead of dispatching through a vtable
+/// on every call.
+fn accessor_cmp<'a, T, L, R>(left: L, right: R) -> impl Fn(usize, usize) -> Ordering + 'a
+where
+    T: Ord,
+    L: ArrayAccessor<Item = T> + 'a,
+    R: ArrayAccessor<Item = T> + 'a,
+{
+    move |i, j| match (left.is_null(i), right.is_null(j)) {
+        (true, true) => Ordering::Equal,
+        (true, false) => Ordering::Less,
+        (false, true) => Ordering::Greater,
+        (false, false) => left.value(i).cmp(&right.value(j)),
+    }
 }
 
-impl<K: Ord, V> BTreeMapExt<K, V> for BTreeMap<K, V> {
-    fn largest_node_less(&self, key: &K) -> Option<(&K, &V)> {
-        self.range((Bound::Unbounded, Bound::Excluded(key)))
-            .next_back()
+/// Views `arr` as `PrimitiveArray<K>` for comparison. Zero-copy (shared buffers)
+/// when `arr` already has type `K`; otherwise — a logical type whose physical
+/// storage is `K::Native`, e.g. `Date32`/`Time32` over `i32` or `Timestamp`/
+/// `Duration` over `i64` — the array data is relabeled to `K` without copying the
+/// values, so all such logical types share one comparison path.
+fn reinterpret_primitive<K: ArrowPrimitiveType>(arr: &dyn Array) -> Result<PrimitiveArray<K>> {
+    if let Some(arr) = arr.as_primitive_opt::<K>() {
+        return Ok(arr.clone());
     }
+    let data = arr
+        .to_data()
+        .into_builder()
+        .data_type(K::DATA_TYPE)
+        .build()
+        .map_err(|e| {
+            Error::internal(format!(
+                "failed to reinterpret {} as {}: {e}",
+                arr.data_type(),
+                K::DATA_TYPE
+            ))
+        })?;
+    Ok(PrimitiveArray::<K>::from(data))
 }
 
-/// An in-memory structure that can quickly satisfy scalar queries using a btree of ScalarValue
-#[derive(Debug, DeepSizeOf, PartialEq, Eq)]
+/// Like [`accessor_cmp`] but for primitive columns, comparing native values with
+/// [`ArrowNativeTypeOp::compare`] (total order, so floats match arrow's NaN-last
+/// `make_comparator` ordering).
+fn primitive_cmp<'a, T>(
+    left: &'a PrimitiveArray<T>,
+    right: &'a PrimitiveArray<T>,
+) -> impl Fn(usize, usize) -> Ordering + 'a
+where
+    T: ArrowPrimitiveType,
+{
+    move |i, j| match (left.is_null(i), right.is_null(j)) {
+        (true, true) => Ordering::Equal,
+        (true, false) => Ordering::Less,
+        (false, true) => Ordering::Greater,
+        (false, false) => left.value(i).compare(right.value(j)),
+    }
+}
+
+/// Satisfies scalar queries by searching the `page_lookup.lance` batch directly.
+///
+/// The batch holds one row per page with columns `min | max | null_count | page_idx`,
+/// sorted ascending by `min` with NULLs first (the order the index is trained in).
+/// Both query paths binary-search the sorted `min` column for a starting row and
+/// scan forward filtering by `max`:
+///
+/// - Equality / `IN` (`candidate_pages_for_values`) dispatch on the query's
+///   *physical storage type* to a monomorphized, inlined comparator: numerics go
+///   through `scan_native` (logical types sharing a native — e.g. `Date32` and
+///   `Int32` — fold to one path), byte-likes through `scan_accessor`. Only types
+///   without a native fast path (struct-backed intervals, booleans) fall back to the
+///   boxed [`make_comparator`] via `scan_fallback`.
+/// - Range searches (`pages_between`) currently use [`make_comparator`] directly.
+#[derive(Debug, PartialEq, DeepSizeOf)]
 pub struct BTreeLookup {
-    tree: BTreeMap<OrderableScalarValue, Vec<PageRecord>>,
-    /// Pages where the value may be null (does not include all_null_pages)
+    /// One row per page (`min | max | null_count | page_idx`), sorted by `min`.
+    batch: RecordBatch,
+    /// Pages with at least one null value (does not include `all_null_pages`).
     null_pages: Vec<u32>,
-    /// Pages that are entirely null
+    /// Pages that are entirely null.
     all_null_pages: Vec<u32>,
+    /// Index of the first row whose `max` is non-null. Entirely-null pages sort to
+    /// the front (NULLs first) and are skipped when searching value ranges.
+    search_start: usize,
 }
 
-impl BTreeLookup {
-    fn empty() -> Self {
-        Self {
-            tree: BTreeMap::new(),
-            null_pages: Vec::new(),
-            all_null_pages: Vec::new(),
-        }
-    }
-}
-
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 enum Matches {
     Some(u32),
     All(u32),
@@ -647,184 +737,460 @@ impl Matches {
 }
 
 impl BTreeLookup {
-    fn new(
-        tree: BTreeMap<OrderableScalarValue, Vec<PageRecord>>,
-        null_pages: Vec<u32>,
-        all_null_pages: Vec<u32>,
-    ) -> Self {
-        Self {
-            tree,
+    /// Build a lookup over the `page_lookup.lance` batch. The batch is retained as
+    /// the source of truth; only the small null-page index lists are precomputed.
+    fn try_new(batch: RecordBatch) -> Result<Self> {
+        let mut null_pages = Vec::new();
+        let mut all_null_pages = Vec::new();
+        let mut search_start = batch.num_rows();
+
+        if batch.num_rows() > 0 {
+            let maxs = batch.column(1);
+            let null_counts = batch
+                .column(2)
+                .as_any()
+                .downcast_ref::<UInt32Array>()
+                .ok_or_else(|| Error::internal("BTree lookup null_count column must be UInt32"))?;
+            let page_numbers = batch
+                .column(3)
+                .as_any()
+                .downcast_ref::<UInt32Array>()
+                .ok_or_else(|| Error::internal("BTree lookup page_idx column must be UInt32"))?;
+
+            for idx in 0..batch.num_rows() {
+                let page_number = page_numbers.values()[idx];
+                // An entirely-null page has a null `max`; it is never searched by value.
+                if maxs.is_null(idx) {
+                    all_null_pages.push(page_number);
+                    continue;
+                }
+                if search_start == batch.num_rows() {
+                    search_start = idx;
+                }
+                if null_counts.values()[idx] > 0 {
+                    null_pages.push(page_number);
+                }
+            }
+        } else {
+            search_start = 0;
+        }
+
+        Ok(Self {
+            batch,
             null_pages,
             all_null_pages,
-        }
+            search_start,
+        })
+    }
+
+    fn page_numbers(&self) -> Result<&UInt32Array> {
+        self.batch
+            .column(3)
+            .as_any()
+            .downcast_ref::<UInt32Array>()
+            .ok_or_else(|| Error::internal("BTree lookup page_idx column must be UInt32"))
     }
 
     // All pages that could have a value equal to val
-    fn pages_eq(&self, query: &OrderableScalarValue) -> Vec<Matches> {
+    fn pages_eq(&self, query: &OrderableScalarValue) -> Result<Vec<Matches>> {
         if query.0.is_null() {
-            self.pages_null()
+            Ok(self.pages_null())
         } else {
-            self.pages_between((Bound::Included(query), Bound::Excluded(query)))
+            let query_arr = query.0.to_array_of_size(1)?;
+            let pages = self.candidate_pages_for_values(query_arr.as_ref())?;
+            Ok(pages.into_iter().map(Matches::Some).collect())
         }
     }
 
     // All pages that could have a value equal to one of the values
-    fn pages_in(&self, values: impl IntoIterator<Item = OrderableScalarValue>) -> Vec<Matches> {
-        // TODO: Right now we convert all Matches::All into Matches::Some.  We could refine this.
-        // It would improve performance on low cardinality data.
-        let page_lists = values
-            .into_iter()
-            .map(|val| {
-                self.pages_eq(&val)
-                    .into_iter()
-                    .map(|matches| matches.page_id())
-            })
-            .collect::<Vec<_>>();
-        let total_size = page_lists.iter().map(|set| set.len()).sum();
-        let mut heap = BinaryHeap::with_capacity(total_size);
-        for page_list in page_lists {
-            heap.extend(page_list);
+    fn pages_in(
+        &self,
+        values: impl IntoIterator<Item = OrderableScalarValue>,
+    ) -> Result<Vec<Matches>> {
+        // Equality lookups never produce a full-page (`Matches::All`) match because a
+        // single value cannot cover an entire page's range, so every candidate is
+        // `Matches::Some`. Refining this for low-cardinality data is the TODO in
+        // `pages_between`.
+        let values = values.into_iter();
+        let mut has_null = false;
+        let mut non_null = Vec::with_capacity(values.size_hint().0);
+        for val in values {
+            if val.0.is_null() {
+                has_null = true;
+            } else {
+                non_null.push(val.0);
+            }
+        }
+
+        // Build a single array holding every queried value so the comparators are
+        // constructed once and reused across all of them, rather than per value.
+        let mut all_pages = if non_null.is_empty() {
+            Vec::new()
+        } else {
+            let query_arr = ScalarValue::iter_to_array(non_null)?;
+            self.candidate_pages_for_values(query_arr.as_ref())?
+        };
+        if has_null {
+            all_pages.extend(self.pages_null().into_iter().map(|m| m.page_id()));
         }
-        let mut all_pages = heap.into_sorted_vec();
+        all_pages.sort_unstable();
         all_pages.dedup();
-        all_pages.into_iter().map(Matches::Some).collect()
+        Ok(all_pages.into_iter().map(Matches::Some).collect())
+    }
+
+    /// Candidate page numbers (deduped, ascending) for an equality search against
+    /// every value in `query`. A page is a candidate when its `[min, max]` range
+    /// could contain the value, i.e. `min <= value <= max`.
+    ///
+    /// The comparators are built once over the whole `query` array and reused for
+    /// each value, so an N-value `IN` costs three comparator constructions instead
+    /// of three per value.
+    fn candidate_pages_for_values(&self, query: &dyn Array) -> Result<Vec<u32>> {
+        let num_rows = self.batch.num_rows();
+        if self.search_start >= num_rows || query.is_empty() {
+            return Ok(vec![]);
+        }
+
+        let mins = self.batch.column(0).as_ref();
+        let maxs = self.batch.column(1).as_ref();
+        let page_ids = self.page_numbers()?.values();
+
+        // Compare against the page columns with a native, monomorphized comparator
+        // that inlines, rather than the boxed `DynComparator` from `make_comparator`
+        // (one vtable call per comparison). Logical types that share a physical
+        // storage type route to one path via a zero-copy reinterpret, so e.g. every
+        // date/time/timestamp/duration type reuses the `i32`/`i64` path instead of
+        // generating its own. Types with no native path (intervals with struct
+        // natives, booleans, ...) take the `make_comparator` fallback. The query
+        // array always matches the column type, so its type selects the branch.
+        use DataType::*;
+        match query.data_type() {
+            Int8 => self.scan_native::<Int8Type>(mins, maxs, query, page_ids),
+            Int16 => self.scan_native::<Int16Type>(mins, maxs, query, page_ids),
+            // i32-backed: Int32, Date32, Time32, Decimal32, year-month intervals.
+            Int32 | Date32 | Time32(_) | Decimal32(_, _) | Interval(IntervalUnit::YearMonth) => {
+                self.scan_native::<Int32Type>(mins, maxs, query, page_ids)
+            }
+            // i64-backed: Int64, Date64, Time64, Timestamp, Duration, Decimal64.
+            Int64 | Date64 | Time64(_) | Timestamp(_, _) | Duration(_) | Decimal64(_, _) => {
+                self.scan_native::<Int64Type>(mins, maxs, query, page_ids)
+            }
+            UInt8 => self.scan_native::<UInt8Type>(mins, maxs, query, page_ids),
+            UInt16 => self.scan_native::<UInt16Type>(mins, maxs, query, page_ids),
+            UInt32 => self.scan_native::<UInt32Type>(mins, maxs, query, page_ids),
+            UInt64 => self.scan_native::<UInt64Type>(mins, maxs, query, page_ids),
+            Float16 => self.scan_native::<Float16Type>(mins, maxs, query, page_ids),
+            Float32 => self.scan_native::<Float32Type>(mins, maxs, query, page_ids),
+            Float64 => self.scan_native::<Float64Type>(mins, maxs, query, page_ids),
+            Decimal128(_, _) => self.scan_native::<Decimal128Type>(mins, maxs, query, page_ids),
+            Decimal256(_, _) => self.scan_native::<Decimal256Type>(mins, maxs, query, page_ids),
+            Utf8 => Ok(self.scan_accessor(
+                mins.as_string::<i32>(),
+                maxs.as_string::<i32>(),
+                query.as_string::<i32>(),
+                page_ids,
+            )),
+            LargeUtf8 => Ok(self.scan_accessor(
+                mins.as_string::<i64>(),
+                maxs.as_string::<i64>(),
+                query.as_string::<i64>(),
+                page_ids,
+            )),
+            Binary => Ok(self.scan_accessor(
+                mins.as_binary::<i32>(),
+                maxs.as_binary::<i32>(),
+                query.as_binary::<i32>(),
+                page_ids,
+            )),
+            LargeBinary => Ok(self.scan_accessor(
+                mins.as_binary::<i64>(),
+                maxs.as_binary::<i64>(),
+                query.as_binary::<i64>(),
+                page_ids,
+            )),
+            FixedSizeBinary(_) => Ok(self.scan_accessor(
+                mins.as_fixed_size_binary(),
+                maxs.as_fixed_size_binary(),
+                query.as_fixed_size_binary(),
+                page_ids,
+            )),
+            _ => self.scan_fallback(mins, maxs, query, page_ids),
+        }
+    }
+
+    /// Native-comparator equality scan for a primitive physical type `K`. The page
+    /// columns and `query` are reinterpreted to `PrimitiveArray<K>` (zero-copy when
+    /// already that type) and compared with [`primitive_cmp`].
+    fn scan_native<K: ArrowPrimitiveType>(
+        &self,
+        mins: &dyn Array,
+        maxs: &dyn Array,
+        query: &dyn Array,
+        page_ids: &[u32],
+    ) -> Result<Vec<u32>> {
+        let mins = reinterpret_primitive::<K>(mins)?;
+        let maxs = reinterpret_primitive::<K>(maxs)?;
+        let query = reinterpret_primitive::<K>(query)?;
+        Ok(self.scan_equality_pages(
+            query.len(),
+            page_ids,
+            |idx| maxs.is_null(idx),
+            primitive_cmp(&mins, &query),
+            primitive_cmp(&maxs, &query),
+            primitive_cmp(&mins, &mins),
+        ))
+    }
+
+    /// Native-comparator equality scan for byte-like columns (`Utf8`/`Binary`/
+    /// `FixedSizeBinary` and their large variants), compared lexicographically via
+    /// [`accessor_cmp`].
+    fn scan_accessor<T, A>(&self, mins: A, maxs: A, query: A, page_ids: &[u32]) -> Vec<u32>
+    where
+        T: Ord,
+        A: ArrayAccessor<Item = T> + Copy,
+    {
+        self.scan_equality_pages(
+            query.len(),
+            page_ids,
+            |idx| maxs.is_null(idx),
+            accessor_cmp(mins, query),
+            accessor_cmp(maxs, query),
+            accessor_cmp(mins, mins),
+        )
+    }
+
+    /// Fallback equality scan for types without a native path (intervals with struct
+    /// natives, booleans, ...), using arrow's boxed `make_comparator`.
+    fn scan_fallback(
+        &self,
+        mins: &dyn Array,
+        maxs: &dyn Array,
+        query: &dyn Array,
+        page_ids: &[u32],
+    ) -> Result<Vec<u32>> {
+        // The batch is sorted ascending by `min` with NULLs first; compare the query
+        // values the same way so the binary searches stay consistent.
+        let opts = SortOptions {
+            descending: false,
+            nulls_first: true,
+        };
+        let cmp_min = make_comparator(mins, query, opts)?;
+        let cmp_max = make_comparator(maxs, query, opts)?;
+        let cmp_min_min = make_comparator(mins, mins, opts)?;
+        Ok(self.scan_equality_pages(
+            query.len(),
+            page_ids,
+            |idx| maxs.is_null(idx),
+            cmp_min,
+            cmp_max,
+            cmp_min_min,
+        ))
+    }
+
+    /// Binary-search + forward-scan the page batch for equality candidates.
+    ///
+    /// Monomorphized over the comparator closures so a typed-native comparator
+    /// inlines (no per-call vtable dispatch). The closures encode NULLs-first,
+    /// ascending order:
+    ///   * `max_is_null(i)` — whether page `i`'s `max` is null (an all-null page)
+    ///   * `cmp_min(i, j)` — page `i`'s `min` vs query value `j`
+    ///   * `cmp_max(i, j)` — page `i`'s `max` vs query value `j`
+    ///   * `cmp_min_min(i, anchor)` — two page `min`s, to expand left onto a straddle
+    fn scan_equality_pages(
+        &self,
+        num_query: usize,
+        page_ids: &[u32],
+        max_is_null: impl Fn(usize) -> bool,
+        cmp_min: impl Fn(usize, usize) -> Ordering,
+        cmp_max: impl Fn(usize, usize) -> Ordering,
+        cmp_min_min: impl Fn(usize, usize) -> Ordering,
+    ) -> Vec<u32> {
+        let num_rows = self.batch.num_rows();
+        // High-cardinality lookups hit ~one page per value; presize to avoid the
+        // element-by-element `RawVec` growth that profiling flagged.
+        let mut pages = Vec::with_capacity(num_query);
+        for j in 0..num_query {
+            // Start row: peek a little to the left of the value. A query for 7 must
+            // still reach a page like [5, 10], so we include every page whose `min`
+            // equals the largest `min` strictly less than the value.
+            let p = partition_point(0, num_rows, |i| cmp_min(i, j) == Ordering::Less);
+            let start = if p == 0 {
+                self.search_start
+            } else {
+                let anchor = p - 1;
+                partition_point(0, p, |i| cmp_min_min(i, anchor) == Ordering::Less)
+            }
+            .max(self.search_start);
+
+            // End row: pages whose `min` exceeds the value cannot match.
+            let end = partition_point(start, num_rows, |i| cmp_min(i, j) != Ordering::Greater);
+
+            // The window splits at `p` (first row with `min >= value`):
+            //   * `[start, p)` — the peek-left/straddle region (`min < value`). A page
+            //     here matches only if its `max` reaches the value, so it needs the
+            //     filter, and it may include a null-`min`/null-`max` straddle page.
+            //   * `[p, end)` — rows with `min == value`. These always match (`max >=
+            //     min == value`) and can't have a null `max` (all-null pages sort to
+            //     the front, before `search_start <= start`), so we copy them in one
+            //     slice instead of pushing per row.
+            let bulk_start = p.max(start);
+            for (offset, &page_id) in page_ids[start..bulk_start].iter().enumerate() {
+                let idx = start + offset;
+                // All-null pages are only matched by IS NULL queries.
+                if max_is_null(idx) {
+                    continue;
+                }
+                // Candidate when the page's `max` reaches the value (`max >= value`).
+                if cmp_max(idx, j) != Ordering::Less {
+                    pages.push(page_id);
+                }
+            }
+            pages.extend_from_slice(&page_ids[bulk_start..end]);
+        }
+
+        pages.sort_unstable();
+        pages.dedup();
+        pages
     }
 
     // All pages that could have a value in the range
     fn pages_between(
         &self,
         range: (Bound<&OrderableScalarValue>, Bound<&OrderableScalarValue>),
-    ) -> Vec<Matches> {
-        // We need to grab a little bit left of the given range because the query might be 7
-        // and the first page might be something like 5-10.
-        let lower_bound = match range.0 {
-            Bound::Unbounded => Bound::Unbounded,
-            // It doesn't matter if the bound is exclusive or inclusive.  We are going to grab
-            // the first node whose min is strictly less than the given bound.  Then we grab
-            // all nodes greater than or equal to that
-            //
-            // We have to peek a bit to the left because we might have something like a lower
-            // bound of 7 and there is a page [5-10] we want to search for.
-            Bound::Included(lower) => self
-                .tree
-                .largest_node_less(lower)
-                .map(|val| Bound::Included(val.0))
-                .unwrap_or(Bound::Unbounded),
-            Bound::Excluded(lower) => self
-                .tree
-                .largest_node_less(lower)
-                .map(|val| Bound::Included(val.0))
-                .unwrap_or(Bound::Unbounded),
+    ) -> Result<Vec<Matches>> {
+        let num_rows = self.batch.num_rows();
+        // No searchable (non-all-null) pages.
+        if self.search_start >= num_rows {
+            return Ok(vec![]);
+        }
+
+        let mins = self.batch.column(0).as_ref();
+        let maxs = self.batch.column(1).as_ref();
+        let page_numbers = self.page_numbers()?;
+
+        // The batch is sorted ascending by `min` with NULLs first; compare bounds
+        // the same way so the binary searches and the null `min` of a straddling
+        // page are handled consistently.
+        let opts = SortOptions {
+            descending: false,
+            nulls_first: true,
+        };
+        // Bounds become 1-row arrays of the column type so arrow's type-dispatched
+        // comparator can compare them against the `min`/`max` columns.
+        let lower_arr = match range.0 {
+            Bound::Unbounded => None,
+            Bound::Included(v) | Bound::Excluded(v) => Some(v.0.to_array_of_size(1)?),
         };
-        let upper_bound = match range.1 {
-            Bound::Unbounded => Bound::Unbounded,
-            Bound::Included(upper) => Bound::Included(upper),
-            // Even if the upper bound is excluded we need to include it on an [x, x) query.  This is because the
-            // query might be [x, x).  Our lower bound might find some [a-x] bucket and we still
-            // want to include any [x, z] bucket.
-            //
-            // We could be slightly more accurate here and only include the upper bound if the lower bound
-            // is defined, inclusive, and equal to the upper bound.  However, let's keep it simple for now.  This
-            // should only affect the probably rare case that our query is a true range query and the value
-            // matches an upper bound.  This will all be moot if/when we merge pages.
-            Bound::Excluded(upper) => Bound::Included(upper),
+        let upper_arr = match range.1 {
+            Bound::Unbounded => None,
+            Bound::Included(v) | Bound::Excluded(v) => Some(v.0.to_array_of_size(1)?),
         };
 
-        match (lower_bound, upper_bound) {
-            (Bound::Excluded(lower), Bound::Excluded(upper))
-            | (Bound::Excluded(lower), Bound::Included(upper))
-            | (Bound::Included(lower), Bound::Excluded(upper)) => {
-                // It's not really clear what (Included(5), Excluded(5)) would mean so we
-                // interpret it as an empty range which matches rust's BTreeMap behavior
-                if lower >= upper {
-                    return vec![];
+        // Start row: peek a little to the left of the lower bound. A query for 7
+        // must still reach a page like [5, 10], so we include every page whose
+        // `min` equals the largest `min` strictly less than the lower bound.
+        let start = match &lower_arr {
+            None => self.search_start,
+            Some(lower) => {
+                let cmp = make_comparator(mins, lower.as_ref(), opts)?;
+                // first row with min >= lower
+                let p = partition_point(0, num_rows, |i| cmp(i, 0) == Ordering::Less);
+                if p == 0 {
+                    self.search_start
+                } else {
+                    // first row sharing the straddling page's `min`
+                    let straddle = mins.slice(p - 1, 1);
+                    let cmp = make_comparator(mins, straddle.as_ref(), opts)?;
+                    partition_point(0, p, |i| cmp(i, 0) == Ordering::Less)
                 }
             }
-            (Bound::Included(lower), Bound::Included(upper)) => {
-                if lower > upper {
-                    return vec![];
-                }
+        }
+        .max(self.search_start);
+
+        // End row: pages whose `min` exceeds the upper bound cannot match. The
+        // upper bound is treated as inclusive even when the query bound is
+        // exclusive, so an [x, x) query still reaches a page whose `min` == x.
+        let end = match &upper_arr {
+            None => num_rows,
+            Some(upper) => {
+                let cmp = make_comparator(mins, upper.as_ref(), opts)?;
+                partition_point(start, num_rows, |i| cmp(i, 0) != Ordering::Greater)
             }
-            _ => {}
+        };
+
+        if start >= end {
+            return Ok(vec![]);
         }
 
+        // Comparators reused across the candidate rows.
+        let cmp_max_lower = lower_arr
+            .as_ref()
+            .map(|l| make_comparator(maxs, l.as_ref(), opts))
+            .transpose()?;
+        let cmp_min_lower = lower_arr
+            .as_ref()
+            .map(|l| make_comparator(mins, l.as_ref(), opts))
+            .transpose()?;
+        let cmp_max_upper = upper_arr
+            .as_ref()
+            .map(|u| make_comparator(maxs, u.as_ref(), opts))
+            .transpose()?;
+
         let mut matches = Vec::new();
+        for idx in start..end {
+            // All-null pages are only matched by IS NULL queries.
+            if maxs.is_null(idx) {
+                continue;
+            }
 
-        for (min, page_records) in self.tree.range((lower_bound, upper_bound)) {
-            for page_record in page_records {
-                match lower_bound {
-                    Bound::Unbounded => {}
-                    Bound::Included(lower) => {
-                        if page_record.max.cmp(lower) == Ordering::Less {
-                            continue;
-                        }
-                    }
-                    Bound::Excluded(lower) => {
-                        if page_record.max.cmp(lower) != Ordering::Greater {
-                            continue;
-                        }
-                    }
-                }
-                // At this point we know the page record matches at least some values.
-                // We should test to see if ALL values are a match.
+            // Candidate filter: the page's `max` reaches the lower bound.
+            let lower_ok = match (range.0, &cmp_max_lower) {
+                (Bound::Unbounded, _) => true,
+                (Bound::Included(_), Some(cmp)) => cmp(idx, 0) != Ordering::Less, // max >= lower
+                (Bound::Excluded(_), Some(cmp)) => cmp(idx, 0) == Ordering::Greater, // max > lower
+                _ => unreachable!("lower bound and its comparator are constructed together"),
+            };
+            if !lower_ok {
+                continue;
+            }
 
-                if min.0.is_null() || page_record.max.0.is_null() {
-                    // If there are nulls then we just use Matches::Some
-                    matches.push(Matches::Some(page_record.page_number));
-                    continue;
-                }
+            let page_number = page_numbers.values()[idx];
 
-                match range.0 {
-                    // range.0 < X therefore if the smallest value is not strictly greater than
-                    // the lower bound we only have partial match
-                    Bound::Excluded(lower) => {
-                        if min.cmp(lower) != Ordering::Greater {
-                            matches.push(Matches::Some(page_record.page_number));
-                            continue;
-                        }
-                    }
-                    // range.0 <= X therefore if the smallest value is not greater than or equal
-                    // to the lower bound we only have partial match
-                    Bound::Included(lower) => {
-                        if min.cmp(lower) == Ordering::Less {
-                            matches.push(Matches::Some(page_record.page_number));
-                            continue;
-                        }
-                    }
-                    Bound::Unbounded => {}
-                }
-                match range.1 {
-                    // X < range.1 therefore if the largest value is not strictly less than
-                    // the upper bound we only have partial match
-                    Bound::Excluded(upper) => {
-                        if page_record.max.cmp(upper) != Ordering::Less {
-                            matches.push(Matches::Some(page_record.page_number));
-                            continue;
-                        }
-                    }
-                    // X <= range.1 therefore if the largest value is not less than or equal to
-                    // the upper bound we only have partial match
-                    Bound::Included(upper) => {
-                        if page_record.max.cmp(upper) == Ordering::Greater {
-                            matches.push(Matches::Some(page_record.page_number));
-                            continue;
-                        }
-                    }
-                    Bound::Unbounded => {}
-                }
-                // The min is greater than the lower bound and the max is less than the upper bound
-                // so we have a full match
-                matches.push(Matches::All(page_record.page_number));
+            // A page with a null `min` straddles the NULL/non-NULL boundary, so it
+            // is only ever a partial match.
+            if mins.is_null(idx) {
+                matches.push(Matches::Some(page_number));
+                continue;
+            }
+
+            // Full match requires the page to sit entirely within the query range.
+            let lower_full = match (range.0, &cmp_min_lower) {
+                (Bound::Unbounded, _) => true,
+                (Bound::Included(_), Some(cmp)) => cmp(idx, 0) != Ordering::Less, // min >= lower
+                (Bound::Excluded(_), Some(cmp)) => cmp(idx, 0) == Ordering::Greater, // min > lower
+                _ => unreachable!("lower bound and its comparator are constructed together"),
+            };
+            let upper_full = match (range.1, &cmp_max_upper) {
+                (Bound::Unbounded, _) => true,
+                (Bound::Included(_), Some(cmp)) => cmp(idx, 0) != Ordering::Greater, // max <= upper
+                (Bound::Excluded(_), Some(cmp)) => cmp(idx, 0) == Ordering::Less,    // max < upper
+                _ => unreachable!("upper bound and its comparator are constructed together"),
+            };
+            if lower_full && upper_full {
+                matches.push(Matches::All(page_number));
+            } else {
+                matches.push(Matches::Some(page_number));
             }
         }
 
-        matches
+        Ok(matches)
     }
 
     fn pages_null(&self) -> Vec<Matches> {
         self.null_pages
             .iter()
-            .map(|page_id| Matches::Some(*page_id))
+            .copied()
+            .map(Matches::Some)
             .chain(self.all_null_pages.iter().copied().map(Matches::All))
             .collect()
     }
@@ -1013,26 +1379,37 @@ impl CacheKey for BTreePageKey {
 /// `BTreeIndex::try_from_serialized` reconstructs the in-memory lookup with
 /// no IO) plus the page batch size and range-partition map.
 #[derive(Debug, Clone)]
-pub struct BTreeIndexState {
+struct BTreeIndexState {
     lookup_batch: RecordBatch,
     batch_size: u64,
     ranges_to_files: Option<Arc<RangeInclusiveMap<u32, (String, u32)>>>,
 }
 
 impl DeepSizeOf for BTreeIndexState {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // `ranges_to_files` is tiny and `RangeInclusiveMap` is not `DeepSizeOf`;
         // the lookup batch dominates, matching how `BTreeIndex` accounts for itself.
-        self.lookup_batch.get_array_memory_size()
+        self.lookup_batch.deep_size_of_children(context)
     }
 }
 
 impl BTreeIndexState {
+    fn from_index(index: &dyn ScalarIndex) -> Result<Self> {
+        let btree = index.as_any().downcast_ref::<BTreeIndex>().ok_or_else(|| {
+            Error::internal("BTreeIndexState::from_index called with a non-BTree index")
+        })?;
+        Ok(Self {
+            lookup_batch: btree.page_lookup.batch.clone(),
+            batch_size: btree.batch_size,
+            ranges_to_files: btree.ranges_to_files.clone(),
+        })
+    }
+
     fn reconstruct(
         &self,
         store: Arc<dyn IndexStore>,
         index_cache: &LanceCache,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Arc<dyn ScalarIndex>> {
         let index = BTreeIndex::try_from_serialized(
             self.lookup_batch.clone(),
@@ -1047,106 +1424,58 @@ impl BTreeIndexState {
 }
 
 impl CacheCodecImpl for BTreeIndexState {
-    /// Wire format (no stability guarantees yet — the cache is rebuilt from
-    /// source on any version mismatch):
+    const TYPE_ID: &'static str = "lance.scalar.BTreeIndexState";
+    const CURRENT_VERSION: u32 = 1;
+
+    /// Wire format:
     /// ```text
-    /// u64 batch_size (LE)
-    /// u8  has_ranges (0 = None, 1 = Some)
-    /// if has_ranges:
-    ///   u32 entry_count (LE)
-    ///   per entry: u32 start | u32 end | u32 offset | u32 path_len | path bytes
-    /// lookup batch (Arrow IPC stream)
+    /// HEADER    : BTreeIndexHeader proto (batch_size + page-range mapping)
+    /// ARROW_IPC : page-lookup batch
     /// ```
-    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()> {
-        writer.write_all(&self.batch_size.to_le_bytes())?;
-        match &self.ranges_to_files {
-            None => writer.write_all(&[0u8])?,
-            Some(ranges) => {
-                writer.write_all(&[1u8])?;
-                let count = u32::try_from(ranges.len()).map_err(|_| {
-                    Error::io("BTreeIndexState: ranges_to_files exceeds u32::MAX entries")
-                })?;
-                writer.write_all(&count.to_le_bytes())?;
-                for (range, (path, page_offset)) in ranges.iter() {
-                    writer.write_all(&range.start().to_le_bytes())?;
-                    writer.write_all(&range.end().to_le_bytes())?;
-                    writer.write_all(&page_offset.to_le_bytes())?;
-                    let path_len = u32::try_from(path.len()).map_err(|_| {
-                        Error::io("BTreeIndexState: ranges_to_files path exceeds u32::MAX bytes")
-                    })?;
-                    writer.write_all(&path_len.to_le_bytes())?;
-                    writer.write_all(path.as_bytes())?;
-                }
-            }
-        }
-        write_ipc_stream(&self.lookup_batch, writer)?;
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let ranges_to_files = match &self.ranges_to_files {
+            None => Vec::new(),
+            Some(ranges) => ranges
+                .iter()
+                .map(|(range, (path, page_offset))| RangeToFile {
+                    start: *range.start(),
+                    end: *range.end(),
+                    page_offset: *page_offset,
+                    path: path.clone(),
+                })
+                .collect(),
+        };
+        let header = BTreeIndexHeader {
+            batch_size: self.batch_size,
+            has_ranges_to_files: self.ranges_to_files.is_some(),
+            ranges_to_files,
+        };
+        w.write_header(&header)?;
+        w.write_ipc(&self.lookup_batch)?;
         Ok(())
     }
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let batch_size = read_u64_le(data, &mut offset)?;
-        let has_ranges = read_u8(data, &mut offset)?;
-        let ranges_to_files = match has_ranges {
-            0 => None,
-            1 => {
-                let count = read_u32_le(data, &mut offset)? as usize;
-                let mut entries = Vec::with_capacity(count);
-                for _ in 0..count {
-                    let start = read_u32_le(data, &mut offset)?;
-                    let end = read_u32_le(data, &mut offset)?;
-                    let page_offset = read_u32_le(data, &mut offset)?;
-                    let path_len = read_u32_le(data, &mut offset)? as usize;
-                    let path = read_bytes(data, &mut offset, path_len)?;
-                    let path = std::str::from_utf8(&path)
-                        .map_err(|e| Error::io(format!("BTreeIndexState path: {e}")))?
-                        .to_string();
-                    entries.push((start..=end, (path, page_offset)));
-                }
-                Some(Arc::new(entries.into_iter().collect()))
-            }
-            other => {
-                return Err(Error::io(format!(
-                    "BTreeIndexState: invalid has_ranges tag {other}"
-                )));
-            }
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: BTreeIndexHeader = r.read_header()?;
+        let ranges_to_files = if header.has_ranges_to_files {
+            let map: RangeInclusiveMap<u32, (String, u32)> = header
+                .ranges_to_files
+                .into_iter()
+                .map(|entry| (entry.start..=entry.end, (entry.path, entry.page_offset)))
+                .collect();
+            Some(Arc::new(map))
+        } else {
+            None
         };
-        let lookup_batch = read_ipc_stream_single_at(data, &mut offset)?;
+        let lookup_batch = r.read_ipc()?;
         Ok(Self {
             lookup_batch,
-            batch_size,
+            batch_size: header.batch_size,
             ranges_to_files,
         })
     }
 }
 
-fn read_bytes(data: &bytes::Bytes, offset: &mut usize, len: usize) -> Result<bytes::Bytes> {
-    if data.len() < *offset + len {
-        return Err(Error::io(format!(
-            "BTreeIndexState: short read of {len} bytes at offset {offset} (have {})",
-            data.len()
-        )));
-    }
-    let slice = data.slice(*offset..*offset + len);
-    *offset += len;
-    Ok(slice)
-}
-
-fn read_u8(data: &bytes::Bytes, offset: &mut usize) -> Result<u8> {
-    let bytes = read_bytes(data, offset, 1)?;
-    Ok(bytes[0])
-}
-
-fn read_u32_le(data: &bytes::Bytes, offset: &mut usize) -> Result<u32> {
-    let bytes = read_bytes(data, offset, 4)?;
-    Ok(u32::from_le_bytes(bytes.as_ref().try_into().unwrap()))
-}
-
-fn read_u64_le(data: &bytes::Bytes, offset: &mut usize) -> Result<u64> {
-    let bytes = read_bytes(data, offset, 8)?;
-    Ok(u64::from_le_bytes(bytes.as_ref().try_into().unwrap()))
-}
-
 /// Cache key for a [`BTreeIndexState`]. The cache it is used with is already
 /// namespaced per-index, so the key string is a constant.
 struct BTreeIndexStateKey;
@@ -1206,27 +1535,15 @@ pub struct BTreeIndex {
     /// - The local page_idx is calculated: `142 - 100 = 42`.
     /// - The system now knows to read page `42` from the file `part_2_page_file.lance`.
     ranges_to_files: Option<Arc<RangeInclusiveMap<u32, (String, u32)>>>,
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
-
-    /// The raw lookup batch this index was built from (the contents of
-    /// `page_lookup.lance`). Retained so the index can be serialized into a
-    /// cache as a [`BTreeIndexState`] without re-reading it from storage.
-    ///
-    /// TODO: this duplicates the min/max values already held in `page_lookup`.
-    /// A follow-up could rewrite `BTreeLookup` to query this batch directly
-    /// (binary search on the sorted `min` column + linear scan, type-dispatched
-    /// per column type), eliminating the duplication and making this batch the
-    /// single source of truth.
-    lookup_batch: RecordBatch,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
 }
 
 impl DeepSizeOf for BTreeIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // We don't include the index cache, or anything stored in it. For example:
-        // sub_index and fri.
-        self.page_lookup.deep_size_of_children(context)
-            + self.store.deep_size_of_children(context)
-            + self.lookup_batch.get_array_memory_size()
+        // sub_index and fri. `page_lookup` owns the lookup batch (the single source
+        // of truth), so accounting for it covers the lookup data.
+        self.page_lookup.deep_size_of_children(context) + self.store.deep_size_of_children(context)
     }
 }
 
@@ -1239,8 +1556,7 @@ impl BTreeIndex {
         index_cache: WeakLanceCache,
         batch_size: u64,
         ranges_to_files: Option<Arc<RangeInclusiveMap<u32, (String, u32)>>>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
-        lookup_batch: RecordBatch,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Self {
         Self {
             page_lookup,
@@ -1250,10 +1566,69 @@ impl BTreeIndex {
             batch_size,
             ranges_to_files,
             frag_reuse_index,
-            lookup_batch,
         }
     }
 
+    /// For each key in `keys`, whether this index contains it — a batched
+    /// existence check returning a mask aligned to `keys`.
+    ///
+    /// The per-key sibling of `search(Equals(..))`, but one call replaces N
+    /// probes: keys are grouped by page using the same page resolution as
+    /// [`ScalarIndex::search`] (`pages_eq`), each touched page is loaded once
+    /// (session-cached), and membership is tested against the page's values via
+    /// `FlatIndex::contains_values`. Avoids the per-key `SearchResult` /
+    /// `RowAddrTreeMap` allocation when the caller only wants a yes/no.
+    ///
+    /// Intended for primary-key dedup, where keys are non-null; a null key maps
+    /// to `false`.
+    pub async fn contains_keys(
+        &self,
+        keys: &[ScalarValue],
+        metrics: &dyn MetricsCollector,
+    ) -> Result<Vec<bool>> {
+        // Group each key (by input position) under every page whose value range
+        // could hold it. Mirrors `search`'s page selection so the two agree.
+        let mut by_page: HashMap<u32, Vec<(usize, OrderableScalarValue)>> = HashMap::new();
+        for (idx, key) in keys.iter().enumerate() {
+            if key.is_null() {
+                continue;
+            }
+            let ov = OrderableScalarValue(key.clone());
+            for matches in self.page_lookup.pages_eq(&ov)? {
+                by_page
+                    .entry(matches.page_id())
+                    .or_default()
+                    .push((idx, ov.clone()));
+            }
+        }
+
+        let index_reader = LazyIndexReader::new(self.store.clone(), self.ranges_to_files.clone());
+        let page_tasks = by_page.into_iter().map(|(page_number, entries)| {
+            let index_reader = index_reader.clone();
+            async move {
+                let page = self.lookup_page(page_number, index_reader, metrics).await?;
+                let needles: Vec<OrderableScalarValue> =
+                    entries.iter().map(|(_, ov)| ov.clone()).collect();
+                let present = page.contains_values(&needles)?;
+                Result::Ok((entries, present))
+            }
+        });
+
+        let mut result = vec![false; keys.len()];
+        let page_results: Vec<_> = stream::iter(page_tasks)
+            .buffer_unordered(get_num_compute_intensive_cpus())
+            .try_collect()
+            .await?;
+        for (entries, present) in page_results {
+            for (idx, ov) in entries {
+                if present.contains(&ov) {
+                    result[idx] = true;
+                }
+            }
+        }
+        Ok(result)
+    }
+
     async fn lookup_page(
         &self,
         page_number: u32,
@@ -1287,11 +1662,28 @@ impl BTreeIndex {
         FlatIndex::try_new(serialized_page)
     }
 
+    /// Compile a sargable predicate into a physical expr against the per-page
+    /// schema ([values, ids]). Built once in `search` and shared across pages so
+    /// a large IN-list is not re-materialized for every page.
+    fn compile_predicate(&self, query: &SargableQuery) -> Result<Arc<dyn PhysicalExpr>> {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(BTREE_VALUES_COLUMN, self.data_type.clone(), true),
+            Field::new(BTREE_IDS_COLUMN, DataType::UInt64, false),
+        ]));
+        let df_schema = DFSchema::try_from(schema)?;
+        Ok(create_physical_expr(
+            &query.to_expr(BTREE_VALUES_COLUMN.to_string()),
+            &df_schema,
+            &ExecutionProps::default(),
+        )?)
+    }
+
     async fn search_page(
         &self,
         query: &SargableQuery,
         matches: Matches,
         index_reader: LazyIndexReader,
+        prebuilt: Option<&Arc<dyn PhysicalExpr>>,
         metrics: &dyn MetricsCollector,
     ) -> Result<NullableRowAddrSet> {
         let subindex = self
@@ -1299,13 +1691,12 @@ impl BTreeIndex {
             .await?;
 
         match matches {
-            Matches::Some(_) => {
-                // TODO: If this is an IN query we can perhaps simplify the subindex query by restricting it to the
-                // values that might be in the page.  E.g. if we are searching for X IN [5, 3, 7] and five is in pages
-                // 1 and 2 and three is in page 2 and seven is in pages 8 and 9, then when searching page 2 we only need
-                // to search for X IN [5, 3]
-                subindex.search(query, metrics)
-            }
+            // For a large IsIn the predicate is compiled once (see `search`) and
+            // reused here, instead of rebuilding the whole IN-list per page.
+            Matches::Some(_) => match prebuilt {
+                Some(expr) => subindex.search_prebuilt(expr, metrics),
+                None => subindex.search(query, metrics),
+            },
             Matches::All(_) => Ok(match query {
                 // This means we hit an all-null page so just grab all row ids as true
                 SargableQuery::IsNull() => subindex.all_ignore_nulls(),
@@ -1321,70 +1712,10 @@ impl BTreeIndex {
         index_cache: &LanceCache,
         batch_size: u64,
         ranges_to_files: Option<Arc<RangeInclusiveMap<u32, (String, u32)>>>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Self> {
-        let mut map = BTreeMap::<OrderableScalarValue, Vec<PageRecord>>::new();
-        // Pages that have at least one null value
-        let mut null_pages = Vec::<u32>::new();
-        // Pages that are entirely null
-        let mut all_null_pages = Vec::<u32>::new();
-
-        if data.num_rows() == 0 {
-            let data_type = data.column(0).data_type().clone();
-            let page_lookup = Arc::new(BTreeLookup::empty());
-            return Ok(Self::new(
-                page_lookup,
-                store,
-                data_type,
-                WeakLanceCache::from(index_cache),
-                batch_size,
-                ranges_to_files,
-                frag_reuse_index,
-                data,
-            ));
-        }
-
-        let mins = data.column(0);
-        let maxs = data.column(1);
-        let null_counts = data
-            .column(2)
-            .as_any()
-            .downcast_ref::<UInt32Array>()
-            .unwrap();
-        let page_numbers = data
-            .column(3)
-            .as_any()
-            .downcast_ref::<UInt32Array>()
-            .unwrap();
-
-        for idx in 0..data.num_rows() {
-            let min = OrderableScalarValue(ScalarValue::try_from_array(&mins, idx)?);
-            let max = OrderableScalarValue(ScalarValue::try_from_array(&maxs, idx)?);
-            let null_count = null_counts.values()[idx];
-            let page_number = page_numbers.values()[idx];
-
-            // If the page is entirely null don't even bother putting it in the tree
-            if max.0.is_null() {
-                all_null_pages.push(page_number);
-                // continue so we don't add it to the null_pages
-                continue;
-            } else {
-                map.entry(min)
-                    .or_default()
-                    .push(PageRecord { max, page_number });
-            }
-
-            if null_count > 0 {
-                null_pages.push(page_number);
-            }
-        }
-
-        let last_max = ScalarValue::try_from_array(&maxs, data.num_rows() - 1)?;
-        map.entry(OrderableScalarValue(last_max)).or_default();
-
-        let data_type = mins.data_type().clone();
-
-        let page_lookup = Arc::new(BTreeLookup::new(map, null_pages, all_null_pages));
+        let data_type = data.column(0).data_type().clone();
+        let page_lookup = Arc::new(BTreeLookup::try_new(data)?);
 
         Ok(Self::new(
             page_lookup,
@@ -1394,13 +1725,12 @@ impl BTreeIndex {
             batch_size,
             ranges_to_files,
             frag_reuse_index,
-            data,
         ))
     }
 
     async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let (page_lookup_file, standalone_partition_page_file) =
@@ -1489,7 +1819,7 @@ impl BTreeIndex {
     }
 
     /// Create a stream of all the data in the index, in the same format used to train the index
-    async fn into_data_stream(self) -> Result<SendableRecordBatchStream> {
+    async fn data_stream(&self) -> Result<SendableRecordBatchStream> {
         let lazy_reader = LazyIndexReader::new(self.store.clone(), self.ranges_to_files.clone());
         let reader = lazy_reader.get().await?;
         let new_schema = Arc::new(self.train_schema());
@@ -1512,25 +1842,66 @@ impl BTreeIndex {
         )))
     }
 
-    async fn combine_old_new(
-        self,
+    /// Merge N source BTree segments plus an additional `new_data` stream into
+    /// a single BTree under `dest_store`, without re-reading the dataset.
+    pub async fn merge_segments(
+        segments: &[Arc<Self>],
         new_data: SendableRecordBatchStream,
-        chunk_size: u64,
-        old_data_filter: Option<OldIndexDataFilter>,
-    ) -> Result<SendableRecordBatchStream> {
-        let value_column_index = new_data.schema().index_of(VALUE_COLUMN_NAME)?;
-
-        let new_input = Arc::new(OneShotExec::new(new_data));
-        let old_stream = self.into_data_stream().await?;
-        let old_stream = match old_data_filter {
-            Some(filter) => filter_row_ids(old_stream, filter),
-            None => old_stream,
+        dest_store: &dyn IndexStore,
+        old_data_filters: &[Option<OldIndexDataFilter>],
+    ) -> Result<CreatedIndex> {
+        let Some(first) = segments.first() else {
+            return Err(Error::invalid_input(
+                "cannot merge BTree index without at least one source segment".to_string(),
+            ));
         };
-        let old_input = Arc::new(OneShotExec::new(old_stream));
-        debug_assert_eq!(
-            old_input.schema().flattened_fields().len(),
-            new_input.schema().flattened_fields().len()
-        );
+
+        if old_data_filters.len() != segments.len() {
+            return Err(Error::invalid_input(format!(
+                "BTree merge: expected one old-data filter per source segment \
+                 (segments={}, filters={})",
+                segments.len(),
+                old_data_filters.len()
+            )));
+        }
+
+        for segment in segments.iter().skip(1) {
+            if segment.data_type != first.data_type {
+                return Err(Error::index(format!(
+                    "cannot merge BTree segments with different value types ({:?} vs {:?})",
+                    first.data_type, segment.data_type
+                )));
+            }
+        }
+
+        let new_schema = new_data.schema();
+        let value_column_index = new_schema.index_of(VALUE_COLUMN_NAME)?;
+        let new_value_type = new_schema.field(value_column_index).data_type();
+        if new_value_type != &first.data_type {
+            return Err(Error::invalid_input(format!(
+                "BTree merge: new_data value column type {:?} does not match \
+                 segment value type {:?}",
+                new_value_type, first.data_type
+            )));
+        }
+
+        let mut inputs: Vec<Arc<dyn ExecutionPlan>> = Vec::with_capacity(segments.len() + 1);
+        for (segment, old_data_filter) in segments.iter().zip(old_data_filters) {
+            if filter_keeps_nothing(old_data_filter) {
+                continue;
+            }
+            let stream = segment.data_stream().await?;
+            let stream = match segment.frag_reuse_index.clone() {
+                Some(frag_reuse_index) => remap_row_ids(stream, frag_reuse_index),
+                None => stream,
+            };
+            let stream = match old_data_filter.clone() {
+                Some(filter) => filter_row_ids(stream, filter),
+                None => stream,
+            };
+            inputs.push(Arc::new(OneShotExec::new(stream)));
+        }
+        inputs.push(Arc::new(OneShotExec::new(new_data)));
 
         let sort_expr = PhysicalSortExpr {
             expr: Arc::new(Column::new(VALUE_COLUMN_NAME, value_column_index)),
@@ -1539,11 +1910,10 @@ impl BTreeIndex {
                 nulls_first: true,
             },
         };
-        // The UnionExec creates multiple partitions but the SortPreservingMergeExec merges
-        // them back into a single partition.
-        let all_data = UnionExec::try_new(vec![old_input, new_input])?;
-        let ordered = Arc::new(SortPreservingMergeExec::new([sort_expr].into(), all_data));
-
+        // UnionExec yields multiple partitions; SortPreservingMergeExec merges
+        // them back into a single partition while preserving value-ordering.
+        let unioned = UnionExec::try_new(inputs)?;
+        let ordered = Arc::new(SortPreservingMergeExec::new([sort_expr].into(), unioned));
         let unchunked = execute_plan(
             ordered,
             LanceExecutionOptions {
@@ -1551,7 +1921,17 @@ impl BTreeIndex {
                 ..Default::default()
             },
         )?;
-        Ok(chunk_concat_stream(unchunked, chunk_size as usize))
+        let merged_stream = chunk_concat_stream(unchunked, first.batch_size as usize);
+
+        let files =
+            train_btree_index(merged_stream, dest_store, first.batch_size, None, None).await?;
+
+        Ok(CreatedIndex {
+            index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
+                .unwrap(),
+            index_version: BTREE_INDEX_VERSION,
+            files,
+        })
     }
 }
 
@@ -1574,6 +1954,28 @@ fn filter_row_ids(
     Box::pin(RecordBatchStreamAdapter::new(schema, filtered))
 }
 
+/// True if `filter` would keep no rows at all (its keep-set is empty), letting
+/// the merge skip reading the segment entirely.
+fn filter_keeps_nothing(filter: &Option<OldIndexDataFilter>) -> bool {
+    match filter {
+        Some(OldIndexDataFilter::Fragments { to_keep, .. }) => to_keep.is_empty(),
+        Some(OldIndexDataFilter::RowIds(valid)) => valid.is_empty(),
+        None => false,
+    }
+}
+
+fn remap_row_ids(
+    stream: SendableRecordBatchStream,
+    frag_reuse_index: Arc<dyn RowIdRemapper>,
+) -> SendableRecordBatchStream {
+    let schema = stream.schema();
+    let remapped = stream.map(move |batch_result| {
+        let batch = batch_result?;
+        Ok(frag_reuse_index.remap_row_ids_record_batch(batch, 1)?)
+    });
+    Box::pin(RecordBatchStreamAdapter::new(schema, remapped))
+}
+
 fn wrap_bound(bound: &Bound<ScalarValue>) -> Bound<OrderableScalarValue> {
     match bound {
         Bound::Unbounded => Bound::Unbounded,
@@ -1612,12 +2014,6 @@ impl Index for BTreeIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::not_supported_source(
-            "BTreeIndex is not vector index".into(),
-        ))
-    }
-
     async fn prewarm(&self) -> Result<()> {
         let index_reader = LazyIndexReader::new(self.store.clone(), self.ranges_to_files.clone());
         let reader = index_reader.get().await?;
@@ -1660,18 +2056,25 @@ impl Index for BTreeIndex {
     }
 
     fn statistics(&self) -> Result<serde_json::Value> {
-        let min = self
-            .page_lookup
-            .tree
-            .first_key_value()
-            .map(|(k, _)| k.clone());
-        let max = self
-            .page_lookup
-            .tree
-            .last_key_value()
-            .map(|(k, _)| k.clone());
+        let lookup = &self.page_lookup;
+        let batch = &lookup.batch;
+        let num_rows = batch.num_rows();
+        // The batch is sorted by `min`, so the smallest searchable value is the
+        // `min` of the first non-all-null page and the largest is the `max` of the
+        // last page.
+        let (min, max) = if lookup.search_start >= num_rows {
+            (None, None)
+        } else {
+            let min = OrderableScalarValue(ScalarValue::try_from_array(
+                batch.column(0),
+                lookup.search_start,
+            )?);
+            let max =
+                OrderableScalarValue(ScalarValue::try_from_array(batch.column(1), num_rows - 1)?);
+            (Some(min), Some(max))
+        };
         serde_json::to_value(&BTreeStatistics {
-            num_pages: self.page_lookup.tree.len() as u32,
+            num_pages: num_rows as u32,
             min,
             max,
         })
@@ -1718,7 +2121,7 @@ impl ScalarIndex for BTreeIndex {
                     "full text search is not supported for BTree index, build a inverted index for it",
                 ));
             }
-            SargableQuery::IsNull() => self.page_lookup.pages_null(),
+            SargableQuery::IsNull() => Ok(self.page_lookup.pages_null()),
             SargableQuery::LikePrefix(prefix) => {
                 // Convert LikePrefix to a range query: [prefix, next_prefix)
                 match prefix {
@@ -1752,7 +2155,7 @@ impl ScalarIndex for BTreeIndex {
                     }
                 }
             }
-        };
+        }?;
 
         // For non-IsNull queries, also include null pages so that null row IDs
         // are tracked in the result. Any comparison with NULL yields NULL, and
@@ -1763,6 +2166,11 @@ impl ScalarIndex for BTreeIndex {
         // We add them as Matches::Some (not Matches::All) so that
         // FlatIndex::search() evaluates the predicate and correctly marks
         // the rows as NULL rather than TRUE.
+        //
+        // TODO: the lookup batch retains a per-page `null_count`. A fully-covered
+        // page with zero nulls is a true Matches::All, while one with nulls needs
+        // Matches::Some only to track the null rows; surfacing `null_count` here
+        // could refine that classification (see #6802).
         if !matches!(query, SargableQuery::IsNull()) {
             let existing: HashSet<u32> = pages.iter().map(|m| m.page_id()).collect();
             for &page_id in self
@@ -1777,13 +2185,27 @@ impl ScalarIndex for BTreeIndex {
             }
         }
 
+        // Compile a large IsIn predicate once and reuse it across every page;
+        // rebuilding the full IN-list per page is O(pages * values) and dominates
+        // the lookup for sets with many values.
+        let prebuilt = match query {
+            SargableQuery::IsIn(_) => Some(self.compile_predicate(query)?),
+            _ => None,
+        };
+
         let lazy_index_reader =
             LazyIndexReader::new(self.store.clone(), self.ranges_to_files.clone());
         let page_tasks = pages
             .into_iter()
             .map(|page_index| {
-                self.search_page(query, page_index, lazy_index_reader.clone(), metrics)
-                    .boxed()
+                self.search_page(
+                    query,
+                    page_index,
+                    lazy_index_reader.clone(),
+                    prebuilt.as_ref(),
+                    metrics,
+                )
+                .boxed()
             })
             .collect::<Vec<_>>();
         debug!("Searching {} btree pages", page_tasks.len());
@@ -1808,7 +2230,7 @@ impl ScalarIndex for BTreeIndex {
 
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         // (part_id, path)
@@ -1830,6 +2252,7 @@ impl ScalarIndex for BTreeIndex {
 
         let mapping = Arc::new(mapping.clone());
         let train_schema = Arc::new(self.train_schema());
+        let mut remapped_files = Vec::new();
 
         // TODO: Could potentially parallelize this across parts, unclear it would be worth it
         for (part_id, page_file) in part_page_files {
@@ -1860,7 +2283,10 @@ impl ScalarIndex for BTreeIndex {
                 remapped_stream,
             ));
 
-            train_btree_index(remapped_stream, dest_store, self.batch_size, None, part_id).await?;
+            let mut files =
+                train_btree_index(remapped_stream, dest_store, self.batch_size, None, part_id)
+                    .await?;
+            remapped_files.append(&mut files);
         }
 
         if let Some(ranges_to_files) = &self.ranges_to_files {
@@ -1872,7 +2298,7 @@ impl ScalarIndex for BTreeIndex {
             let lookup_files = (0..num_parts)
                 .map(|part_id| part_lookup_file_path((part_id as u64) << 32))
                 .collect::<Vec<_>>();
-            merge_metadata_files(
+            let merged_files = merge_metadata_files(
                 dest_store,
                 &page_files,
                 &lookup_files,
@@ -1880,13 +2306,15 @@ impl ScalarIndex for BTreeIndex {
                 noop_progress(),
             )
             .await?;
+            remapped_files.retain(|file| file.path.ends_with("_page_data.lance"));
+            remapped_files.extend(merged_files);
         }
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
                 .unwrap(),
             index_version: BTREE_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: remapped_files,
         })
     }
 
@@ -1896,19 +2324,15 @@ impl ScalarIndex for BTreeIndex {
         dest_store: &dyn IndexStore,
         old_data_filter: Option<OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        // Merge the existing index data with the new data and then retrain the index on the merged stream
-        let merged_data_source = self
-            .clone()
-            .combine_old_new(new_data, self.batch_size, old_data_filter)
-            .await?;
-        train_btree_index(merged_data_source, dest_store, self.batch_size, None, None).await?;
-
-        Ok(CreatedIndex {
-            index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
-                .unwrap(),
-            index_version: BTREE_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
-        })
+        // Updating is the single-segment case of a segment merge: union this
+        // index's data with `new_data`, re-sort on value, and retrain.
+        Self::merge_segments(
+            &[Arc::new(self.clone())],
+            new_data,
+            dest_store,
+            &[old_data_filter],
+        )
+        .await
     }
 
     fn update_criteria(&self) -> UpdateCriteria {
@@ -1973,7 +2397,7 @@ pub trait BTreeSubIndex: Debug + Send + Sync + DeepSizeOf {
     async fn remap_subindex(
         &self,
         serialized: RecordBatch,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
     ) -> Result<RecordBatch>;
 }
 
@@ -2045,7 +2469,7 @@ pub async fn train_btree_index(
     batch_size: u64,
     fragment_ids: Option<Vec<u32>>,
     range_id: Option<u32>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     // Create `partition_id` for distributed index building.
     // This ID serves as a high-level mask (first 32 bits of a u64) to ensure
     // that index partitions generated by different workers do not conflict.
@@ -2110,7 +2534,7 @@ pub async fn train_btree_index(
         );
         batch_idx += 1;
     }
-    sub_index_file.finish().await?;
+    let pages_file = sub_index_file.finish().await?;
     let record_batch = btree_stats_as_batch(encoded_batches, &value_type)?;
     let mut file_schema = record_batch.schema().as_ref().clone();
     file_schema
@@ -2136,13 +2560,11 @@ pub async fn train_btree_index(
         }
     };
     btree_index_file.write_record_batch(record_batch).await?;
-    btree_index_file.finish().await?;
-    Ok(())
+    let lookup_file = btree_index_file.finish().await?;
+    Ok(vec![pages_file, lookup_file])
 }
 
-fn find_single_partition_files(
-    files: &[lance_table::format::IndexFile],
-) -> Result<Option<(&str, &str)>> {
+fn find_single_partition_files(files: &[super::IndexFile]) -> Result<Option<(&str, &str)>> {
     let lookup_files = files
         .iter()
         .filter_map(|file| {
@@ -2195,7 +2617,7 @@ async fn merge_metadata_files(
     part_lookup_files: &[String],
     batch_readhead: Option<usize>,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     if part_lookup_files.is_empty() || part_page_files.is_empty() {
         return Err(Error::internal(
             "No partition files provided for merging".to_string(),
@@ -2271,6 +2693,7 @@ async fn merge_metadata_files(
             progress,
         )
         .await
+        .map(|file| vec![file])
     } else {
         merge_pages_and_lookups(
             store,
@@ -2324,7 +2747,7 @@ async fn merge_range_partitioned_lookups(
     batch_size: u64,
     batch_readhead: Option<usize>,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<IndexFile> {
     let sorted_part_lookup_files = sort_files_by_partition_id(part_lookup_files)?;
     let mut lookup_file = store
         .new_index_file(BTREE_LOOKUP_NAME, lookup_schema)
@@ -2364,12 +2787,12 @@ async fn merge_range_partitioned_lookups(
         serde_json::to_string(&pages_per_file)?,
     );
 
-    lookup_file.finish_with_metadata(metadata).await?;
+    let lookup_file = lookup_file.finish_with_metadata(metadata).await?;
     progress.stage_complete("merge_lookups").await?;
 
     // In this mode, we only clean up lookup files, and page files are untouched.
     cleanup_partition_files(store, part_lookup_files, &[]).await;
-    Ok(())
+    Ok(lookup_file)
 }
 
 /// Merges partition files using a K-way sort-merge algorithm.
@@ -2388,7 +2811,7 @@ async fn merge_pages_and_lookups(
     batch_size: u64,
     batch_readhead: Option<usize>,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     // Create a new global page file
     let partition_id = extract_partition_id(part_lookup_files[0].as_str())?;
     let page_file = page_files_map.get(&partition_id).unwrap();
@@ -2411,7 +2834,7 @@ async fn merge_pages_and_lookups(
         progress.clone(),
     )
     .await?;
-    page_file.finish().await?;
+    let page_file = page_file.finish().await?;
     progress.stage_complete("merge_pages").await?;
 
     let lookup_batch = RecordBatch::try_new(
@@ -2436,7 +2859,7 @@ async fn merge_pages_and_lookups(
         .stage_start("write_lookup_file", Some(1), "files")
         .await?;
     lookup_file.write_record_batch(lookup_batch).await?;
-    lookup_file.finish_with_metadata(metadata).await?;
+    let lookup_file = lookup_file.finish_with_metadata(metadata).await?;
     progress.stage_progress("write_lookup_file", 1).await?;
     progress.stage_complete("write_lookup_file").await?;
 
@@ -2444,7 +2867,7 @@ async fn merge_pages_and_lookups(
     // Only perform deletion after files are successfully written, ensuring debug information is not lost in case of failure
     cleanup_partition_files(store, part_lookup_files, part_page_files).await;
 
-    Ok(())
+    Ok(vec![page_file, lookup_file])
 }
 
 // Adjust local_page_idx_ in each look-up file to create a contiguous global_page_idx
@@ -2787,11 +3210,7 @@ impl TrainingRequest for BTreeTrainingRequest {
 pub struct BTreeIndexPlugin;
 
 #[async_trait]
-impl ScalarIndexPlugin for BTreeIndexPlugin {
-    fn name(&self) -> &str {
-        "BTree"
-    }
-
+impl BasicTrainer for BTreeIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -2807,26 +3226,6 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
         Ok(Box::new(BTreeTrainingRequest::new(params)))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        true
-    }
-
-    fn version(&self) -> u32 {
-        BTREE_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        Some(Box::new(SargableQueryParser::new(
-            index_name,
-            self.name().to_string(),
-            false,
-        )))
-    }
-
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -2853,7 +3252,7 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
                  The `range_id` field will be removed in a future release."
             );
         }
-        train_btree_index(
+        let files = train_btree_index(
             data,
             index_store,
             request
@@ -2868,15 +3267,46 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
             index_details: prost_types::Any::from_msg(&pbold::BTreeIndexDetails::default())
                 .unwrap(),
             index_version: BTREE_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files,
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for BTreeIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "BTree"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        true
+    }
+
+    fn version(&self) -> u32 {
+        BTREE_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        Some(Box::new(SargableQueryParser::new(
+            index_name,
+            self.name().to_string(),
+            false,
+        )))
+    }
 
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(BTreeIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
@@ -2885,7 +3315,7 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
     async fn get_from_cache(
         &self,
         index_store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Option<Arc<dyn ScalarIndex>>> {
         let Some(state) = cache.get_with_key(&BTreeIndexStateKey).await else {
@@ -2896,26 +3326,37 @@ impl ScalarIndexPlugin for BTreeIndexPlugin {
             cache,
             frag_reuse_index,
         )?))
-    }
-
-    async fn put_in_cache(&self, cache: &LanceCache, index: Arc<dyn ScalarIndex>) -> Result<()> {
-        let btree = index.as_any().downcast_ref::<BTreeIndex>().ok_or_else(|| {
-            Error::internal("BTreeIndexPlugin::put_in_cache called with a non-BTree index")
-        })?;
-        let state = BTreeIndexState {
-            lookup_batch: btree.lookup_batch.clone(),
-            batch_size: btree.batch_size,
-            ranges_to_files: btree.ranges_to_files.clone(),
-        };
+    }
+
+    async fn put_in_cache(&self, cache: &LanceCache, index: Arc<dyn ScalarIndex>) -> Result<()> {
+        let state = BTreeIndexState::from_index(index.as_ref())?;
         cache
             .insert_with_key(&BTreeIndexStateKey, Arc::new(state))
             .await;
         Ok(())
     }
+
+    async fn get_or_insert_in_cache(
+        &self,
+        index_store: Arc<dyn IndexStore>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        cache: &LanceCache,
+        load: ScalarIndexLoad<'_>,
+    ) -> Result<Arc<dyn ScalarIndex>> {
+        single_flight_open(
+            cache,
+            BTreeIndexStateKey,
+            load,
+            BTreeIndexState::from_index,
+            move |state| state.reconstruct(index_store, cache, frag_reuse_index),
+        )
+        .await
+    }
 }
 
 #[cfg(test)]
 mod tests {
+    use lance_core::utils::row_addr_remap::RowAddrRemap;
     use std::sync::atomic::Ordering;
     use std::{collections::HashMap, sync::Arc};
 
@@ -2927,10 +3368,10 @@ mod tests {
     };
     use datafusion_common::{DataFusionError, ScalarValue};
     use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};
-    use deepsize::DeepSizeOf;
     use futures::TryStreamExt;
     use futures::stream;
     use lance_core::cache::LanceCache;
+    use lance_core::deepsize::DeepSizeOf;
     use lance_core::utils::tempfile::TempObjDir;
     use lance_datafusion::{chunker::break_stream, datagen::DatafusionDatagenExt};
     use lance_datagen::{ArrayGeneratorExt, BatchCount, RowCount, array, gen_batch};
@@ -2950,12 +3391,29 @@ mod tests {
     };
 
     use super::{
-        BTreeIndexPlugin, BTreeIndexState, BTreePageKey, DEFAULT_BTREE_BATCH_SIZE,
-        OrderableScalarValue, part_lookup_file_path, part_page_data_file_path, train_btree_index,
+        BTreeIndexPlugin, BTreeIndexState, BTreeLookup, BTreePageKey, DEFAULT_BTREE_BATCH_SIZE,
+        Matches, OrderableScalarValue, part_lookup_file_path, part_page_data_file_path,
+        train_btree_index,
     };
     use crate::scalar::registry::ScalarIndexPlugin;
     use arrow_array::RecordBatch;
-    use lance_core::cache::{CacheCodecImpl, CacheKey};
+    use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey};
+
+    /// Serialize a `BTreeIndexState` body (no envelope) for tests.
+    fn serialize_state(state: &BTreeIndexState) -> Vec<u8> {
+        let mut buf = Vec::new();
+        state
+            .serialize(&mut CacheEntryWriter::new(&mut buf))
+            .unwrap();
+        buf
+    }
+
+    /// Deserialize a `BTreeIndexState` body (no envelope) for tests.
+    fn deserialize_state(buf: Vec<u8>) -> lance_core::Result<BTreeIndexState> {
+        let data = bytes::Bytes::from(buf);
+        let mut reader = CacheEntryReader::new(&data, 0, BTreeIndexState::CURRENT_VERSION);
+        BTreeIndexState::deserialize(&mut reader)
+    }
     use rangemap::RangeInclusiveMap;
 
     lance_testing::define_stage_event_progress!(
@@ -2979,6 +3437,37 @@ mod tests {
         assert!(size_of_many_i32 > 128 * 4);
     }
 
+    #[test]
+    fn test_orderable_dictionary_cmp() {
+        use arrow_schema::DataType;
+        use std::cmp::Ordering;
+
+        let dict = |s: &str, key: DataType| {
+            OrderableScalarValue(ScalarValue::Dictionary(
+                Box::new(key),
+                Box::new(ScalarValue::Utf8(Some(s.to_string()))),
+            ))
+        };
+
+        // Dictionary scalars are ordered by their underlying value, regardless
+        // of the key type. This is exercised when loading a scalar index built
+        // on a dictionary-encoded column into a BTreeMap.
+        assert_eq!(
+            dict("a", DataType::Int16).cmp(&dict("b", DataType::Int16)),
+            Ordering::Less
+        );
+        assert_eq!(
+            dict("b", DataType::Int32).cmp(&dict("b", DataType::Int16)),
+            Ordering::Equal
+        );
+
+        // A non-null dictionary value sorts after null.
+        assert_eq!(
+            dict("a", DataType::Int16).cmp(&OrderableScalarValue(ScalarValue::Null)),
+            Ordering::Greater
+        );
+    }
+
     #[tokio::test]
     async fn test_null_ids() {
         let tmpdir = TempObjDir::default();
@@ -3016,7 +3505,7 @@ mod tests {
 
         // Remap with a no-op mapping.  The remapped index should be identical to the original
         index
-            .remap(&HashMap::default(), remap_store.as_ref())
+            .remap(&RowAddrRemap::empty(), remap_store.as_ref())
             .await
             .unwrap();
 
@@ -3096,6 +3585,86 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_contains_keys_matches_search() {
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // 1000 distinct Int32 values [0, 1000), spread across many small pages
+        // (batch_size 64) so the keys below exercise multi-page grouping.
+        let data = gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .col("_rowid", array::step::<UInt64Type>())
+            .into_df_exec(RowCount::from(100), BatchCount::from(10));
+        let schema = data.schema();
+        let sort_expr = PhysicalSortExpr::new_default(col("value", schema.as_ref()).unwrap());
+        let plan = Arc::new(SortExec::new([sort_expr].into(), data));
+        let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+        let stream = break_stream(stream, 64);
+        let stream = stream.map_err(DataFusionError::from);
+        let stream =
+            Box::pin(RecordBatchStreamAdapter::new(schema, stream)) as SendableRecordBatchStream;
+
+        train_btree_index(stream, test_store.as_ref(), 64, None, None)
+            .await
+            .unwrap();
+        let index = BTreeIndex::load(test_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // Present (range ends, mid, and adjacent values that straddle page
+        // boundaries), interleaved with absent (below/above range, and a gap).
+        let keys: Vec<i32> = vec![0, 999, 500, 1, 998, -1, 1000, 1500, 250, 251, 7, 64, 63, 65];
+        let scalar_keys: Vec<ScalarValue> =
+            keys.iter().map(|k| ScalarValue::Int32(Some(*k))).collect();
+
+        let batched = index
+            .contains_keys(&scalar_keys, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+
+        // Oracle: the per-key Equals search the batched path replaces.
+        let mut oracle = Vec::with_capacity(keys.len());
+        for k in &scalar_keys {
+            let result = index
+                .search(&SargableQuery::Equals(k.clone()), &NoOpMetricsCollector)
+                .await
+                .unwrap();
+            oracle.push(!result.row_addrs().is_empty());
+        }
+        assert_eq!(
+            batched, oracle,
+            "contains_keys must agree with per-key Equals search; keys={keys:?}"
+        );
+
+        // And both must match ground truth: [0, 1000) present, others absent.
+        let expected: Vec<bool> = keys.iter().map(|k| (0..1000).contains(k)).collect();
+        assert_eq!(batched, expected);
+
+        // Empty input → empty mask.
+        assert!(
+            index
+                .contains_keys(&[], &NoOpMetricsCollector)
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        // A null key maps to false (and must not panic).
+        let with_null = vec![ScalarValue::Int32(Some(5)), ScalarValue::Int32(None)];
+        assert_eq!(
+            index
+                .contains_keys(&with_null, &NoOpMetricsCollector)
+                .await
+                .unwrap(),
+            vec![true, false]
+        );
+    }
+
     #[tokio::test]
     async fn test_page_cache() {
         let tmpdir = TempObjDir::default();
@@ -4447,7 +5016,7 @@ mod tests {
 
         // Remap with a no-op mapping.  The remapped index should be identical to the original
         ranged_index
-            .remap(&HashMap::default(), remap_store.as_ref())
+            .remap(&RowAddrRemap::empty(), remap_store.as_ref())
             .await
             .unwrap();
 
@@ -4758,12 +5327,8 @@ mod tests {
             mapping.insert(old_id, None);
         }
 
-        let mut new_id_counter = 100_000;
-
         // Remap all other rows
-        for old_id in (0..1000).chain(10000..15000) {
-            let new_id = new_id_counter;
-            new_id_counter += 1;
+        for (new_id, old_id) in (100_000..).zip((0..1000).chain(10000..15000)) {
             mapping.insert(old_id, Some(new_id));
         }
 
@@ -4775,7 +5340,10 @@ mod tests {
         ));
 
         // Remap the index with our deletion mapping
-        index.remap(&mapping, remap_store.as_ref()).await.unwrap();
+        index
+            .remap(&RowAddrRemap::direct(mapping), remap_store.as_ref())
+            .await
+            .unwrap();
 
         let remapped_index = BTreeIndex::load(remap_store.clone(), None, &LanceCache::no_cache())
             .await
@@ -4943,10 +5511,621 @@ mod tests {
         .unwrap()
     }
 
+    fn osv(v: i32) -> OrderableScalarValue {
+        OrderableScalarValue(ScalarValue::Int32(Some(v)))
+    }
+
+    /// The rewritten [`BTreeLookup`] searches the lookup batch directly, so this
+    /// exercises the binary-search bounds, duplicate `min` values, a partial-null
+    /// (null `min`) straddling page, and the `Matches::Some`/`All` classification.
+    #[test]
+    fn test_btree_lookup_pages_between() {
+        // Pages sorted by `min`, NULLs first. Page 0 straddles the NULL/non-NULL
+        // boundary; pages 2 and 3 share a `min` of 20.
+        let batch = record_batch!(
+            ("min", Int32, [None, Some(10), Some(20), Some(20), Some(40)]),
+            (
+                "max",
+                Int32,
+                [Some(5), Some(20), Some(20), Some(30), Some(50)]
+            ),
+            ("null_count", UInt32, [2, 0, 0, 0, 0]),
+            ("page_idx", UInt32, [0, 1, 2, 3, 4])
+        )
+        .unwrap();
+        let lookup = BTreeLookup::try_new(batch).unwrap();
+        assert_eq!(lookup.null_pages, vec![0]);
+        assert!(lookup.all_null_pages.is_empty());
+        assert_eq!(lookup.search_start, 0);
+
+        let between = |lo: i32, hi: i32| {
+            let mut m = lookup
+                .pages_between((
+                    std::ops::Bound::Included(&osv(lo)),
+                    std::ops::Bound::Included(&osv(hi)),
+                ))
+                .unwrap();
+            m.sort_by_key(|m| m.page_id());
+            m
+        };
+
+        // Equality only ever yields partial (Some) matches.
+        assert_eq!(lookup.pages_eq(&osv(15)).unwrap(), vec![Matches::Some(1)]);
+        assert_eq!(
+            lookup.pages_eq(&osv(20)).unwrap(),
+            vec![Matches::Some(1), Matches::Some(2), Matches::Some(3)]
+        );
+        assert!(lookup.pages_eq(&osv(35)).unwrap().is_empty());
+
+        // [20, 25]: page 2 ([20, 20]) sits entirely inside -> All; pages 1 and 3
+        // only partially overlap -> Some. The null-min page 0 (max 5) is excluded.
+        assert_eq!(
+            between(20, 25),
+            vec![Matches::Some(1), Matches::All(2), Matches::Some(3)]
+        );
+
+        // A query below all non-null data still reaches the straddling page 0,
+        // which is only ever a partial match because its `min` is NULL.
+        assert_eq!(between(0, 5), vec![Matches::Some(0)]);
+
+        // Unbounded above: page 4 ([40, 50]) is fully covered from 40 onward.
+        assert_eq!(
+            lookup
+                .pages_between((
+                    std::ops::Bound::Included(&osv(40)),
+                    std::ops::Bound::Unbounded
+                ))
+                .unwrap(),
+            vec![Matches::All(4)]
+        );
+
+        // Empty / inverted ranges select nothing.
+        assert!(between(31, 39).is_empty());
+        assert!(
+            lookup
+                .pages_between((
+                    std::ops::Bound::Included(&osv(25)),
+                    std::ops::Bound::Included(&osv(15))
+                ))
+                .unwrap()
+                .is_empty()
+        );
+    }
+
+    /// Exercises the native byte comparator path (`accessor_cmp`) for
+    /// variable-length `Binary` and fixed-width `FixedSizeBinary` (e.g. UUID)
+    /// columns, including the null-min straddle page and duplicate `min`s.
+    #[test]
+    fn test_btree_lookup_pages_eq_bytes() {
+        use arrow_array::{
+            ArrayRef, BinaryArray, FixedSizeBinaryArray, LargeBinaryArray, LargeStringArray,
+            UInt32Array,
+        };
+        use arrow_schema::{DataType, Field, Schema};
+
+        // 2-byte big-endian keys, so lexicographic byte order matches numeric
+        // order. Same layout as the int test: page 0 is a null-min straddle,
+        // pages 2 and 3 share `min` 20, and 35 falls in a gap.
+        fn be(v: u16) -> [u8; 2] {
+            v.to_be_bytes()
+        }
+        let mins = [None, Some(10u16), Some(20), Some(20), Some(40)];
+        let maxs = [Some(5u16), Some(20), Some(20), Some(30), Some(50)];
+        let null_count = UInt32Array::from(vec![2u32, 0, 0, 0, 0]);
+        let page_idx = UInt32Array::from(vec![0u32, 1, 2, 3, 4]);
+
+        let assert_byte_lookup =
+            |min_arr: ArrayRef, max_arr: ArrayRef, sv: &dyn Fn(u16) -> ScalarValue| {
+                let batch = RecordBatch::try_new(
+                    Arc::new(Schema::new(vec![
+                        Field::new("min", min_arr.data_type().clone(), true),
+                        Field::new("max", max_arr.data_type().clone(), true),
+                        Field::new("null_count", DataType::UInt32, false),
+                        Field::new("page_idx", DataType::UInt32, false),
+                    ])),
+                    vec![
+                        min_arr,
+                        max_arr,
+                        Arc::new(null_count.clone()),
+                        Arc::new(page_idx.clone()),
+                    ],
+                )
+                .unwrap();
+                let lookup = BTreeLookup::try_new(batch).unwrap();
+
+                let eq = |v: u16| {
+                    let mut p: Vec<u32> = lookup
+                        .pages_eq(&OrderableScalarValue(sv(v)))
+                        .unwrap()
+                        .into_iter()
+                        .map(|m| m.page_id())
+                        .collect();
+                    p.sort_unstable();
+                    p
+                };
+                assert_eq!(eq(15), vec![1]); // only page 1 ([10, 20])
+                assert_eq!(eq(20), vec![1, 2, 3]); // shared min of 2 & 3, max of 1
+                assert!(eq(35).is_empty()); // gap between pages 3 and 4
+                assert_eq!(eq(5), vec![0]); // reaches the null-min straddle via its max
+
+                // IN merges and dedups across values.
+                let mut in_pages: Vec<u32> = lookup
+                    .pages_in([5u16, 15].into_iter().map(|v| OrderableScalarValue(sv(v))))
+                    .unwrap()
+                    .into_iter()
+                    .map(|m| m.page_id())
+                    .collect();
+                in_pages.sort_unstable();
+                assert_eq!(in_pages, vec![0, 1]);
+            };
+
+        let fsb = |arr: &[Option<u16>]| -> ArrayRef {
+            Arc::new(
+                FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                    arr.iter().copied().map(|o| o.map(be)),
+                    2,
+                )
+                .unwrap(),
+            )
+        };
+        assert_byte_lookup(fsb(&mins), fsb(&maxs), &|v| {
+            ScalarValue::FixedSizeBinary(2, Some(be(v).to_vec()))
+        });
+
+        let bin = |arr: &[Option<u16>]| -> ArrayRef {
+            Arc::new(BinaryArray::from_iter(
+                arr.iter().copied().map(|o| o.map(|v| be(v).to_vec())),
+            ))
+        };
+        assert_byte_lookup(bin(&mins), bin(&maxs), &|v| {
+            ScalarValue::Binary(Some(be(v).to_vec()))
+        });
+
+        let lbin = |arr: &[Option<u16>]| -> ArrayRef {
+            Arc::new(LargeBinaryArray::from_iter(
+                arr.iter().copied().map(|o| o.map(|v| be(v).to_vec())),
+            ))
+        };
+        assert_byte_lookup(lbin(&mins), lbin(&maxs), &|v| {
+            ScalarValue::LargeBinary(Some(be(v).to_vec()))
+        });
+
+        // `LargeUtf8` over zero-padded decimal strings, whose lexicographic order
+        // matches the numeric order of the keys.
+        let lstr = |arr: &[Option<u16>]| -> ArrayRef {
+            Arc::new(LargeStringArray::from_iter(
+                arr.iter().copied().map(|o| o.map(|v| format!("{v:02}"))),
+            ))
+        };
+        assert_byte_lookup(lstr(&mins), lstr(&maxs), &|v| {
+            ScalarValue::LargeUtf8(Some(format!("{v:02}")))
+        });
+    }
+
+    /// Exercises the physical-type reinterpret path: temporal columns (`Date32`
+    /// over `i32`, `Timestamp` over `i64`) are compared through the integer native
+    /// path without a dedicated per-type branch.
+    #[test]
+    fn test_btree_lookup_pages_eq_temporal() {
+        use arrow_array::{ArrayRef, Date32Array, TimestampMicrosecondArray, UInt32Array};
+        use arrow_schema::{DataType, Field, Schema};
+
+        let null_count = UInt32Array::from(vec![2u32, 0, 0, 0, 0]);
+        let page_idx = UInt32Array::from(vec![0u32, 1, 2, 3, 4]);
+
+        let assert_lookup =
+            |min_arr: ArrayRef, max_arr: ArrayRef, sv: &dyn Fn(i64) -> ScalarValue| {
+                let batch = RecordBatch::try_new(
+                    Arc::new(Schema::new(vec![
+                        Field::new("min", min_arr.data_type().clone(), true),
+                        Field::new("max", max_arr.data_type().clone(), true),
+                        Field::new("null_count", DataType::UInt32, false),
+                        Field::new("page_idx", DataType::UInt32, false),
+                    ])),
+                    vec![
+                        min_arr,
+                        max_arr,
+                        Arc::new(null_count.clone()),
+                        Arc::new(page_idx.clone()),
+                    ],
+                )
+                .unwrap();
+                let lookup = BTreeLookup::try_new(batch).unwrap();
+                let eq = |v: i64| {
+                    let mut p: Vec<u32> = lookup
+                        .pages_eq(&OrderableScalarValue(sv(v)))
+                        .unwrap()
+                        .into_iter()
+                        .map(|m| m.page_id())
+                        .collect();
+                    p.sort_unstable();
+                    p
+                };
+                assert_eq!(eq(15), vec![1]); // only page 1 ([10, 20])
+                assert_eq!(eq(20), vec![1, 2, 3]); // shared min of 2 & 3, max of 1
+                assert!(eq(35).is_empty()); // gap between pages 3 and 4
+                assert_eq!(eq(5), vec![0]); // reaches the null-min straddle via its max
+            };
+
+        // Timestamp (i64-backed) → Int64 native path.
+        assert_lookup(
+            Arc::new(TimestampMicrosecondArray::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(TimestampMicrosecondArray::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::TimestampMicrosecond(Some(v), None),
+        );
+
+        // Date32 (i32-backed) → Int32 native path.
+        assert_lookup(
+            Arc::new(Date32Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(Date32Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::Date32(Some(v as i32)),
+        );
+    }
+
+    /// Exercises the remaining physical-type dispatch arms that the temporal and
+    /// byte tests don't reach: every integer width and signedness, `Float16`, and
+    /// the 128-/256-bit decimal paths. All share the temporal test's numeric layout
+    /// (mins `[_, 10, 20, 20, 40]`, maxs `[5, 20, 20, 30, 50]`) so the assertions are
+    /// identical; only the array/scalar type varies.
+    #[test]
+    fn test_btree_lookup_pages_eq_numeric_widths() {
+        use arrow::datatypes::i256;
+        use arrow_array::{
+            ArrayRef, Decimal128Array, Decimal256Array, Float16Array, Int8Array, Int16Array,
+            UInt8Array, UInt16Array, UInt32Array, UInt64Array,
+        };
+        use arrow_schema::{DataType, Field, Schema};
+        use half::f16;
+
+        let null_count = UInt32Array::from(vec![2u32, 0, 0, 0, 0]);
+        let page_idx = UInt32Array::from(vec![0u32, 1, 2, 3, 4]);
+        let assert_lookup =
+            |min_arr: ArrayRef, max_arr: ArrayRef, sv: &dyn Fn(i64) -> ScalarValue| {
+                let batch = RecordBatch::try_new(
+                    Arc::new(Schema::new(vec![
+                        Field::new("min", min_arr.data_type().clone(), true),
+                        Field::new("max", max_arr.data_type().clone(), true),
+                        Field::new("null_count", DataType::UInt32, false),
+                        Field::new("page_idx", DataType::UInt32, false),
+                    ])),
+                    vec![
+                        min_arr,
+                        max_arr,
+                        Arc::new(null_count.clone()),
+                        Arc::new(page_idx.clone()),
+                    ],
+                )
+                .unwrap();
+                let lookup = BTreeLookup::try_new(batch).unwrap();
+                let eq = |v: i64| {
+                    let mut p: Vec<u32> = lookup
+                        .pages_eq(&OrderableScalarValue(sv(v)))
+                        .unwrap()
+                        .into_iter()
+                        .map(|m| m.page_id())
+                        .collect();
+                    p.sort_unstable();
+                    p
+                };
+                assert_eq!(eq(15), vec![1]); // only page 1 ([10, 20])
+                assert_eq!(eq(20), vec![1, 2, 3]); // shared min of 2 & 3, max of 1
+                assert!(eq(35).is_empty()); // gap between pages 3 and 4
+                assert_eq!(eq(5), vec![0]); // reaches the null-min straddle via its max
+            };
+
+        assert_lookup(
+            Arc::new(Int8Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(Int8Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::Int8(Some(v as i8)),
+        );
+        assert_lookup(
+            Arc::new(Int16Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(Int16Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::Int16(Some(v as i16)),
+        );
+        assert_lookup(
+            Arc::new(UInt8Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(UInt8Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::UInt8(Some(v as u8)),
+        );
+        assert_lookup(
+            Arc::new(UInt16Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(UInt16Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::UInt16(Some(v as u16)),
+        );
+        assert_lookup(
+            Arc::new(UInt32Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(UInt32Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::UInt32(Some(v as u32)),
+        );
+        assert_lookup(
+            Arc::new(UInt64Array::from(vec![
+                None,
+                Some(10),
+                Some(20),
+                Some(20),
+                Some(40),
+            ])),
+            Arc::new(UInt64Array::from(vec![
+                Some(5),
+                Some(20),
+                Some(20),
+                Some(30),
+                Some(50),
+            ])),
+            &|v| ScalarValue::UInt64(Some(v as u64)),
+        );
+
+        let f = |v: f64| f16::from_f64(v);
+        assert_lookup(
+            Arc::new(Float16Array::from(vec![
+                None,
+                Some(f(10.0)),
+                Some(f(20.0)),
+                Some(f(20.0)),
+                Some(f(40.0)),
+            ])),
+            Arc::new(Float16Array::from(vec![
+                Some(f(5.0)),
+                Some(f(20.0)),
+                Some(f(20.0)),
+                Some(f(30.0)),
+                Some(f(50.0)),
+            ])),
+            &|v| ScalarValue::Float16(Some(f(v as f64))),
+        );
+
+        // Decimal128 (i128 native path). Comparison is on the raw integer, so a
+        // scale of 0 lets the values double as plain integers.
+        let dec128 = |vals: Vec<Option<i128>>| -> ArrayRef {
+            Arc::new(
+                Decimal128Array::from(vals)
+                    .with_precision_and_scale(18, 0)
+                    .unwrap(),
+            )
+        };
+        assert_lookup(
+            dec128(vec![None, Some(10), Some(20), Some(20), Some(40)]),
+            dec128(vec![Some(5), Some(20), Some(20), Some(30), Some(50)]),
+            &|v| ScalarValue::Decimal128(Some(v as i128), 18, 0),
+        );
+
+        // Decimal256 (i256 native path).
+        let dec256 = |vals: Vec<Option<i128>>| -> ArrayRef {
+            Arc::new(
+                Decimal256Array::from(
+                    vals.into_iter()
+                        .map(|o| o.map(i256::from_i128))
+                        .collect::<Vec<_>>(),
+                )
+                .with_precision_and_scale(40, 0)
+                .unwrap(),
+            )
+        };
+        assert_lookup(
+            dec256(vec![None, Some(10), Some(20), Some(20), Some(40)]),
+            dec256(vec![Some(5), Some(20), Some(20), Some(30), Some(50)]),
+            &|v| ScalarValue::Decimal256(Some(i256::from_i128(v as i128)), 40, 0),
+        );
+    }
+
+    /// Exercises the NULL paths of the lookup directly: `pages_eq(NULL)` and
+    /// `pages_in` with a NULL in the value list (and a NULL-only list), including
+    /// the partial-null (`Some`) vs entirely-null (`All`) page classification.
+    #[test]
+    fn test_btree_lookup_pages_null() {
+        // Page 0 is entirely null (null max -> All); page 1 is a partial-null
+        // straddle (max 5, null_count > 0 -> Some); page 2 also carries a null.
+        let batch = record_batch!(
+            ("min", Int32, [None, None, Some(10), Some(20), Some(40)]),
+            ("max", Int32, [None, Some(5), Some(20), Some(30), Some(50)]),
+            ("null_count", UInt32, [3, 2, 1, 0, 0]),
+            ("page_idx", UInt32, [0, 1, 2, 3, 4])
+        )
+        .unwrap();
+        let lookup = BTreeLookup::try_new(batch).unwrap();
+        assert_eq!(lookup.all_null_pages, vec![0]);
+        assert_eq!(lookup.null_pages, vec![1, 2]);
+
+        // pages_eq(NULL) short-circuits to the null pages: partial-null pages are
+        // `Some`, the entirely-null page is `All`.
+        assert_eq!(
+            lookup
+                .pages_eq(&OrderableScalarValue(ScalarValue::Int32(None)))
+                .unwrap(),
+            vec![Matches::Some(1), Matches::Some(2), Matches::All(0)]
+        );
+
+        let in_ids = |vals: Vec<Option<i32>>| {
+            let mut p: Vec<u32> = lookup
+                .pages_in(
+                    vals.into_iter()
+                        .map(|v| OrderableScalarValue(ScalarValue::Int32(v))),
+                )
+                .unwrap()
+                .into_iter()
+                .map(|m| m.page_id())
+                .collect();
+            p.sort_unstable();
+            p
+        };
+        // Baseline: a non-null value only -> just its value page.
+        assert_eq!(in_ids(vec![Some(45)]), vec![4]);
+        // A NULL in the list unions in every null page (0, 1, 2).
+        assert_eq!(in_ids(vec![Some(45), None]), vec![0, 1, 2, 4]);
+        // A NULL-only list (empty non-null set) returns exactly the null pages.
+        assert_eq!(in_ids(vec![None]), vec![0, 1, 2]);
+    }
+
+    /// A 0-row page_lookup batch (an index over an empty dataset) must yield no
+    /// candidates for any query rather than panicking on the binary-search bounds.
+    #[test]
+    fn test_btree_lookup_empty_batch() {
+        use arrow_schema::{DataType, Field, Schema};
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("page_idx", DataType::UInt32, false),
+        ]));
+        let lookup = BTreeLookup::try_new(RecordBatch::new_empty(schema)).unwrap();
+        assert_eq!(lookup.search_start, 0);
+        assert!(lookup.null_pages.is_empty());
+        assert!(lookup.all_null_pages.is_empty());
+
+        assert!(lookup.pages_eq(&osv(5)).unwrap().is_empty());
+        assert!(lookup.pages_in([osv(5)]).unwrap().is_empty());
+        assert!(
+            lookup
+                .pages_between((
+                    std::ops::Bound::Included(&osv(0)),
+                    std::ops::Bound::Included(&osv(100)),
+                ))
+                .unwrap()
+                .is_empty()
+        );
+        assert!(lookup.pages_null().is_empty());
+    }
+
+    /// A straddle page (null `min`, non-null `max`) can sort ahead of an entirely-
+    /// null page within the leading NULL-`min` group. When it does, `search_start`
+    /// points at the straddle and the all-null page falls inside the forward-scan
+    /// window, so both the equality and range scans must skip it (it matches only
+    /// IS NULL).
+    #[test]
+    fn test_btree_lookup_skips_all_null_page_in_scan_window() {
+        // Page 0: straddle (null min, max 5). Page 1: entirely null (null min/max).
+        let batch = record_batch!(
+            ("min", Int32, [None, None, Some(10), Some(20), Some(40)]),
+            ("max", Int32, [Some(5), None, Some(20), Some(30), Some(50)]),
+            ("null_count", UInt32, [2, 3, 0, 0, 0]),
+            ("page_idx", UInt32, [0, 1, 2, 3, 4])
+        )
+        .unwrap();
+        let lookup = BTreeLookup::try_new(batch).unwrap();
+        assert_eq!(lookup.search_start, 0); // straddle page 0 has a non-null max
+        assert_eq!(lookup.all_null_pages, vec![1]);
+        assert_eq!(lookup.null_pages, vec![0]);
+
+        // Equality for 5 peeks left across the all-null page 1 (index 1, inside the
+        // scan window) and must skip it, reaching only the straddle page 0.
+        assert_eq!(
+            lookup
+                .pages_eq(&osv(5))
+                .unwrap()
+                .into_iter()
+                .map(|m| m.page_id())
+                .collect::<Vec<_>>(),
+            vec![0]
+        );
+
+        // The same all-null page sits inside the range scan window and is skipped:
+        // page 0 (straddle) is a partial match, pages 2-4 are fully covered.
+        let mut between = lookup
+            .pages_between((
+                std::ops::Bound::Included(&osv(0)),
+                std::ops::Bound::Included(&osv(100)),
+            ))
+            .unwrap();
+        between.sort_by_key(|m| m.page_id());
+        assert_eq!(
+            between,
+            vec![
+                Matches::Some(0),
+                Matches::All(2),
+                Matches::All(3),
+                Matches::All(4),
+            ]
+        );
+    }
+
     fn assert_state_roundtrips(state: &BTreeIndexState) {
-        let mut buf = Vec::new();
-        state.serialize(&mut buf).unwrap();
-        let restored = BTreeIndexState::deserialize(&bytes::Bytes::from(buf)).unwrap();
+        let restored = deserialize_state(serialize_state(state)).unwrap();
         assert_eq!(restored.lookup_batch, state.lookup_batch);
         assert_eq!(restored.batch_size, state.batch_size);
         assert_eq!(restored.ranges_to_files, state.ranges_to_files);
@@ -5011,13 +6190,11 @@ mod tests {
 
         // Round-trip the state through the codec and reconstruct an index from it.
         let state = BTreeIndexState {
-            lookup_batch: index.lookup_batch.clone(),
+            lookup_batch: index.page_lookup.batch.clone(),
             batch_size: index.batch_size,
             ranges_to_files: index.ranges_to_files.clone(),
         };
-        let mut buf = Vec::new();
-        state.serialize(&mut buf).unwrap();
-        let restored = BTreeIndexState::deserialize(&bytes::Bytes::from(buf)).unwrap();
+        let restored = deserialize_state(serialize_state(&state)).unwrap();
         let reconstructed = restored
             .reconstruct(test_store.clone(), &LanceCache::no_cache(), None)
             .unwrap();
@@ -5053,23 +6230,62 @@ mod tests {
         assert_eq!(expected, actual);
     }
 
+    /// The lookup batch must decode zero-copy through the full envelope even
+    /// though the proto header pushes the IPC section to a non-aligned offset.
+    #[test]
+    fn test_btree_index_state_lookup_is_zero_copy() {
+        use lance_core::cache::CacheCodec;
+        const ALIGN: usize = 64;
+
+        let ranges: RangeInclusiveMap<u32, (String, u32)> =
+            [(0..=99, ("part_0_page_file.lance".to_string(), 0))]
+                .into_iter()
+                .collect();
+        let state = BTreeIndexState {
+            lookup_batch: sample_lookup_batch(),
+            batch_size: 8192,
+            ranges_to_files: Some(Arc::new(ranges)),
+        };
+
+        let codec = CacheCodec::from_impl::<BTreeIndexState>();
+        let any: Arc<dyn std::any::Any + Send + Sync> = Arc::new(state);
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+
+        let mut v = vec![0u8; buf.len() + ALIGN];
+        let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+        v[pad..pad + buf.len()].copy_from_slice(&buf);
+        let data = bytes::Bytes::from(v).slice(pad..pad + buf.len());
+
+        let restored = codec.deserialize(&data).hit().unwrap();
+        let restored = restored.downcast::<BTreeIndexState>().unwrap();
+
+        let base = data.as_ptr() as usize;
+        let end = base + data.len();
+        for col in restored.lookup_batch.columns() {
+            for buffer in col.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= base && ptr < end,
+                    "lookup batch buffer was realigned out of the input — misaligned IPC section",
+                );
+            }
+        }
+    }
+
     #[test]
-    fn test_btree_index_state_rejects_invalid_has_ranges_tag() {
-        // u64 batch_size (any) then a bad has_ranges tag.
+    fn test_btree_index_state_rejects_truncated_header() {
+        // A header length prefix that overruns the buffer must error rather
+        // than panic or silently misread it.
         let mut buf = Vec::new();
-        buf.extend_from_slice(&1000u64.to_le_bytes());
-        buf.push(7u8);
-        let err = BTreeIndexState::deserialize(&bytes::Bytes::from(buf)).unwrap_err();
-        let msg = err.to_string();
-        assert!(
-            msg.contains("has_ranges") && msg.contains("7"),
-            "expected error to mention the bad has_ranges tag, got: {msg}"
-        );
+        buf.extend_from_slice(&100u32.to_le_bytes()); // claims a 100-byte header
+        buf.extend_from_slice(&[0u8; 4]); // but only 4 bytes follow
+        assert!(deserialize_state(buf).is_err());
     }
 
     #[tokio::test]
     async fn test_btree_index_state_reconstruct_applies_frag_reuse_index() {
-        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails};
+        use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle};
         use std::collections::HashMap;
         use uuid::Uuid;
 
@@ -5093,7 +6309,7 @@ mod tests {
             .await
             .unwrap();
         let state = BTreeIndexState {
-            lookup_batch: index.lookup_batch.clone(),
+            lookup_batch: index.page_lookup.batch.clone(),
             batch_size: index.batch_size,
             ranges_to_files: index.ranges_to_files.clone(),
         };
@@ -5101,11 +6317,12 @@ mod tests {
         // Remap row 0 -> row 5000 (outside the original [0, 1000) range so no collision).
         // Querying for value == 0 should now return row 5000, confirming reconstruct threaded
         // the FragReuseIndex through to the rebuilt BTreeIndex.
-        let frag_reuse_index = Arc::new(FragReuseIndex::new(
-            Uuid::new_v4(),
-            vec![HashMap::from([(0u64, Some(5000u64))])],
-            FragReuseIndexDetails { versions: vec![] },
-        ));
+        let frag_reuse_index: Arc<dyn crate::scalar::RowIdRemapper> =
+            Arc::new(FragReuseIndexHandle(Arc::new(FragReuseIndex::new(
+                Uuid::new_v4(),
+                vec![HashMap::from([(0u64, Some(5000u64))])],
+                FragReuseIndexDetails { versions: vec![] },
+            ))));
         let reconstructed = state
             .reconstruct(
                 test_store.clone(),
diff --git a/rust/lance-index/src/scalar/btree/flat.rs b/rust/lance-index/src/scalar/btree/flat.rs
index 4240753772b..1af5a6a69e9 100644
--- a/rust/lance-index/src/scalar/btree/flat.rs
+++ b/rust/lance-index/src/scalar/btree/flat.rs
@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashMap;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::collections::BTreeSet;
 use std::{ops::Bound, sync::Arc};
 
 use arrow_array::Array;
@@ -11,19 +12,20 @@ use arrow_array::{
 
 use datafusion_common::DFSchema;
 use datafusion_expr::execution_props::ExecutionProps;
-use datafusion_physical_expr::create_physical_expr;
-use deepsize::DeepSizeOf;
+use datafusion_physical_expr::{PhysicalExpr, create_physical_expr};
 use lance_arrow::RecordBatchExt;
-use lance_arrow::ipc::{read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream};
 use lance_core::Result;
-use lance_core::cache::CacheCodecImpl;
+use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::address::RowAddress;
 use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use roaring::RoaringBitmap;
 use tracing::instrument;
 
+use datafusion_common::ScalarValue;
+
 use crate::metrics::MetricsCollector;
-use crate::scalar::btree::BTREE_VALUES_COLUMN;
+use crate::scalar::btree::{BTREE_VALUES_COLUMN, OrderableScalarValue};
 use crate::scalar::{AnyQuery, SargableQuery};
 
 const VALUES_COL_IDX: usize = 0;
@@ -43,7 +45,7 @@ pub struct FlatIndex {
 }
 
 impl DeepSizeOf for FlatIndex {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         self.data.get_array_memory_size()
     }
 }
@@ -83,6 +85,46 @@ impl FlatIndex {
         self.data.column(IDS_COL_IDX)
     }
 
+    fn values(&self) -> &ArrayRef {
+        self.data.column(VALUES_COL_IDX)
+    }
+
+    /// Which of `needles` are present in this page.
+    ///
+    /// Batched existence sibling of [`Self::search`]: it runs the same `IsIn`
+    /// predicate over the page's `values` column, but returns the matched
+    /// *values* rather than row addresses — so the caller can map each result
+    /// back to the input key it asked about. The page scan stays vectorized;
+    /// only the (small) matched subset is lifted into `ScalarValue`.
+    ///
+    /// Nulls: a null `values` entry never matches a (non-null) primary-key
+    /// needle, so it is simply absent from the result.
+    pub(crate) fn contains_values(
+        &self,
+        needles: &[OrderableScalarValue],
+    ) -> Result<BTreeSet<OrderableScalarValue>> {
+        if needles.is_empty() {
+            return Ok(BTreeSet::new());
+        }
+        let query = SargableQuery::IsIn(needles.iter().map(|v| v.0.clone()).collect());
+        let expr = query.to_expr(BTREE_VALUES_COLUMN.to_string());
+        let expr = create_physical_expr(&expr, &self.df_schema, &ExecutionProps::default())?;
+        let predicate = expr.evaluate(&self.data)?;
+        let predicate = predicate.into_array(self.data.num_rows())?;
+        let predicate = predicate
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .expect("Predicate should return boolean array");
+        let matched = arrow_select::filter::filter(self.values(), predicate)?;
+        (0..matched.len())
+            .map(|i| {
+                Ok(OrderableScalarValue(ScalarValue::try_from_array(
+                    &matched, i,
+                )?))
+            })
+            .collect()
+    }
+
     pub fn all(&self) -> NullableRowAddrSet {
         // Some rows will be in both sets but that is ok, null trumps true
         NullableRowAddrSet::new(self.all_addrs_map.clone(), self.null_addrs_map.clone())
@@ -92,10 +134,7 @@ impl FlatIndex {
         NullableRowAddrSet::new(self.all_addrs_map.clone(), Default::default())
     }
 
-    pub fn remap_batch(
-        batch: RecordBatch,
-        mapping: &HashMap<u64, Option<u64>>,
-    ) -> Result<RecordBatch> {
+    pub fn remap_batch(batch: RecordBatch, mapping: &RowAddrRemap) -> Result<RecordBatch> {
         let row_ids = batch.column(IDS_COL_IDX).as_primitive::<UInt64Type>();
         let val_idx_and_new_id = row_ids
             .values()
@@ -103,8 +142,7 @@ impl FlatIndex {
             .enumerate()
             .filter_map(|(idx, old_id)| {
                 mapping
-                    .get(old_id)
-                    .copied()
+                    .get(*old_id)
                     .unwrap_or(Some(*old_id))
                     .map(|new_id| (idx, new_id))
             })
@@ -180,13 +218,13 @@ impl FlatIndex {
                         ));
                     }
                 }
-                (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) => {
-                    if lower.is_null() {
-                        return Ok(NullableRowAddrSet::new(
-                            Default::default(),
-                            self.all_addrs_map.clone(),
-                        ));
-                    }
+                (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded)
+                    if lower.is_null() =>
+                {
+                    return Ok(NullableRowAddrSet::new(
+                        Default::default(),
+                        self.all_addrs_map.clone(),
+                    ));
                 }
                 _ => {}
             },
@@ -196,7 +234,22 @@ impl FlatIndex {
         // No shortcut possible, need to actually evaluate the query
         let expr = query.to_expr(BTREE_VALUES_COLUMN.to_string());
         let expr = create_physical_expr(&expr, &self.df_schema, &ExecutionProps::default())?;
+        self.eval_expr(&expr)
+    }
 
+    /// Evaluate a predicate compiled once by the caller. Lets a large IsIn that
+    /// spans many pages build the physical expr a single time instead of
+    /// rebuilding the whole IN-list per page (the dominant cost of a big lookup).
+    pub fn search_prebuilt(
+        &self,
+        expr: &Arc<dyn PhysicalExpr>,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<NullableRowAddrSet> {
+        metrics.record_comparisons(self.data.num_rows());
+        self.eval_expr(expr)
+    }
+
+    fn eval_expr(&self, expr: &Arc<dyn PhysicalExpr>) -> Result<NullableRowAddrSet> {
         let predicate = expr.evaluate(&self.data)?;
         let predicate = predicate.into_array(self.data.num_rows())?;
         let predicate = predicate
@@ -236,32 +289,38 @@ impl FlatIndex {
 }
 
 impl CacheCodecImpl for FlatIndex {
-    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()> {
+    const TYPE_ID: &'static str = "lance.scalar.FlatIndex";
+    const CURRENT_VERSION: u32 = 1;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         // Format:
-        // [len-prefixed all_addrs_map][len-prefixed null_addrs_map][batch IPC stream]
-        writer.write_all(&(self.all_addrs_map.serialized_size() as u64).to_le_bytes())?;
-        self.all_addrs_map.serialize_into(&mut *writer)?;
+        // RAW_BLOB  : all_addrs_map (roaring tree map)
+        // RAW_BLOB  : null_addrs_map (roaring tree map)
+        // ARROW_IPC : data batch
+        let mut all_addrs_bytes = Vec::with_capacity(self.all_addrs_map.serialized_size());
+        self.all_addrs_map.serialize_into(&mut all_addrs_bytes)?;
+        w.write_raw(&all_addrs_bytes)?;
 
-        writer.write_all(&(self.null_addrs_map.serialized_size() as u64).to_le_bytes())?;
-        self.null_addrs_map.serialize_into(&mut *writer)?;
+        let mut null_addrs_bytes = Vec::with_capacity(self.null_addrs_map.serialized_size());
+        self.null_addrs_map.serialize_into(&mut null_addrs_bytes)?;
+        w.write_raw(&null_addrs_bytes)?;
 
-        write_ipc_stream(self.data.as_ref(), writer)?;
+        w.write_ipc(self.data.as_ref())?;
 
         Ok(())
     }
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self>
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self>
     where
         Self: Sized,
     {
-        let mut offset = 0;
-        let all_addrs_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let all_addrs_bytes = r.read_raw()?;
         let all_addrs_map = RowAddrTreeMap::deserialize_from(all_addrs_bytes.as_ref())?;
 
-        let null_addrs_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let null_addrs_bytes = r.read_raw()?;
         let null_addrs_map = RowAddrTreeMap::deserialize_from(null_addrs_bytes.as_ref())?;
 
-        let batch = read_ipc_stream_single_at(data, &mut offset)?;
+        let batch = r.read_ipc()?;
 
         let df_schema = DFSchema::try_from(batch.schema())?;
 
@@ -284,7 +343,11 @@ mod tests {
     use super::*;
     use arrow_array::{record_batch, types::Int32Type};
     use datafusion_common::ScalarValue;
+    use lance_core::utils::row_addr_remap::GroupInput;
     use lance_datagen::{RowCount, array, gen_batch};
+    use roaring::RoaringTreemap;
+    use rstest::rstest;
+    use std::collections::HashMap;
 
     fn example_index() -> FlatIndex {
         let batch = gen_batch()
@@ -309,8 +372,12 @@ mod tests {
 
     fn assert_roundtrips(index: &FlatIndex) {
         let mut buf = Vec::new();
-        index.serialize(&mut buf).unwrap();
-        let restored = FlatIndex::deserialize(&bytes::Bytes::from(buf)).unwrap();
+        index
+            .serialize(&mut CacheEntryWriter::new(&mut buf))
+            .unwrap();
+        let data = bytes::Bytes::from(buf);
+        let mut reader = CacheEntryReader::new(&data, 0, FlatIndex::CURRENT_VERSION);
+        let restored = FlatIndex::deserialize(&mut reader).unwrap();
 
         assert_eq!(restored.data, index.data);
         assert_eq!(restored.all_addrs_map, index.all_addrs_map);
@@ -335,6 +402,41 @@ mod tests {
         assert_roundtrips(&FlatIndex::try_new(empty).unwrap());
     }
 
+    /// The data batch must decode zero-copy through the full envelope-bearing
+    /// [`CacheCodec`], even though the two roaring blobs and the envelope push
+    /// the IPC section to a non-aligned starting offset.
+    #[test]
+    fn test_flat_index_data_is_zero_copy() {
+        use lance_core::cache::CacheCodec;
+        const ALIGN: usize = 64;
+
+        let index = example_index();
+        let codec = CacheCodec::from_impl::<FlatIndex>();
+        let any: Arc<dyn std::any::Any + Send + Sync> = Arc::new(index);
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+
+        let mut v = vec![0u8; buf.len() + ALIGN];
+        let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+        v[pad..pad + buf.len()].copy_from_slice(&buf);
+        let data = bytes::Bytes::from(v).slice(pad..pad + buf.len());
+
+        let restored = codec.deserialize(&data).hit().unwrap();
+        let restored = restored.downcast::<FlatIndex>().unwrap();
+
+        let base = data.as_ptr() as usize;
+        let end = base + data.len();
+        for col in restored.data.columns() {
+            for buffer in col.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= base && ptr < end,
+                    "data batch buffer was realigned out of the input — misaligned IPC section",
+                );
+            }
+        }
+    }
+
     #[tokio::test]
     async fn test_equality() {
         check_index(&SargableQuery::Equals(ScalarValue::from(100)), &[0]).await;
@@ -389,9 +491,10 @@ mod tests {
         // 3 -> delete
         // Keep remaining as is
         let mapping = HashMap::<u64, Option<u64>>::from_iter(vec![(0, Some(2000)), (3, None)]);
-        let remapped =
-            FlatIndex::try_new(FlatIndex::remap_batch((*index.data).clone(), &mapping).unwrap())
-                .unwrap();
+        let remapped = FlatIndex::try_new(
+            FlatIndex::remap_batch((*index.data).clone(), &RowAddrRemap::direct(mapping)).unwrap(),
+        )
+        .unwrap();
 
         let expected = FlatIndex::try_new(
             gen_batch()
@@ -404,18 +507,22 @@ mod tests {
         assert_eq!(remapped.data, expected.data);
     }
 
-    // It's possible, during compaction, that an entire page of values is deleted.  We just serialize
-    // it as an empty record batch.
-    #[tokio::test]
-    async fn test_remap_to_nothing() {
+    // An entire page (frag 0) is deleted during compaction. remap_batch must
+    // drop every row regardless of which RowAddrRemap mode expresses it.
+    // example_index holds row ids 5, 0, 3, 100, all in frag 0.
+    #[rstest]
+    #[case::compact(RowAddrRemap::compact([GroupInput {
+        rewritten_old_row_addrs: RoaringTreemap::new(),
+        old_frag_ids: vec![0],
+        new_frags: vec![],
+    }])
+    .unwrap())]
+    #[case::explicit(RowAddrRemap::direct(
+        [5u64, 0, 3, 100].into_iter().map(|id| (id, None)).collect(),
+    ))]
+    fn test_remap_to_nothing(#[case] remap: RowAddrRemap) {
         let index = example_index();
-        let mapping = HashMap::<u64, Option<u64>>::from_iter(vec![
-            (5, None),
-            (0, None),
-            (3, None),
-            (100, None),
-        ]);
-        let remapped = FlatIndex::remap_batch((*index.data).clone(), &mapping).unwrap();
+        let remapped = FlatIndex::remap_batch((*index.data).clone(), &remap).unwrap();
         assert_eq!(remapped.num_rows(), 0);
     }
 
diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs
index 187d5be999f..9dae8bab67c 100644
--- a/rust/lance-index/src/scalar/expression.rs
+++ b/rust/lance-index/src/scalar/expression.rs
@@ -22,6 +22,7 @@ use super::{GeoQuery, RelationQuery};
 use lance_core::{Error, Result};
 use lance_datafusion::{expr::safe_coerce_scalar, planner::Planner};
 use lance_select::{IndexExprResult, NullableIndexExprResult, NullableRowAddrMask};
+use roaring::RoaringBitmap;
 use tracing::instrument;
 
 const MAX_DEPTH: usize = 500;
@@ -178,6 +179,18 @@ impl MultiQueryParser {
     pub fn add(&mut self, other: Box<dyn ScalarQueryParser>) {
         self.parsers.push(other);
     }
+
+    /// Pick the first underlying parser whose `is_valid_reference` accepts `expr`.
+    pub fn select(
+        &self,
+        expr: &Expr,
+        data_type: &DataType,
+    ) -> Option<(&dyn ScalarQueryParser, DataType)> {
+        self.parsers.iter().find_map(|p| {
+            p.is_valid_reference(expr, data_type)
+                .map(|dt| (p.as_ref(), dt))
+        })
+    }
 }
 
 impl ScalarQueryParser for MultiQueryParser {
@@ -256,6 +269,7 @@ pub struct SargableQueryParser {
     index_name: String,
     index_type: String,
     needs_recheck: bool,
+    supports_like_prefix: bool,
 }
 
 impl SargableQueryParser {
@@ -264,8 +278,17 @@ impl SargableQueryParser {
             index_name,
             index_type,
             needs_recheck,
+            supports_like_prefix: true,
         }
     }
+
+    /// Bitmap (and similar) indexes cannot answer prefix queries; disabling
+    /// `LikePrefix` emission makes `LIKE`/`starts_with` predicates fall back to
+    /// ordinary filtering instead of failing at search time.
+    pub fn without_like_prefix(mut self) -> Self {
+        self.supports_like_prefix = false;
+        self
+    }
 }
 
 impl ScalarQueryParser for SargableQueryParser {
@@ -379,12 +402,24 @@ impl ScalarQueryParser for SargableQueryParser {
     ) -> Option<IndexedExpression> {
         // Handle starts_with(col, 'prefix') -> convert to LikePrefix query
         if func.name() == "starts_with" && args.len() == 2 {
+            // Indexes that cannot answer prefix queries (e.g. bitmap) fall back to
+            // ordinary filtering rather than emitting a query they would reject.
+            if !self.supports_like_prefix {
+                return None;
+            }
             // Extract the prefix from the second argument
             let prefix = match &args[1] {
                 Expr::Literal(ScalarValue::Utf8(Some(s)), _) => ScalarValue::Utf8(Some(s.clone())),
                 Expr::Literal(ScalarValue::LargeUtf8(Some(s)), _) => {
                     ScalarValue::LargeUtf8(Some(s.clone()))
                 }
+                // Lance stores `Utf8View` columns as `Utf8` (normalized at write time), so a
+                // `Utf8View` literal is normalized to `Utf8` to match the indexed data: the
+                // BTree compares the query bound against `Utf8` page statistics at the Arrow
+                // level, which rejects a `Utf8View` bound.
+                Expr::Literal(ScalarValue::Utf8View(Some(s)), _) => {
+                    ScalarValue::Utf8(Some(s.clone()))
+                }
                 _ => return None,
             };
 
@@ -412,19 +447,29 @@ impl ScalarQueryParser for SargableQueryParser {
             return None;
         }
 
+        // Indexes that cannot answer prefix queries (e.g. bitmap) fall back to
+        // ordinary filtering rather than emitting a query they would reject.
+        if !self.supports_like_prefix {
+            return None;
+        }
+
         // Extract the pattern string
         let pattern_str = match pattern {
-            ScalarValue::Utf8(Some(s)) => s.as_str(),
-            ScalarValue::LargeUtf8(Some(s)) => s.as_str(),
+            ScalarValue::Utf8(Some(s))
+            | ScalarValue::LargeUtf8(Some(s))
+            | ScalarValue::Utf8View(Some(s)) => s.as_str(),
             _ => return None,
         };
 
         // Try to extract a prefix from the LIKE pattern
         let (prefix, needs_refine) = extract_like_leading_prefix(pattern_str, like.escape_char)?;
 
-        // Create the prefix ScalarValue with the same type as the pattern
+        // Create the prefix ScalarValue with the same type as the pattern. `Utf8View` is
+        // normalized to `Utf8` because Lance stores `Utf8View` columns as `Utf8`, and the
+        // downstream BTree compares the query bound against `Utf8` page statistics at the
+        // Arrow level (a `Utf8View` bound would fail that comparison).
         let prefix_value = match pattern {
-            ScalarValue::Utf8(_) => ScalarValue::Utf8(Some(prefix)),
+            ScalarValue::Utf8(_) | ScalarValue::Utf8View(_) => ScalarValue::Utf8(Some(prefix)),
             ScalarValue::LargeUtf8(_) => ScalarValue::LargeUtf8(Some(prefix)),
             _ => return None,
         };
@@ -436,6 +481,7 @@ impl ScalarQueryParser for SargableQueryParser {
             index_type: self.index_type.clone(),
             query: Arc::new(query),
             needs_recheck: self.needs_recheck,
+            fragment_bitmap: None,
         }));
 
         // If the pattern has wildcards beyond simple prefix, add refine expression
@@ -779,20 +825,28 @@ impl ScalarQueryParser for LabelListQueryParser {
     }
 }
 
-/// A parser for indices that handle string contains queries
+/// A parser for indices that handle string `contains` queries, and -- when
+/// `supports_regex` is set -- `regexp_like` / `regexp_match` queries.
 #[derive(Debug, Clone)]
 pub struct TextQueryParser {
     index_name: String,
     index_type: String,
     needs_recheck: bool,
+    supports_regex: bool,
 }
 
 impl TextQueryParser {
-    pub fn new(index_name: String, index_type: String, needs_recheck: bool) -> Self {
+    pub fn new(
+        index_name: String,
+        index_type: String,
+        needs_recheck: bool,
+        supports_regex: bool,
+    ) -> Self {
         Self {
             index_name,
             index_type,
             needs_recheck,
+            supports_regex,
         }
     }
 }
@@ -835,31 +889,156 @@ impl ScalarQueryParser for TextQueryParser {
         func: &ScalarUDF,
         args: &[Expr],
     ) -> Option<IndexedExpression> {
-        if args.len() != 2 {
+        // The first argument is the indexed column; the second is the substring
+        // / pattern. `contains` takes exactly two arguments; the regex functions
+        // optionally take a third flags argument.
+        if args.len() < 2 {
             return None;
         }
-        let scalar = maybe_scalar(&args[1], data_type)?;
-        match scalar {
-            ScalarValue::Utf8(Some(scalar_str)) | ScalarValue::LargeUtf8(Some(scalar_str)) => {
-                if func.name() == "contains" {
-                    let query = TextQuery::StringContains(scalar_str);
-                    Some(IndexedExpression::index_query_with_recheck(
-                        column.to_string(),
-                        self.index_name.clone(),
-                        self.index_type.clone(),
-                        Arc::new(query),
-                        self.needs_recheck,
-                    ))
-                } else {
+        // A non-string pattern cannot be handled.
+        let (ScalarValue::Utf8(Some(pattern)) | ScalarValue::LargeUtf8(Some(pattern))) =
+            maybe_scalar(&args[1], data_type)?
+        else {
+            return None;
+        };
+
+        let query = match func.name() {
+            "contains" if args.len() == 2 => TextQuery::StringContains(pattern),
+            "regexp_like" | "regexp_match" if self.supports_regex => {
+                let pattern = match args.get(2) {
+                    Some(flags_expr) => apply_regex_flags(&pattern, flags_expr)?,
+                    None => pattern,
+                };
+                // If the pattern yields no usable trigram (e.g. `a.b`), leave it
+                // to a full scan instead of routing it to the index, which could
+                // only answer with an unsupported "recheck everything" result.
+                if !crate::scalar::ngram::regex_can_use_index(&pattern) {
+                    return None;
+                }
+                TextQuery::Regex(pattern)
+            }
+            _ => return None,
+        };
+
+        Some(IndexedExpression::index_query_with_recheck(
+            column.to_string(),
+            self.index_name.clone(),
+            self.index_type.clone(),
+            Arc::new(query),
+            self.needs_recheck,
+        ))
+    }
+
+    fn visit_like(
+        &self,
+        column: &str,
+        like: &Like,
+        pattern: &ScalarValue,
+    ) -> Option<IndexedExpression> {
+        // Infix LIKE is accelerated only by the ngram index (via its regex
+        // machinery). A plain-literal `regexp_like(col, 'foo')` is rewritten to
+        // `col LIKE '%foo%'` before it reaches the index, so this is the path
+        // that accelerates those. ILIKE is skipped because its case folding does
+        // not match the index's normalization.
+        if !self.supports_regex || like.case_insensitive {
+            return None;
+        }
+        let pattern_str = match pattern {
+            ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => s.as_str(),
+            _ => return None,
+        };
+        // Translate the LIKE pattern into a loose regex used only for candidate
+        // generation; the original LIKE stays as the recheck filter, so the
+        // regex only needs to be a sound superset.
+        let regex = like_to_regex(pattern_str, like.escape_char)?;
+        if !crate::scalar::ngram::regex_can_use_index(&regex) {
+            return None;
+        }
+        Some(IndexedExpression {
+            scalar_query: Some(ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: column.to_string(),
+                index_name: self.index_name.clone(),
+                index_type: self.index_type.clone(),
+                query: Arc::new(TextQuery::Regex(regex)),
+                needs_recheck: self.needs_recheck,
+                fragment_bitmap: None,
+            })),
+            refine_expr: Some(Expr::Like(like.clone())),
+        })
+    }
+}
+
+/// Translate a LIKE pattern into a regular expression used purely for ngram
+/// candidate generation: `%` becomes `.*`, `_` becomes `.`, and literal
+/// characters are regex-escaped. Returns `None` when no literal run is long
+/// enough to yield a trigram (the index could not help, so a full scan is left
+/// to handle it).
+fn like_to_regex(pattern: &str, escape: Option<char>) -> Option<String> {
+    let mut regex = String::new();
+    let mut run = 0usize;
+    let mut longest_run = 0usize;
+    let mut chars = pattern.chars();
+    while let Some(c) = chars.next() {
+        let literal = if Some(c) == escape {
+            // The next character is escaped, i.e. a literal.
+            chars.next()
+        } else {
+            match c {
+                '%' => {
+                    regex.push_str(".*");
+                    run = 0;
+                    None
+                }
+                '_' => {
+                    regex.push('.');
+                    run = 0;
                     None
                 }
+                other => Some(other),
             }
-            _ => {
-                // If the scalar is not a string, we cannot handle it
-                None
+        };
+        if let Some(lit) = literal {
+            if regex_syntax::is_meta_character(lit) {
+                regex.push('\\');
+            }
+            regex.push(lit);
+            // Only runs of alphanumeric characters can produce a trigram.
+            if lit.is_alphanumeric() {
+                run += 1;
+                longest_run = longest_run.max(run);
+            } else {
+                run = 0;
             }
         }
     }
+    (longest_run >= 3).then_some(regex)
+}
+
+/// Fold the supported `regexp_like` / `regexp_match` flags into an inline prefix
+/// on the pattern (e.g. flags `"i"` -> `"(?i)pattern"`). Returns `None` for a
+/// non-literal flags argument or an unrecognized flag, so the caller leaves the
+/// predicate to a full recheck rather than risk changing its semantics.
+fn apply_regex_flags(pattern: &str, flags_expr: &Expr) -> Option<String> {
+    let (Expr::Literal(ScalarValue::Utf8(Some(flags)), _)
+    | Expr::Literal(ScalarValue::LargeUtf8(Some(flags)), _)) = flags_expr
+    else {
+        return None;
+    };
+    let mut inline = String::new();
+    for flag in flags.chars() {
+        // Only flags expressible as an inline `(?...)` group in the regex crate
+        // (which the recheck uses) are safe to fold.
+        if ['i', 's', 'm', 'x'].contains(&flag) {
+            inline.push(flag);
+        } else {
+            return None;
+        }
+    }
+    if inline.is_empty() {
+        Some(pattern.to_string())
+    } else {
+        Some(format!("(?{inline}){pattern}"))
+    }
 }
 
 /// A parser for indices that handle queries with the contains_tokens function
@@ -1074,7 +1253,8 @@ impl IndexedExpression {
                 index_name,
                 index_type,
                 query,
-                needs_recheck: false, // Default to false, will be set by parser
+                needs_recheck: false,  // Default to false, will be set by parser
+                fragment_bitmap: None, // Filled in by `apply_scalar_indices`
             })),
             refine_expr: None,
         }
@@ -1095,6 +1275,7 @@ impl IndexedExpression {
                 index_type,
                 query,
                 needs_recheck,
+                fragment_bitmap: None, // Filled in by `apply_scalar_indices`
             })),
             refine_expr: None,
         }
@@ -1221,6 +1402,20 @@ pub trait ScalarIndexLoader: Send + Sync {
         index_name: &str,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn ScalarIndex>>;
+
+    /// Translate an address-domain index result into the row-id domain
+    ///
+    /// Address-domain indices (see [`ScalarIndex::results_are_row_addresses`])
+    /// report matches as physical row addresses. The default returns `result`
+    /// unchanged, which is correct when addresses and row ids coincide (no
+    /// stable row ids). A dataset with stable row ids overrides this to remap
+    /// addresses to stable row ids via its per-fragment row-id sequences.
+    async fn row_addr_result_to_row_ids(
+        &self,
+        result: NullableIndexExprResult,
+    ) -> Result<NullableIndexExprResult> {
+        Ok(result)
+    }
 }
 
 /// This represents a search into a scalar index
@@ -1236,10 +1431,21 @@ pub struct ScalarIndexSearch {
     pub query: Arc<dyn AnyQuery>,
     /// If true, the query results are inexact and will need a recheck
     pub needs_recheck: bool,
+    /// The fragments the underlying index has entries for.
+    ///
+    /// `None` means coverage is unknown (e.g. constructed outside of scanner
+    /// planning, or from a legacy code path). Optimizer rules that need to
+    /// decide whether the index covers the dataset must treat `None` as
+    /// "refuse to use" — the bitmap is the only way to safely answer that
+    /// question synchronously without an async metadata load.
+    pub fragment_bitmap: Option<RoaringBitmap>,
 }
 
 impl PartialEq for ScalarIndexSearch {
     fn eq(&self, other: &Self) -> bool {
+        // `fragment_bitmap` is metadata derived from the dataset state, not
+        // part of the query identity, so it intentionally does not participate
+        // in equality.
         self.column == other.column
             && self.index_name == other.index_name
             && self.query.as_ref().eq(other.query.as_ref())
@@ -1270,6 +1476,175 @@ impl PartialEq for ScalarIndexExpr {
     }
 }
 
+/// Returns the tighter (more restrictive) lower bound.
+/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value.
+fn tighter_lower_bound(a: &Bound<ScalarValue>, b: &Bound<ScalarValue>) -> Bound<ScalarValue> {
+    match (a, b) {
+        (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(),
+        (Bound::Included(va), Bound::Included(vb)) => {
+            if va >= vb {
+                Bound::Included(va.clone())
+            } else {
+                Bound::Included(vb.clone())
+            }
+        }
+        (Bound::Excluded(va), Bound::Excluded(vb)) => {
+            if va >= vb {
+                Bound::Excluded(va.clone())
+            } else {
+                Bound::Excluded(vb.clone())
+            }
+        }
+        (Bound::Excluded(va), Bound::Included(vb)) => {
+            if va >= vb {
+                Bound::Excluded(va.clone())
+            } else {
+                Bound::Included(vb.clone())
+            }
+        }
+        (Bound::Included(va), Bound::Excluded(vb)) => {
+            if vb >= va {
+                Bound::Excluded(vb.clone())
+            } else {
+                Bound::Included(va.clone())
+            }
+        }
+    }
+}
+
+/// Returns the tighter (more restrictive) upper bound.
+/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value.
+fn tighter_upper_bound(a: &Bound<ScalarValue>, b: &Bound<ScalarValue>) -> Bound<ScalarValue> {
+    match (a, b) {
+        (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(),
+        (Bound::Included(va), Bound::Included(vb)) => {
+            if va <= vb {
+                Bound::Included(va.clone())
+            } else {
+                Bound::Included(vb.clone())
+            }
+        }
+        (Bound::Excluded(va), Bound::Excluded(vb)) => {
+            if va <= vb {
+                Bound::Excluded(va.clone())
+            } else {
+                Bound::Excluded(vb.clone())
+            }
+        }
+        (Bound::Excluded(va), Bound::Included(vb)) => {
+            if va <= vb {
+                Bound::Excluded(va.clone())
+            } else {
+                Bound::Included(vb.clone())
+            }
+        }
+        (Bound::Included(va), Bound::Excluded(vb)) => {
+            if vb <= va {
+                Bound::Excluded(vb.clone())
+            } else {
+                Bound::Included(va.clone())
+            }
+        }
+    }
+}
+
+impl ScalarIndexExpr {
+    /// Optimize the expression tree by merging range queries on the same index.
+    ///
+    /// This collects all leaf Range queries from the AND tree, groups them by
+    /// index name, merges overlapping ranges into a single closed-range query,
+    /// and rebuilds the tree. This handles the case where `log_time >= X` and
+    /// `log_time <= Y` end up in different branches of a nested AND tree.
+    pub fn optimize(self) -> Self {
+        match self {
+            Self::And(_, _) => self.optimize_and_tree(),
+            Self::Or(lhs, rhs) => Self::Or(Box::new(lhs.optimize()), Box::new(rhs.optimize())),
+            Self::Not(inner) => Self::Not(Box::new(inner.optimize())),
+            other => other,
+        }
+    }
+
+    /// Flatten an AND tree, merge ranges on same index, rebuild.
+    fn optimize_and_tree(self) -> Self {
+        let mut leaves = Vec::new();
+        self.collect_and_leaves(&mut leaves);
+
+        // Try to merge Range queries on the same index
+        let mut merged_indices: Vec<bool> = vec![false; leaves.len()];
+        let mut result_leaves: Vec<Self> = Vec::new();
+
+        for i in 0..leaves.len() {
+            if merged_indices[i] {
+                continue;
+            }
+            let mut current = leaves[i].clone();
+
+            // Try to merge with subsequent leaves on the same index
+            for j in (i + 1)..leaves.len() {
+                if merged_indices[j] {
+                    continue;
+                }
+                if let Some(merged) = try_merge_range_pair(&current, &leaves[j]) {
+                    current = merged;
+                    merged_indices[j] = true;
+                }
+            }
+
+            result_leaves.push(current);
+        }
+
+        // Rebuild the AND tree from remaining leaves
+        let mut iter = result_leaves.into_iter();
+        let first = iter.next().expect("AND tree must have at least one leaf");
+        iter.fold(first, |acc, leaf| Self::And(Box::new(acc), Box::new(leaf)))
+    }
+
+    /// Recursively collect all leaf nodes from an AND tree.
+    fn collect_and_leaves(self, leaves: &mut Vec<Self>) {
+        match self {
+            Self::And(lhs, rhs) => {
+                lhs.collect_and_leaves(leaves);
+                rhs.collect_and_leaves(leaves);
+            }
+            other => leaves.push(other),
+        }
+    }
+}
+
+/// Try to merge two ScalarIndexExpr nodes if they are both Range queries on the same index.
+fn try_merge_range_pair(lhs: &ScalarIndexExpr, rhs: &ScalarIndexExpr) -> Option<ScalarIndexExpr> {
+    let (ScalarIndexExpr::Query(l), ScalarIndexExpr::Query(r)) = (lhs, rhs) else {
+        return None;
+    };
+    if l.index_name != r.index_name || l.column != r.column {
+        return None;
+    }
+
+    let l_query = l.query.as_any().downcast_ref::<SargableQuery>()?;
+    let r_query = r.query.as_any().downcast_ref::<SargableQuery>()?;
+
+    let (SargableQuery::Range(l_low, l_high), SargableQuery::Range(r_low, r_high)) =
+        (l_query, r_query)
+    else {
+        return None;
+    };
+
+    let merged_low = tighter_lower_bound(l_low, r_low);
+    let merged_high = tighter_upper_bound(l_high, r_high);
+
+    Some(ScalarIndexExpr::Query(ScalarIndexSearch {
+        column: l.column.clone(),
+        index_name: l.index_name.clone(),
+        index_type: l.index_type.clone(),
+        query: Arc::new(SargableQuery::Range(merged_low, merged_high)),
+        needs_recheck: l.needs_recheck || r.needs_recheck,
+        // Both queries target the same index (checked above), so they share
+        // the same fragment coverage; carry it over to keep the merged query
+        // usable by coverage-dependent optimizer rules.
+        fragment_bitmap: l.fragment_bitmap.clone(),
+    }))
+}
+
 impl std::fmt::Display for ScalarIndexExpr {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -1287,12 +1662,16 @@ impl std::fmt::Display for ScalarIndexExpr {
     }
 }
 
-impl From<SearchResult> for NullableIndexExprResult {
-    fn from(result: SearchResult) -> Self {
-        match result {
-            SearchResult::Exact(mask) => Self::exact(NullableRowAddrMask::AllowList(mask)),
-            SearchResult::AtMost(mask) => Self::at_most(NullableRowAddrMask::AllowList(mask)),
-            SearchResult::AtLeast(mask) => Self::at_least(NullableRowAddrMask::AllowList(mask)),
+fn search_result_to_nullable(result: SearchResult) -> NullableIndexExprResult {
+    match result {
+        SearchResult::Exact(mask) => {
+            NullableIndexExprResult::exact(NullableRowAddrMask::AllowList(mask))
+        }
+        SearchResult::AtMost(mask) => {
+            NullableIndexExprResult::at_most(NullableRowAddrMask::AllowList(mask))
+        }
+        SearchResult::AtLeast(mask) => {
+            NullableIndexExprResult::at_least(NullableRowAddrMask::AllowList(mask))
         }
     }
 }
@@ -1332,7 +1711,15 @@ impl ScalarIndexExpr {
                     .load_index(&search.column, &search.index_name, metrics)
                     .await?;
                 let search_result = index.search(search.query.as_ref(), metrics).await?;
-                Ok(search_result.into())
+                let result = search_result_to_nullable(search_result);
+                if index.results_are_row_addresses() {
+                    // Translate address-domain results to the row-id domain
+                    // before combining or scanning; otherwise stable-row-id
+                    // datasets silently drop matches (issue #7434).
+                    index_loader.row_addr_result_to_row_ids(result).await
+                } else {
+                    Ok(result)
+                }
             }
         }
     }
@@ -1437,8 +1824,8 @@ fn maybe_indexed_column<'b>(
 ) -> Option<(String, DataType, &'b dyn ScalarQueryParser)> {
     // First try to extract the full nested column path for get_field expressions
     if let Some(nested_path) = extract_nested_column_path(expr)
-        && let Some((data_type, parser)) = index_info.get_index(&nested_path)
-        && let Some(data_type) = parser.is_valid_reference(expr, data_type)
+        && let Some((data_type, multi)) = index_info.get_index(&nested_path)
+        && let Some((parser, data_type)) = multi.select(expr, data_type)
     {
         return Some((nested_path, data_type, parser));
     }
@@ -1446,12 +1833,9 @@ fn maybe_indexed_column<'b>(
     match expr {
         Expr::Column(col) => {
             let col = col.name.as_str();
-            let (data_type, parser) = index_info.get_index(col)?;
-            if let Some(data_type) = parser.is_valid_reference(expr, data_type) {
-                Some((col.to_string(), data_type, parser))
-            } else {
-                None
-            }
+            let (data_type, multi) = index_info.get_index(col)?;
+            let (parser, data_type) = multi.select(expr, data_type)?;
+            Some((col.to_string(), data_type, parser))
         }
         Expr::ScalarFunction(udf) => {
             if udf.args.is_empty() {
@@ -1459,12 +1843,9 @@ fn maybe_indexed_column<'b>(
             }
             // For non-get_field functions, fall back to old behavior
             let col = maybe_column(&udf.args[0])?;
-            let (data_type, parser) = index_info.get_index(col)?;
-            if let Some(data_type) = parser.is_valid_reference(expr, data_type) {
-                Some((col.to_string(), data_type, parser))
-            } else {
-                None
-            }
+            let (data_type, multi) = index_info.get_index(col)?;
+            let (parser, data_type) = multi.select(expr, data_type)?;
+            Some((col.to_string(), data_type, parser))
         }
         _ => None,
     }
@@ -1483,7 +1864,7 @@ fn maybe_scalar(expr: &Expr, expected_type: &DataType) -> Option<ScalarValue> {
         // In this case we need to extract the value, apply the cast, and then test the casted value
         Expr::Cast(cast) => match cast.expr.as_ref() {
             Expr::Literal(value, _) => {
-                let casted = value.cast_to(&cast.data_type).ok()?;
+                let casted = value.cast_to(cast.field.data_type()).ok()?;
                 safe_coerce_scalar(&casted, expected_type)
             }
             _ => None,
@@ -1798,7 +2179,18 @@ fn visit_node(
         Expr::IsFalse(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, false)),
         Expr::IsTrue(expr) => Ok(visit_is_bool(expr.as_ref(), index_info, true)),
         Expr::IsNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, false)),
-        Expr::IsNotNull(expr) => Ok(visit_is_null(expr.as_ref(), index_info, true)),
+        Expr::IsNotNull(expr) => {
+            // `regexp_match(col, pat)` returns a list and is coerced to
+            // `IsNotNull(regexp_match(...))` before it reaches here. Unwrap that
+            // so the regex acceleration applies; everything else is a genuine
+            // IS NOT NULL check.
+            if let Expr::ScalarFunction(scalar_fn) = expr.as_ref()
+                && scalar_fn.func.name() == "regexp_match"
+            {
+                return Ok(visit_scalar_fn(scalar_fn, index_info));
+            }
+            Ok(visit_is_null(expr.as_ref(), index_info, true))
+        }
         Expr::Not(expr) => visit_not(expr.as_ref(), index_info, depth),
         Expr::BinaryExpr(binary_expr) => visit_binary_expr(binary_expr, index_info, depth),
         Expr::ScalarFunction(scalar_fn) => Ok(visit_scalar_fn(scalar_fn, index_info)),
@@ -1818,7 +2210,17 @@ fn visit_node(
 pub trait IndexInformationProvider {
     /// Check if an index exists for `col` and, if so, return the data type of col
     /// as well as a query parser that can parse queries for that column
-    fn get_index(&self, col: &str) -> Option<(&DataType, &dyn ScalarQueryParser)>;
+    fn get_index(&self, col: &str) -> Option<(&DataType, &MultiQueryParser)>;
+
+    /// The set of fragments covered by `(column, index_name)`.
+    ///
+    /// Returns `None` when the provider doesn't know — callers must treat
+    /// that as "coverage unknown" rather than "covers everything". The
+    /// default implementation always returns `None`, so providers that
+    /// haven't been updated cannot accidentally claim full coverage.
+    fn fragment_bitmap(&self, _column: &str, _index_name: &str) -> Option<RoaringBitmap> {
+        None
+    }
 }
 
 /// Attempt to split a filter expression into a search of scalar indexes and an
@@ -1827,7 +2229,31 @@ pub fn apply_scalar_indices(
     expr: Expr,
     index_info: &dyn IndexInformationProvider,
 ) -> Result<IndexedExpression> {
-    Ok(visit_node(&expr, index_info, 0)?.unwrap_or(IndexedExpression::refine_only(expr)))
+    let mut result =
+        visit_node(&expr, index_info, 0)?.unwrap_or(IndexedExpression::refine_only(expr));
+    if let Some(query) = result.scalar_query.as_mut() {
+        populate_fragment_bitmaps(query, index_info);
+    }
+    Ok(result)
+}
+
+/// Walk a [`ScalarIndexExpr`] and fill in `fragment_bitmap` on each leaf from
+/// the `index_info` provider. Leaves the bitmap as `None` if the provider
+/// can't answer.
+fn populate_fragment_bitmaps(
+    expr: &mut ScalarIndexExpr,
+    index_info: &dyn IndexInformationProvider,
+) {
+    match expr {
+        ScalarIndexExpr::Not(inner) => populate_fragment_bitmaps(inner, index_info),
+        ScalarIndexExpr::And(lhs, rhs) | ScalarIndexExpr::Or(lhs, rhs) => {
+            populate_fragment_bitmaps(lhs, index_info);
+            populate_fragment_bitmaps(rhs, index_info);
+        }
+        ScalarIndexExpr::Query(search) => {
+            search.fragment_bitmap = index_info.fragment_bitmap(&search.column, &search.index_name);
+        }
+    }
 }
 
 #[derive(Clone, Default, Debug)]
@@ -1966,11 +2392,18 @@ mod tests {
 
     struct ColInfo {
         data_type: DataType,
-        parser: Box<dyn ScalarQueryParser>,
+        parser: Box<MultiQueryParser>,
     }
 
     impl ColInfo {
         fn new(data_type: DataType, parser: Box<dyn ScalarQueryParser>) -> Self {
+            Self {
+                data_type,
+                parser: Box::new(MultiQueryParser::single(parser)),
+            }
+        }
+
+        fn with_multi(data_type: DataType, parser: Box<MultiQueryParser>) -> Self {
             Self { data_type, parser }
         }
     }
@@ -1992,7 +2425,7 @@ mod tests {
     }
 
     impl IndexInformationProvider for MockIndexInfoProvider {
-        fn get_index(&self, col: &str) -> Option<(&DataType, &dyn ScalarQueryParser)> {
+        fn get_index(&self, col: &str) -> Option<(&DataType, &MultiQueryParser)> {
             self.indexed_columns
                 .get(col)
                 .map(|col_info| (&col_info.data_type, col_info.parser.as_ref()))
@@ -2019,9 +2452,10 @@ mod tests {
         let state = ctx.state();
         let mut expr = state.create_logical_expr(expr, &df_schema).unwrap();
         if optimize {
-            let simplify_context = SimplifyContext::default()
+            let simplify_context = SimplifyContext::builder()
                 .with_schema(Arc::new(df_schema))
-                .with_query_execution_start_time(Some(Utc::now()));
+                .with_query_execution_start_time(Some(Utc::now()))
+                .build();
             let simplifier =
                 datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
             expr = simplifier.simplify(expr).unwrap();
@@ -2422,6 +2856,7 @@ mod tests {
             index_type: "BTree".to_string(),
             query: Arc::new(SargableQuery::Equals(ScalarValue::UInt32(Some(10)))),
             needs_recheck: false,
+            fragment_bitmap: None,
         }));
         let right = Box::new(ScalarIndexExpr::Query(ScalarIndexSearch {
             column: "color".to_string(),
@@ -2431,6 +2866,7 @@ mod tests {
                 "blue".to_string(),
             )))),
             needs_recheck: false,
+            fragment_bitmap: None,
         }));
         check(
             &index_info,
@@ -2639,6 +3075,59 @@ mod tests {
         assert!(matches!(negated.upper, NullableRowAddrMask::BlockList(_)));
     }
 
+    #[test]
+    fn test_like_to_regex() {
+        // `%` -> `.*`, `_` -> `.`, with a literal run of at least three chars.
+        assert_eq!(like_to_regex("%foo%", None).as_deref(), Some(".*foo.*"));
+        assert_eq!(like_to_regex("foo%bar", None).as_deref(), Some("foo.*bar"));
+        assert_eq!(like_to_regex("foo_bar", None).as_deref(), Some("foo.bar"));
+        assert_eq!(like_to_regex("foobar", None).as_deref(), Some("foobar"));
+
+        // Regex metacharacters in the literal portion are escaped.
+        assert_eq!(
+            like_to_regex("%a.bcd%", None).as_deref(),
+            Some(".*a\\.bcd.*")
+        );
+
+        // No literal run of three alphanumeric characters -> no index help.
+        assert_eq!(like_to_regex("%ab%", None), None);
+        assert_eq!(like_to_regex("%a%b%c%", None), None);
+        assert_eq!(like_to_regex("%", None), None);
+
+        // The escape character makes the following character a literal.
+        assert_eq!(
+            like_to_regex(r"%foo\%bar%", Some('\\')).as_deref(),
+            Some(".*foo%bar.*")
+        );
+    }
+
+    #[test]
+    fn test_apply_regex_flags() {
+        fn flags(s: &str) -> Expr {
+            Expr::Literal(ScalarValue::Utf8(Some(s.to_string())), None)
+        }
+
+        // Empty flags leave the pattern untouched (no inline group emitted).
+        assert_eq!(apply_regex_flags("foo", &flags("")).as_deref(), Some("foo"));
+        // Supported flags are folded into an inline `(?...)` prefix.
+        assert_eq!(
+            apply_regex_flags("foo", &flags("i")).as_deref(),
+            Some("(?i)foo")
+        );
+        assert_eq!(
+            apply_regex_flags("foo", &flags("is")).as_deref(),
+            Some("(?is)foo")
+        );
+        // An unrecognized flag bails out so the caller leaves the predicate to a
+        // full recheck rather than risk changing its semantics.
+        assert_eq!(apply_regex_flags("foo", &flags("g")), None);
+        // A non-string (hence non-literal-flags) argument cannot be folded.
+        assert_eq!(
+            apply_regex_flags("foo", &Expr::Literal(ScalarValue::Int32(Some(1)), None)),
+            None
+        );
+    }
+
     #[test]
     fn test_extract_like_leading_prefix() {
         // Simple prefix patterns (no recheck needed)
@@ -2840,9 +3329,10 @@ mod tests {
             .unwrap();
 
         // Apply DataFusion simplification (this may convert starts_with to LIKE)
-        let simplify_context = SimplifyContext::default()
+        let simplify_context = SimplifyContext::builder()
             .with_schema(Arc::new(df_schema))
-            .with_query_execution_start_time(Some(Utc::now()));
+            .with_query_execution_start_time(Some(Utc::now()))
+            .build();
         let simplifier =
             datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context);
         let simplified_expr = simplifier.simplify(expr).unwrap();
@@ -2956,6 +3446,166 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_sargable_query_parser_utf8view() {
+        // Follow-up to PR #7310 / #7139: the BTree `SargableQueryParser` must accept
+        // `Utf8View` prefixes for `starts_with` and infix-free LIKE, not only `Utf8` /
+        // `LargeUtf8`. DataFusion can coerce the predicate literal to `ScalarValue::Utf8View`;
+        // dropping that variant silently skips the index. The `Utf8View` prefix is normalized
+        // to `Utf8` (Lance stores `Utf8View` columns as `Utf8`), so the emitted query is a
+        // `LikePrefix(Utf8(..))`. `visit_scalar_function` / `visit_like` are exercised directly
+        // so the test does not depend on the planner's coercion choices, and the `Utf8` case
+        // is a parity control: the pre-existing path must keep behaving identically.
+        let parser = SargableQueryParser::new("color_idx".to_string(), "BTree".to_string(), false);
+
+        let assert_like_prefix =
+            |indexed: &IndexedExpression, expected: &ScalarValue, needs_refine: bool| {
+                assert_eq!(
+                    indexed.refine_expr.is_some(),
+                    needs_refine,
+                    "unexpected refine_expr presence"
+                );
+                let Some(ScalarIndexExpr::Query(search)) = &indexed.scalar_query else {
+                    panic!("expected a scalar index query");
+                };
+                match search
+                    .query
+                    .as_any()
+                    .downcast_ref::<SargableQuery>()
+                    .expect("query should be a SargableQuery")
+                {
+                    SargableQuery::LikePrefix(prefix) => assert_eq!(prefix, expected),
+                    _ => panic!("expected a LikePrefix query"),
+                }
+            };
+
+        // starts_with(col, <Utf8View prefix>) -> LikePrefix(Utf8). Reuse a real
+        // `starts_with` UDF parsed from SQL, then swap in a `Utf8View` literal argument.
+        let schema = Schema::new(vec![Field::new("color", DataType::Utf8View, false)]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+        let Expr::ScalarFunction(starts_with) = state
+            .create_logical_expr("starts_with(color, 'foo')", &df_schema)
+            .unwrap()
+        else {
+            panic!("expected starts_with to parse as a scalar function");
+        };
+        let args = vec![
+            starts_with.args[0].clone(),
+            Expr::Literal(ScalarValue::Utf8View(Some("foo".to_string())), None),
+        ];
+        let indexed = parser
+            .visit_scalar_function(
+                "color",
+                &DataType::Utf8View,
+                starts_with.func.as_ref(),
+                &args,
+            )
+            .expect("starts_with should use the BTree index");
+        assert_like_prefix(&indexed, &ScalarValue::Utf8(Some("foo".to_string())), false);
+
+        // col LIKE <Utf8View pattern>. `visit_like` is called directly so the test does not
+        // depend on DataFusion's LIKE type coercion choosing `Utf8View` for the pattern.
+        let like = |pattern: ScalarValue| {
+            Like::new(
+                false,
+                Box::new(Expr::Column(Column::new_unqualified("color"))),
+                Box::new(Expr::Literal(pattern, None)),
+                None,
+                false,
+            )
+        };
+
+        // Pure prefix: routed to the index with no recheck needed.
+        let pattern = ScalarValue::Utf8View(Some("foo%".to_string()));
+        let indexed = parser
+            .visit_like("color", &like(pattern.clone()), &pattern)
+            .expect("LIKE prefix should use the BTree index");
+        assert_like_prefix(&indexed, &ScalarValue::Utf8(Some("foo".to_string())), false);
+
+        // Wildcards beyond the leading prefix keep the original LIKE as a recheck.
+        let pattern = ScalarValue::Utf8View(Some("foo%bar%".to_string()));
+        let indexed = parser
+            .visit_like("color", &like(pattern.clone()), &pattern)
+            .expect("LIKE prefix should use the BTree index");
+        assert_like_prefix(&indexed, &ScalarValue::Utf8(Some("foo".to_string())), true);
+
+        // Parity control: the pre-existing `Utf8` path is unchanged.
+        let pattern = ScalarValue::Utf8(Some("foo%".to_string()));
+        let indexed = parser
+            .visit_like("color", &like(pattern.clone()), &pattern)
+            .expect("LIKE prefix should use the BTree index");
+        assert_like_prefix(&indexed, &ScalarValue::Utf8(Some("foo".to_string())), false);
+    }
+
+    #[test]
+    fn test_sargable_query_parser_without_like_prefix() {
+        // Bitmap indexes configure the parser with `without_like_prefix`: a bitmap index
+        // cannot answer `LikePrefix` queries (its `search` rejects them), so `starts_with` /
+        // `LIKE 'prefix%'` must not be turned into an index query. Returning `None` lets the
+        // predicate fall back to ordinary filtering instead of failing at search time.
+        let bitmap_parser =
+            SargableQueryParser::new("color_idx".to_string(), "BITMAP".to_string(), false)
+                .without_like_prefix();
+        let btree_parser =
+            SargableQueryParser::new("color_idx".to_string(), "BTree".to_string(), false);
+
+        let schema = Schema::new(vec![Field::new("color", DataType::Utf8, false)]);
+        let df_schema: DFSchema = schema.try_into().unwrap();
+        let ctx = get_session_context(&LanceExecutionOptions::default());
+        let state = ctx.state();
+        let Expr::ScalarFunction(starts_with) = state
+            .create_logical_expr("starts_with(color, 'foo')", &df_schema)
+            .unwrap()
+        else {
+            panic!("expected starts_with to parse as a scalar function");
+        };
+
+        let pattern = ScalarValue::Utf8(Some("foo%".to_string()));
+        let like = Like::new(
+            false,
+            Box::new(Expr::Column(Column::new_unqualified("color"))),
+            Box::new(Expr::Literal(pattern.clone(), None)),
+            None,
+            false,
+        );
+
+        // Bitmap parser: both prefix paths fall back (return `None`).
+        assert!(
+            bitmap_parser
+                .visit_scalar_function(
+                    "color",
+                    &DataType::Utf8,
+                    starts_with.func.as_ref(),
+                    &starts_with.args,
+                )
+                .is_none(),
+            "bitmap parser must not emit a LikePrefix for starts_with"
+        );
+        assert!(
+            bitmap_parser.visit_like("color", &like, &pattern).is_none(),
+            "bitmap parser must not emit a LikePrefix for LIKE"
+        );
+
+        // A prefix-capable parser (e.g. BTree) still emits the index query.
+        assert!(
+            btree_parser
+                .visit_scalar_function(
+                    "color",
+                    &DataType::Utf8,
+                    starts_with.func.as_ref(),
+                    &starts_with.args,
+                )
+                .is_some(),
+            "BTree parser should still emit a LikePrefix for starts_with"
+        );
+        assert!(
+            btree_parser.visit_like("color", &like, &pattern).is_some(),
+            "BTree parser should still emit a LikePrefix for LIKE"
+        );
+    }
+
     #[test]
     fn test_serialize_index_expr_result_round_trip() {
         use lance_select::{RowAddrMask, RowAddrTreeMap};
@@ -3106,4 +3756,901 @@ mod tests {
         assert_eq!(round_tripped.upper, RowAddrMask::from_allowed(upper_addrs));
         assert_eq!(round_tripped_frags, fragments_covered);
     }
+
+    /// Regression test: when two JSON indices target different paths on the same
+    /// column, a query against one path must be routed to its own index instead
+    /// of being intercepted by whichever parser was registered first.
+    #[test]
+    fn test_multi_json_indices_route_by_path() {
+        // Build a MultiQueryParser containing two JSON sub-parsers: one for
+        // path "$.a" and one for path "$.b".
+        let mut multi = MultiQueryParser::single(Box::new(JsonQueryParser::new(
+            "$.a".to_string(),
+            Box::new(SargableQueryParser::new(
+                "json_a_idx".to_string(),
+                "Json".to_string(),
+                false,
+            )),
+        )));
+        multi.add(Box::new(JsonQueryParser::new(
+            "$.b".to_string(),
+            Box::new(SargableQueryParser::new(
+                "json_b_idx".to_string(),
+                "Json".to_string(),
+                false,
+            )),
+        )));
+
+        let index_info = MockIndexInfoProvider::new(vec![(
+            "json",
+            ColInfo::with_multi(DataType::LargeBinary, Box::new(multi)),
+        )]);
+
+        // Query against path "$.b" must hit the "$.b" index.
+        let expected_b = IndexedExpression::index_query(
+            "json".to_string(),
+            "json_b_idx".to_string(),
+            "Json".to_string(),
+            Arc::new(JsonQuery::new(
+                Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                    "foo".to_string(),
+                )))),
+                "$.b".to_string(),
+            )),
+        );
+        check(
+            &index_info,
+            "json_extract(json, '$.b') = 'foo'",
+            Some(expected_b),
+            false,
+        );
+
+        // Query against path "$.a" must hit the "$.a" index.
+        let expected_a = IndexedExpression::index_query(
+            "json".to_string(),
+            "json_a_idx".to_string(),
+            "Json".to_string(),
+            Arc::new(JsonQuery::new(
+                Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                    "foo".to_string(),
+                )))),
+                "$.a".to_string(),
+            )),
+        );
+        check(
+            &index_info,
+            "json_extract(json, '$.a') = 'foo'",
+            Some(expected_a),
+            false,
+        );
+
+        // Query against an unindexed path must not bind to either index.
+        check_no_index(&index_info, "json_extract(json, '$.c') = 'foo'");
+    }
+
+    #[test]
+    fn test_optimize_nested_and_tree() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // Simulate: AND(AND(fqdn@idx_fqdn, log_time >= X @idx_time), AND(log_time <= Y @idx_time, channel@idx_ch))
+        let fqdn_query = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "fqdn".to_string(),
+            index_name: "fqdn_idx".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                "eng.bhd.0068".to_string(),
+            )))),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let time_gte = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "log_time".to_string(),
+            index_name: "log_time_idx".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(100))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let time_lte = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "log_time".to_string(),
+            index_name: "log_time_idx".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(200))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let channel_query = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "channel".to_string(),
+            index_name: "channel_idx".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                "/ados/node/monitor".to_string(),
+            )))),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        // Build nested AND: AND(AND(fqdn, time_gte), AND(time_lte, channel))
+        let nested = ScalarIndexExpr::And(
+            Box::new(ScalarIndexExpr::And(
+                Box::new(fqdn_query),
+                Box::new(time_gte),
+            )),
+            Box::new(ScalarIndexExpr::And(
+                Box::new(time_lte),
+                Box::new(channel_query),
+            )),
+        );
+
+        // Optimize should merge time_gte + time_lte into a single Range
+        let optimized = nested.optimize();
+
+        // The optimized tree should contain a merged Range(Included(100), Included(200))
+        // and the other queries as separate leaves
+        // Let's verify by collecting leaves
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+
+        // Should have 3 leaves: fqdn, merged_time, channel
+        assert_eq!(
+            leaves.len(),
+            3,
+            "Expected 3 leaves after merge, got: {:?}",
+            leaves
+        );
+
+        // Find the merged time query
+        let time_query = leaves
+            .iter()
+            .find(|l| {
+                if let ScalarIndexExpr::Query(s) = l {
+                    s.index_name == "log_time_idx"
+                } else {
+                    false
+                }
+            })
+            .expect("Should have a log_time query");
+
+        if let ScalarIndexExpr::Query(s) = time_query {
+            let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+            assert_eq!(
+                *range,
+                SargableQuery::Range(
+                    Bound::Included(ScalarValue::Int64(Some(100))),
+                    Bound::Included(ScalarValue::Int64(Some(200))),
+                ),
+                "Range should be merged into closed interval"
+            );
+        }
+    }
+
+    #[test]
+    fn test_optimize_no_merge_different_indexes() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // Two range queries on DIFFERENT indexes should NOT be merged
+        let range_a = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "a".to_string(),
+            index_name: "idx_a".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(10))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let range_b = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "b".to_string(),
+            index_name: "idx_b".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(100))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let expr = ScalarIndexExpr::And(Box::new(range_a), Box::new(range_b));
+        let optimized = expr.optimize();
+
+        // Should remain as two separate leaves (not merged)
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+        assert_eq!(leaves.len(), 2);
+    }
+
+    #[test]
+    fn test_optimize_no_merge_non_range() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // Equals + Range on same index should NOT merge
+        let eq_query = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Equals(ScalarValue::Int64(Some(42)))),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let range_query = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(100))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let expr = ScalarIndexExpr::And(Box::new(eq_query), Box::new(range_query));
+        let optimized = expr.optimize();
+
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+        assert_eq!(leaves.len(), 2, "Equals + Range should not merge");
+    }
+
+    #[test]
+    fn test_optimize_exclusive_bounds() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // x > 10 AND x < 20 → Range(Excluded(10), Excluded(20))
+        let gt = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Excluded(ScalarValue::Int64(Some(10))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let lt = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Excluded(ScalarValue::Int64(Some(20))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let expr = ScalarIndexExpr::And(Box::new(gt), Box::new(lt));
+        let optimized = expr.optimize();
+
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+        assert_eq!(leaves.len(), 1);
+
+        if let ScalarIndexExpr::Query(s) = &leaves[0] {
+            let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+            assert_eq!(
+                *range,
+                SargableQuery::Range(
+                    Bound::Excluded(ScalarValue::Int64(Some(10))),
+                    Bound::Excluded(ScalarValue::Int64(Some(20))),
+                )
+            );
+        } else {
+            panic!("Expected a Query leaf");
+        }
+    }
+
+    #[test]
+    fn test_optimize_preserves_or_and_not() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // AND(OR(a, b), Range_gte, Range_lte)
+        // OR node should be preserved, ranges should merge
+        let or_node = ScalarIndexExpr::Or(
+            Box::new(ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: "c".to_string(),
+                index_name: "idx_c".to_string(),
+                index_type: "".to_string(),
+                query: Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                    "a".to_string(),
+                )))),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            })),
+            Box::new(ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: "c".to_string(),
+                index_name: "idx_c".to_string(),
+                index_type: "".to_string(),
+                query: Arc::new(SargableQuery::Equals(ScalarValue::Utf8(Some(
+                    "b".to_string(),
+                )))),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            })),
+        );
+        let range_gte = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "t".to_string(),
+            index_name: "idx_t".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(5))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let range_lte = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "t".to_string(),
+            index_name: "idx_t".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(50))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let expr = ScalarIndexExpr::And(
+            Box::new(ScalarIndexExpr::And(Box::new(or_node), Box::new(range_gte))),
+            Box::new(range_lte),
+        );
+        let optimized = expr.optimize();
+
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+        // Should have 2 leaves: OR node (preserved) + merged range
+        assert_eq!(leaves.len(), 2);
+
+        // One leaf should be the merged range
+        let merged = leaves
+            .iter()
+            .find(|l| matches!(l, ScalarIndexExpr::Query(s) if s.index_name == "idx_t"))
+            .expect("Should have merged range");
+
+        if let ScalarIndexExpr::Query(s) = merged {
+            let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+            assert_eq!(
+                *range,
+                SargableQuery::Range(
+                    Bound::Included(ScalarValue::Int64(Some(5))),
+                    Bound::Included(ScalarValue::Int64(Some(50))),
+                )
+            );
+        }
+
+        // Other leaf should be the OR node
+        assert!(
+            leaves
+                .iter()
+                .any(|l| matches!(l, ScalarIndexExpr::Or(_, _)))
+        );
+    }
+
+    #[test]
+    fn test_optimize_needs_recheck_preserved() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // If either range has needs_recheck=true, merged result should too
+        let range_a = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(1))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: true,
+            fragment_bitmap: None,
+        });
+        let range_b = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(99))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let expr = ScalarIndexExpr::And(Box::new(range_a), Box::new(range_b));
+        let optimized = expr.optimize();
+
+        let mut leaves = Vec::new();
+        optimized.collect_and_leaves(&mut leaves);
+        assert_eq!(leaves.len(), 1);
+
+        if let ScalarIndexExpr::Query(s) = &leaves[0] {
+            assert!(
+                s.needs_recheck,
+                "Merged query should preserve needs_recheck=true"
+            );
+        } else {
+            panic!("Expected a Query leaf");
+        }
+    }
+
+    #[test]
+    fn test_optimize_single_query_passthrough() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        // A single query (not in AND) should pass through unchanged
+        let single = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "x".to_string(),
+            index_name: "idx_x".to_string(),
+            index_type: "".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Included(ScalarValue::Int64(Some(1))),
+                Bound::Unbounded,
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+
+        let optimized = single.clone().optimize();
+        assert_eq!(optimized, single);
+    }
+
+    #[test]
+    fn test_optimize_complex_nested_cases() {
+        use super::{ScalarIndexExpr, ScalarIndexSearch};
+        use crate::scalar::SargableQuery;
+        use datafusion_common::ScalarValue;
+        use std::ops::Bound;
+        use std::sync::Arc;
+
+        let make_range =
+            |col: &str, idx: &str, low: Bound<ScalarValue>, high: Bound<ScalarValue>| {
+                ScalarIndexExpr::Query(ScalarIndexSearch {
+                    column: col.to_string(),
+                    index_name: idx.to_string(),
+                    index_type: "".to_string(),
+                    query: Arc::new(SargableQuery::Range(low, high)),
+                    needs_recheck: false,
+                    fragment_bitmap: None,
+                })
+            };
+        let make_eq = |col: &str, idx: &str, val: ScalarValue| {
+            ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: col.to_string(),
+                index_name: idx.to_string(),
+                index_type: "".to_string(),
+                query: Arc::new(SargableQuery::Equals(val)),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            })
+        };
+
+        // Case 1: Multiple ranges on same index should all merge
+        // x >= 10 AND x <= 200 AND x >= 50 AND x <= 100 → Range(50, 100)
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_range(
+                        "x",
+                        "idx_x",
+                        Bound::Included(ScalarValue::Int64(Some(10))),
+                        Bound::Unbounded,
+                    )),
+                    Box::new(make_range(
+                        "x",
+                        "idx_x",
+                        Bound::Unbounded,
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                    )),
+                )),
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_range(
+                        "x",
+                        "idx_x",
+                        Bound::Included(ScalarValue::Int64(Some(50))),
+                        Bound::Unbounded,
+                    )),
+                    Box::new(make_range(
+                        "x",
+                        "idx_x",
+                        Bound::Unbounded,
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                    )),
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(leaves.len(), 1, "All 4 ranges should merge into 1");
+
+            if let ScalarIndexExpr::Query(s) = &leaves[0] {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(50))),
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                    )
+                );
+            } else {
+                panic!("Expected merged Query");
+            }
+        }
+
+        // Case 2: NOT(range) should NOT be merged with sibling range
+        // AND(NOT(x >= 10), x <= 20) → stays as 2 leaves
+        {
+            let not_node = ScalarIndexExpr::Not(Box::new(make_range(
+                "x",
+                "idx_x",
+                Bound::Included(ScalarValue::Int64(Some(10))),
+                Bound::Unbounded,
+            )));
+            let range_lte = make_range(
+                "x",
+                "idx_x",
+                Bound::Unbounded,
+                Bound::Included(ScalarValue::Int64(Some(20))),
+            );
+
+            let expr = ScalarIndexExpr::And(Box::new(not_node), Box::new(range_lte));
+            let optimized = expr.optimize();
+
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(
+                leaves.len(),
+                2,
+                "NOT(range) should not merge with sibling range"
+            );
+            assert!(leaves.iter().any(|l| matches!(l, ScalarIndexExpr::Not(_))));
+        }
+
+        // Case 3: Ranges inside OR branches get optimized independently
+        // OR(AND(x >= 10, x <= 20), AND(x >= 30, x <= 40))
+        {
+            let branch1 = ScalarIndexExpr::And(
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(10))),
+                    Bound::Unbounded,
+                )),
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Unbounded,
+                    Bound::Included(ScalarValue::Int64(Some(20))),
+                )),
+            );
+            let branch2 = ScalarIndexExpr::And(
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(30))),
+                    Bound::Unbounded,
+                )),
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Unbounded,
+                    Bound::Included(ScalarValue::Int64(Some(40))),
+                )),
+            );
+
+            let expr = ScalarIndexExpr::Or(Box::new(branch1), Box::new(branch2));
+            let optimized = expr.optimize();
+
+            // Top level should still be OR
+            if let ScalarIndexExpr::Or(lhs, rhs) = &optimized {
+                // Each branch should be a single merged range
+                let mut left_leaves = Vec::new();
+                lhs.clone().collect_and_leaves(&mut left_leaves);
+                assert_eq!(
+                    left_leaves.len(),
+                    1,
+                    "Left OR branch should have merged range"
+                );
+                if let ScalarIndexExpr::Query(s) = &left_leaves[0] {
+                    let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                    assert_eq!(
+                        *range,
+                        SargableQuery::Range(
+                            Bound::Included(ScalarValue::Int64(Some(10))),
+                            Bound::Included(ScalarValue::Int64(Some(20))),
+                        )
+                    );
+                }
+
+                let mut right_leaves = Vec::new();
+                rhs.clone().collect_and_leaves(&mut right_leaves);
+                assert_eq!(
+                    right_leaves.len(),
+                    1,
+                    "Right OR branch should have merged range"
+                );
+                if let ScalarIndexExpr::Query(s) = &right_leaves[0] {
+                    let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                    assert_eq!(
+                        *range,
+                        SargableQuery::Range(
+                            Bound::Included(ScalarValue::Int64(Some(30))),
+                            Bound::Included(ScalarValue::Int64(Some(40))),
+                        )
+                    );
+                }
+            } else {
+                panic!("Expected OR at top level");
+            }
+        }
+
+        // Case 4: Multiple indexes, each with its own range pair
+        // AND(a >= 1, a <= 10, b >= 100, b <= 200)
+        // → merged into: AND(a in [1,10], b in [100,200])
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_range(
+                        "a",
+                        "idx_a",
+                        Bound::Included(ScalarValue::Int64(Some(1))),
+                        Bound::Unbounded,
+                    )),
+                    Box::new(make_range(
+                        "a",
+                        "idx_a",
+                        Bound::Unbounded,
+                        Bound::Included(ScalarValue::Int64(Some(10))),
+                    )),
+                )),
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_range(
+                        "b",
+                        "idx_b",
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                        Bound::Unbounded,
+                    )),
+                    Box::new(make_range(
+                        "b",
+                        "idx_b",
+                        Bound::Unbounded,
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                    )),
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(
+                leaves.len(),
+                2,
+                "Two indexes should each merge independently"
+            );
+
+            let a_leaf = leaves
+                .iter()
+                .find(|l| matches!(l, ScalarIndexExpr::Query(s) if s.index_name == "idx_a"))
+                .expect("Should have idx_a");
+            if let ScalarIndexExpr::Query(s) = a_leaf {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(1))),
+                        Bound::Included(ScalarValue::Int64(Some(10))),
+                    )
+                );
+            }
+
+            let b_leaf = leaves
+                .iter()
+                .find(|l| matches!(l, ScalarIndexExpr::Query(s) if s.index_name == "idx_b"))
+                .expect("Should have idx_b");
+            if let ScalarIndexExpr::Query(s) = b_leaf {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                    )
+                );
+            }
+        }
+
+        // Case 5: Mix of Equals and Range on different indexes
+        // AND(fqdn = 'x', time >= 100, time <= 200, channel = 'y')
+        // → AND(fqdn = 'x', time in [100,200], channel = 'y')
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_eq(
+                        "fqdn",
+                        "fqdn_idx",
+                        ScalarValue::Utf8(Some("x".to_string())),
+                    )),
+                    Box::new(make_range(
+                        "time",
+                        "time_idx",
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                        Bound::Unbounded,
+                    )),
+                )),
+                Box::new(ScalarIndexExpr::And(
+                    Box::new(make_range(
+                        "time",
+                        "time_idx",
+                        Bound::Unbounded,
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                    )),
+                    Box::new(make_eq(
+                        "channel",
+                        "ch_idx",
+                        ScalarValue::Utf8(Some("y".to_string())),
+                    )),
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(leaves.len(), 3, "fqdn + merged_time + channel = 3 leaves");
+
+            let time_leaf = leaves
+                .iter()
+                .find(|l| matches!(l, ScalarIndexExpr::Query(s) if s.index_name == "time_idx"))
+                .expect("Should have time query");
+            if let ScalarIndexExpr::Query(s) = time_leaf {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                    )
+                );
+            }
+        }
+
+        // Case 6: Overlapping closed ranges → takes intersection
+        // Range(10, 50) AND Range(30, 80) → Range(30, 50)
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(10))),
+                    Bound::Included(ScalarValue::Int64(Some(50))),
+                )),
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(30))),
+                    Bound::Included(ScalarValue::Int64(Some(80))),
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(leaves.len(), 1);
+            if let ScalarIndexExpr::Query(s) = &leaves[0] {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(30))),
+                        Bound::Included(ScalarValue::Int64(Some(50))),
+                    )
+                );
+            }
+        }
+
+        // Case 7: Mixed Included/Excluded on same value
+        // x >= 5 AND x > 5 → Excluded(5) (stricter)
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(5))),
+                    Bound::Unbounded,
+                )),
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Excluded(ScalarValue::Int64(Some(5))),
+                    Bound::Unbounded,
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(leaves.len(), 1);
+            if let ScalarIndexExpr::Query(s) = &leaves[0] {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Excluded(ScalarValue::Int64(Some(5))),
+                        Bound::Unbounded,
+                    )
+                );
+            }
+        }
+
+        // Case 8: Empty range (lower > upper) — still valid, just produces empty results
+        // x >= 200 AND x <= 100 → Range(Included(200), Included(100))
+        // BTree search will find 0 pages matching this, which is correct.
+        {
+            let expr = ScalarIndexExpr::And(
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Included(ScalarValue::Int64(Some(200))),
+                    Bound::Unbounded,
+                )),
+                Box::new(make_range(
+                    "x",
+                    "idx_x",
+                    Bound::Unbounded,
+                    Bound::Included(ScalarValue::Int64(Some(100))),
+                )),
+            );
+
+            let optimized = expr.optimize();
+            let mut leaves = Vec::new();
+            optimized.collect_and_leaves(&mut leaves);
+            assert_eq!(leaves.len(), 1);
+            if let ScalarIndexExpr::Query(s) = &leaves[0] {
+                let range = s.query.as_any().downcast_ref::<SargableQuery>().unwrap();
+                // Merged: lower=Included(200), upper=Included(100) — empty range, but valid
+                assert_eq!(
+                    *range,
+                    SargableQuery::Range(
+                        Bound::Included(ScalarValue::Int64(Some(200))),
+                        Bound::Included(ScalarValue::Int64(Some(100))),
+                    )
+                );
+            }
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs
new file mode 100644
index 00000000000..c79e4a33274
--- /dev/null
+++ b/rust/lance-index/src/scalar/fmindex.rs
@@ -0,0 +1,3714 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! FM-Index for exact substring search (following the Infini-gram Mini paper)
+//!
+//! The FM-Index is a compressed full-text index based on the Burrows-Wheeler Transform (BWT).
+//! It supports exact substring matching via backward search and returns exact row ids.
+//!
+//! Architecture (matching the paper):
+//!   - Huffman-shaped Wavelet Tree over BWT for entropy-compressed rank queries (~0.26N)
+//!   - Sampled Suffix Array every D-th position for locate (~N/D × 8 bytes)
+//!   - doc_start_positions for mapping text positions to documents (tiny)
+//!   - No doc_array — documents are resolved via SA sampling + LF-mapping + binary search
+//!
+//! Total index size: ~0.44N (matching paper's claim)
+//!
+//! Storage layout (v10 - blocked, partitioned):
+//!   - BWT wavelet tree bitvectors in blocks of BLOCK_WORDS (32KB each)
+//!   - SA samples stored as packed binary blocks after wavelet blocks
+//!   - Row IDs and doc_start_positions in metadata
+//!   - File metadata: c_table, huffman_codes, tree topology
+
+use std::cmp::Reverse;
+use std::collections::{BinaryHeap, HashMap};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, OnceLock};
+
+use arrow_array::RecordBatch;
+use arrow_schema::{DataType, Field};
+use async_trait::async_trait;
+use datafusion::execution::SendableRecordBatchStream;
+use futures::{StreamExt, TryStreamExt};
+use lance_core::cache::LanceCache;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
+use lance_core::{Error, ROW_ADDR, Result};
+use roaring::RoaringBitmap;
+
+use crate::metrics::MetricsCollector;
+use crate::pb;
+use crate::scalar::expression::{ScalarQueryParser, TextQueryParser};
+use crate::scalar::registry::{
+    BasicTrainer, DefaultTrainingRequest, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering,
+    TrainingRequest, VALUE_COLUMN_NAME,
+};
+use crate::scalar::{
+    AnyQuery, BuiltinIndexType, CreatedIndex, IndexFile, IndexStore, OldIndexDataFilter,
+    RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, TextQuery, UpdateCriteria,
+};
+use crate::{Index, IndexType};
+
+const FMINDEX_INDEX_VERSION: u32 = 10;
+const BLOCK_WORDS: usize = 4096;
+const PARTITION_SIZE: usize = 10_000;
+const DEFAULT_PARTITION_SIZE_BYTES: usize = 16 * 1024 * 1024;
+const DEFAULT_DEMAND_PAGE_TARGET_BYTES: usize = 512 * 1024;
+const DEFAULT_PREWARM_CHUNK_TARGET_BYTES: usize = 8 * 1024 * 1024;
+const FMINDEX_PARTITION_FINGERPRINT_KEY: &str = "partition_fingerprint";
+const SENTINEL_BYTE: u8 = 0xFF;
+
+/// SA sampling rate. Store every D-th SA entry. Locate walks at most D LF steps.
+const SA_SAMPLE_RATE: usize = 32;
+
+static LANCE_FMINDEX_NUM_WORKERS: std::sync::LazyLock<usize> = std::sync::LazyLock::new(|| {
+    std::env::var("LANCE_FMINDEX_NUM_WORKERS")
+        .unwrap_or_else(|_| get_num_compute_intensive_cpus().to_string())
+        .parse()
+        .expect("failed to parse LANCE_FMINDEX_NUM_WORKERS")
+});
+static LANCE_FMINDEX_PARTITION_ROWS: std::sync::LazyLock<usize> = std::sync::LazyLock::new(|| {
+    std::env::var("LANCE_FMINDEX_PARTITION_ROWS")
+        .unwrap_or_else(|_| PARTITION_SIZE.to_string())
+        .parse()
+        .expect("failed to parse LANCE_FMINDEX_PARTITION_ROWS")
+});
+static LANCE_FMINDEX_PARTITION_BYTES: std::sync::LazyLock<usize> = std::sync::LazyLock::new(|| {
+    std::env::var("LANCE_FMINDEX_PARTITION_BYTES")
+        .unwrap_or_else(|_| DEFAULT_PARTITION_SIZE_BYTES.to_string())
+        .parse()
+        .expect("failed to parse LANCE_FMINDEX_PARTITION_BYTES")
+});
+static LANCE_FMINDEX_WRITE_QUEUE_SIZE: std::sync::LazyLock<usize> =
+    std::sync::LazyLock::new(|| {
+        std::env::var("LANCE_FMINDEX_WRITE_QUEUE_SIZE")
+            .unwrap_or_else(|_| "1".to_string())
+            .parse()
+            .expect("failed to parse LANCE_FMINDEX_WRITE_QUEUE_SIZE")
+    });
+static LANCE_FMINDEX_RESUME_EXISTING_PARTITIONS: std::sync::LazyLock<bool> =
+    std::sync::LazyLock::new(|| {
+        std::env::var("LANCE_FMINDEX_RESUME_EXISTING_PARTITIONS")
+            .map(|value| {
+                matches!(
+                    value.as_str(),
+                    "1" | "true" | "TRUE" | "True" | "yes" | "YES"
+                )
+            })
+            .unwrap_or(false)
+    });
+static LANCE_FMINDEX_PREWARM_CHUNK_BYTES: std::sync::LazyLock<usize> =
+    std::sync::LazyLock::new(|| {
+        std::env::var("LANCE_FMINDEX_PREWARM_CHUNK_BYTES")
+            .unwrap_or_else(|_| DEFAULT_PREWARM_CHUNK_TARGET_BYTES.to_string())
+            .parse()
+            .expect("failed to parse LANCE_FMINDEX_PREWARM_CHUNK_BYTES")
+    });
+static LANCE_FMINDEX_DEMAND_PAGE_BYTES: std::sync::LazyLock<usize> =
+    std::sync::LazyLock::new(|| {
+        std::env::var("LANCE_FMINDEX_DEMAND_PAGE_BYTES")
+            .unwrap_or_else(|_| DEFAULT_DEMAND_PAGE_TARGET_BYTES.to_string())
+            .parse()
+            .expect("failed to parse LANCE_FMINDEX_DEMAND_PAGE_BYTES")
+    });
+static LANCE_FMINDEX_PREWARM_CHUNK_CONCURRENCY: std::sync::LazyLock<usize> =
+    std::sync::LazyLock::new(|| {
+        std::env::var("LANCE_FMINDEX_PREWARM_CHUNK_CONCURRENCY")
+            .unwrap_or_else(|_| "1".to_string())
+            .parse()
+            .expect("failed to parse LANCE_FMINDEX_PREWARM_CHUNK_CONCURRENCY")
+    });
+
+fn fmindex_partition_path(partition_id: u64) -> String {
+    format!("part_{partition_id}_fm.lance")
+}
+
+fn fmindex_partition_id_from_path(path: &str) -> Option<u64> {
+    path.strip_prefix("part_")
+        .and_then(|r| r.strip_suffix("_fm.lance"))
+        .and_then(|s| s.parse::<u64>().ok())
+}
+
+fn fmindex_num_workers() -> usize {
+    (*LANCE_FMINDEX_NUM_WORKERS).max(1)
+}
+
+fn fmindex_partition_rows() -> usize {
+    (*LANCE_FMINDEX_PARTITION_ROWS).max(1)
+}
+
+fn fmindex_partition_bytes() -> usize {
+    (*LANCE_FMINDEX_PARTITION_BYTES).max(1)
+}
+
+fn fmindex_write_queue_size() -> usize {
+    (*LANCE_FMINDEX_WRITE_QUEUE_SIZE).max(1)
+}
+
+fn fmindex_resume_existing_partitions() -> bool {
+    *LANCE_FMINDEX_RESUME_EXISTING_PARTITIONS
+}
+
+fn fmindex_prewarm_chunk_bytes() -> usize {
+    (*LANCE_FMINDEX_PREWARM_CHUNK_BYTES).max(BLOCK_WORDS * 8)
+}
+
+fn fmindex_demand_page_bytes() -> usize {
+    (*LANCE_FMINDEX_DEMAND_PAGE_BYTES).max(BLOCK_WORDS * 8)
+}
+
+fn fmindex_demand_page_rows() -> usize {
+    (fmindex_demand_page_bytes() / (BLOCK_WORDS * 8)).max(1)
+}
+
+fn fmindex_prewarm_chunk_concurrency() -> usize {
+    (*LANCE_FMINDEX_PREWARM_CHUNK_CONCURRENCY).max(1)
+}
+
+// ── Bitvector with O(1) rank ─────────────────────────────────────────────────
+
+const SUPERBLOCK_BITS: usize = 512;
+const WORDS_PER_SUPERBLOCK: usize = SUPERBLOCK_BITS / 64;
+
+#[derive(Debug, Clone)]
+struct RankBitVec {
+    words: Vec<u64>,
+    superblocks: Vec<u32>,
+    len: usize,
+}
+
+#[allow(dead_code)]
+impl RankBitVec {
+    fn new(len: usize) -> Self {
+        Self {
+            words: vec![0u64; len.div_ceil(64)],
+            superblocks: Vec::new(),
+            len,
+        }
+    }
+
+    #[inline]
+    fn set(&mut self, pos: usize) {
+        self.words[pos / 64] |= 1u64 << (pos % 64);
+    }
+
+    #[inline]
+    fn get(&self, pos: usize) -> bool {
+        (self.words[pos / 64] >> (pos % 64)) & 1 != 0
+    }
+
+    fn build_rank_index(&mut self) {
+        let num_sb = self.words.len().div_ceil(WORDS_PER_SUPERBLOCK) + 1;
+        self.superblocks = Vec::with_capacity(num_sb);
+        let mut cum = 0u32;
+        for (i, chunk) in self.words.chunks(WORDS_PER_SUPERBLOCK).enumerate() {
+            self.superblocks.push(if i == 0 { 0 } else { cum });
+            for &w in chunk {
+                cum += w.count_ones();
+            }
+        }
+        self.superblocks.push(cum);
+    }
+
+    #[inline]
+    fn rank1(&self, pos: usize) -> usize {
+        if pos == 0 {
+            return 0;
+        }
+        let word_idx = pos / 64;
+        let bit_idx = pos % 64;
+        let sb_idx = word_idx / WORDS_PER_SUPERBLOCK;
+        let mut count = self.superblocks[sb_idx] as usize;
+        for i in (sb_idx * WORDS_PER_SUPERBLOCK)..word_idx {
+            count += self.words[i].count_ones() as usize;
+        }
+        if bit_idx > 0 {
+            count += (self.words[word_idx] & ((1u64 << bit_idx) - 1)).count_ones() as usize;
+        }
+        count
+    }
+
+    #[inline]
+    fn rank0(&self, pos: usize) -> usize {
+        pos - self.rank1(pos)
+    }
+
+    fn deep_size(&self) -> usize {
+        self.words.len() * 8 + self.superblocks.len() * 4
+    }
+}
+
+// ── Huffman-shaped Wavelet Tree ──────────────────────────────────────────────
+
+#[derive(Debug, Clone, Default)]
+struct HuffmanCode {
+    bits: u32,
+    length: u8,
+    node_path: Vec<usize>,
+}
+
+#[derive(Debug, Clone)]
+enum WaveletChild {
+    Node(usize),
+    Leaf(u8),
+}
+
+#[derive(Debug, Clone)]
+struct HuffmanWaveletTree {
+    nodes: Vec<RankBitVec>,
+    codes: [HuffmanCode; 256],
+    children: Vec<(WaveletChild, WaveletChild)>,
+    len: usize,
+}
+
+#[derive(Debug)]
+enum HuffNode {
+    Leaf(u8),
+    Internal { left: Box<Self>, right: Box<Self> },
+}
+
+impl PartialEq for HuffNode {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+impl Eq for HuffNode {}
+impl PartialOrd for HuffNode {
+    fn partial_cmp(&self, o: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(o))
+    }
+}
+impl Ord for HuffNode {
+    fn cmp(&self, _: &Self) -> std::cmp::Ordering {
+        std::cmp::Ordering::Equal
+    }
+}
+
+#[allow(dead_code)]
+impl HuffmanWaveletTree {
+    fn build(data: &[u8]) -> Self {
+        let n = data.len();
+        if n == 0 {
+            return Self {
+                nodes: Vec::new(),
+                codes: std::array::from_fn(|_| HuffmanCode::default()),
+                children: Vec::new(),
+                len: 0,
+            };
+        }
+
+        let mut freq = [0u64; 256];
+        for &b in data {
+            freq[b as usize] += 1;
+        }
+
+        let mut heap: BinaryHeap<(Reverse<u64>, Reverse<usize>, Box<HuffNode>)> = BinaryHeap::new();
+        let mut tie = 0;
+        for (v, &f) in freq.iter().enumerate() {
+            if f > 0 {
+                heap.push((Reverse(f), Reverse(tie), Box::new(HuffNode::Leaf(v as u8))));
+                tie += 1;
+            }
+        }
+        if heap.len() == 1 {
+            let (f, _, node) = heap.pop().unwrap();
+            heap.push((Reverse(0), Reverse(tie), Box::new(HuffNode::Leaf(255))));
+            tie += 1;
+            heap.push((f, Reverse(tie), node));
+            tie += 1;
+        }
+        while heap.len() > 1 {
+            let (Reverse(f1), _, l) = heap.pop().unwrap();
+            let (Reverse(f2), _, r) = heap.pop().unwrap();
+            heap.push((
+                Reverse(f1 + f2),
+                Reverse(tie),
+                Box::new(HuffNode::Internal { left: l, right: r }),
+            ));
+            tie += 1;
+        }
+        let root = heap.pop().unwrap().2;
+
+        let mut codes: [HuffmanCode; 256] = std::array::from_fn(|_| HuffmanCode::default());
+        let mut node_count = 0;
+        let mut children_map: Vec<(WaveletChild, WaveletChild)> = Vec::new();
+
+        fn assign(
+            node: &HuffNode,
+            bits: u32,
+            len: u8,
+            path: &mut Vec<usize>,
+            nid: &mut usize,
+            codes: &mut [HuffmanCode; 256],
+            cm: &mut Vec<(WaveletChild, WaveletChild)>,
+        ) -> WaveletChild {
+            match node {
+                HuffNode::Leaf(b) => {
+                    codes[*b as usize] = HuffmanCode {
+                        bits,
+                        length: len,
+                        node_path: path.clone(),
+                    };
+                    WaveletChild::Leaf(*b)
+                }
+                HuffNode::Internal { left, right } => {
+                    let my = *nid;
+                    *nid += 1;
+                    path.push(my);
+                    cm.push((WaveletChild::Leaf(0), WaveletChild::Leaf(0)));
+                    let lc = assign(left, bits << 1, len + 1, path, nid, codes, cm);
+                    let rc = assign(right, (bits << 1) | 1, len + 1, path, nid, codes, cm);
+                    cm[my] = (lc, rc);
+                    path.pop();
+                    WaveletChild::Node(my)
+                }
+            }
+        }
+        assign(
+            &root,
+            0,
+            0,
+            &mut Vec::new(),
+            &mut node_count,
+            &mut codes,
+            &mut children_map,
+        );
+
+        let mut node_sizes = vec![0usize; node_count];
+        for &b in data {
+            for &nid in &codes[b as usize].node_path {
+                node_sizes[nid] += 1;
+            }
+        }
+        let mut nodes: Vec<RankBitVec> = node_sizes.iter().map(|&sz| RankBitVec::new(sz)).collect();
+        let mut cursors = vec![0usize; node_count];
+        for &b in data {
+            let code = &codes[b as usize];
+            for (level, &nid) in code.node_path.iter().enumerate() {
+                if (code.bits >> (code.length - 1 - level as u8)) & 1 == 1 {
+                    nodes[nid].set(cursors[nid]);
+                }
+                cursors[nid] += 1;
+            }
+        }
+        for n in &mut nodes {
+            n.build_rank_index();
+        }
+        Self {
+            nodes,
+            codes,
+            children: children_map,
+            len: n,
+        }
+    }
+
+    /// Retrieve the byte at position `pos` in the original BWT.
+    #[inline]
+    fn access(&self, mut pos: usize) -> u8 {
+        if self.nodes.is_empty() {
+            return 0;
+        }
+        let mut node_idx = 0;
+        loop {
+            let bit = self.nodes[node_idx].get(pos);
+            let (ref left, ref right) = self.children[node_idx];
+            if bit {
+                pos = self.nodes[node_idx].rank1(pos);
+                match right {
+                    WaveletChild::Leaf(b) => return *b,
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            } else {
+                pos = self.nodes[node_idx].rank0(pos);
+                match left {
+                    WaveletChild::Leaf(b) => return *b,
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            }
+        }
+    }
+
+    /// Count occurrences of byte `c` in positions `[0, pos)`.
+    #[inline]
+    fn rank(&self, c: u8, pos: usize) -> usize {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return 0;
+        }
+        let (mut lo, mut hi) = (0, pos);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                lo = self.nodes[nid].rank0(lo);
+                hi = self.nodes[nid].rank0(hi);
+            } else {
+                lo = self.nodes[nid].rank1(lo);
+                hi = self.nodes[nid].rank1(hi);
+            }
+        }
+        hi - lo
+    }
+
+    #[inline]
+    fn rank_pair(&self, c: u8, lo: usize, hi: usize) -> (usize, usize) {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return (0, 0);
+        }
+        let (mut s, mut l, mut h) = (0, lo, hi);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                s = self.nodes[nid].rank0(s);
+                l = self.nodes[nid].rank0(l);
+                h = self.nodes[nid].rank0(h);
+            } else {
+                s = self.nodes[nid].rank1(s);
+                l = self.nodes[nid].rank1(l);
+                h = self.nodes[nid].rank1(h);
+            }
+        }
+        (l - s, h - s)
+    }
+
+    fn deep_size(&self) -> usize {
+        self.nodes.iter().map(|n| n.deep_size()).sum::<usize>()
+            + self
+                .codes
+                .iter()
+                .map(|c| c.node_path.len() * 8)
+                .sum::<usize>()
+            + self.children.len() * 24
+    }
+}
+
+// ── Suffix Array ─────────────────────────────────────────────────────────────
+
+fn build_suffix_array(text: &[u8]) -> Vec<usize> {
+    let n = text.len();
+    if n == 0 {
+        return Vec::new();
+    }
+    if n > i32::MAX as usize {
+        let mut sa = vec![0i64; n];
+        assert_eq!(libsais_rs::libsais64(text, &mut sa, 0, None), 0);
+        sa.iter().map(|&x| x as usize).collect()
+    } else {
+        let mut sa = vec![0i32; n];
+        assert_eq!(libsais_rs::libsais(text, &mut sa, 0, None), 0);
+        sa.iter().map(|&x| x as usize).collect()
+    }
+}
+
+// ── Lazy Block Loading ───────────────────────────────────────────────────────
+
+const BLOCK_BITS: usize = BLOCK_WORDS * 64;
+
+struct LazyRankBitVec {
+    prefix_ranks: Vec<u64>,
+    blocks: Vec<OnceLock<Vec<u64>>>,
+    reader: Arc<dyn crate::scalar::IndexReader>,
+    block_row_offset: usize,
+    len: usize,
+}
+
+impl std::fmt::Debug for LazyRankBitVec {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LazyRankBitVec")
+            .field("len", &self.len)
+            .finish()
+    }
+}
+
+impl LazyRankBitVec {
+    fn new(
+        prefix_ranks: Vec<u64>,
+        num_blocks: usize,
+        reader: Arc<dyn crate::scalar::IndexReader>,
+        offset: usize,
+        len: usize,
+    ) -> Self {
+        Self {
+            prefix_ranks,
+            blocks: (0..num_blocks).map(|_| OnceLock::new()).collect(),
+            reader,
+            block_row_offset: offset,
+            len,
+        }
+    }
+
+    async fn load_block_if_needed(&self, idx: usize) -> Result<()> {
+        if idx >= self.blocks.len() {
+            return Err(Error::index(format!(
+                "FM-Index block {idx} is out of range for {} blocks",
+                self.blocks.len()
+            )));
+        }
+        if self.blocks[idx].get().is_none() {
+            self.load_page_containing(idx).await?;
+        }
+        Ok(())
+    }
+
+    async fn load_page_containing(&self, idx: usize) -> Result<()> {
+        let page_rows = fmindex_demand_page_rows();
+        let start = (idx / page_rows) * page_rows;
+        let end = (start + page_rows).min(self.blocks.len());
+        if self.blocks[start..end]
+            .iter()
+            .all(|block| block.get().is_some())
+        {
+            return Ok(());
+        }
+
+        let batch = self
+            .reader
+            .read_range(
+                self.block_row_offset + start..self.block_row_offset + end,
+                Some(&["words"]),
+            )
+            .await?;
+        if batch.num_rows() != end - start {
+            return Err(Error::index(format!(
+                "expected {} FM-Index block rows, got {}",
+                end - start,
+                batch.num_rows()
+            )));
+        }
+
+        let col = Self::words_column(&batch)?;
+        for row in 0..batch.num_rows() {
+            let block_idx = start + row;
+            if self.blocks[block_idx].get().is_none() {
+                let words = Self::decode_words(col.value(row));
+                let _ = self.blocks[block_idx].set(words);
+            }
+        }
+        Ok(())
+    }
+
+    async fn load_block_for_rank(&self, pos: usize) -> Result<()> {
+        if pos > self.len {
+            return Err(Error::invalid_input(format!(
+                "FM-Index rank position {pos} exceeds bitvector length {}",
+                self.len
+            )));
+        }
+        if pos == 0 {
+            return Ok(());
+        }
+        if pos == self.len && pos.is_multiple_of(BLOCK_BITS) {
+            if let Some(last_idx) = self.blocks.len().checked_sub(1) {
+                return self.load_block_if_needed(last_idx).await;
+            }
+            return Ok(());
+        }
+        if pos.is_multiple_of(BLOCK_BITS) {
+            return Ok(());
+        }
+        self.load_block_if_needed(pos / BLOCK_BITS).await
+    }
+
+    async fn load_block_for_access(&self, pos: usize) -> Result<()> {
+        self.load_block_if_needed(pos / BLOCK_BITS).await
+    }
+
+    #[inline]
+    fn ensure_block(&self, idx: usize) -> &[u64] {
+        self.blocks[idx].get_or_init(|| {
+            tokio::task::block_in_place(|| {
+                tokio::runtime::Handle::current().block_on(self.load_block(idx))
+            })
+            .unwrap_or_else(|e| panic!("FM-Index block load failed: {e}"))
+        })
+    }
+
+    async fn load_block(&self, idx: usize) -> Result<Vec<u64>> {
+        let row = self.block_row_offset + idx;
+        let batch = self
+            .reader
+            .read_range(row..row + 1, Some(&["words"]))
+            .await?;
+        let col = Self::words_column(&batch)?;
+        Ok(Self::decode_words(col.value(0)))
+    }
+
+    fn words_column(batch: &RecordBatch) -> Result<&arrow_array::LargeBinaryArray> {
+        batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow_array::LargeBinaryArray>()
+            .ok_or_else(|| Error::invalid_input("expected LargeBinary words column"))
+    }
+
+    fn decode_words(raw: &[u8]) -> Vec<u64> {
+        raw.chunks_exact(8)
+            .map(|c| u64::from_le_bytes(c.try_into().unwrap()))
+            .collect()
+    }
+
+    fn terminal_rank1(&self) -> usize {
+        if self.len == 0 {
+            return 0;
+        }
+        let Some(last_idx) = self.blocks.len().checked_sub(1) else {
+            return 0;
+        };
+        let mut count = self.prefix_ranks.get(last_idx).copied().unwrap_or(0) as usize;
+        let block = self.ensure_block(last_idx);
+        let local = self.len - last_idx * BLOCK_BITS;
+        let full_words = local / 64;
+        let trailing_bits = local % 64;
+        for w in &block[..full_words] {
+            count += w.count_ones() as usize;
+        }
+        if trailing_bits > 0 {
+            count += (block[full_words] & ((1u64 << trailing_bits) - 1)).count_ones() as usize;
+        }
+        count
+    }
+
+    #[inline]
+    fn rank1(&self, pos: usize) -> usize {
+        debug_assert!(pos <= self.len);
+        if pos == 0 {
+            return 0;
+        }
+        let bi = pos / BLOCK_BITS;
+        let local = pos % BLOCK_BITS;
+        if local == 0 {
+            if let Some(prefix_rank) = self.prefix_ranks.get(bi) {
+                return *prefix_rank as usize;
+            }
+            if pos == self.len {
+                return self.terminal_rank1();
+            }
+        }
+        let mut count = self.prefix_ranks[bi] as usize;
+        let block = self.ensure_block(bi);
+        let wi = local / 64;
+        let bit = local % 64;
+        for w in &block[..wi] {
+            count += w.count_ones() as usize;
+        }
+        if bit > 0 {
+            count += (block[wi] & ((1u64 << bit) - 1)).count_ones() as usize;
+        }
+        count
+    }
+
+    #[inline]
+    fn rank0(&self, pos: usize) -> usize {
+        pos - self.rank1(pos)
+    }
+
+    #[inline]
+    fn get(&self, pos: usize) -> bool {
+        debug_assert!(pos < self.len);
+        let bi = pos / BLOCK_BITS;
+        let local = pos % BLOCK_BITS;
+        let block = self.ensure_block(bi);
+        (block[local / 64] >> (local % 64)) & 1 != 0
+    }
+
+    fn deep_size(&self) -> usize {
+        let loaded: usize = self
+            .blocks
+            .iter()
+            .filter_map(|b| b.get())
+            .map(|w| w.len() * 8)
+            .sum();
+        self.prefix_ranks.len() * 8 + loaded
+    }
+}
+
+struct LazyHuffmanWaveletTree {
+    nodes: Vec<LazyRankBitVec>,
+    codes: [HuffmanCode; 256],
+    children: Vec<(WaveletChild, WaveletChild)>,
+    len: usize,
+}
+
+impl std::fmt::Debug for LazyHuffmanWaveletTree {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LazyHuffmanWaveletTree")
+            .field("len", &self.len)
+            .finish()
+    }
+}
+
+impl LazyHuffmanWaveletTree {
+    /// Pre-load all wavelet tree blocks into memory.
+    async fn load_all(&self) -> Result<()> {
+        if self.nodes.is_empty() {
+            return Ok(());
+        }
+
+        #[derive(Clone, Copy)]
+        struct RowRange {
+            start: usize,
+            end: usize,
+            node_idx: usize,
+        }
+
+        let mut ranges = self
+            .nodes
+            .iter()
+            .enumerate()
+            .filter(|(_, node)| !node.blocks.is_empty())
+            .map(|(node_idx, node)| RowRange {
+                start: node.block_row_offset,
+                end: node.block_row_offset + node.blocks.len(),
+                node_idx,
+            })
+            .collect::<Vec<_>>();
+        ranges.sort_by_key(|range| range.start);
+        let Some(total_rows) = ranges.iter().map(|range| range.end).max() else {
+            return Ok(());
+        };
+        let ranges = Arc::new(ranges);
+
+        let chunk_loaded = |start: usize, end: usize| {
+            ranges.iter().all(|range| {
+                let overlap_start = start.max(range.start);
+                let overlap_end = end.min(range.end);
+                if overlap_start >= overlap_end {
+                    return true;
+                }
+                let node = &self.nodes[range.node_idx];
+                node.blocks[overlap_start - range.start..overlap_end - range.start]
+                    .iter()
+                    .all(|block| block.get().is_some())
+            })
+        };
+
+        let chunk_rows = (fmindex_prewarm_chunk_bytes() / (BLOCK_WORDS * 8)).max(1);
+        let reader = Arc::clone(&self.nodes[0].reader);
+        let chunks = (0..total_rows)
+            .step_by(chunk_rows)
+            .map(|start| {
+                let end = (start + chunk_rows).min(total_rows);
+                (start, end)
+            })
+            .filter(|(start, end)| !chunk_loaded(*start, *end))
+            .collect::<Vec<_>>();
+
+        futures::stream::iter(chunks)
+            .map(|(start, end)| {
+                let reader = Arc::clone(&reader);
+                async move {
+                    let batch = reader.read_range(start..end, Some(&["words"])).await?;
+                    Result::<_>::Ok((start, end, batch))
+                }
+            })
+            .buffer_unordered(fmindex_prewarm_chunk_concurrency())
+            .try_for_each(|(start, end, batch)| {
+                let ranges = Arc::clone(&ranges);
+                let nodes = &self.nodes;
+                async move {
+                    if batch.num_rows() != end - start {
+                        return Err(Error::index(format!(
+                            "expected {} FM-Index block rows, got {}",
+                            end - start,
+                            batch.num_rows()
+                        )));
+                    }
+
+                    let col = LazyRankBitVec::words_column(&batch)?;
+                    let mut range_idx = ranges.partition_point(|range| range.end <= start);
+                    for row in 0..batch.num_rows() {
+                        let absolute_row = start + row;
+                        while range_idx < ranges.len() && absolute_row >= ranges[range_idx].end {
+                            range_idx += 1;
+                        }
+                        if range_idx == ranges.len() || absolute_row < ranges[range_idx].start {
+                            continue;
+                        }
+
+                        let range = ranges[range_idx];
+                        let block_idx = absolute_row - range.start;
+                        let node = &nodes[range.node_idx];
+                        if node.blocks[block_idx].get().is_none() {
+                            let words = LazyRankBitVec::decode_words(col.value(row));
+                            let _ = node.blocks[block_idx].set(words);
+                        }
+                    }
+
+                    Ok(())
+                }
+            })
+            .await?;
+
+        Ok(())
+    }
+
+    #[inline]
+    fn access(&self, mut pos: usize) -> u8 {
+        if self.nodes.is_empty() {
+            return 0;
+        }
+        let mut node_idx = 0;
+        loop {
+            let bit = self.nodes[node_idx].get(pos);
+            let (ref left, ref right) = self.children[node_idx];
+            if bit {
+                pos = self.nodes[node_idx].rank1(pos);
+                match right {
+                    WaveletChild::Leaf(b) => return *b,
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            } else {
+                pos = self.nodes[node_idx].rank0(pos);
+                match left {
+                    WaveletChild::Leaf(b) => return *b,
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            }
+        }
+    }
+
+    async fn access_async(&self, mut pos: usize) -> Result<u8> {
+        if self.nodes.is_empty() {
+            return Ok(0);
+        }
+        let mut node_idx = 0;
+        loop {
+            self.nodes[node_idx].load_block_for_access(pos).await?;
+            let bit = self.nodes[node_idx].get(pos);
+            let (ref left, ref right) = self.children[node_idx];
+            if bit {
+                pos = self.nodes[node_idx].rank1(pos);
+                match right {
+                    WaveletChild::Leaf(b) => return Ok(*b),
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            } else {
+                pos = self.nodes[node_idx].rank0(pos);
+                match left {
+                    WaveletChild::Leaf(b) => return Ok(*b),
+                    WaveletChild::Node(next) => node_idx = *next,
+                }
+            }
+        }
+    }
+
+    #[inline]
+    fn rank(&self, c: u8, pos: usize) -> usize {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return 0;
+        }
+        let (mut lo, mut hi) = (0, pos);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                lo = self.nodes[nid].rank0(lo);
+                hi = self.nodes[nid].rank0(hi);
+            } else {
+                lo = self.nodes[nid].rank1(lo);
+                hi = self.nodes[nid].rank1(hi);
+            }
+        }
+        hi - lo
+    }
+
+    async fn rank_async(&self, c: u8, pos: usize) -> Result<usize> {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return Ok(0);
+        }
+        let (mut lo, mut hi) = (0, pos);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            self.nodes[nid].load_block_for_rank(lo).await?;
+            self.nodes[nid].load_block_for_rank(hi).await?;
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                lo = self.nodes[nid].rank0(lo);
+                hi = self.nodes[nid].rank0(hi);
+            } else {
+                lo = self.nodes[nid].rank1(lo);
+                hi = self.nodes[nid].rank1(hi);
+            }
+        }
+        Ok(hi - lo)
+    }
+
+    #[inline]
+    fn rank_pair(&self, c: u8, lo: usize, hi: usize) -> (usize, usize) {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return (0, 0);
+        }
+        let (mut s, mut l, mut h) = (0, lo, hi);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                s = self.nodes[nid].rank0(s);
+                l = self.nodes[nid].rank0(l);
+                h = self.nodes[nid].rank0(h);
+            } else {
+                s = self.nodes[nid].rank1(s);
+                l = self.nodes[nid].rank1(l);
+                h = self.nodes[nid].rank1(h);
+            }
+        }
+        (l - s, h - s)
+    }
+
+    async fn rank_pair_async(&self, c: u8, lo: usize, hi: usize) -> Result<(usize, usize)> {
+        let code = &self.codes[c as usize];
+        if code.length == 0 {
+            return Ok((0, 0));
+        }
+        let (mut s, mut l, mut h) = (0, lo, hi);
+        for (level, &nid) in code.node_path.iter().enumerate() {
+            self.nodes[nid].load_block_for_rank(s).await?;
+            self.nodes[nid].load_block_for_rank(l).await?;
+            self.nodes[nid].load_block_for_rank(h).await?;
+            if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 {
+                s = self.nodes[nid].rank0(s);
+                l = self.nodes[nid].rank0(l);
+                h = self.nodes[nid].rank0(h);
+            } else {
+                s = self.nodes[nid].rank1(s);
+                l = self.nodes[nid].rank1(l);
+                h = self.nodes[nid].rank1(h);
+            }
+        }
+        Ok((l - s, h - s))
+    }
+
+    fn deep_size(&self) -> usize {
+        self.nodes.iter().map(|n| n.deep_size()).sum::<usize>()
+            + self
+                .codes
+                .iter()
+                .map(|c| c.node_path.len() * 8)
+                .sum::<usize>()
+    }
+}
+
+// ── FM-Index (in-memory, build-time) ─────────────────────────────────────────
+
+#[derive(Debug, Clone)]
+pub struct FMIndex {
+    wavelet: HuffmanWaveletTree,
+    row_ids: Vec<u64>,
+    /// Sampled SA: sa_samples[i] = SA[i * SA_SAMPLE_RATE]. Size: N/D × 8 bytes.
+    sa_samples: Vec<u64>,
+    /// Starting byte offset of each document in the concatenated text.
+    doc_start_positions: Vec<u64>,
+    c_table: Vec<usize>,
+    alphabet_size: usize,
+}
+
+impl DeepSizeOf for FMIndex {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
+        self.wavelet.deep_size()
+            + self.row_ids.len() * 8
+            + self.sa_samples.len() * 8
+            + self.doc_start_positions.len() * 8
+            + self.c_table.len() * std::mem::size_of::<usize>()
+    }
+}
+
+#[allow(dead_code)]
+impl FMIndex {
+    fn build(texts: &[(u64, &[u8])]) -> Result<Self> {
+        if texts.is_empty() {
+            return Ok(Self {
+                wavelet: HuffmanWaveletTree {
+                    nodes: Vec::new(),
+                    codes: std::array::from_fn(|_| HuffmanCode::default()),
+                    children: Vec::new(),
+                    len: 0,
+                },
+                row_ids: Vec::new(),
+                sa_samples: Vec::new(),
+                doc_start_positions: Vec::new(),
+                c_table: vec![0; 257],
+                alphabet_size: 256,
+            });
+        }
+
+        let mut concat = Vec::new();
+        let mut doc_row_ids = Vec::new();
+        let mut doc_starts: Vec<u64> = Vec::new();
+        for (row_id, text) in texts {
+            doc_starts.push(concat.len() as u64);
+            doc_row_ids.push(*row_id);
+            concat.extend_from_slice(text);
+            concat.push(SENTINEL_BYTE); // \xFF separator between documents
+        }
+        // Append unique terminator \x00 so SA-IS produces a proper suffix array
+        // with a single-cycle LF-mapping permutation.
+        concat.push(0x00);
+        let n = concat.len();
+        let sa = build_suffix_array(&concat);
+
+        let bwt: Vec<u8> = sa
+            .iter()
+            .map(|&pos| {
+                if pos == 0 {
+                    concat[n - 1]
+                } else {
+                    concat[pos - 1]
+                }
+            })
+            .collect();
+
+        let mut counts = vec![0usize; 257];
+        for &b in &concat {
+            counts[b as usize + 1] += 1;
+        }
+        for i in 1..257 {
+            counts[i] += counts[i - 1];
+        }
+
+        // Sampled SA: store every D-th entry
+        let sa_samples: Vec<u64> = sa
+            .iter()
+            .step_by(SA_SAMPLE_RATE)
+            .map(|&pos| pos as u64)
+            .collect();
+
+        let wavelet = HuffmanWaveletTree::build(&bwt);
+
+        Ok(Self {
+            wavelet,
+            row_ids: doc_row_ids,
+            sa_samples,
+            doc_start_positions: doc_starts,
+            c_table: counts,
+            alphabet_size: 256,
+        })
+    }
+
+    /// Locate: resolve SA[pos] by walking LF-mapping until hitting a sampled position.
+    /// For large data (N >> SA_SAMPLE_RATE), converges within SA_SAMPLE_RATE steps.
+    /// For small data with short LF cycles, may need up to N steps.
+    #[inline]
+    fn locate(&self, mut pos: usize) -> usize {
+        let mut steps = 0;
+        let n = self.wavelet.len;
+        loop {
+            if pos.is_multiple_of(SA_SAMPLE_RATE) && (pos / SA_SAMPLE_RATE) < self.sa_samples.len()
+            {
+                return (self.sa_samples[pos / SA_SAMPLE_RATE] as usize + steps) % n;
+            }
+            let c = self.wavelet.access(pos);
+            pos = self.c_table[c as usize] + self.wavelet.rank(c, pos);
+            steps += 1;
+            if steps >= n {
+                log::warn!("FM-Index SA locate exceeded {n} steps, possible index corruption");
+                return 0;
+            }
+        }
+    }
+
+    /// Map a text position to document index via binary search on doc_start_positions.
+    #[inline]
+    fn doc_for_position(&self, text_pos: usize) -> usize {
+        let tp = text_pos as u64;
+        match self.doc_start_positions.binary_search(&tp) {
+            Ok(idx) => idx,
+            Err(idx) => idx - 1,
+        }
+    }
+
+    fn backward_search(&self, pattern: &[u8]) -> (usize, usize) {
+        if pattern.is_empty() || self.wavelet.len == 0 {
+            return (0, 0);
+        }
+        let (mut lo, mut hi) = (0, self.wavelet.len);
+        for &b in pattern.iter().rev() {
+            let c = self.c_table[b as usize];
+            let (occ_lo, occ_hi) = self.wavelet.rank_pair(b, lo, hi);
+            lo = c + occ_lo;
+            hi = c + occ_hi;
+            if lo >= hi {
+                return (0, 0);
+            }
+        }
+        (lo, hi)
+    }
+
+    #[cfg(test)]
+    fn search(&self, pattern: &[u8]) -> RoaringBitmap {
+        let (lo, hi) = self.backward_search(pattern);
+        if lo >= hi {
+            return RoaringBitmap::new();
+        }
+        let mut result = RoaringBitmap::new();
+        for i in lo..hi {
+            let text_pos = self.locate(i);
+            let doc_idx = self.doc_for_position(text_pos);
+            result.insert(self.row_ids[doc_idx] as u32);
+        }
+        result
+    }
+
+    /// Search returning full u64 row addresses (preserving fragment ID in upper bits).
+    fn search_row_addrs(&self, pattern: &[u8]) -> Vec<u64> {
+        let (lo, hi) = self.backward_search(pattern);
+        if lo >= hi {
+            return Vec::new();
+        }
+        let mut seen = std::collections::HashSet::new();
+        let mut result = Vec::new();
+        for i in lo..hi {
+            let text_pos = self.locate(i);
+            let doc_idx = self.doc_for_position(text_pos);
+            let row_addr = self.row_ids[doc_idx];
+            if seen.insert(row_addr) {
+                result.push(row_addr);
+            }
+        }
+        result
+    }
+
+    fn serialize_huffman_codes(&self) -> Vec<u8> {
+        let mut buf = Vec::new();
+        for code in &self.wavelet.codes {
+            buf.extend_from_slice(&code.bits.to_le_bytes());
+            buf.push(code.length);
+            buf.extend_from_slice(&(code.node_path.len() as u16).to_le_bytes());
+            for &nid in &code.node_path {
+                buf.extend_from_slice(&(nid as u32).to_le_bytes());
+            }
+        }
+        buf
+    }
+
+    fn deserialize_huffman_codes(data: &[u8]) -> [HuffmanCode; 256] {
+        let mut codes: [HuffmanCode; 256] = std::array::from_fn(|_| HuffmanCode::default());
+        let mut cur = 0;
+        for code in &mut codes {
+            let bits = u32::from_le_bytes(data[cur..cur + 4].try_into().unwrap());
+            cur += 4;
+            let length = data[cur];
+            cur += 1;
+            let plen = u16::from_le_bytes(data[cur..cur + 2].try_into().unwrap()) as usize;
+            cur += 2;
+            let mut node_path = Vec::with_capacity(plen);
+            for _ in 0..plen {
+                node_path.push(u32::from_le_bytes(data[cur..cur + 4].try_into().unwrap()) as usize);
+                cur += 4;
+            }
+            *code = HuffmanCode {
+                bits,
+                length,
+                node_path,
+            };
+        }
+        codes
+    }
+
+    fn serialize_tree_topology(&self) -> Vec<u8> {
+        let mut buf = Vec::new();
+        buf.extend_from_slice(&(self.wavelet.children.len() as u32).to_le_bytes());
+        for (left, right) in &self.wavelet.children {
+            for child in [left, right] {
+                match child {
+                    WaveletChild::Node(id) => {
+                        buf.push(0);
+                        buf.extend_from_slice(&(*id as u32).to_le_bytes());
+                    }
+                    WaveletChild::Leaf(b) => {
+                        buf.push(1);
+                        buf.extend_from_slice(&(*b as u32).to_le_bytes());
+                    }
+                }
+            }
+        }
+        buf
+    }
+
+    fn deserialize_tree_topology(data: &[u8]) -> Vec<(WaveletChild, WaveletChild)> {
+        let mut cur = 0;
+        let count = u32::from_le_bytes(data[cur..cur + 4].try_into().unwrap()) as usize;
+        cur += 4;
+        let mut children = Vec::with_capacity(count);
+        for _ in 0..count {
+            let mut read_child = || {
+                let t = data[cur];
+                cur += 1;
+                let v = u32::from_le_bytes(data[cur..cur + 4].try_into().unwrap());
+                cur += 4;
+                if t == 0 {
+                    WaveletChild::Node(v as usize)
+                } else {
+                    WaveletChild::Leaf(v as u8)
+                }
+            };
+            let l = read_child();
+            let r = read_child();
+            children.push((l, r));
+        }
+        children
+    }
+
+    fn serialize_c_table(&self) -> Vec<u8> {
+        self.c_table
+            .iter()
+            .flat_map(|&v| (v as u64).to_le_bytes())
+            .collect()
+    }
+
+    fn deserialize_c_table(data: &[u8]) -> Vec<usize> {
+        data.chunks_exact(8)
+            .map(|c| u64::from_le_bytes(c.try_into().unwrap()) as usize)
+            .collect()
+    }
+
+    fn u64_to_bytes(data: &[u64]) -> Vec<u8> {
+        data.iter().flat_map(|v| v.to_le_bytes()).collect()
+    }
+
+    fn build_wavelet_batch(&self) -> Result<RecordBatch> {
+        use arrow_array::{LargeBinaryArray, UInt32Array, UInt64Array};
+        let mut nid_b = Vec::new();
+        let mut bid_b = Vec::new();
+        let mut words_b: Vec<Vec<u8>> = Vec::new();
+        let mut pr_b = Vec::new();
+        let mut bl_b = Vec::new();
+
+        for (i, node) in self.wavelet.nodes.iter().enumerate() {
+            let mut pr: u64 = 0;
+            if node.words.is_empty() {
+                nid_b.push(i as u32);
+                bid_b.push(0u32);
+                words_b.push(Vec::new());
+                pr_b.push(0u64);
+                bl_b.push(node.len as u64);
+            } else {
+                for (bi, chunk) in node.words.chunks(BLOCK_WORDS).enumerate() {
+                    nid_b.push(i as u32);
+                    bid_b.push(bi as u32);
+                    words_b.push(Self::u64_to_bytes(chunk));
+                    pr_b.push(pr);
+                    bl_b.push(node.len as u64);
+                    pr += chunk.iter().map(|w| w.count_ones() as u64).sum::<u64>();
+                }
+            }
+        }
+        let refs: Vec<&[u8]> = words_b.iter().map(|v| v.as_slice()).collect();
+        let schema = Arc::new(Self::block_schema());
+        Ok(RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(UInt32Array::from(nid_b)),
+                Arc::new(UInt32Array::from(bid_b)),
+                Arc::new(LargeBinaryArray::from(refs)),
+                Arc::new(UInt64Array::from(pr_b)),
+                Arc::new(UInt64Array::from(bl_b)),
+            ],
+        )?)
+    }
+
+    fn block_schema() -> arrow_schema::Schema {
+        arrow_schema::Schema::new(vec![
+            Field::new("node_id", DataType::UInt32, false),
+            Field::new("block_id", DataType::UInt32, false),
+            Field::new("words", DataType::LargeBinary, false),
+            Field::new("prefix_rank", DataType::UInt64, false),
+            Field::new("bit_len", DataType::UInt64, false),
+        ])
+    }
+}
+
+// ── Lazy FM-Index ────────────────────────────────────────────────────────────
+
+#[derive(Debug)]
+struct LazyFMIndex {
+    wavelet: LazyHuffmanWaveletTree,
+    row_ids: Vec<u64>,
+    sa_samples: Vec<u64>,
+    doc_start_positions: Vec<u64>,
+    c_table: Vec<usize>,
+    fully_prewarmed: AtomicBool,
+}
+
+impl LazyFMIndex {
+    /// Pre-load all wavelet tree blocks before sync search operations.
+    async fn prewarm(&self) -> Result<()> {
+        if self.fully_prewarmed.load(Ordering::Acquire) {
+            return Ok(());
+        }
+        self.wavelet.load_all().await.inspect(|_| {
+            self.fully_prewarmed.store(true, Ordering::Release);
+        })
+    }
+
+    fn backward_search(&self, pattern: &[u8]) -> (usize, usize) {
+        if pattern.is_empty() || self.wavelet.len == 0 {
+            return (0, 0);
+        }
+        let (mut lo, mut hi) = (0, self.wavelet.len);
+        for &b in pattern.iter().rev() {
+            let c = self.c_table[b as usize];
+            let (occ_lo, occ_hi) = self.wavelet.rank_pair(b, lo, hi);
+            lo = c + occ_lo;
+            hi = c + occ_hi;
+            if lo >= hi {
+                return (0, 0);
+            }
+        }
+        (lo, hi)
+    }
+
+    async fn backward_search_async(&self, pattern: &[u8]) -> Result<(usize, usize)> {
+        if pattern.is_empty() || self.wavelet.len == 0 {
+            return Ok((0, 0));
+        }
+        let (mut lo, mut hi) = (0, self.wavelet.len);
+        for &b in pattern.iter().rev() {
+            let c = self.c_table[b as usize];
+            let (occ_lo, occ_hi) = self.wavelet.rank_pair_async(b, lo, hi).await?;
+            lo = c + occ_lo;
+            hi = c + occ_hi;
+            if lo >= hi {
+                return Ok((0, 0));
+            }
+        }
+        Ok((lo, hi))
+    }
+
+    #[inline]
+    fn locate(&self, mut pos: usize) -> usize {
+        let mut steps = 0;
+        let n = self.wavelet.len;
+        loop {
+            if pos.is_multiple_of(SA_SAMPLE_RATE) && (pos / SA_SAMPLE_RATE) < self.sa_samples.len()
+            {
+                return (self.sa_samples[pos / SA_SAMPLE_RATE] as usize + steps) % n;
+            }
+            let c = self.wavelet.access(pos);
+            pos = self.c_table[c as usize] + self.wavelet.rank(c, pos);
+            steps += 1;
+            if steps >= n {
+                log::warn!("FM-Index SA locate exceeded {n} steps, possible index corruption");
+                return 0;
+            }
+        }
+    }
+
+    async fn locate_async(&self, mut pos: usize) -> Result<usize> {
+        let mut steps = 0;
+        let n = self.wavelet.len;
+        loop {
+            if pos.is_multiple_of(SA_SAMPLE_RATE) && (pos / SA_SAMPLE_RATE) < self.sa_samples.len()
+            {
+                return Ok((self.sa_samples[pos / SA_SAMPLE_RATE] as usize + steps) % n);
+            }
+            let c = self.wavelet.access_async(pos).await?;
+            pos = self.c_table[c as usize] + self.wavelet.rank_async(c, pos).await?;
+            steps += 1;
+            if steps >= n {
+                log::warn!("FM-Index SA locate exceeded {n} steps, possible index corruption");
+                return Ok(0);
+            }
+        }
+    }
+
+    #[inline]
+    fn doc_for_position(&self, text_pos: usize) -> usize {
+        let tp = text_pos as u64;
+        match self.doc_start_positions.binary_search(&tp) {
+            Ok(idx) => idx,
+            Err(idx) => idx - 1,
+        }
+    }
+
+    #[cfg(test)]
+    fn search(&self, pattern: &[u8]) -> RoaringBitmap {
+        let (lo, hi) = self.backward_search(pattern);
+        if lo >= hi {
+            return RoaringBitmap::new();
+        }
+        let mut result = RoaringBitmap::new();
+        for i in lo..hi {
+            let text_pos = self.locate(i);
+            let doc_idx = self.doc_for_position(text_pos);
+            result.insert(self.row_ids[doc_idx] as u32);
+        }
+        result
+    }
+
+    /// Search returning full u64 row addresses (preserving fragment ID in upper bits).
+    fn search_row_addrs(&self, pattern: &[u8]) -> Vec<u64> {
+        let (lo, hi) = self.backward_search(pattern);
+        if lo >= hi {
+            return Vec::new();
+        }
+        let mut seen = std::collections::HashSet::new();
+        let mut result = Vec::new();
+        for i in lo..hi {
+            let text_pos = self.locate(i);
+            let doc_idx = self.doc_for_position(text_pos);
+            let row_addr = self.row_ids[doc_idx];
+            if seen.insert(row_addr) {
+                result.push(row_addr);
+            }
+        }
+        result
+    }
+
+    async fn search_row_addrs_async(&self, pattern: &[u8]) -> Result<Vec<u64>> {
+        let (lo, hi) = self.backward_search_async(pattern).await?;
+        if lo >= hi {
+            return Ok(Vec::new());
+        }
+        let mut seen = std::collections::HashSet::new();
+        let mut result = Vec::new();
+        for i in lo..hi {
+            let text_pos = self.locate_async(i).await?;
+            let doc_idx = self.doc_for_position(text_pos);
+            let row_addr = self.row_ids[doc_idx];
+            if seen.insert(row_addr) {
+                result.push(row_addr);
+            }
+        }
+        Ok(result)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn from_reader(
+        reader: Arc<dyn crate::scalar::IndexReader>,
+        num_bwt_nodes: usize,
+        huffman_codes: [HuffmanCode; 256],
+        children: Vec<(WaveletChild, WaveletChild)>,
+        c_table: Vec<usize>,
+        bwt_len: usize,
+        total_wavelet_rows: usize,
+        num_sa_blocks: usize,
+        sa_samples_len: usize,
+        row_ids: Vec<u64>,
+        doc_start_positions: Vec<u64>,
+    ) -> Result<Self> {
+        use arrow_array::UInt64Array;
+
+        let meta = reader
+            .read_range(
+                0..total_wavelet_rows,
+                Some(&["node_id", "prefix_rank", "bit_len"]),
+            )
+            .await?;
+        let nid_col = meta
+            .column_by_name("node_id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<arrow_array::UInt32Array>()
+            .unwrap();
+        let pr_col = meta
+            .column_by_name("prefix_rank")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+        let bl_col = meta
+            .column_by_name("bit_len")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+
+        struct NM {
+            prs: Vec<u64>,
+            offset: usize,
+            blen: usize,
+        }
+        let mut nms: Vec<NM> = (0..num_bwt_nodes)
+            .map(|_| NM {
+                prs: Vec::new(),
+                offset: 0,
+                blen: 0,
+            })
+            .collect();
+        for row in 0..meta.num_rows() {
+            let nid = nid_col.value(row) as usize;
+            if nid >= num_bwt_nodes {
+                continue;
+            }
+            let nm = &mut nms[nid];
+            if nm.prs.is_empty() {
+                nm.offset = row;
+            }
+            nm.prs.push(pr_col.value(row));
+            nm.blen = bl_col.value(row) as usize;
+        }
+
+        let mut bwt_nodes = Vec::with_capacity(num_bwt_nodes);
+        for nm in &nms {
+            bwt_nodes.push(LazyRankBitVec::new(
+                nm.prs.clone(),
+                nm.prs.len(),
+                reader.clone(),
+                nm.offset,
+                nm.blen,
+            ));
+        }
+        let wavelet = LazyHuffmanWaveletTree {
+            nodes: bwt_nodes,
+            codes: huffman_codes,
+            children,
+            len: bwt_len,
+        };
+
+        // Read SA samples from packed binary blocks
+        let mut sa_samples = Vec::with_capacity(sa_samples_len);
+        let sa_batch = reader
+            .read_range(
+                total_wavelet_rows..total_wavelet_rows + num_sa_blocks,
+                Some(&["words"]),
+            )
+            .await?;
+        let words_col = sa_batch
+            .column_by_name("words")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<arrow_array::LargeBinaryArray>()
+            .unwrap();
+        for i in 0..sa_batch.num_rows() {
+            let raw = words_col.value(i);
+            for chunk in raw.chunks_exact(8) {
+                sa_samples.push(u64::from_le_bytes(chunk.try_into().unwrap()));
+            }
+        }
+        sa_samples.truncate(sa_samples_len);
+
+        Ok(Self {
+            wavelet,
+            row_ids,
+            sa_samples,
+            doc_start_positions,
+            c_table,
+            fully_prewarmed: AtomicBool::new(false),
+        })
+    }
+
+    fn deep_size(&self) -> usize {
+        self.wavelet.deep_size()
+            + self.row_ids.len() * 8
+            + self.sa_samples.len() * 8
+            + self.doc_start_positions.len() * 8
+            + self.c_table.len() * std::mem::size_of::<usize>()
+    }
+}
+
+// ── FMIndexScalarIndex ───────────────────────────────────────────────────────
+
+#[derive(Debug)]
+struct FMIndexPartition {
+    #[allow(dead_code)]
+    id: u64,
+    fm: LazyFMIndex,
+}
+
+#[derive(Debug)]
+pub struct FMIndexScalarIndex {
+    partitions: Vec<Arc<FMIndexPartition>>,
+    io_parallelism: usize,
+}
+
+impl DeepSizeOf for FMIndexScalarIndex {
+    fn deep_size_of_children(&self, _ctx: &mut lance_core::deepsize::Context) -> usize {
+        self.partitions.iter().map(|p| p.fm.deep_size()).sum()
+    }
+}
+
+impl FMIndexScalarIndex {
+    async fn load_partition(
+        store: &dyn IndexStore,
+        filename: &str,
+        pid: u64,
+    ) -> Result<FMIndexPartition> {
+        let reader = store.open_index_file(filename).await?;
+        let md = &reader.schema().metadata;
+
+        let parse = |key: &str| -> Result<usize> {
+            md.get(key)
+                .ok_or_else(|| Error::invalid_input(format!("missing {key}")))?
+                .parse()
+                .map_err(|e| Error::invalid_input(format!("invalid {key}: {e}")))
+        };
+
+        let num_bwt_nodes = parse("num_bwt_nodes")?;
+        let bwt_len = parse("bwt_len")?;
+        let num_sa_blocks = parse("num_sa_blocks")?;
+        let sa_samples_len = parse("sa_samples_len")?;
+        let total_wavelet_rows = parse("total_wavelet_rows")?;
+
+        let c_table = FMIndex::deserialize_c_table(&hex_decode(
+            md.get("c_table")
+                .ok_or_else(|| Error::invalid_input("missing c_table"))?,
+        )?);
+        let huffman_codes = FMIndex::deserialize_huffman_codes(&hex_decode(
+            md.get("huffman_codes")
+                .ok_or_else(|| Error::invalid_input("missing huffman_codes"))?,
+        )?);
+        let children = FMIndex::deserialize_tree_topology(&hex_decode(
+            md.get("tree_topology")
+                .ok_or_else(|| Error::invalid_input("missing tree_topology"))?,
+        )?);
+
+        // row_ids and doc_start_positions stored in metadata (small)
+        let row_ids_hex = md
+            .get("row_ids")
+            .ok_or_else(|| Error::invalid_input("missing row_ids"))?;
+        let row_ids_bytes = hex_decode(row_ids_hex)?;
+        let row_ids: Vec<u64> = row_ids_bytes
+            .chunks_exact(8)
+            .map(|c| u64::from_le_bytes(c.try_into().unwrap()))
+            .collect();
+
+        let doc_starts_hex = md
+            .get("doc_start_positions")
+            .ok_or_else(|| Error::invalid_input("missing doc_start_positions"))?;
+        let doc_starts_bytes = hex_decode(doc_starts_hex)?;
+        let doc_start_positions: Vec<u64> = doc_starts_bytes
+            .chunks_exact(8)
+            .map(|c| u64::from_le_bytes(c.try_into().unwrap()))
+            .collect();
+
+        let fm = Box::pin(LazyFMIndex::from_reader(
+            reader,
+            num_bwt_nodes,
+            huffman_codes,
+            children,
+            c_table,
+            bwt_len,
+            total_wavelet_rows,
+            num_sa_blocks,
+            sa_samples_len,
+            row_ids,
+            doc_start_positions,
+        ))
+        .await?;
+        Ok(FMIndexPartition { id: pid, fm })
+    }
+
+    async fn load(
+        store: Arc<dyn IndexStore>,
+        _fri: Option<Arc<dyn RowIdRemapper>>,
+        _cache: &LanceCache,
+    ) -> Result<Arc<Self>> {
+        let files = store.list_files_with_sizes().await?;
+        let mut pfiles: Vec<(u64, String)> = Vec::new();
+        for f in &files {
+            if let Some(id) = fmindex_partition_id_from_path(&f.path) {
+                pfiles.push((id, f.path.clone()));
+            }
+        }
+        if pfiles.is_empty() {
+            return Err(Error::invalid_input("no FM-Index partition files found"));
+        }
+        pfiles.sort_by_key(|(id, _)| *id);
+        let io_parallelism = store.io_parallelism().max(1);
+        let mut parts = futures::stream::iter(pfiles)
+            .map(|(id, name)| {
+                let store = Arc::clone(&store);
+                async move {
+                    let partition = Self::load_partition(store.as_ref(), &name, id).await?;
+                    Result::<_>::Ok((id, Arc::new(partition)))
+                }
+            })
+            .buffer_unordered(io_parallelism)
+            .try_collect::<Vec<_>>()
+            .await?;
+        parts.sort_by_key(|(id, _)| *id);
+        let parts = parts
+            .into_iter()
+            .map(|(_, partition)| partition)
+            .collect::<Vec<_>>();
+        Ok(Arc::new(Self {
+            partitions: parts,
+            io_parallelism,
+        }))
+    }
+
+    fn partition_parallelism(&self) -> usize {
+        self.io_parallelism.max(1).min(self.partitions.len().max(1))
+    }
+
+    async fn prewarm_partitions(&self) -> Result<()> {
+        futures::stream::iter(self.partitions.iter().cloned())
+            .map(|partition| async move { partition.fm.prewarm().await })
+            .buffer_unordered(self.partition_parallelism())
+            .try_collect::<Vec<_>>()
+            .await?;
+        Ok(())
+    }
+
+    async fn search_string_contains(&self, pattern: &[u8]) -> Result<SearchResult> {
+        use lance_select::RowAddrTreeMap;
+
+        let pattern: Arc<[u8]> = Arc::from(pattern);
+        let tree = futures::stream::iter(self.partitions.iter().cloned())
+            .map(|partition| {
+                let pattern = Arc::clone(&pattern);
+                async move {
+                    if partition.fm.fully_prewarmed.load(Ordering::Acquire) {
+                        spawn_cpu(move || {
+                            Result::<Vec<u64>>::Ok(partition.fm.search_row_addrs(pattern.as_ref()))
+                        })
+                        .await
+                    } else {
+                        partition.fm.search_row_addrs_async(pattern.as_ref()).await
+                    }
+                }
+            })
+            .buffer_unordered(self.partition_parallelism())
+            .try_fold(RowAddrTreeMap::new(), |mut tree, row_addrs| async move {
+                for row_addr in row_addrs {
+                    tree.insert(row_addr);
+                }
+                Result::Ok(tree)
+            })
+            .await?;
+
+        Ok(SearchResult::Exact(lance_select::NullableRowAddrSet::new(
+            tree,
+            Default::default(),
+        )))
+    }
+}
+
+#[async_trait]
+impl Index for FMIndexScalarIndex {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+    fn as_index(self: Arc<Self>) -> Arc<dyn Index> {
+        self
+    }
+    async fn prewarm(&self) -> Result<()> {
+        self.prewarm_partitions().await
+    }
+    fn statistics(&self) -> Result<serde_json::Value> {
+        Ok(serde_json::json!({
+            "type": "Fm",
+            "num_partitions": self.partitions.len(),
+            "total_bwt_len": self.partitions.iter().map(|p| p.fm.wavelet.len).sum::<usize>(),
+            "total_docs": self.partitions.iter().map(|p| p.fm.row_ids.len()).sum::<usize>(),
+        }))
+    }
+    fn index_type(&self) -> IndexType {
+        IndexType::Fm
+    }
+    async fn calculate_included_frags(&self) -> Result<RoaringBitmap> {
+        let mut frags = RoaringBitmap::new();
+        for p in &self.partitions {
+            for &rid in &p.fm.row_ids {
+                frags.insert((rid >> 32) as u32);
+            }
+        }
+        Ok(frags)
+    }
+}
+
+#[async_trait]
+impl ScalarIndex for FMIndexScalarIndex {
+    async fn search(
+        &self,
+        query: &dyn AnyQuery,
+        _metrics: &dyn MetricsCollector,
+    ) -> Result<SearchResult> {
+        let tq = query
+            .as_any()
+            .downcast_ref::<TextQuery>()
+            .ok_or_else(|| Error::invalid_input("Fm only supports TextQuery"))?;
+        match tq {
+            TextQuery::StringContains(pattern) => {
+                self.search_string_contains(pattern.as_bytes()).await
+            }
+            // Regex queries are routed only to the ngram index (the FM-index's
+            // query parser advertises `supports_regex = false`), so this is
+            // unreachable in practice; reject it explicitly rather than silently.
+            TextQuery::Regex(_) => Err(Error::invalid_input(
+                "FMIndex does not support regular expression queries",
+            )),
+        }
+    }
+    fn can_remap(&self) -> bool {
+        false
+    }
+    async fn remap(&self, _: &RowAddrRemap, _: &dyn IndexStore) -> Result<CreatedIndex> {
+        Err(Error::not_supported("Fm does not support remap"))
+    }
+    async fn update(
+        &self,
+        new_data: SendableRecordBatchStream,
+        dest: &dyn IndexStore,
+        _old_data_filter: Option<OldIndexDataFilter>,
+    ) -> Result<CreatedIndex> {
+        let files = write_partitioned_fmindex_stream(new_data, dest).await?;
+        Ok(CreatedIndex {
+            index_details: prost_types::Any::from_msg(&pb::FmIndexDetails {}).unwrap(),
+            index_version: FMINDEX_INDEX_VERSION,
+            files,
+        })
+    }
+    fn update_criteria(&self) -> UpdateCriteria {
+        UpdateCriteria::requires_old_data(
+            TrainingCriteria::new(TrainingOrdering::None).with_row_addr(),
+        )
+    }
+    fn derive_index_params(&self) -> Result<ScalarIndexParams> {
+        Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::Fm))
+    }
+}
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+#[derive(Debug)]
+struct FMIndexPartitionJob {
+    partition_id: u64,
+    texts: Vec<(u64, Vec<u8>)>,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct FMIndexPartitionConfig {
+    num_workers: usize,
+    max_rows: usize,
+    max_bytes: usize,
+    queue_size: usize,
+    resume_existing: bool,
+}
+
+impl FMIndexPartitionConfig {
+    fn from_env() -> Self {
+        Self {
+            num_workers: fmindex_num_workers(),
+            max_rows: fmindex_partition_rows(),
+            max_bytes: fmindex_partition_bytes(),
+            queue_size: fmindex_write_queue_size(),
+            resume_existing: fmindex_resume_existing_partitions(),
+        }
+    }
+
+    fn normalized(self) -> Self {
+        Self {
+            num_workers: self.num_workers.max(1),
+            max_rows: self.max_rows.max(1),
+            max_bytes: self.max_bytes.max(1),
+            queue_size: self.queue_size.max(1),
+            resume_existing: self.resume_existing,
+        }
+    }
+}
+
+async fn write_partitioned_fmindex_stream(
+    stream: SendableRecordBatchStream,
+    store: &dyn IndexStore,
+) -> Result<Vec<IndexFile>> {
+    write_partitioned_fmindex_stream_with_config(stream, store, FMIndexPartitionConfig::from_env())
+        .await
+}
+
+async fn write_partitioned_fmindex_stream_with_config(
+    mut stream: SendableRecordBatchStream,
+    store: &dyn IndexStore,
+    config: FMIndexPartitionConfig,
+) -> Result<Vec<IndexFile>> {
+    let config = config.normalized();
+    log::info!(
+        "building FMIndex with {} workers, partition rows {}, partition bytes {}",
+        config.num_workers,
+        config.max_rows,
+        config.max_bytes
+    );
+
+    let (sender, receiver): (
+        async_channel::Sender<FMIndexPartitionJob>,
+        async_channel::Receiver<FMIndexPartitionJob>,
+    ) = async_channel::bounded(config.queue_size);
+    let store = store.clone_arc();
+    let mut completed_files = if config.resume_existing {
+        store
+            .list_files_with_sizes()
+            .await?
+            .into_iter()
+            .filter_map(|file| fmindex_partition_id_from_path(&file.path).map(|id| (id, file)))
+            .collect::<HashMap<_, _>>()
+    } else {
+        HashMap::new()
+    };
+    if !completed_files.is_empty() {
+        log::info!(
+            "resuming FMIndex build with {} existing partition files",
+            completed_files.len()
+        );
+    }
+    let mut files = Vec::new();
+    let mut worker_tasks = Vec::with_capacity(config.num_workers);
+    for _ in 0..config.num_workers {
+        let receiver = receiver.clone();
+        let store = store.clone();
+        worker_tasks.push(tokio::task::spawn(async move {
+            let mut files = Vec::new();
+            while let Ok(job) = receiver.recv().await {
+                files.push(
+                    write_fmindex_partition_owned(job.texts, store.clone(), job.partition_id)
+                        .await?,
+                );
+            }
+            Result::Ok(files)
+        }));
+    }
+    drop(receiver);
+
+    let producer_result = async {
+        let mut partition = Vec::with_capacity(config.max_rows.min(PARTITION_SIZE));
+        let mut partition_bytes = 0usize;
+        let mut partition_id = 0;
+
+        while let Some(batch) = stream.next().await {
+            let batch = batch?;
+            // Prefer _rowaddr (global row address) over _rowid to ensure stable,
+            // globally unique identifiers across segments.
+            let row_addrs: &arrow_array::UInt64Array = batch
+                .column_by_name(ROW_ADDR)
+                .or_else(|| batch.column_by_name("_rowid"))
+                .and_then(|c| c.as_any().downcast_ref())
+                .ok_or_else(|| {
+                    Error::invalid_input("Fm training data must include _rowaddr or _rowid column")
+                })?;
+            // Use the named value column; fall back to column(0) for legacy streams
+            let value_col = batch
+                .column_by_name(VALUE_COLUMN_NAME)
+                .unwrap_or_else(|| batch.column(0));
+            for i in 0..batch.num_rows() {
+                let rid = row_addrs.value(i);
+                if let Some(bytes) = extract_sanitized_text_bytes(value_col.as_ref(), i)? {
+                    partition_bytes = partition_bytes.saturating_add(bytes.len().saturating_add(1));
+                    partition.push((rid, bytes));
+                    if fmindex_partition_limit_reached(
+                        partition.len(),
+                        partition_bytes,
+                        config.max_rows,
+                        config.max_bytes,
+                    ) {
+                        finish_fmindex_partition(
+                            &sender,
+                            store.as_ref(),
+                            &mut partition,
+                            &mut partition_bytes,
+                            partition_id,
+                            config.max_rows,
+                            &mut FMIndexPartitionResumeState {
+                                completed_files: &mut completed_files,
+                                output_files: &mut files,
+                            },
+                        )
+                        .await?;
+                        partition_id += 1;
+                    }
+                }
+            }
+        }
+
+        if !partition.is_empty() {
+            finish_fmindex_partition(
+                &sender,
+                store.as_ref(),
+                &mut partition,
+                &mut partition_bytes,
+                partition_id,
+                config.max_rows,
+                &mut FMIndexPartitionResumeState {
+                    completed_files: &mut completed_files,
+                    output_files: &mut files,
+                },
+            )
+            .await?;
+        }
+
+        Result::Ok(())
+    }
+    .await;
+    drop(sender);
+
+    let mut worker_error = None;
+    for worker_task in worker_tasks {
+        match worker_task.await {
+            Ok(Ok(worker_files)) => files.extend(worker_files),
+            Ok(Err(err)) => {
+                if worker_error.is_none() {
+                    worker_error = Some(err);
+                }
+            }
+            Err(err) => {
+                if worker_error.is_none() {
+                    worker_error = Some(Error::execution(format!(
+                        "FMIndex partition worker failed: {err}"
+                    )));
+                }
+            }
+        }
+    }
+    if let Some(err) = worker_error {
+        return Err(err);
+    }
+    producer_result?;
+
+    for (partition_id, file) in completed_files.drain() {
+        log::info!(
+            "deleting stale FMIndex partition {partition_id} from previous build: {}",
+            file.path
+        );
+        store.delete_index_file(&file.path).await?;
+    }
+    if files.is_empty() {
+        return Ok(vec![write_empty_fmindex_partition(store.as_ref()).await?]);
+    }
+
+    files.sort_unstable_by_key(|(partition_id, _)| *partition_id);
+    let files = files.into_iter().map(|(_, file)| file).collect();
+
+    Ok(files)
+}
+
+fn fmindex_partition_limit_reached(
+    rows: usize,
+    bytes: usize,
+    max_rows: usize,
+    max_bytes: usize,
+) -> bool {
+    // Byte limits are soft for a single oversized document because FMIndex
+    // partitions cannot split a document without changing search semantics.
+    rows >= max_rows || bytes >= max_bytes
+}
+
+struct FMIndexPartitionResumeState<'a> {
+    completed_files: &'a mut HashMap<u64, IndexFile>,
+    output_files: &'a mut Vec<(u64, IndexFile)>,
+}
+
+async fn finish_fmindex_partition(
+    sender: &async_channel::Sender<FMIndexPartitionJob>,
+    store: &dyn IndexStore,
+    partition: &mut Vec<(u64, Vec<u8>)>,
+    partition_bytes: &mut usize,
+    partition_id: u64,
+    max_rows: usize,
+    resume_state: &mut FMIndexPartitionResumeState<'_>,
+) -> Result<()> {
+    let texts = std::mem::replace(partition, Vec::with_capacity(max_rows.min(PARTITION_SIZE)));
+    *partition_bytes = 0;
+    if let Some(file) = resume_state.completed_files.remove(&partition_id) {
+        let fingerprint = fmindex_partition_fingerprint(&texts);
+        if fmindex_partition_matches(store, &file, &fingerprint).await? {
+            resume_state.output_files.push((partition_id, file));
+            return Ok(());
+        }
+        log::info!(
+            "rebuilding stale FMIndex partition {partition_id}: {}",
+            file.path
+        );
+        store.delete_index_file(&file.path).await?;
+    }
+    sender
+        .send(FMIndexPartitionJob {
+            partition_id,
+            texts,
+        })
+        .await
+        .map_err(|err| {
+            Error::execution(format!(
+                "failed to schedule FMIndex partition {partition_id}: {err}"
+            ))
+        })
+}
+
+fn fmindex_partition_fingerprint(texts: &[(u64, Vec<u8>)]) -> String {
+    const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
+    const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;
+
+    fn update(hash: &mut u64, bytes: &[u8]) {
+        for byte in bytes {
+            *hash ^= *byte as u64;
+            *hash = hash.wrapping_mul(FNV_PRIME);
+        }
+    }
+
+    let mut hash = FNV_OFFSET;
+    update(&mut hash, b"lance-fmindex-partition-v1");
+    update(&mut hash, &(texts.len() as u64).to_le_bytes());
+    for (row_id, bytes) in texts {
+        update(&mut hash, &row_id.to_le_bytes());
+        update(&mut hash, &(bytes.len() as u64).to_le_bytes());
+        update(&mut hash, bytes);
+    }
+    format!("{hash:016x}")
+}
+
+async fn fmindex_partition_matches(
+    store: &dyn IndexStore,
+    file: &IndexFile,
+    expected_fingerprint: &str,
+) -> Result<bool> {
+    let reader = store.open_index_file(&file.path).await?;
+    Ok(reader
+        .schema()
+        .metadata
+        .get(FMINDEX_PARTITION_FINGERPRINT_KEY)
+        .is_some_and(|fingerprint| fingerprint == expected_fingerprint))
+}
+
+fn sanitize_text_bytes(bytes: &[u8]) -> Vec<u8> {
+    bytes
+        .iter()
+        .map(|&b| {
+            if b == SENTINEL_BYTE || b == 0x00 {
+                b' '
+            } else {
+                b
+            }
+        })
+        .collect()
+}
+
+fn extract_sanitized_text_bytes(
+    array: &dyn arrow_array::Array,
+    index: usize,
+) -> Result<Option<Vec<u8>>> {
+    if array.is_null(index) {
+        return Ok(None);
+    }
+    match array.data_type() {
+        DataType::Utf8 => Ok(Some(sanitize_text_bytes(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::StringArray>()
+                .unwrap()
+                .value(index)
+                .as_bytes(),
+        ))),
+        DataType::LargeUtf8 => Ok(Some(sanitize_text_bytes(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::LargeStringArray>()
+                .unwrap()
+                .value(index)
+                .as_bytes(),
+        ))),
+        DataType::Binary => Ok(Some(sanitize_text_bytes(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::BinaryArray>()
+                .unwrap()
+                .value(index),
+        ))),
+        DataType::LargeBinary => Ok(Some(sanitize_text_bytes(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::LargeBinaryArray>()
+                .unwrap()
+                .value(index),
+        ))),
+        _ => Err(Error::invalid_input(format!(
+            "Fm does not support data type: {:?}",
+            array.data_type()
+        ))),
+    }
+}
+
+#[cfg(test)]
+fn extract_text_bytes(array: &dyn arrow_array::Array, index: usize) -> Result<Option<Vec<u8>>> {
+    if array.is_null(index) {
+        return Ok(None);
+    }
+    match array.data_type() {
+        DataType::Utf8 => Ok(Some(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::StringArray>()
+                .unwrap()
+                .value(index)
+                .as_bytes()
+                .to_vec(),
+        )),
+        DataType::LargeUtf8 => Ok(Some(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::LargeStringArray>()
+                .unwrap()
+                .value(index)
+                .as_bytes()
+                .to_vec(),
+        )),
+        DataType::Binary => Ok(Some(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::BinaryArray>()
+                .unwrap()
+                .value(index)
+                .to_vec(),
+        )),
+        DataType::LargeBinary => Ok(Some(
+            array
+                .as_any()
+                .downcast_ref::<arrow_array::LargeBinaryArray>()
+                .unwrap()
+                .value(index)
+                .to_vec(),
+        )),
+        _ => Err(Error::invalid_input(format!(
+            "Fm does not support data type: {:?}",
+            array.data_type()
+        ))),
+    }
+}
+
+fn hex_encode(data: &[u8]) -> String {
+    data.iter().map(|b| format!("{b:02x}")).collect()
+}
+fn hex_decode(s: &str) -> Result<Vec<u8>> {
+    if !s.len().is_multiple_of(2) {
+        return Err(Error::invalid_input("invalid hex length"));
+    }
+    (0..s.len())
+        .step_by(2)
+        .map(|i| {
+            u8::from_str_radix(&s[i..i + 2], 16)
+                .map_err(|e| Error::invalid_input(format!("invalid hex: {e}")))
+        })
+        .collect()
+}
+
+/// Write an FM-Index partition to storage.
+///
+/// Layout:
+///   - Wavelet block rows (BWT nodes)
+///   - SA sample blocks (packed u64 in LargeBinary)
+///   - Metadata: c_table, huffman_codes, tree_topology, row_ids, doc_start_positions
+async fn write_fmindex(
+    fm: &FMIndex,
+    store: &dyn IndexStore,
+    filename: &str,
+    partition_fingerprint: Option<&str>,
+) -> Result<IndexFile> {
+    let schema = Arc::new(FMIndex::block_schema());
+
+    let mut writer = store.new_index_file(filename, schema.clone()).await?;
+
+    // 1. Wavelet blocks
+    let wb = fm.build_wavelet_batch()?;
+    let nw = wb.num_rows();
+    writer.write_record_batch(wb).await?;
+
+    // 2. SA samples packed as binary blocks
+    let u64s_per_block = BLOCK_WORDS; // 4096 u64s per block = 32KB
+    let mut sa_nid = Vec::new();
+    let mut sa_bid = Vec::new();
+    let mut sa_words: Vec<Vec<u8>> = Vec::new();
+    let mut sa_pr = Vec::new();
+    let mut sa_bl = Vec::new();
+    for (bi, chunk) in fm.sa_samples.chunks(u64s_per_block).enumerate() {
+        sa_nid.push(u32::MAX);
+        sa_bid.push(bi as u32);
+        sa_words.push(FMIndex::u64_to_bytes(chunk));
+        sa_pr.push(0u64);
+        sa_bl.push(fm.sa_samples.len() as u64);
+    }
+    let num_sa_blocks = sa_nid.len();
+    if num_sa_blocks > 0 {
+        let refs: Vec<&[u8]> = sa_words.iter().map(|v| v.as_slice()).collect();
+        writer
+            .write_record_batch(RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(arrow_array::UInt32Array::from(sa_nid)),
+                    Arc::new(arrow_array::UInt32Array::from(sa_bid)),
+                    Arc::new(arrow_array::LargeBinaryArray::from(refs)),
+                    Arc::new(arrow_array::UInt64Array::from(sa_pr)),
+                    Arc::new(arrow_array::UInt64Array::from(sa_bl)),
+                ],
+            )?)
+            .await?;
+    }
+
+    // Metadata
+    let mut metadata = HashMap::new();
+    metadata.insert("num_bwt_nodes".into(), fm.wavelet.nodes.len().to_string());
+    metadata.insert("bwt_len".into(), fm.wavelet.len.to_string());
+    metadata.insert("num_sa_blocks".into(), num_sa_blocks.to_string());
+    metadata.insert("sa_samples_len".into(), fm.sa_samples.len().to_string());
+    metadata.insert("total_wavelet_rows".into(), nw.to_string());
+    metadata.insert("sa_sample_rate".into(), SA_SAMPLE_RATE.to_string());
+    metadata.insert("alphabet_size".into(), fm.alphabet_size.to_string());
+    if let Some(fingerprint) = partition_fingerprint {
+        metadata.insert(
+            FMINDEX_PARTITION_FINGERPRINT_KEY.into(),
+            fingerprint.to_string(),
+        );
+    }
+    metadata.insert("c_table".into(), hex_encode(&fm.serialize_c_table()));
+    metadata.insert(
+        "huffman_codes".into(),
+        hex_encode(&fm.serialize_huffman_codes()),
+    );
+    metadata.insert(
+        "tree_topology".into(),
+        hex_encode(&fm.serialize_tree_topology()),
+    );
+    // row_ids in metadata (10K × 8 = 80KB per partition — small)
+    let row_ids_bytes: Vec<u8> = fm.row_ids.iter().flat_map(|&v| v.to_le_bytes()).collect();
+    metadata.insert("row_ids".into(), hex_encode(&row_ids_bytes));
+    // doc_start_positions in metadata (10K × 8 = 80KB per partition — small)
+    let doc_starts_bytes: Vec<u8> = fm
+        .doc_start_positions
+        .iter()
+        .flat_map(|&v| v.to_le_bytes())
+        .collect();
+    metadata.insert("doc_start_positions".into(), hex_encode(&doc_starts_bytes));
+
+    writer.finish_with_metadata(metadata).await
+}
+
+#[cfg(test)]
+async fn write_partitioned_fmindex(
+    texts: &[(u64, Vec<u8>)],
+    store: &dyn IndexStore,
+) -> Result<Vec<IndexFile>> {
+    if texts.is_empty() {
+        return Ok(vec![write_empty_fmindex_partition(store).await?]);
+    }
+    let mut files = Vec::new();
+    for (pid, chunk) in texts.chunks(PARTITION_SIZE).enumerate() {
+        files.push(write_fmindex_partition(chunk, store, pid as u64).await?);
+    }
+    Ok(files)
+}
+
+#[cfg(test)]
+async fn write_fmindex_partition(
+    texts: &[(u64, Vec<u8>)],
+    store: &dyn IndexStore,
+    partition_id: u64,
+) -> Result<IndexFile> {
+    let fingerprint = fmindex_partition_fingerprint(texts);
+    let refs: Vec<(u64, &[u8])> = texts.iter().map(|(id, t)| (*id, t.as_slice())).collect();
+    let fm = FMIndex::build(&refs)?;
+    write_fmindex(
+        &fm,
+        store,
+        &fmindex_partition_path(partition_id),
+        Some(&fingerprint),
+    )
+    .await
+}
+
+async fn write_fmindex_partition_owned(
+    texts: Vec<(u64, Vec<u8>)>,
+    store: Arc<dyn IndexStore>,
+    partition_id: u64,
+) -> Result<(u64, IndexFile)> {
+    let fingerprint = fmindex_partition_fingerprint(&texts);
+    let fm = spawn_cpu(move || {
+        let refs: Vec<(u64, &[u8])> = texts.iter().map(|(id, t)| (*id, t.as_slice())).collect();
+        FMIndex::build(&refs)
+    })
+    .await?;
+    let file = write_fmindex(
+        &fm,
+        store.as_ref(),
+        &fmindex_partition_path(partition_id),
+        Some(&fingerprint),
+    )
+    .await?;
+    Ok((partition_id, file))
+}
+
+async fn write_empty_fmindex_partition(store: &dyn IndexStore) -> Result<IndexFile> {
+    let fm = FMIndex::build(&[])?;
+    let fingerprint = fmindex_partition_fingerprint(&[]);
+    write_fmindex(&fm, store, &fmindex_partition_path(0), Some(&fingerprint)).await
+}
+
+// ── Plugin ───────────────────────────────────────────────────────────────────
+
+#[derive(Debug, Default)]
+pub struct FMIndexPlugin;
+
+#[async_trait]
+impl BasicTrainer for FMIndexPlugin {
+    fn new_training_request(
+        &self,
+        _params: &str,
+        field: &Field,
+    ) -> Result<Box<dyn TrainingRequest>> {
+        match field.data_type() {
+            DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary => {}
+            _ => {
+                return Err(Error::invalid_input(format!(
+                    "FM-Index does not support {:?}",
+                    field.data_type()
+                )));
+            }
+        }
+        Ok(Box::new(DefaultTrainingRequest::new(
+            TrainingCriteria::new(TrainingOrdering::None).with_row_addr(),
+        )))
+    }
+    async fn train_index(
+        &self,
+        data: SendableRecordBatchStream,
+        store: &dyn IndexStore,
+        _req: Box<dyn TrainingRequest>,
+        _fids: Option<Vec<u32>>,
+        _progress: Arc<dyn crate::progress::IndexBuildProgress>,
+    ) -> Result<CreatedIndex> {
+        let files = write_partitioned_fmindex_stream(data, store).await?;
+        Ok(CreatedIndex {
+            index_details: prost_types::Any::from_msg(&pb::FmIndexDetails {}).unwrap(),
+            index_version: FMINDEX_INDEX_VERSION,
+            files,
+        })
+    }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for FMIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "Fm"
+    }
+    fn provides_exact_answer(&self) -> bool {
+        true
+    }
+    fn version(&self) -> u32 {
+        FMINDEX_INDEX_VERSION
+    }
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        Some(Box::new(TextQueryParser::new(
+            index_name,
+            self.name().to_string(),
+            // needs_recheck: the FM-index returns exact substring matches.
+            false,
+            // supports_regex: regex acceleration is only implemented for ngram.
+            false,
+        )))
+    }
+    async fn load_index(
+        &self,
+        store: Arc<dyn IndexStore>,
+        details: &prost_types::Any,
+        fri: Option<Arc<dyn RowIdRemapper>>,
+        cache: &LanceCache,
+    ) -> Result<Arc<dyn ScalarIndex>> {
+        let _ = details.to_msg::<pb::FmIndexDetails>().unwrap_or_default();
+        Ok(FMIndexScalarIndex::load(store, fri, cache).await? as Arc<dyn ScalarIndex>)
+    }
+    async fn load_statistics(
+        &self,
+        _: Arc<dyn IndexStore>,
+        _: &prost_types::Any,
+    ) -> Result<Option<serde_json::Value>> {
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::{BinaryArray, LargeBinaryArray, LargeStringArray, StringArray, UInt64Array};
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+    use futures::stream;
+    use lance_core::{ROW_ADDR, cache::LanceCache};
+    use lance_io::object_store::ObjectStore;
+    use object_store::path::Path;
+    use std::sync::Arc;
+
+    use crate::scalar::lance_format::LanceIndexStore;
+    use crate::scalar::registry::BasicTrainer;
+
+    #[derive(Debug, Clone)]
+    struct FailNewFileStore {
+        inner: Arc<dyn IndexStore>,
+    }
+
+    impl DeepSizeOf for FailNewFileStore {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
+            0
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl IndexStore for FailNewFileStore {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(self.clone())
+        }
+
+        fn io_parallelism(&self) -> usize {
+            self.inner.io_parallelism()
+        }
+
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.with_io_priority(io_priority),
+            })
+        }
+
+        async fn new_index_file(
+            &self,
+            _name: &str,
+            _schema: Arc<arrow_schema::Schema>,
+        ) -> Result<Box<dyn crate::scalar::IndexWriter>> {
+            Err(Error::execution(
+                "injected FMIndex write failure".to_string(),
+            ))
+        }
+
+        async fn open_index_file(&self, name: &str) -> Result<Arc<dyn crate::scalar::IndexReader>> {
+            self.inner.open_index_file(name).await
+        }
+
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            self.inner.copy_index_file(name, dest_store).await
+        }
+
+        async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile> {
+            self.inner.rename_index_file(name, new_name).await
+        }
+
+        async fn delete_index_file(&self, name: &str) -> Result<()> {
+            self.inner.delete_index_file(name).await
+        }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+            self.inner.list_files_with_sizes().await
+        }
+    }
+
+    fn loaded_wavelet_blocks(index: &FMIndexScalarIndex) -> usize {
+        index
+            .partitions
+            .iter()
+            .flat_map(|partition| partition.fm.wavelet.nodes.iter())
+            .map(|node| {
+                node.blocks
+                    .iter()
+                    .filter(|block| block.get().is_some())
+                    .count()
+            })
+            .sum()
+    }
+
+    fn total_wavelet_blocks(index: &FMIndexScalarIndex) -> usize {
+        index
+            .partitions
+            .iter()
+            .flat_map(|partition| partition.fm.wavelet.nodes.iter())
+            .map(|node| node.blocks.len())
+            .sum()
+    }
+
+    #[test]
+    fn test_fmindex_build_and_search() {
+        let texts: Vec<(u64, &[u8])> = vec![
+            (0, b"hello world"),
+            (1, b"hello rust"),
+            (2, b"goodbye world"),
+        ];
+        let fm = FMIndex::build(&texts).unwrap();
+
+        let r = fm.search(b"hello");
+        assert!(r.contains(0));
+        assert!(r.contains(1));
+        assert!(!r.contains(2));
+
+        let r = fm.search(b"world");
+        assert!(r.contains(0));
+        assert!(!r.contains(1));
+        assert!(r.contains(2));
+
+        let r = fm.search(b"goodbye");
+        assert!(!r.contains(0));
+        assert!(!r.contains(1));
+        assert!(r.contains(2));
+
+        assert!(fm.search(b"xyz").is_empty());
+    }
+
+    #[test]
+    fn test_fmindex_empty() {
+        let fm = FMIndex::build(&[]).unwrap();
+        assert!(fm.search(b"anything").is_empty());
+    }
+
+    #[test]
+    fn test_fmindex_single_char_search() {
+        let texts: Vec<(u64, &[u8])> = vec![(0, b"abc"), (1, b"def")];
+        let fm = FMIndex::build(&texts).unwrap();
+        assert!(fm.search(b"a").contains(0));
+        assert!(!fm.search(b"a").contains(1));
+        assert!(!fm.search(b"d").contains(0));
+        assert!(fm.search(b"d").contains(1));
+    }
+
+    #[test]
+    fn test_fmindex_repeated_pattern() {
+        let texts: Vec<(u64, &[u8])> = vec![(0, b"ababab"), (1, b"cdcd")];
+        let fm = FMIndex::build(&texts).unwrap();
+        assert!(fm.search(b"ab").contains(0));
+        assert!(!fm.search(b"ab").contains(1));
+        assert!(!fm.search(b"cd").contains(0));
+        assert!(fm.search(b"cd").contains(1));
+    }
+
+    #[test]
+    fn test_early_exit_all_docs_match() {
+        let texts: Vec<(u64, &[u8])> = vec![(0, b"the cat"), (1, b"the dog"), (2, b"the bird")];
+        let fm = FMIndex::build(&texts).unwrap();
+        assert_eq!(fm.search(b"the").len(), 3);
+    }
+
+    #[test]
+    fn test_locate_correctness() {
+        let texts: Vec<(u64, &[u8])> = vec![
+            (0, b"the quick brown fox jumps over the lazy dog"),
+            (1, b"pack my box with five dozen liquor jugs"),
+            (2, b"how vexingly quick daft zebras jump"),
+        ];
+        let fm = FMIndex::build(&texts).unwrap();
+
+        let r = fm.search(b"quick");
+        assert!(r.contains(0));
+        assert!(!r.contains(1));
+        assert!(r.contains(2));
+
+        let r = fm.search(b"the");
+        assert!(r.contains(0));
+        assert!(!r.contains(1));
+        assert!(!r.contains(2));
+
+        let r = fm.search(b"jump");
+        assert!(r.contains(0));
+        assert!(r.contains(2));
+    }
+
+    #[test]
+    fn test_many_documents() {
+        let docs: Vec<Vec<u8>> = (0..100)
+            .map(|i| format!("document number {} with hello world data xyz", i).into_bytes())
+            .collect();
+        let texts: Vec<(u64, &[u8])> = docs
+            .iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d.as_slice()))
+            .collect();
+        let fm = FMIndex::build(&texts).unwrap();
+
+        assert_eq!(fm.search(b"hello world").len(), 100);
+        assert_eq!(fm.search(b"document number 42").len(), 1);
+        assert_eq!(fm.search(b"nonexistent").len(), 0);
+    }
+
+    #[test]
+    fn test_index_size_ratio() {
+        let docs: Vec<Vec<u8>> = (0..200)
+            .map(|i| {
+                format!(
+                    "document {} with enough text to test size ratio properly end",
+                    i
+                )
+                .into_bytes()
+            })
+            .collect();
+        let texts: Vec<(u64, &[u8])> = docs
+            .iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d.as_slice()))
+            .collect();
+        let fm = FMIndex::build(&texts).unwrap();
+
+        let text_size: usize = docs.iter().map(|d| d.len()).sum();
+        let wavelet_size = fm.wavelet.deep_size();
+        let sa_size = fm.sa_samples.len() * 8;
+        let total = wavelet_size + sa_size;
+
+        let ratio = total as f64 / text_size as f64;
+        assert!(
+            ratio < 1.5,
+            "index should be much smaller than text, got ratio={ratio:.2}"
+        );
+    }
+
+    #[test]
+    fn test_wavelet_access_consistency() {
+        let docs: Vec<Vec<u8>> = (0..50)
+            .map(|i| format!("document {i} hello world test").into_bytes())
+            .collect();
+        let texts: Vec<(u64, &[u8])> = docs
+            .iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d.as_slice()))
+            .collect();
+
+        let mut concat = Vec::new();
+        for (_, text) in &texts {
+            concat.extend_from_slice(text);
+            concat.push(SENTINEL_BYTE);
+        }
+        concat.push(0x00);
+        let sa = build_suffix_array(&concat);
+        let n = concat.len();
+        let bwt: Vec<u8> = sa
+            .iter()
+            .map(|&pos| {
+                if pos == 0 {
+                    concat[n - 1]
+                } else {
+                    concat[pos - 1]
+                }
+            })
+            .collect();
+        let wavelet = HuffmanWaveletTree::build(&bwt);
+
+        for (i, &expected) in bwt.iter().enumerate().take(n.min(500)) {
+            assert_eq!(wavelet.access(i), expected, "access mismatch at {i}");
+        }
+    }
+
+    #[test]
+    fn test_serialization_roundtrip() {
+        let texts: Vec<(u64, &[u8])> = vec![
+            (10, b"alpha beta gamma"),
+            (20, b"beta gamma delta"),
+            (30, b"gamma delta epsilon"),
+        ];
+        let fm = FMIndex::build(&texts).unwrap();
+
+        // Test huffman codes roundtrip
+        let hc_bytes = fm.serialize_huffman_codes();
+        let hc = FMIndex::deserialize_huffman_codes(&hc_bytes);
+        for (i, (loaded, original)) in hc.iter().zip(fm.wavelet.codes.iter()).enumerate() {
+            assert_eq!(loaded.bits, original.bits, "bits mismatch at {i}");
+            assert_eq!(loaded.length, original.length, "length mismatch at {i}");
+            assert_eq!(loaded.node_path, original.node_path, "path mismatch at {i}");
+        }
+
+        // Test tree topology roundtrip
+        let topo_bytes = fm.serialize_tree_topology();
+        let topo = FMIndex::deserialize_tree_topology(&topo_bytes);
+        assert_eq!(topo.len(), fm.wavelet.children.len());
+
+        // Test c_table roundtrip
+        let ct_bytes = fm.serialize_c_table();
+        let ct = FMIndex::deserialize_c_table(&ct_bytes);
+        assert_eq!(ct, fm.c_table);
+    }
+
+    #[test]
+    fn test_hex_roundtrip() {
+        let data = vec![0u8, 1, 127, 255, 42];
+        let encoded = hex_encode(&data);
+        let decoded = hex_decode(&encoded).unwrap();
+        assert_eq!(data, decoded);
+    }
+
+    #[test]
+    fn test_partition_limit_reached_by_rows_or_bytes() {
+        assert!(fmindex_partition_limit_reached(10, 5, 10, 100));
+        assert!(fmindex_partition_limit_reached(3, 100, 10, 100));
+        assert!(!fmindex_partition_limit_reached(3, 99, 10, 100));
+    }
+
+    #[test]
+    fn test_sentinel_sanitization() {
+        // Text containing \xFF should be sanitized to space during training.
+        let texts: Vec<(u64, &[u8])> = vec![(0, b"hello\xFFworld")];
+        let fm = FMIndex::build(&texts).unwrap();
+        // Build itself does not sanitize, but search should still work.
+        let r = fm.search(b"hello");
+        assert!(r.contains(0));
+    }
+
+    #[test]
+    fn test_wavelet_rank_pair_consistency() {
+        let docs: Vec<Vec<u8>> = (0..30)
+            .map(|i| format!("doc {i} with repeated words hello world test data").into_bytes())
+            .collect();
+        let texts: Vec<(u64, &[u8])> = docs
+            .iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d.as_slice()))
+            .collect();
+        let fm = FMIndex::build(&texts).unwrap();
+
+        let n = fm.wavelet.len;
+        for b in [b'a', b'e', b' ', SENTINEL_BYTE] {
+            for &(lo, hi) in &[(0usize, 1usize), (0, n), (n / 4, n / 2)] {
+                if lo >= n || hi > n || lo >= hi {
+                    continue;
+                }
+                let (pl, ph) = fm.wavelet.rank_pair(b, lo, hi);
+                let rl = fm.wavelet.rank(b, lo);
+                let rh = fm.wavelet.rank(b, hi);
+                assert_eq!(pl, rl, "rank_pair lo mismatch for b={b} [{lo},{hi})");
+                assert_eq!(ph, rh, "rank_pair hi mismatch for b={b} [{lo},{hi})");
+            }
+        }
+    }
+
+    #[test]
+    fn test_large_sa_sampling() {
+        // Test with enough documents to have multiple SA sample points
+        let docs: Vec<Vec<u8>> = (0..50)
+            .map(|i| {
+                format!(
+                    "document number {} with lots of text to ensure we have enough bytes for multiple SA samples across the suffix array positions",
+                    i
+                )
+                .into_bytes()
+            })
+            .collect();
+        let texts: Vec<(u64, &[u8])> = docs
+            .iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d.as_slice()))
+            .collect();
+        let fm = FMIndex::build(&texts).unwrap();
+
+        assert!(fm.sa_samples.len() > 1, "should have multiple SA samples");
+        assert_eq!(fm.search(b"document number 25").len(), 1);
+        assert_eq!(fm.search(b"document number").len(), 50);
+        assert_eq!(fm.search(b"nonexistent pattern").len(), 0);
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_write_and_load_roundtrip() {
+        let texts: Vec<(u64, &[u8])> = vec![
+            (0, b"hello world foo bar"),
+            (1, b"hello rust baz qux"),
+            (2, b"goodbye world quux"),
+        ];
+        let fm = FMIndex::build(&texts).unwrap();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Write
+        write_fmindex(&fm, store.as_ref(), &fmindex_partition_path(0), None)
+            .await
+            .unwrap();
+
+        // Load
+        let part =
+            FMIndexScalarIndex::load_partition(store.as_ref(), &fmindex_partition_path(0), 0)
+                .await
+                .unwrap();
+
+        // Verify search results match
+        let r = part.fm.search(b"hello");
+        assert!(r.contains(0));
+        assert!(r.contains(1));
+        assert!(!r.contains(2));
+
+        let r = part.fm.search(b"world");
+        assert!(r.contains(0));
+        assert!(!r.contains(1));
+        assert!(r.contains(2));
+
+        assert!(part.fm.search(b"xyz").is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_partitioned_write_and_load() {
+        let docs: Vec<Vec<u8>> = (0..30)
+            .map(|i| format!("document {i} hello world test data").into_bytes())
+            .collect();
+        let texts: Vec<(u64, Vec<u8>)> = docs
+            .into_iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d))
+            .collect();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_partitioned_fmindex(&texts, store.as_ref())
+            .await
+            .unwrap();
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // Search across partitions
+        let r = index
+            .search(
+                &TextQuery::StringContains("hello world".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(30));
+            }
+            _ => panic!("expected exact result"),
+        }
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("document 15".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(1));
+            }
+            _ => panic!("expected exact result"),
+        }
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("nonexistent".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(0));
+            }
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_public_prewarm_loads_lazy_blocks() {
+        let docs: Vec<Vec<u8>> = (0..30)
+            .map(|i| format!("document {i} hello world test data").into_bytes())
+            .collect();
+        let texts: Vec<(u64, Vec<u8>)> = docs
+            .into_iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d))
+            .collect();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_partitioned_fmindex(&texts, store.as_ref())
+            .await
+            .unwrap();
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let total_blocks = total_wavelet_blocks(index.as_ref());
+        assert!(total_blocks > 0);
+        assert_eq!(loaded_wavelet_blocks(index.as_ref()), 0);
+
+        index.prewarm().await.unwrap();
+        assert_eq!(loaded_wavelet_blocks(index.as_ref()), total_blocks);
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("hello world".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(30));
+            }
+            _ => panic!("expected exact result"),
+        }
+        assert_eq!(loaded_wavelet_blocks(index.as_ref()), total_blocks);
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_lazy_rank_terminal_block_boundary() {
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let schema = Arc::new(arrow_schema::Schema::new(vec![Field::new(
+            "words",
+            DataType::LargeBinary,
+            false,
+        )]));
+        let words = vec![u64::MAX; BLOCK_WORDS];
+        let bytes = FMIndex::u64_to_bytes(&words);
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(LargeBinaryArray::from(vec![bytes.as_slice()]))],
+        )
+        .unwrap();
+        let mut writer = store.new_index_file("rank.lance", schema).await.unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+
+        let reader = store.open_index_file("rank.lance").await.unwrap();
+        let bitvec = LazyRankBitVec::new(vec![0], 1, reader, 0, BLOCK_BITS);
+        bitvec.load_block_for_rank(BLOCK_BITS).await.unwrap();
+
+        assert_eq!(bitvec.rank1(BLOCK_BITS), BLOCK_BITS);
+        assert_eq!(bitvec.rank0(BLOCK_BITS), 0);
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_contains_search_does_not_full_prewarm_partitions() {
+        let docs: Vec<Vec<u8>> = (0..30)
+            .map(|i| format!("document {i} hello world test data").into_bytes())
+            .collect();
+        let texts: Vec<(u64, Vec<u8>)> = docs
+            .into_iter()
+            .enumerate()
+            .map(|(i, d)| (i as u64, d))
+            .collect();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_partitioned_fmindex(&texts, store.as_ref())
+            .await
+            .unwrap();
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        assert_eq!(loaded_wavelet_blocks(index.as_ref()), 0);
+        assert!(
+            index
+                .partitions
+                .iter()
+                .all(|partition| !partition.fm.fully_prewarmed.load(Ordering::Acquire))
+        );
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("document 15".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(1));
+            }
+            _ => panic!("expected exact result"),
+        }
+
+        assert!(
+            index
+                .partitions
+                .iter()
+                .all(|partition| !partition.fm.fully_prewarmed.load(Ordering::Acquire))
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_plugin_train_and_load() {
+        let docs = vec!["hello world", "hello rust", "goodbye world"];
+        let row_addrs: Vec<u64> = vec![0, 1, 2];
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(docs)),
+                Arc::new(UInt64Array::from(row_addrs)),
+            ],
+        )
+        .unwrap();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let req = FMIndexPlugin
+            .new_training_request("", &arrow_schema::Field::new("val", DataType::Utf8, false))
+            .unwrap();
+        let created = FMIndexPlugin
+            .train_index(
+                Box::pin(stream),
+                store.as_ref(),
+                req,
+                None,
+                Arc::new(crate::progress::NoopIndexBuildProgress),
+            )
+            .await
+            .unwrap();
+
+        let index = FMIndexPlugin
+            .load_index(store, &created.index_details, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("hello".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(2));
+            }
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_stream_train_splits_partition_by_bytes() {
+        let docs = vec!["abcd", "efgh", "ijkl"];
+        let row_addrs: Vec<u64> = vec![0, 1, 2];
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(docs)),
+                Arc::new(UInt64Array::from(row_addrs)),
+            ],
+        )
+        .unwrap();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
+        let files = write_partitioned_fmindex_stream_with_config(
+            Box::pin(stream),
+            store.as_ref(),
+            FMIndexPartitionConfig {
+                num_workers: 2,
+                max_rows: 100,
+                max_bytes: 5,
+                queue_size: 1,
+                resume_existing: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(files.len(), 3);
+        assert_eq!(files[0].path, fmindex_partition_path(0));
+        assert_eq!(files[1].path, fmindex_partition_path(1));
+        assert_eq!(files[2].path, fmindex_partition_path(2));
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let r = index
+            .search(
+                &TextQuery::StringContains("efgh".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(1));
+            }
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_stream_train_resumes_existing_partitions() {
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let first_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["first partition"])),
+                Arc::new(UInt64Array::from(vec![0])),
+            ],
+        )
+        .unwrap();
+        let first_stream =
+            RecordBatchStreamAdapter::new(schema.clone(), stream::iter(vec![Ok(first_batch)]));
+        let first_files = write_partitioned_fmindex_stream_with_config(
+            Box::pin(first_stream),
+            store.as_ref(),
+            FMIndexPartitionConfig {
+                num_workers: 1,
+                max_rows: 1,
+                max_bytes: 1024,
+                queue_size: 1,
+                resume_existing: false,
+            },
+        )
+        .await
+        .unwrap();
+        assert_eq!(first_files.len(), 1);
+        assert_eq!(first_files[0].path, fmindex_partition_path(0));
+
+        let resumed_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![
+                    "first partition",
+                    "second partition",
+                ])),
+                Arc::new(UInt64Array::from(vec![0, 1])),
+            ],
+        )
+        .unwrap();
+        let resumed_stream =
+            RecordBatchStreamAdapter::new(schema.clone(), stream::iter(vec![Ok(resumed_batch)]));
+        let resumed_files = write_partitioned_fmindex_stream_with_config(
+            Box::pin(resumed_stream),
+            store.as_ref(),
+            FMIndexPartitionConfig {
+                num_workers: 1,
+                max_rows: 1,
+                max_bytes: 1024,
+                queue_size: 1,
+                resume_existing: true,
+            },
+        )
+        .await
+        .unwrap();
+
+        let resumed_paths = resumed_files
+            .iter()
+            .map(|file| file.path.clone())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            resumed_paths,
+            vec![fmindex_partition_path(0), fmindex_partition_path(1)]
+        );
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let r = index
+            .search(
+                &TextQuery::StringContains("partition".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(2));
+            }
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_stream_train_rebuilds_stale_resume_partitions() {
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let stale_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["stale zero", "stale one"])),
+                Arc::new(UInt64Array::from(vec![0, 1])),
+            ],
+        )
+        .unwrap();
+        let stale_stream =
+            RecordBatchStreamAdapter::new(schema.clone(), stream::iter(vec![Ok(stale_batch)]));
+        write_partitioned_fmindex_stream_with_config(
+            Box::pin(stale_stream),
+            store.as_ref(),
+            FMIndexPartitionConfig {
+                num_workers: 1,
+                max_rows: 1,
+                max_bytes: 1024,
+                queue_size: 1,
+                resume_existing: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let fresh_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["fresh zero"])),
+                Arc::new(UInt64Array::from(vec![0])),
+            ],
+        )
+        .unwrap();
+        let fresh_stream =
+            RecordBatchStreamAdapter::new(schema.clone(), stream::iter(vec![Ok(fresh_batch)]));
+        let fresh_files = write_partitioned_fmindex_stream_with_config(
+            Box::pin(fresh_stream),
+            store.as_ref(),
+            FMIndexPartitionConfig {
+                num_workers: 1,
+                max_rows: 10,
+                max_bytes: 1024,
+                queue_size: 1,
+                resume_existing: true,
+            },
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(fresh_files.len(), 1);
+        assert_eq!(fresh_files[0].path, fmindex_partition_path(0));
+        let remaining_paths = store
+            .list_files_with_sizes()
+            .await
+            .unwrap()
+            .iter()
+            .map(|file| file.path.clone())
+            .collect::<Vec<_>>();
+        assert_eq!(remaining_paths, vec![fmindex_partition_path(0)]);
+
+        let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let r = index
+            .search(
+                &TextQuery::StringContains("fresh zero".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => assert_eq!(set.len(), Some(1)),
+            _ => panic!("expected exact result"),
+        }
+
+        let r = index
+            .search(
+                &TextQuery::StringContains("stale one".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => assert_eq!(set.len(), Some(0)),
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_stream_train_propagates_worker_write_error() {
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec!["hello"])),
+                Arc::new(UInt64Array::from(vec![0])),
+            ],
+        )
+        .unwrap();
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let inner = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        )) as Arc<dyn IndexStore>;
+        let store = FailNewFileStore { inner };
+
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(vec![Ok(batch)]));
+        let err = write_partitioned_fmindex_stream_with_config(
+            Box::pin(stream),
+            &store,
+            FMIndexPartitionConfig {
+                num_workers: 1,
+                max_rows: 1,
+                max_bytes: 1024,
+                queue_size: 1,
+                resume_existing: false,
+            },
+        )
+        .await
+        .unwrap_err();
+
+        assert!(format!("{err}").contains("injected FMIndex write failure"));
+    }
+
+    #[tokio::test]
+    async fn test_fail_new_file_store_with_io_priority_preserves_failure() {
+        // `with_io_priority` re-wraps in `FailNewFileStore`, so the reprioritized
+        // store must keep injecting the `new_index_file` failure.
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let inner = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        )) as Arc<dyn IndexStore>;
+        let store = FailNewFileStore { inner };
+
+        let reprioritized = store.with_io_priority(7);
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "x",
+            DataType::UInt64,
+            false,
+        )]));
+        let err = reprioritized
+            .new_index_file("test", schema)
+            .await
+            .err()
+            .expect("new_index_file should fail");
+        assert!(format!("{err}").contains("injected FMIndex write failure"));
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_plugin_train_streams_multiple_partitions() {
+        fn training_batch(
+            schema: Arc<arrow_schema::Schema>,
+            start: usize,
+            len: usize,
+        ) -> RecordBatch {
+            let docs = vec!["x"; len];
+            let row_addrs: Vec<u64> = (start..start + len).map(|i| i as u64).collect();
+            RecordBatch::try_new(
+                schema,
+                vec![
+                    Arc::new(StringArray::from(docs)),
+                    Arc::new(UInt64Array::from(row_addrs)),
+                ],
+            )
+            .unwrap()
+        }
+
+        let total_rows = PARTITION_SIZE + 5;
+        let first_batch_rows = PARTITION_SIZE - 3;
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            arrow_schema::Field::new(
+                crate::scalar::registry::VALUE_COLUMN_NAME,
+                DataType::Utf8,
+                false,
+            ),
+            arrow_schema::Field::new(ROW_ADDR, DataType::UInt64, false),
+        ]));
+        let batches = vec![
+            Ok(training_batch(schema.clone(), 0, first_batch_rows)),
+            Ok(training_batch(
+                schema.clone(),
+                first_batch_rows,
+                total_rows - first_batch_rows,
+            )),
+        ];
+
+        let tempdir = tempfile::tempdir().unwrap();
+        let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            index_dir,
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let stream = RecordBatchStreamAdapter::new(schema, stream::iter(batches));
+        let req = FMIndexPlugin
+            .new_training_request("", &arrow_schema::Field::new("val", DataType::Utf8, false))
+            .unwrap();
+        let created = FMIndexPlugin
+            .train_index(
+                Box::pin(stream),
+                store.as_ref(),
+                req,
+                None,
+                Arc::new(crate::progress::NoopIndexBuildProgress),
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(created.files.len(), 2);
+        assert_eq!(created.files[0].path, fmindex_partition_path(0));
+        assert_eq!(created.files[1].path, fmindex_partition_path(1));
+
+        let index = FMIndexPlugin
+            .load_index(store, &created.index_details, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let r = index
+            .search(
+                &TextQuery::StringContains("x".to_string()),
+                &crate::metrics::NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        match r {
+            SearchResult::Exact(set) => {
+                assert_eq!(set.len(), Some(total_rows as u64));
+            }
+            _ => panic!("expected exact result"),
+        }
+    }
+
+    #[test]
+    fn test_build_wavelet_batch() {
+        let texts: Vec<(u64, &[u8])> = vec![(0, b"hello world"), (1, b"test data")];
+        let fm = FMIndex::build(&texts).unwrap();
+        let batch = fm.build_wavelet_batch().unwrap();
+        assert!(batch.num_rows() > 0);
+        assert_eq!(batch.num_columns(), 5);
+    }
+
+    #[test]
+    fn test_extract_text_bytes_types() {
+        let utf8 = StringArray::from(vec!["hello"]);
+        assert_eq!(
+            extract_text_bytes(&utf8, 0).unwrap(),
+            Some(b"hello".to_vec())
+        );
+
+        let large_utf8 = LargeStringArray::from(vec!["world"]);
+        assert_eq!(
+            extract_text_bytes(&large_utf8, 0).unwrap(),
+            Some(b"world".to_vec())
+        );
+
+        let binary = BinaryArray::from(vec![b"bytes" as &[u8]]);
+        assert_eq!(
+            extract_text_bytes(&binary, 0).unwrap(),
+            Some(b"bytes".to_vec())
+        );
+        let binary_with_sentinels = BinaryArray::from(vec![b"a\xFFb\0c" as &[u8]]);
+        assert_eq!(
+            extract_sanitized_text_bytes(&binary_with_sentinels, 0).unwrap(),
+            Some(b"a b c".to_vec())
+        );
+
+        let large_binary = LargeBinaryArray::from(vec![b"large" as &[u8]]);
+        assert_eq!(
+            extract_text_bytes(&large_binary, 0).unwrap(),
+            Some(b"large".to_vec())
+        );
+
+        // Null handling
+        let nullable = StringArray::from(vec![None::<&str>]);
+        assert_eq!(extract_text_bytes(&nullable, 0).unwrap(), None);
+    }
+
+    #[test]
+    fn test_fmindex_statistics() {
+        let rt = tokio::runtime::Builder::new_current_thread()
+            .build()
+            .unwrap();
+        rt.block_on(async {
+            let docs: Vec<Vec<u8>> = (0..10).map(|i| format!("doc {i}").into_bytes()).collect();
+            let texts: Vec<(u64, Vec<u8>)> = docs
+                .into_iter()
+                .enumerate()
+                .map(|(i, d)| (i as u64, d))
+                .collect();
+
+            let tempdir = tempfile::tempdir().unwrap();
+            let index_dir = Path::from_filesystem_path(tempdir.path()).unwrap();
+            let store = Arc::new(LanceIndexStore::new(
+                Arc::new(ObjectStore::local()),
+                index_dir,
+                Arc::new(LanceCache::no_cache()),
+            ));
+
+            write_partitioned_fmindex(&texts, store.as_ref())
+                .await
+                .unwrap();
+            let index = FMIndexScalarIndex::load(store, None, &LanceCache::no_cache())
+                .await
+                .unwrap();
+
+            let stats = index.statistics().unwrap();
+            assert_eq!(stats["type"], "Fm");
+            assert_eq!(stats["total_docs"], 10);
+            assert!(stats["total_bwt_len"].as_u64().unwrap() > 0);
+        });
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs
index 6adf4457f05..926bfad6a69 100644
--- a/rust/lance-index/src/scalar/inverted.rs
+++ b/rust/lance-index/src/scalar/inverted.rs
@@ -4,9 +4,11 @@
 pub mod builder;
 mod cache_codec;
 mod encoding;
+mod impact;
 mod index;
 mod iter;
 pub mod json;
+mod lazy_docset;
 pub mod parser;
 pub mod query;
 mod scorer;
@@ -90,7 +92,7 @@ pub async fn build_global_bm25_scorer(
     let (mut total_tokens, mut num_docs, first_token_docs) =
         first_index.bm25_stats_for_terms(&terms).await?;
     let mut token_docs = HashMap::with_capacity(terms.len());
-    for (term, count) in terms.iter().cloned().zip(first_token_docs.into_iter()) {
+    for (term, count) in terms.iter().cloned().zip(first_token_docs) {
         token_docs.insert(term, count);
     }
 
@@ -99,7 +101,7 @@ pub async fn build_global_bm25_scorer(
             index.bm25_stats_for_terms(&terms).await?;
         total_tokens += segment_total_tokens;
         num_docs += segment_num_docs;
-        for (term, count) in terms.iter().zip(segment_token_docs.into_iter()) {
+        for (term, count) in terms.iter().zip(segment_token_docs) {
             *token_docs
                 .get_mut(term)
                 .expect("global scorer terms should already be initialized") += count;
@@ -113,12 +115,11 @@ use lance_core::Error;
 
 use crate::pbold;
 use crate::progress::IndexBuildProgress;
-use crate::{
-    frag_reuse::FragReuseIndex,
-    scalar::{
-        CreatedIndex, ScalarIndex,
-        expression::{FtsQueryParser, ScalarQueryParser},
-        registry::{ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest},
+use crate::scalar::{
+    CreatedIndex, RowIdRemapper, ScalarIndex,
+    expression::{FtsQueryParser, ScalarQueryParser},
+    registry::{
+        BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
     },
 };
 
@@ -146,15 +147,17 @@ impl InvertedIndexPlugin {
             }
         });
 
+        params.validate_format_version()?;
+        let format_version = params.resolved_format_version();
         let details = pbold::InvertedIndexDetails::try_from(&params)?;
         let mut inverted_index =
             InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask)
                 .with_progress(progress);
-        inverted_index.update(data, index_store, None).await?;
+        let files = inverted_index.update(data, index_store, None).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
-            index_version: current_fts_format_version().index_version(),
-            files: Some(index_store.list_files_with_sizes().await?),
+            index_version: format_version.index_version(),
+            files,
         })
     }
 
@@ -192,11 +195,7 @@ impl TrainingRequest for InvertedIndexTrainingRequest {
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for InvertedIndexPlugin {
-    fn name(&self) -> &str {
-        "Inverted"
-    }
-
+impl BasicTrainer for InvertedIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -214,37 +213,10 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
                 .into()))
         }
 
-        let params = serde_json::from_str::<InvertedIndexParams>(params)?;
+        let params = InvertedIndexParams::from_training_json(params)?;
         Ok(Box::new(InvertedIndexTrainingRequest::new(params)))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        false
-    }
-
-    fn version(&self) -> u32 {
-        max_supported_fts_format_version().index_version()
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        let Ok(index_details) = _index_details.to_msg::<pbold::InvertedIndexDetails>() else {
-            return None;
-        };
-
-        if Self::can_accelerate_queries(&index_details) {
-            Some(Box::new(FtsQueryParser::new(
-                index_name,
-                self.name().to_string(),
-            )))
-        } else {
-            None
-        }
-    }
-
     /// Train a new index
     ///
     /// The provided data must fulfill all the criteria returned by `training_criteria`.
@@ -279,6 +251,44 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
         )
         .await
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for InvertedIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "Inverted"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        false
+    }
+
+    fn version(&self) -> u32 {
+        max_supported_fts_format_version().index_version()
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        let Ok(index_details) = _index_details.to_msg::<pbold::InvertedIndexDetails>() else {
+            return None;
+        };
+
+        if Self::can_accelerate_queries(&index_details) {
+            Some(Box::new(FtsQueryParser::new(
+                index_name,
+                self.name().to_string(),
+            )))
+        } else {
+            None
+        }
+    }
 
     /// Load an index from storage
     ///
@@ -288,7 +298,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(
@@ -307,6 +317,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::scalar::{BuiltinIndexType, ScalarIndexParams};
 
     #[test]
     fn test_plugin_version_tracks_max_supported_format() {
@@ -316,4 +327,36 @@ mod tests {
             max_supported_fts_format_version().index_version()
         );
     }
+
+    #[test]
+    fn test_new_training_request_defaults_missing_block_size_to_128() {
+        let plugin = InvertedIndexPlugin;
+        let field = Field::new("text", DataType::Utf8, true);
+
+        let cases = [
+            (
+                ScalarIndexParams::for_builtin(BuiltinIndexType::Inverted),
+                false,
+            ),
+            (ScalarIndexParams::new("inverted".to_string()), false),
+            (
+                ScalarIndexParams::new("inverted".to_string())
+                    .with_params(&serde_json::json!({ "with_position": true })),
+                true,
+            ),
+        ];
+
+        for (params, expected_with_position) in cases {
+            let request = plugin
+                .new_training_request(params.params.as_deref().unwrap_or("{}"), &field)
+                .unwrap();
+            let request = request
+                .as_any()
+                .downcast_ref::<InvertedIndexTrainingRequest>()
+                .unwrap();
+
+            assert_eq!(request.parameters.posting_block_size(), DEFAULT_BLOCK_SIZE);
+            assert_eq!(request.parameters.has_positions(), expected_with_position);
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs
index b8b6b52c6af..41804e94fd1 100644
--- a/rust/lance-index/src/scalar/inverted/builder.rs
+++ b/rust/lance-index/src/scalar/inverted/builder.rs
@@ -1,49 +1,47 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use super::encoding::encode_group_starts;
 use super::{InvertedIndexParams, index::*};
 use crate::scalar::inverted::document_tokenizer::DocType;
 use crate::scalar::inverted::json::JsonTextStream;
+use crate::scalar::inverted::tokenizer::LEGACY_BLOCK_SIZE;
 use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
 #[cfg(test)]
 use crate::scalar::lance_format::LanceIndexStore;
-use crate::scalar::{IndexStore, OldIndexDataFilter};
+use crate::scalar::{IndexFile, IndexStore, OldIndexDataFilter};
 use crate::vector::graph::OrderedFloat;
 use crate::{progress::IndexBuildProgress, progress::noop_progress};
 use arrow::array::AsArray;
 use arrow::datatypes;
-use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array};
+use arrow_array::{Array, BinaryArray, RecordBatch};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
-use bitpacking::{BitPacker, BitPacker4x};
-use bytes::Bytes;
-use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream};
-use deepsize::DeepSizeOf;
+use datafusion::execution::SendableRecordBatchStream;
 use fst::Streamer;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::{StreamExt, TryStreamExt};
 use lance_arrow::json::JSON_EXT_NAME;
 use lance_arrow::{ARROW_EXT_NAME_KEY, iter_str_array};
+use lance_bitpacking::{BitPacker, BitPacker4x};
 use lance_core::cache::LanceCache;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::LanceOptionExt;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use lance_core::utils::tokio::{IO_CORE_RESERVATION, get_num_compute_intensive_cpus, spawn_cpu};
-use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result};
+use lance_core::{Error, ROW_ID, Result};
 use lance_io::object_store::ObjectStore;
 use lance_select::RowSetOps;
 use object_store::path::Path;
 use roaring::RoaringBitmap;
 use smallvec::SmallVec;
 use std::collections::HashMap;
-use std::pin::Pin;
 use std::str::FromStr;
 use std::sync::Arc;
 use std::sync::LazyLock;
-use std::task::{Context, Poll};
 use std::{fmt::Debug, sync::atomic::AtomicU64};
 use tracing::instrument;
 
-// the number of elements in each block
-// each block contains 128 row ids and 128 frequencies
-// WARNING: changing this value will break the compatibility with existing indexes
+// The legacy bitpacking block size. Position streams still use this block size;
+// FTS posting blocks choose their physical bitpacker from the configured
+// InvertedIndexParams::block_size.
 pub const BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
 
 // The default number of workers to use for FTS builds.
@@ -74,92 +72,8 @@ static LANCE_FTS_POSTING_BATCH_ROWS: LazyLock<usize> = LazyLock::new(|| {
         .parse()
         .expect("failed to parse LANCE_FTS_POSTING_BATCH_ROWS")
 });
-// Target serialized byte size of a posting-list cache group. Consecutive
-// posting lists are grouped into a single cache entry until their combined
-// serialized size reaches this target, amortizing per-entry overhead across
-// small (Zipfian-rare) terms. See issue #7040.
-static LANCE_FTS_POSTING_GROUP_TARGET_BYTES: LazyLock<usize> = LazyLock::new(|| {
-    std::env::var("LANCE_FTS_POSTING_GROUP_TARGET_BYTES")
-        .unwrap_or_else(|_| "4096".to_string())
-        .parse()
-        .expect("failed to parse LANCE_FTS_POSTING_GROUP_TARGET_BYTES")
-});
-// Maximum number of posting lists in a single cache group, regardless of byte
-// size. Caps the work and memory of a single group read for corpora with many
-// tiny terms.
-static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock<usize> = LazyLock::new(|| {
-    std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
-        .unwrap_or_else(|_| "256".to_string())
-        .parse()
-        .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
-});
 const MAX_RETAINED_TOKEN_IDS: usize = 8 * 1024;
 
-/// Write-time configuration controlling how consecutive posting lists are
-/// grouped into a single read-path cache entry (issue #7040). Defaults come
-/// from the `LANCE_FTS_POSTING_GROUP_*` environment variables.
-#[derive(Debug, Clone, Copy)]
-pub(crate) struct PostingGroupConfig {
-    pub(crate) target_bytes: usize,
-    pub(crate) max_tokens: usize,
-}
-
-impl Default for PostingGroupConfig {
-    fn default() -> Self {
-        Self {
-            target_bytes: (*LANCE_FTS_POSTING_GROUP_TARGET_BYTES).max(1),
-            max_tokens: (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1),
-        }
-    }
-}
-
-/// Accumulates posting-list group boundaries at write time. Tokens are pushed
-/// in row order; a group is cut once its serialized bytes reach
-/// `target_bytes` or it holds `max_tokens` posting lists. A posting list
-/// larger than the target that *starts* a group occupies that group alone (the
-/// clamp case); one encountered mid-group is absorbed and closes that group, so
-/// a single term is never split across groups.
-#[derive(Debug)]
-pub(crate) struct PostingGroupAccumulator {
-    config: PostingGroupConfig,
-    starts: Vec<u32>,
-    next_token: u32,
-    current_bytes: usize,
-    current_tokens: usize,
-}
-
-impl PostingGroupAccumulator {
-    pub(crate) fn new(config: PostingGroupConfig) -> Self {
-        Self {
-            config,
-            starts: Vec::new(),
-            next_token: 0,
-            current_bytes: 0,
-            current_tokens: 0,
-        }
-    }
-
-    /// Record the next posting list in row order, given its serialized byte size.
-    pub(crate) fn push(&mut self, posting_bytes: usize) {
-        if self.current_tokens == 0 {
-            self.starts.push(self.next_token);
-        }
-        self.current_bytes += posting_bytes;
-        self.current_tokens += 1;
-        self.next_token += 1;
-        if self.current_bytes >= self.config.target_bytes
-            || self.current_tokens >= self.config.max_tokens
-        {
-            self.current_bytes = 0;
-            self.current_tokens = 0;
-        }
-    }
-
-    pub(crate) fn into_starts(self) -> Vec<u32> {
-        self.starts
-    }
-}
-
 fn default_num_workers() -> usize {
     let total_cpus = get_num_compute_intensive_cpus() + *IO_CORE_RESERVATION;
     std::cmp::max(1, total_cpus / 2)
@@ -181,25 +95,44 @@ fn resolve_worker_memory_limit_bytes(params: &InvertedIndexParams, num_workers:
         .unwrap_or(default_worker_memory_limit_bytes)
 }
 
-fn merge_all_tail_partitions(tails: Vec<TailPartition>) -> Result<Option<InnerBuilder>> {
-    if tails.is_empty() {
-        return Ok(None);
+/// Merge the workers' leftover tail builders into as few partitions as the
+/// memory budget allows. Folding unconditionally would collapse every build
+/// whose workers never hit the flush threshold into a single partition, which
+/// destroys intra-query parallelism; splitting by the same per-partition
+/// budget as the flush path makes the final partition count converge to
+/// roughly total_builder_memory / memory_limit_bytes regardless of worker
+/// layout.
+fn merge_all_tail_partitions(
+    tails: Vec<TailPartition>,
+    memory_limit_bytes: u64,
+) -> Result<Vec<InnerBuilder>> {
+    let mut merged_builders: Vec<InnerBuilder> = Vec::new();
+    let mut merged: Option<InnerBuilder> = None;
+    for tail in tails {
+        let builder = tail.builder;
+        if builder.is_empty() {
+            continue;
+        }
+        match &mut merged {
+            Some(current) => {
+                let would_exceed_memory =
+                    current.memory_size().saturating_add(builder.memory_size())
+                        >= memory_limit_bytes;
+                let would_exceed_doc_ids =
+                    current.docs.len().saturating_add(builder.docs.len()) > u32::MAX as usize;
+                if would_exceed_memory || would_exceed_doc_ids {
+                    merged_builders.push(std::mem::replace(current, builder));
+                } else {
+                    current.merge_from(builder)?;
+                }
+            }
+            None => merged = Some(builder),
+        }
     }
-    merge_tail_partition_group(tails).map(Some)
-}
-
-fn merge_tail_partition_group(group: Vec<TailPartition>) -> Result<InnerBuilder> {
-    let mut group = group.into_iter();
-    let mut merged = group
-        .next()
-        .ok_or_else(|| {
-            Error::invalid_input("cannot merge an empty tail partition group".to_owned())
-        })?
-        .builder;
-    for tail in group {
-        merged.merge_from(tail.builder)?;
-    }
-    Ok(merged)
+    if let Some(builder) = merged {
+        merged_builders.push(builder);
+    }
+    Ok(merged_builders)
 }
 
 #[derive(Debug)]
@@ -246,6 +179,7 @@ impl InvertedIndexBuilder {
         fragment_mask: Option<u64>,
         deleted_fragments: RoaringBitmap,
     ) -> Self {
+        let format_version = params.resolved_format_version();
         Self {
             params,
             partitions,
@@ -253,16 +187,19 @@ impl InvertedIndexBuilder {
             src_store: store,
             token_set_format,
             fragment_mask,
-            format_version: current_fts_format_version(),
-            posting_tail_codec: current_fts_format_version().posting_tail_codec(),
+            format_version,
+            posting_tail_codec: format_version.posting_tail_codec(),
             progress: noop_progress(),
             deleted_fragments,
         }
     }
 
     pub fn with_posting_tail_codec(mut self, posting_tail_codec: PostingTailCodec) -> Self {
-        self.format_version =
-            InvertedListFormatVersion::from_posting_tail_codec(posting_tail_codec);
+        self.format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            posting_tail_codec,
+            self.params.block_size,
+        )
+        .expect("invalid posting tail codec for posting block size");
         self.posting_tail_codec = posting_tail_codec;
         self
     }
@@ -288,7 +225,8 @@ impl InvertedIndexBuilder {
         new_data: SendableRecordBatchStream,
         dest_store: &dyn IndexStore,
         old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let schema = new_data.schema();
         let doc_col = schema.field(0).name();
 
@@ -305,15 +243,15 @@ impl InvertedIndexBuilder {
         self.progress
             .stage_start("tokenize_docs", None, "rows")
             .await?;
-        self.update_index(new_data, dest_store).await?;
+        let mut files = self.update_index(new_data, dest_store).await?;
 
         if let Some(OldIndexDataFilter::Fragments { to_remove, .. }) = old_data_filter {
             self.deleted_fragments.extend(to_remove);
         }
 
         self.progress.stage_complete("tokenize_docs").await?;
-        self.write(dest_store).await?;
-        Ok(())
+        files.extend(self.write(dest_store).await?);
+        Ok(files)
     }
 
     pub async fn update_from_segments(
@@ -322,7 +260,8 @@ impl InvertedIndexBuilder {
         dest_store: &dyn IndexStore,
         old_segments: &[Arc<InvertedIndex>],
         old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let schema = new_data.schema();
         let doc_col = schema.field(0).name();
 
@@ -332,7 +271,8 @@ impl InvertedIndexBuilder {
             self.params.lance_tokenizer = Some(doc_type.as_ref().to_string());
         }
 
-        self.merge_existing_segments(dest_store, old_segments, old_data_filter.as_ref())
+        let mut files = self
+            .merge_existing_segments(dest_store, old_segments, old_data_filter.as_ref())
             .await?;
 
         let new_data = document_input(new_data, doc_col)?;
@@ -340,11 +280,11 @@ impl InvertedIndexBuilder {
         self.progress
             .stage_start("tokenize_docs", None, "rows")
             .await?;
-        self.update_index(new_data, dest_store).await?;
+        files.extend(self.update_index(new_data, dest_store).await?);
         self.progress.stage_complete("tokenize_docs").await?;
 
-        self.write(dest_store).await?;
-        Ok(())
+        files.extend(self.write(dest_store).await?);
+        Ok(files)
     }
 
     async fn merge_existing_segments(
@@ -352,10 +292,11 @@ impl InvertedIndexBuilder {
         dest_store: &dyn IndexStore,
         old_segments: &[Arc<InvertedIndex>],
         old_data_filter: Option<&crate::scalar::OldIndexDataFilter>,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         let num_workers = resolve_num_workers(&self.params);
         let memory_limit_bytes = resolve_worker_memory_limit_bytes(&self.params, num_workers);
         let mut merged: Option<InnerBuilder> = None;
+        let mut files = Vec::new();
         for index in old_segments {
             if old_data_filter.is_none() {
                 self.deleted_fragments
@@ -382,7 +323,7 @@ impl InvertedIndexBuilder {
                             > u32::MAX as usize;
                         if would_exceed_memory || would_exceed_doc_ids {
                             let builder = std::mem::replace(merged, partition_builder);
-                            self.write_new_partition(dest_store, builder).await?;
+                            files.extend(self.write_new_partition(dest_store, builder).await?);
                         } else {
                             merged.merge_from(partition_builder)?;
                         }
@@ -393,21 +334,31 @@ impl InvertedIndexBuilder {
         }
 
         if let Some(builder) = merged {
-            self.write_new_partition(dest_store, builder).await?;
+            files.extend(self.write_new_partition(dest_store, builder).await?);
         }
-        Ok(())
+        Ok(files)
     }
 
     async fn write_new_partition(
         &mut self,
         dest_store: &dyn IndexStore,
         mut builder: InnerBuilder,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         let partition_id = self.next_partition_id() | self.fragment_mask.unwrap_or(0);
         builder.set_id(partition_id);
-        builder.write(dest_store).await?;
+        let files = builder
+            .write_to(dest_store, self.partition_write_target())
+            .await?;
         self.new_partitions.push(partition_id);
-        Ok(())
+        Ok(files)
+    }
+
+    fn partition_write_target(&self) -> PartitionWriteTarget {
+        if self.fragment_mask.is_some() {
+            PartitionWriteTarget::Staged
+        } else {
+            PartitionWriteTarget::Final
+        }
     }
 
     fn next_partition_id(&self) -> u64 {
@@ -424,7 +375,7 @@ impl InvertedIndexBuilder {
         &mut self,
         stream: SendableRecordBatchStream,
         dest_store: &dyn IndexStore,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         let num_workers = resolve_num_workers(&self.params);
         let tokenizer = self.params.build()?;
         let with_position = self.params.with_position;
@@ -436,6 +387,7 @@ impl InvertedIndexBuilder {
             fragment_mask: self.fragment_mask,
             token_set_format: self.token_set_format,
             worker_memory_limit_bytes,
+            block_size: self.params.block_size,
         };
         let next_id = self.next_partition_id();
         let id_alloc = Arc::new(AtomicU64::new(next_id));
@@ -507,22 +459,40 @@ impl InvertedIndexBuilder {
             // wait for the workers to finish
             let start = std::time::Instant::now();
             let mut tail_partitions = Vec::new();
+            let mut files = Vec::new();
             for index_task in index_tasks {
                 let output = index_task.await??;
                 self.new_partitions.extend(output.partitions);
+                files.extend(output.files);
                 if let Some(tail_partition) = output.tail_partition {
                     tail_partitions.push(tail_partition);
                 }
             }
-            let merged_tail_partitions =
-                spawn_cpu(move || merge_all_tail_partitions(tail_partitions)).await?;
-            if let Some(builder) = merged_tail_partitions {
-                self.new_partitions.push(builder.id());
-                let mut builder = builder;
-                builder.write(dest_store.as_ref()).await?;
+            let merged_tail_partitions = spawn_cpu(move || {
+                merge_all_tail_partitions(tail_partitions, worker_memory_limit_bytes)
+            })
+            .await?;
+            // Tail partitions hold most of the data when workers rarely hit the
+            // flush threshold; writing them one at a time serializes the
+            // posting-list compression of nearly the whole index behind a
+            // single producer thread. Compress and write them concurrently.
+            let write_target = self.partition_write_target();
+            let mut tail_writes =
+                futures::stream::iter(merged_tail_partitions.into_iter().map(|mut builder| {
+                    let dest_store = dest_store.clone();
+                    async move {
+                        let partition_id = builder.id();
+                        let files = builder.write_to(dest_store.as_ref(), write_target).await?;
+                        Result::Ok((partition_id, files))
+                    }
+                }))
+                .buffer_unordered(get_num_compute_intensive_cpus().clamp(1, 16));
+            while let Some((partition_id, partition_files)) = tail_writes.try_next().await? {
+                self.new_partitions.push(partition_id);
+                files.extend(partition_files);
             }
             log::info!("wait workers indexing elapsed: {:?}", start.elapsed());
-            Result::Ok(())
+            Result::Ok(files)
         };
 
         index_build.await
@@ -530,10 +500,11 @@ impl InvertedIndexBuilder {
 
     pub async fn remap(
         &mut self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         src_store: Arc<dyn IndexStore>,
         dest_store: &dyn IndexStore,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
+        let mut files = Vec::new();
         for part in self.partitions.iter() {
             let part = InvertedPartition::load(
                 src_store.clone(),
@@ -545,20 +516,29 @@ impl InvertedIndexBuilder {
             .await?;
             let mut builder = part.into_builder().await?;
             builder.remap(mapping).await?;
-            builder.write(dest_store).await?;
+            files.extend(
+                builder
+                    .write_to(dest_store, self.partition_write_target())
+                    .await?,
+            );
         }
         if self.fragment_mask.is_none() {
-            self.write_metadata(dest_store, &self.partitions).await?;
+            files.push(self.write_metadata(dest_store, &self.partitions).await?);
         } else {
-            // in distributed mode, the part_temp_metadata is written by the worker
+            // in distributed mode, the staged partition metadata is written by the worker
             for &partition_id in &self.partitions {
-                self.write_part_metadata(dest_store, partition_id).await?;
+                files.push(self.write_part_metadata(dest_store, partition_id).await?);
             }
         }
-        Ok(())
+        Ok(files)
     }
 
-    async fn write_metadata(&self, dest_store: &dyn IndexStore, partitions: &[u64]) -> Result<()> {
+    async fn write_metadata(
+        &self,
+        dest_store: &dyn IndexStore,
+        partitions: &[u64],
+    ) -> Result<IndexFile> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let mut serialized_deleted_fragments =
             Vec::with_capacity(self.deleted_fragments.serialized_size());
         self.deleted_fragments
@@ -575,6 +555,14 @@ impl InvertedIndexBuilder {
                 POSTING_TAIL_CODEC_KEY.to_owned(),
                 self.posting_tail_codec.as_str().to_owned(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                self.format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                self.params.block_size.to_string(),
+            ),
         ]);
 
         if self.params.with_position && self.format_version.uses_shared_position_stream() {
@@ -607,8 +595,7 @@ impl InvertedIndexBuilder {
             .new_index_file(METADATA_FILE, metadata_file_schema)
             .await?;
         writer.write_record_batch(record_batch).await?;
-        writer.finish_with_metadata(metadata).await?;
-        Ok(())
+        writer.finish_with_metadata(metadata).await
     }
 
     /// Write partition metadata file for a single partition
@@ -619,7 +606,8 @@ impl InvertedIndexBuilder {
         &self,
         dest_store: &dyn IndexStore,
         partition: u64, // Modify parameter type
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
+        validate_format_version_block_size(self.format_version, self.params.block_size)?;
         let partitions = vec![partition];
         let mut metadata = HashMap::from_iter(vec![
             ("partitions".to_owned(), serde_json::to_string(&partitions)?),
@@ -632,6 +620,14 @@ impl InvertedIndexBuilder {
                 POSTING_TAIL_CODEC_KEY.to_owned(),
                 self.posting_tail_codec.as_str().to_owned(),
             ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                self.format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                self.params.block_size.to_string(),
+            ),
         ]);
         if self.params.with_position && self.format_version.uses_shared_position_stream() {
             metadata.insert(
@@ -652,30 +648,30 @@ impl InvertedIndexBuilder {
         let mut writer = dest_store
             .new_index_file(&file_name, Arc::new(Schema::empty()))
             .await?;
-        writer.finish_with_metadata(metadata).await?;
-        Ok(())
+        writer.finish_with_metadata(metadata).await
     }
 
     async fn write_metadata_with_progress(
         &self,
         dest_store: &dyn IndexStore,
         partitions: &[u64],
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         let total = if self.fragment_mask.is_none() {
             Some(1)
         } else {
             Some(partitions.len() as u64)
         };
+        let mut files = Vec::new();
         self.progress
             .stage_start("write_metadata", total, "files")
             .await?;
         if self.fragment_mask.is_none() {
-            self.write_metadata(dest_store, partitions).await?;
+            files.push(self.write_metadata(dest_store, partitions).await?);
             self.progress.stage_progress("write_metadata", 1).await?;
         } else {
             let mut completed = 0;
             for &partition_id in partitions {
-                self.write_part_metadata(dest_store, partition_id).await?;
+                files.push(self.write_part_metadata(dest_store, partition_id).await?);
                 completed += 1;
                 self.progress
                     .stage_progress("write_metadata", completed)
@@ -683,10 +679,10 @@ impl InvertedIndexBuilder {
             }
         }
         self.progress.stage_complete("write_metadata").await?;
-        Ok(())
+        Ok(files)
     }
 
-    async fn write(&self, dest_store: &dyn IndexStore) -> Result<()> {
+    async fn write(&self, dest_store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
         let mut partitions = Vec::with_capacity(self.partitions.len() + self.new_partitions.len());
         partitions.extend_from_slice(&self.partitions);
         partitions.extend_from_slice(&self.new_partitions);
@@ -700,22 +696,38 @@ impl InvertedIndexBuilder {
             )
             .await?;
         let mut copied = 0;
+        let mut files = Vec::new();
+        let target = self.partition_write_target();
         for part in self.partitions.iter() {
-            self.src_store
-                .as_ref()
-                .expect("existing partitions require a source store")
-                .copy_index_file(&token_file_path(*part), dest_store)
-                .await?;
-            self.src_store
-                .as_ref()
-                .expect("existing partitions require a source store")
-                .copy_index_file(&posting_file_path(*part), dest_store)
-                .await?;
-            self.src_store
-                .as_ref()
-                .expect("existing partitions require a source store")
-                .copy_index_file(&doc_file_path(*part), dest_store)
-                .await?;
+            files.push(
+                self.src_store
+                    .as_ref()
+                    .expect("existing partitions require a source store")
+                    .copy_index_file_to(
+                        &token_file_path(*part),
+                        &target.token_path(*part),
+                        dest_store,
+                    )
+                    .await?,
+            );
+            files.push(
+                self.src_store
+                    .as_ref()
+                    .expect("existing partitions require a source store")
+                    .copy_index_file_to(
+                        &posting_file_path(*part),
+                        &target.posting_path(*part),
+                        dest_store,
+                    )
+                    .await?,
+            );
+            files.push(
+                self.src_store
+                    .as_ref()
+                    .expect("existing partitions require a source store")
+                    .copy_index_file_to(&doc_file_path(*part), &target.doc_path(*part), dest_store)
+                    .await?,
+            );
             copied += 1;
             self.progress
                 .stage_progress("copy_partitions", copied)
@@ -729,9 +741,11 @@ impl InvertedIndexBuilder {
         }
         self.progress.stage_complete("copy_partitions").await?;
 
-        self.write_metadata_with_progress(dest_store, &partitions)
-            .await?;
-        Ok(())
+        files.extend(
+            self.write_metadata_with_progress(dest_store, &partitions)
+                .await?,
+        );
+        Ok(files)
     }
 }
 
@@ -750,10 +764,10 @@ pub struct InnerBuilder {
     token_set_format: TokenSetFormat,
     format_version: InvertedListFormatVersion,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     pub(crate) tokens: TokenSet,
     pub(crate) posting_lists: Vec<PostingListBuilder>,
     pub(crate) docs: DocSet,
-    pub(crate) group_config: PostingGroupConfig,
 }
 
 impl InnerBuilder {
@@ -772,16 +786,51 @@ impl InnerBuilder {
         token_set_format: TokenSetFormat,
         format_version: InvertedListFormatVersion,
     ) -> Self {
+        Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        block_size: usize,
+    ) -> Self {
+        let format_version = default_fts_format_version_for_block_size(block_size)
+            .expect("invalid posting list block size");
+        Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            block_size,
+        )
+    }
+
+    pub fn new_with_format_version_and_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        format_version: InvertedListFormatVersion,
+        block_size: usize,
+    ) -> Self {
+        validate_format_version_block_size(format_version, block_size)
+            .expect("invalid FTS format version for posting block size");
         Self {
             id,
             with_position,
             token_set_format,
             format_version,
             posting_tail_codec: format_version.posting_tail_codec(),
+            block_size,
             tokens: TokenSet::default(),
             posting_lists: Vec::new(),
             docs: DocSet::default(),
-            group_config: PostingGroupConfig::default(),
         }
     }
 
@@ -791,13 +840,34 @@ impl InnerBuilder {
         token_set_format: TokenSetFormat,
         posting_tail_codec: PostingTailCodec,
     ) -> Self {
-        let format_version = if posting_tail_codec == PostingTailCodec::Fixed32 {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
-        let mut builder =
-            Self::new_with_format_version(id, with_position, token_set_format, format_version);
+        Self::new_with_posting_tail_codec_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            posting_tail_codec,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_posting_tail_codec_and_block_size(
+        id: u64,
+        with_position: bool,
+        token_set_format: TokenSetFormat,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Self {
+        let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            posting_tail_codec,
+            block_size,
+        )
+        .expect("invalid posting tail codec for posting block size");
+        let mut builder = Self::new_with_format_version_and_block_size(
+            id,
+            with_position,
+            token_set_format,
+            format_version,
+            block_size,
+        );
         builder.posting_tail_codec = posting_tail_codec;
         builder
     }
@@ -829,7 +899,7 @@ impl InnerBuilder {
         self.posting_lists = posting_lists;
     }
 
-    pub async fn remap(&mut self, mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    pub async fn remap(&mut self, mapping: &RowAddrRemap) -> Result<()> {
         // for the docs, we need to remove the rows that are removed from the doc set,
         // and update the row ids of the rows that are updated
         let removed = self.docs.remap(mapping);
@@ -868,7 +938,7 @@ impl InnerBuilder {
                 mapping.insert(*row_id, None);
             }
         }
-        self.remap(&mapping).await
+        self.remap(&RowAddrRemap::direct(mapping)).await
     }
 
     pub fn merge_from(&mut self, other: Self) -> Result<()> {
@@ -878,10 +948,10 @@ impl InnerBuilder {
             token_set_format,
             format_version,
             posting_tail_codec,
+            block_size,
             tokens,
             posting_lists,
             docs,
-            group_config: _,
         } = other;
 
         if self.with_position != with_position {
@@ -908,6 +978,12 @@ impl InnerBuilder {
                 self.posting_tail_codec, posting_tail_codec
             )));
         }
+        if self.block_size != block_size {
+            return Err(Error::index(format!(
+                "cannot merge partitions with mismatched FTS block sizes: {} vs {}",
+                self.block_size, block_size
+            )));
+        }
 
         let mut token_id_map = vec![u32::MAX; posting_lists.len()];
         match tokens.tokens {
@@ -933,7 +1009,11 @@ impl InnerBuilder {
             self.docs.append(*row_id, *num_tokens);
         }
         self.posting_lists.resize_with(self.tokens.len(), || {
-            PostingListBuilder::new_with_posting_tail_codec(with_position, self.posting_tail_codec)
+            PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                with_position,
+                self.posting_tail_codec,
+                self.block_size,
+            )
         });
 
         for (token_id, posting_list) in posting_lists.into_iter().enumerate() {
@@ -968,12 +1048,25 @@ impl InnerBuilder {
             + posting_lists_size
     }
 
-    pub async fn write(&mut self, store: &dyn IndexStore) -> Result<()> {
+    pub async fn write(&mut self, store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
+        self.write_to(store, PartitionWriteTarget::Final).await
+    }
+
+    async fn write_to(
+        &mut self,
+        store: &dyn IndexStore,
+        target: PartitionWriteTarget,
+    ) -> Result<Vec<IndexFile>> {
         let docs = Arc::new(std::mem::take(&mut self.docs));
-        self.write_posting_lists(store, docs.clone()).await?;
-        self.write_tokens(store).await?;
-        self.write_docs(store, docs).await?;
-        Ok(())
+        let files = vec![
+            self.write_posting_lists(store, docs.clone(), &target.posting_path(self.id))
+                .await?,
+            self.write_tokens(store, &target.token_path(self.id))
+                .await?,
+            self.write_docs(store, docs, &target.doc_path(self.id))
+                .await?,
+        ];
+        Ok(files)
     }
 
     #[instrument(level = "debug", skip_all)]
@@ -981,12 +1074,17 @@ impl InnerBuilder {
         &mut self,
         store: &dyn IndexStore,
         docs: Arc<DocSet>,
-    ) -> Result<()> {
+        path: &str,
+    ) -> Result<IndexFile> {
         let id = self.id;
         let mut writer = store
             .new_index_file(
-                &posting_file_path(self.id),
-                inverted_list_schema_for_version(self.with_position, self.format_version),
+                path,
+                inverted_list_schema_for_version_with_block_size(
+                    self.with_position,
+                    self.format_version,
+                    self.block_size,
+                ),
             )
             .await?;
         let posting_lists = std::mem::take(&mut self.posting_lists);
@@ -999,48 +1097,73 @@ impl InnerBuilder {
         );
         let with_position = self.with_position;
         let format_version = self.format_version;
-        let group_config = self.group_config;
-        let schema = inverted_list_schema_for_version(self.with_position, self.format_version);
+        let schema = inverted_list_schema_for_version_with_block_size(
+            self.with_position,
+            self.format_version,
+            self.block_size,
+        );
         let docs_for_batches = docs.clone();
         let schema_for_batches = schema.clone();
         let batch_rows = *LANCE_FTS_POSTING_BATCH_ROWS;
         let (tx, rx) = async_channel::bounded(*LANCE_FTS_WRITE_QUEUE_SIZE);
-        let producer = spawn_cpu(move || {
+        // The producer builds posting-list batches on the CPU pool and hands them
+        // to the writer through a bounded channel. Each batch is built inside its
+        // own `spawn_cpu` call and dispatched with an async `send().await` instead
+        // of a blocking send: when the channel is full the producer *yields* its
+        // task rather than parking the pool thread it is running on. Parking would
+        // deadlock on hosts whose CPU pool has a single thread: once the consumer
+        // accumulates enough data to flush an encoded column, the flush also needs
+        // that pool. The parked producer and the starved consumer would then wait
+        // on each other forever.
+        let producer = tokio::spawn(async move {
             let mut batch_builder = PostingListBatchBuilder::new(
-                schema_for_batches.clone(),
+                schema_for_batches,
                 with_position,
                 format_version,
                 batch_rows,
-                group_config,
             );
-            for posting_list in posting_lists {
-                posting_list.append_to_batch_with_docs(
-                    &docs_for_batches,
-                    &mut batch_builder,
-                    format_version,
-                )?;
-                if batch_builder.len() < batch_rows {
-                    continue;
-                }
-
-                let batch = batch_builder.finish()?;
-                if let Err(err) = tx.send_blocking(batch) {
-                    return Err(Error::execution(format!(
-                        "failed to send posting list batch to writer: {err}"
-                    )));
-                }
-            }
+            let mut posting_lists = posting_lists.into_iter();
+            loop {
+                let docs_for_batches = docs_for_batches.clone();
+                // Build the next batch on the CPU pool. The builder and the
+                // remaining posting lists are moved in and handed back so state
+                // persists across batches.
+                let (next_builder, next_posting_lists, batch) = spawn_cpu(move || {
+                    let mut batch_builder = batch_builder;
+                    let mut posting_lists = posting_lists;
+                    let mut batch = None;
+                    for posting_list in posting_lists.by_ref() {
+                        posting_list.append_to_batch_with_docs(
+                            &docs_for_batches,
+                            &mut batch_builder,
+                            format_version,
+                        )?;
+                        if batch_builder.len() >= batch_rows {
+                            batch = Some(batch_builder.finish()?);
+                            break;
+                        }
+                    }
+                    if batch.is_none() && !batch_builder.is_empty() {
+                        batch = Some(batch_builder.finish()?);
+                    }
+                    Result::Ok((batch_builder, posting_lists, batch))
+                })
+                .await?;
+                batch_builder = next_builder;
+                posting_lists = next_posting_lists;
 
-            if !batch_builder.is_empty() {
-                let batch = batch_builder.finish()?;
-                if let Err(err) = tx.send_blocking(batch) {
-                    return Err(Error::execution(format!(
-                        "failed to send posting list batch to writer: {err}"
-                    )));
+                let Some(batch) = batch else {
+                    // No more batches: the posting lists are exhausted.
+                    break;
+                };
+                if tx.send(batch).await.is_err() {
+                    // The receiver is gone, which only happens when the writer
+                    // failed; stop producing and let that error surface there.
+                    break;
                 }
             }
 
-            Result::Ok(batch_builder.into_group_starts())
+            Result::Ok(())
         });
 
         while let Ok(batch) = rx.recv().await {
@@ -1052,48 +1175,59 @@ impl InnerBuilder {
             }
         }
         drop(rx);
-        let group_starts = producer.await?;
-
-        // Persist the posting-list cache-group boundaries as a global buffer,
-        // recording its 1-indexed id in schema metadata so the reader can group
-        // small posting lists into a single cache entry (issue #7040). Empty
-        // partitions skip this entirely and fall back to the per-token path.
-        let mut extra_metadata = HashMap::new();
-        if !group_starts.is_empty() {
-            let encoded = encode_group_starts(&group_starts);
-            let buffer_id = writer.add_global_buffer(Bytes::from(encoded)).await?;
-            extra_metadata.insert(
-                POSTING_GROUP_OFFSETS_BUF_KEY.to_owned(),
-                buffer_id.to_string(),
-            );
-        }
-        writer.finish_with_metadata(extra_metadata).await?;
-        Ok(())
+        producer.await??;
+        writer.finish().await
     }
 
     #[instrument(level = "debug", skip_all)]
-    async fn write_tokens(&mut self, store: &dyn IndexStore) -> Result<()> {
+    async fn write_tokens(&mut self, store: &dyn IndexStore, path: &str) -> Result<IndexFile> {
         log::info!("writing tokens of partition {}", self.id);
         let tokens = std::mem::take(&mut self.tokens);
         let batch = tokens.to_batch(self.token_set_format)?;
-        let mut writer = store
-            .new_index_file(&token_file_path(self.id), batch.schema())
-            .await?;
+        let mut writer = store.new_index_file(path, batch.schema()).await?;
         writer.write_record_batch(batch).await?;
-        writer.finish().await?;
-        Ok(())
+        writer.finish().await
     }
 
     #[instrument(level = "debug", skip_all)]
-    async fn write_docs(&mut self, store: &dyn IndexStore, docs: Arc<DocSet>) -> Result<()> {
+    async fn write_docs(
+        &mut self,
+        store: &dyn IndexStore,
+        docs: Arc<DocSet>,
+        path: &str,
+    ) -> Result<IndexFile> {
         log::info!("writing docs of partition {}", self.id);
         let batch = docs.to_batch()?;
-        let mut writer = store
-            .new_index_file(&doc_file_path(self.id), batch.schema())
-            .await?;
+        let mut writer = store.new_index_file(path, batch.schema()).await?;
         writer.write_record_batch(batch).await?;
-        writer.finish().await?;
-        Ok(())
+        writer.finish().await
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PartitionWriteTarget {
+    Final,
+    Staged,
+}
+
+impl PartitionWriteTarget {
+    fn file_path(self, partition_id: u64, suffix: &str) -> String {
+        match self {
+            Self::Final => partition_file_path(partition_id, suffix),
+            Self::Staged => staged_partition_file_path(partition_id, suffix),
+        }
+    }
+
+    fn token_path(self, partition_id: u64) -> String {
+        self.file_path(partition_id, TOKENS_FILE)
+    }
+
+    fn posting_path(self, partition_id: u64) -> String {
+        self.file_path(partition_id, INVERT_LIST_FILE)
+    }
+
+    fn doc_path(self, partition_id: u64) -> String {
+        self.file_path(partition_id, DOCS_FILE)
     }
 }
 
@@ -1103,6 +1237,7 @@ struct IndexWorker {
     id_alloc: Arc<AtomicU64>,
     builder: InnerBuilder,
     partitions: Vec<u64>,
+    files: Vec<IndexFile>,
     schema: SchemaRef,
     memory_size: u64,
     worker_memory_limit_bytes: u64,
@@ -1119,9 +1254,15 @@ struct TailPartition {
 
 struct WorkerOutput {
     partitions: Vec<u64>,
+    files: Vec<IndexFile>,
     tail_partition: Option<TailPartition>,
 }
 
+enum DocumentSource<'a> {
+    Text(&'a str),
+    StringList(&'a dyn Array),
+}
+
 #[derive(Debug, Clone, Copy)]
 struct IndexWorkerConfig {
     with_position: bool,
@@ -1129,6 +1270,7 @@ struct IndexWorkerConfig {
     fragment_mask: Option<u64>,
     token_set_format: TokenSetFormat,
     worker_memory_limit_bytes: u64,
+    block_size: usize,
 }
 
 impl IndexWorker {
@@ -1172,19 +1314,25 @@ impl IndexWorker {
         id_alloc: Arc<AtomicU64>,
         config: IndexWorkerConfig,
     ) -> Result<Self> {
-        let schema = inverted_list_schema_for_version(config.with_position, config.format_version);
+        let schema = inverted_list_schema_for_version_with_block_size(
+            config.with_position,
+            config.format_version,
+            config.block_size,
+        );
 
         Ok(Self {
             tokenizer,
             dest_store,
-            builder: InnerBuilder::new_with_format_version(
+            builder: InnerBuilder::new_with_format_version_and_block_size(
                 id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
                     | config.fragment_mask.unwrap_or(0),
                 config.with_position,
                 config.token_set_format,
                 config.format_version,
+                config.block_size,
             ),
             partitions: Vec::new(),
+            files: Vec::new(),
             id_alloc,
             schema,
             memory_size: 0,
@@ -1206,150 +1354,259 @@ impl IndexWorker {
 
     async fn process_batch(&mut self, batch: RecordBatch) -> Result<()> {
         let doc_col = batch.column(0);
-        let doc_iter = iter_str_array(doc_col);
         let row_id_col = batch[ROW_ID].as_primitive::<datatypes::UInt64Type>();
-        let docs = doc_iter
-            .zip(row_id_col.values().iter())
-            .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id)));
+        match doc_col.data_type() {
+            DataType::Utf8 | DataType::LargeUtf8 => {
+                let docs = iter_str_array(doc_col.as_ref())
+                    .zip(row_id_col.values().iter())
+                    .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id)));
+
+                for (doc, row_id) in docs {
+                    self.process_document(row_id, DocumentSource::Text(doc), false)
+                        .await?;
+                }
+            }
+            DataType::List(_) => {
+                self.process_string_list_batch::<i32>(doc_col, row_id_col)
+                    .await?;
+            }
+            DataType::LargeList(_) => {
+                self.process_string_list_batch::<i64>(doc_col, row_id_col)
+                    .await?;
+            }
+            data_type => {
+                return Err(Error::index(format!(
+                    "expect data type String, LargeString, List(String), or LargeList(String) but got {}",
+                    data_type
+                )));
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn process_string_list_batch<Offset: arrow::array::OffsetSizeTrait>(
+        &mut self,
+        doc_col: &Arc<dyn Array>,
+        row_id_col: &arrow_array::PrimitiveArray<datatypes::UInt64Type>,
+    ) -> Result<()> {
+        let docs = doc_col.as_list::<Offset>();
+        match docs.value_type() {
+            datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => {}
+            data_type => {
+                return Err(Error::index(format!(
+                    "expect list item data type String or LargeString but got {}",
+                    data_type
+                )));
+            }
+        }
+
+        for (doc, row_id) in docs.iter().zip(row_id_col.values().iter()) {
+            let Some(doc) = doc else {
+                continue;
+            };
+
+            self.process_document(*row_id, DocumentSource::StringList(doc.as_ref()), true)
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    fn checked_token_position(row_id: u64, token_position: usize) -> Result<u32> {
+        u32::try_from(token_position).map_err(|_| {
+            Error::invalid_input(format!(
+                "token position overflow for row_id={row_id}: token_position={token_position}"
+            ))
+        })
+    }
+
+    fn materialize_string_list(elements: &dyn Array) -> String {
+        let mut doc = String::new();
+        for element in iter_str_array(elements).flatten() {
+            if !doc.is_empty() {
+                doc.push(' ');
+            }
+            doc.push_str(element);
+        }
+        doc
+    }
 
+    async fn process_document(
+        &mut self,
+        row_id: u64,
+        document: DocumentSource<'_>,
+        skip_empty_document: bool,
+    ) -> Result<()> {
         let with_position = self.has_position();
-        for (doc, row_id) in docs {
-            let builder_was_empty = self.builder.docs.is_empty();
-            let old_temporary_memory_size = self.temporary_memory_size();
-            let old_token_memory_size = self.builder.tokens.memory_size() as u64;
-            let doc_id = self.builder.docs.len() as u32;
-            let mut token_num: u32 = 0;
-            let mut posting_memory_delta = 0i64;
-            if with_position {
+        let builder_was_empty = self.builder.docs.is_empty();
+        let old_temporary_memory_size = self.temporary_memory_size();
+        let old_token_memory_size = self.builder.tokens.memory_size() as u64;
+        let doc_id = self.builder.docs.len() as u32;
+        let mut token_num: u32 = 0;
+        let mut doc_length_bytes = 0usize;
+        let mut posting_memory_delta = 0i64;
+        if with_position {
+            {
                 if self.token_ids.capacity() < self.last_token_count {
                     self.token_ids
                         .reserve(self.last_token_count - self.token_ids.capacity());
                 }
                 self.token_ids.clear();
+                let tokenizer = &mut self.tokenizer;
                 let builder = &mut self.builder;
                 let token_ids = &mut self.token_ids;
                 let memory_size = &mut self.memory_size;
                 let posting_tail_codec = builder.posting_tail_codec;
 
-                let mut token_stream = self.tokenizer.token_stream_for_doc(doc);
-                while token_stream.advance() {
-                    let token = token_stream.token_mut();
-                    let token_text = std::mem::take(&mut token.text);
-                    let token_id = builder.tokens.add(token_text);
-                    if token_id as usize == builder.posting_lists.len() {
-                        let old_posting_lists_overhead_size = (builder.posting_lists.capacity()
-                            * std::mem::size_of::<PostingListBuilder>())
-                            as u64;
-                        builder.posting_lists.push(
-                            PostingListBuilder::new_with_posting_tail_codec(
-                                true,
-                                posting_tail_codec,
-                            ),
-                        );
-                        let new_posting_lists_overhead_size = (builder.posting_lists.capacity()
-                            * std::mem::size_of::<PostingListBuilder>())
-                            as u64;
-                        Self::adjust_tracked_value(
-                            memory_size,
-                            old_posting_lists_overhead_size,
-                            new_posting_lists_overhead_size,
-                        );
+                let block_size = builder.block_size;
+                let mut process_text = |text: &str| -> Result<()> {
+                    doc_length_bytes += text.len();
+                    let mut token_stream = tokenizer.token_stream_for_doc(text);
+                    while token_stream.advance() {
+                        let token = token_stream.token();
+                        let position = Self::checked_token_position(row_id, token.position)?;
+                        let token_id = builder.tokens.get_or_add(&token.text);
+                        if token_id as usize == builder.posting_lists.len() {
+                            let old_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                                * std::mem::size_of::<PostingListBuilder>())
+                                as u64;
+                            builder.posting_lists.push(
+                                PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                                    true,
+                                    posting_tail_codec,
+                                    block_size,
+                                ),
+                            );
+                            let new_posting_lists_overhead_size = (builder.posting_lists.capacity()
+                                * std::mem::size_of::<PostingListBuilder>())
+                                as u64;
+                            Self::adjust_tracked_value(
+                                memory_size,
+                                old_posting_lists_overhead_size,
+                                new_posting_lists_overhead_size,
+                            );
+                        }
+                        let posting_list = &mut builder.posting_lists[token_id as usize];
+                        let old_posting_memory_size = posting_list.size();
+                        if posting_list.add_occurrence(doc_id, position)? {
+                            token_ids.push(token_id);
+                        }
+                        let new_posting_memory_size = posting_list.size();
+                        posting_memory_delta +=
+                            new_posting_memory_size as i64 - old_posting_memory_size as i64;
+                        token_num += 1;
+                    }
+                    Ok(())
+                };
+
+                match document {
+                    DocumentSource::Text(doc) => {
+                        process_text(doc)?;
                     }
-                    let posting_list = &mut builder.posting_lists[token_id as usize];
-                    let old_posting_memory_size = posting_list.size();
-                    if posting_list.add_occurrence(doc_id, token.position as u32)? {
-                        token_ids.push(token_id);
+                    DocumentSource::StringList(elements) => {
+                        let doc = Self::materialize_string_list(elements);
+                        process_text(&doc)?;
                     }
-                    let new_posting_memory_size = posting_list.size();
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                    token_num += 1;
                 }
-            } else {
+            }
+        } else {
+            {
                 if self.token_ids.capacity() < self.last_token_count {
                     self.token_ids
                         .reserve(self.last_token_count - self.token_ids.capacity());
                 }
                 self.token_ids.clear();
 
-                let mut token_stream = self.tokenizer.token_stream_for_doc(doc);
-                while token_stream.advance() {
-                    let token = token_stream.token_mut();
-                    let token_text = std::mem::take(&mut token.text);
-                    let token_id = self.builder.tokens.add(token_text);
-                    self.token_ids.push(token_id);
-                    token_num += 1;
+                let tokenizer = &mut self.tokenizer;
+                let builder = &mut self.builder;
+                let token_ids = &mut self.token_ids;
+                let mut process_text = |text: &str| {
+                    doc_length_bytes += text.len();
+                    let mut token_stream = tokenizer.token_stream_for_doc(text);
+                    while token_stream.advance() {
+                        let token_id = builder.tokens.get_or_add(&token_stream.token().text);
+                        token_ids.push(token_id);
+                        token_num += 1;
+                    }
+                };
+
+                match document {
+                    DocumentSource::Text(doc) => process_text(doc),
+                    DocumentSource::StringList(elements) => {
+                        let doc = Self::materialize_string_list(elements);
+                        process_text(&doc);
+                    }
                 }
             }
+        }
+        self.adjust_tracked_memory_size(
+            old_token_memory_size,
+            self.builder.tokens.memory_size() as u64,
+        );
+
+        if skip_empty_document && token_num == 0 {
+            self.last_token_count = 0;
+            self.trim_temporary_buffers();
             self.adjust_tracked_memory_size(
-                old_token_memory_size,
-                self.builder.tokens.memory_size() as u64,
+                old_temporary_memory_size,
+                self.temporary_memory_size(),
             );
+            return Ok(());
+        }
 
-            if !with_position {
-                let old_posting_lists_overhead_size = self.posting_lists_overhead_size();
-                self.builder
-                    .posting_lists
-                    .resize_with(self.builder.tokens.len(), || {
-                        PostingListBuilder::new_with_posting_tail_codec(
-                            false,
-                            self.builder.posting_tail_codec,
-                        )
-                    });
-                let new_posting_lists_overhead_size = self.posting_lists_overhead_size();
-                Self::adjust_tracked_value(
-                    &mut self.memory_size,
-                    old_posting_lists_overhead_size,
-                    new_posting_lists_overhead_size,
-                );
-            }
-
-            let old_doc_memory_size = self.builder.docs.memory_size() as u64;
-            let appended_doc_id = self.builder.docs.append(row_id, token_num);
-            debug_assert_eq!(appended_doc_id, doc_id);
-            self.adjust_tracked_memory_size(
-                old_doc_memory_size,
-                self.builder.docs.memory_size() as u64,
+        if !with_position {
+            let old_posting_lists_overhead_size = self.posting_lists_overhead_size();
+            self.builder
+                .posting_lists
+                .resize_with(self.builder.tokens.len(), || {
+                    PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                        false,
+                        self.builder.posting_tail_codec,
+                        self.builder.block_size,
+                    )
+                });
+            let new_posting_lists_overhead_size = self.posting_lists_overhead_size();
+            Self::adjust_tracked_value(
+                &mut self.memory_size,
+                old_posting_lists_overhead_size,
+                new_posting_lists_overhead_size,
             );
-            self.total_doc_length += doc.len();
-
-            if with_position {
-                for &token_id in &self.token_ids {
-                    let (old_posting_memory_size, new_posting_memory_size) = {
-                        let posting_list = &mut self.builder.posting_lists[token_id as usize];
-                        let old_posting_memory_size = posting_list.size();
-                        posting_list.finish_open_doc(doc_id)?;
-                        let new_posting_memory_size = posting_list.size();
-                        (old_posting_memory_size, new_posting_memory_size)
-                    };
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                }
-                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
-            } else if token_num > 0 {
-                self.token_ids.sort_unstable();
-                let mut iter = self.token_ids.iter();
-                let mut current = *iter.next().unwrap();
-                let mut count = 1u32;
-                for &token_id in iter {
-                    if token_id == current {
-                        count += 1;
-                        continue;
-                    }
+        }
 
-                    let (old_posting_memory_size, new_posting_memory_size) = {
-                        let posting_list = &mut self.builder.posting_lists[current as usize];
-                        let old_posting_memory_size = posting_list.size();
-                        posting_list.add(doc_id, PositionRecorder::Count(count));
-                        let new_posting_memory_size = posting_list.size();
-                        (old_posting_memory_size, new_posting_memory_size)
-                    };
-                    posting_memory_delta +=
-                        new_posting_memory_size as i64 - old_posting_memory_size as i64;
+        let old_doc_memory_size = self.builder.docs.memory_size() as u64;
+        let appended_doc_id = self.builder.docs.append(row_id, token_num);
+        debug_assert_eq!(appended_doc_id, doc_id);
+        self.adjust_tracked_memory_size(
+            old_doc_memory_size,
+            self.builder.docs.memory_size() as u64,
+        );
+        self.total_doc_length += doc_length_bytes;
 
-                    current = token_id;
-                    count = 1;
+        if with_position {
+            for &token_id in &self.token_ids {
+                let (old_posting_memory_size, new_posting_memory_size) = {
+                    let posting_list = &mut self.builder.posting_lists[token_id as usize];
+                    let old_posting_memory_size = posting_list.size();
+                    posting_list.finish_open_doc(doc_id)?;
+                    let new_posting_memory_size = posting_list.size();
+                    (old_posting_memory_size, new_posting_memory_size)
+                };
+                posting_memory_delta +=
+                    new_posting_memory_size as i64 - old_posting_memory_size as i64;
+            }
+            Self::apply_delta(&mut self.memory_size, posting_memory_delta);
+        } else if token_num > 0 {
+            self.token_ids.sort_unstable();
+            let mut iter = self.token_ids.iter();
+            let mut current = *iter.next().unwrap();
+            let mut count = 1u32;
+            for &token_id in iter {
+                if token_id == current {
+                    count += 1;
+                    continue;
                 }
+
                 let (old_posting_memory_size, new_posting_memory_size) = {
                     let posting_list = &mut self.builder.posting_lists[current as usize];
                     let old_posting_memory_size = posting_list.size();
@@ -1359,27 +1616,35 @@ impl IndexWorker {
                 };
                 posting_memory_delta +=
                     new_posting_memory_size as i64 - old_posting_memory_size as i64;
-                Self::apply_delta(&mut self.memory_size, posting_memory_delta);
-            }
-            self.last_token_count = self.token_ids.len();
-            self.trim_temporary_buffers();
-            self.adjust_tracked_memory_size(
-                old_temporary_memory_size,
-                self.temporary_memory_size(),
-            );
 
-            if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes {
-                return Err(Error::invalid_input(format!(
-                    "single document row_id={} exceeds worker memory limit: {} > {} bytes",
-                    row_id, self.memory_size, self.worker_memory_limit_bytes
-                )));
+                current = token_id;
+                count = 1;
             }
+            let (old_posting_memory_size, new_posting_memory_size) = {
+                let posting_list = &mut self.builder.posting_lists[current as usize];
+                let old_posting_memory_size = posting_list.size();
+                posting_list.add(doc_id, PositionRecorder::Count(count));
+                let new_posting_memory_size = posting_list.size();
+                (old_posting_memory_size, new_posting_memory_size)
+            };
+            posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64;
+            Self::apply_delta(&mut self.memory_size, posting_memory_delta);
+        }
+        self.last_token_count = self.token_ids.len();
+        self.trim_temporary_buffers();
+        self.adjust_tracked_memory_size(old_temporary_memory_size, self.temporary_memory_size());
+
+        if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes {
+            return Err(Error::invalid_input(format!(
+                "single document row_id={} exceeds worker memory limit: {} > {} bytes",
+                row_id, self.memory_size, self.worker_memory_limit_bytes
+            )));
+        }
 
-            if self.builder.docs.len() as u32 == u32::MAX
-                || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes)
-            {
-                self.flush().await?;
-            }
+        if self.builder.docs.len() as u32 == u32::MAX
+            || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes)
+        {
+            self.flush().await?;
         }
 
         Ok(())
@@ -1398,21 +1663,28 @@ impl IndexWorker {
         self.memory_size = self.temporary_memory_size();
         let with_position = self.has_position();
         let format_version = self.builder.format_version;
+        let block_size = self.builder.block_size;
         let builder = std::mem::replace(
             &mut self.builder,
-            InnerBuilder::new_with_format_version(
+            InnerBuilder::new_with_format_version_and_block_size(
                 self.id_alloc
                     .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
                     | self.fragment_mask.unwrap_or(0),
                 with_position,
                 self.token_set_format,
                 format_version,
+                block_size,
             ),
         );
         let written_partition_id = builder.id();
         let mut builder = builder;
-        builder
-            .write(self.dest_store.as_ref())
+        let target = if self.fragment_mask.is_some() {
+            PartitionWriteTarget::Staged
+        } else {
+            PartitionWriteTarget::Final
+        };
+        let files = builder
+            .write_to(self.dest_store.as_ref(), target)
             .await
             .map_err(|err| {
                 Error::execution(format!(
@@ -1420,6 +1692,7 @@ impl IndexWorker {
                     written_partition_id
                 ))
             })?;
+        self.files.extend(files);
         self.partitions.push(written_partition_id);
         Ok(())
     }
@@ -1434,6 +1707,7 @@ impl IndexWorker {
         };
         Ok(WorkerOutput {
             partitions: self.partitions,
+            files: self.files,
             tail_partition,
         })
     }
@@ -1519,17 +1793,56 @@ pub fn inverted_list_schema_for_version(
     with_position: bool,
     format_version: InvertedListFormatVersion,
 ) -> SchemaRef {
+    inverted_list_schema_for_version_with_block_size(
+        with_position,
+        format_version,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+pub fn inverted_list_schema_for_version_with_block_size(
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+) -> SchemaRef {
+    inverted_list_schema_for_version_with_block_size_and_impacts(
+        with_position,
+        format_version,
+        block_size,
+        true,
+    )
+}
+
+pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts(
+    with_position: bool,
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+    with_impacts: bool,
+) -> SchemaRef {
+    validate_format_version_block_size(format_version, block_size)
+        .expect("invalid FTS format version for posting block size");
     match format_version {
-        InvertedListFormatVersion::V1 => inverted_list_schema_v1(with_position),
-        InvertedListFormatVersion::V2 => inverted_list_schema_with_tail_codec_and_position_codec(
-            with_position,
-            PostingTailCodec::VarintDelta,
-            Some(PositionStreamCodec::PackedDelta),
-        ),
+        InvertedListFormatVersion::V1 => {
+            inverted_list_schema_v1(with_position, block_size, with_impacts)
+        }
+        InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => {
+            inverted_list_schema_with_tail_codec_and_position_codec(
+                with_position,
+                format_version,
+                PostingTailCodec::VarintDelta,
+                Some(PositionStreamCodec::PackedDelta),
+                block_size,
+                with_impacts,
+            )
+        }
     }
 }
 
-fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
+fn inverted_list_schema_v1(
+    with_position: bool,
+    block_size: usize,
+    with_impacts: bool,
+) -> SchemaRef {
     let mut fields = vec![
         arrow_schema::Field::new(
             POSTING_COL,
@@ -1543,6 +1856,17 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
         arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false),
         arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false),
     ];
+    if with_impacts {
+        fields.push(arrow_schema::Field::new(
+            IMPACT_COL,
+            datatypes::DataType::List(Arc::new(Field::new(
+                "item",
+                datatypes::DataType::LargeBinary,
+                true,
+            ))),
+            false,
+        ));
+    }
     if with_position {
         fields.push(arrow_schema::Field::new(
             POSITION_COL,
@@ -1558,24 +1882,44 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef {
             false,
         ));
     }
-    Arc::new(arrow_schema::Schema::new(fields))
+    Arc::new(arrow_schema::Schema::new_with_metadata(
+        fields,
+        HashMap::from([
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                InvertedListFormatVersion::V1.index_version().to_string(),
+            ),
+        ]),
+    ))
 }
 
 pub fn inverted_list_schema_with_tail_codec(
     with_position: bool,
     posting_tail_codec: PostingTailCodec,
 ) -> SchemaRef {
+    let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+        posting_tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+    .expect("invalid posting tail codec for posting block size");
     inverted_list_schema_with_tail_codec_and_position_codec(
         with_position,
+        format_version,
         posting_tail_codec,
         Some(PositionStreamCodec::PackedDelta),
+        LEGACY_BLOCK_SIZE,
+        false,
     )
 }
 
 fn inverted_list_schema_with_tail_codec_and_position_codec(
     with_position: bool,
+    format_version: InvertedListFormatVersion,
     posting_tail_codec: PostingTailCodec,
     position_codec: Option<PositionStreamCodec>,
+    block_size: usize,
+    with_impacts: bool,
 ) -> SchemaRef {
     let mut fields = vec![
         // we compress the posting lists (including row ids and frequencies),
@@ -1592,6 +1936,17 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
         arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false),
         arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false),
     ];
+    if with_impacts {
+        fields.push(arrow_schema::Field::new(
+            IMPACT_COL,
+            datatypes::DataType::List(Arc::new(Field::new(
+                "item",
+                datatypes::DataType::LargeBinary,
+                true,
+            ))),
+            false,
+        ));
+    }
     if with_position {
         fields.push(arrow_schema::Field::new(
             COMPRESSED_POSITION_COL,
@@ -1612,6 +1967,11 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
         POSTING_TAIL_CODEC_KEY.to_owned(),
         posting_tail_codec.as_str().to_owned(),
     )]);
+    metadata.insert(
+        FTS_FORMAT_VERSION_KEY.to_owned(),
+        format_version.index_version().to_string(),
+    );
+    metadata.insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string());
     if let Some(position_codec) = position_codec.filter(|_| with_position) {
         metadata.insert(
             POSITIONS_LAYOUT_KEY.to_owned(),
@@ -1625,129 +1985,6 @@ fn inverted_list_schema_with_tail_codec_and_position_codec(
     Arc::new(arrow_schema::Schema::new_with_metadata(fields, metadata))
 }
 
-/// Flatten the string list stream into a string stream
-pub struct FlattenStream {
-    /// Inner record batch stream with 2 columns:
-    /// 1. doc_col: List(Utf8) or List(LargeUtf8)
-    /// 2. row_id_col: UInt64
-    inner: SendableRecordBatchStream,
-    field_type: DataType,
-    data_type: DataType,
-}
-
-impl FlattenStream {
-    pub fn new(input: SendableRecordBatchStream) -> Self {
-        let schema = input.schema();
-        let field = schema.field(0);
-        let data_type = match field.data_type() {
-            DataType::List(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8,
-            DataType::List(f) if matches!(f.data_type(), DataType::LargeUtf8) => {
-                DataType::LargeUtf8
-            }
-            DataType::LargeList(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8,
-            DataType::LargeList(f) if matches!(f.data_type(), DataType::LargeUtf8) => {
-                DataType::LargeUtf8
-            }
-            _ => panic!(
-                "expect data type List(Utf8) or List(LargeUtf8) but got {:?}",
-                field.data_type()
-            ),
-        };
-        Self {
-            inner: input,
-            field_type: field.data_type().clone(),
-            data_type,
-        }
-    }
-}
-
-impl Stream for FlattenStream {
-    type Item = datafusion_common::Result<RecordBatch>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        match Pin::new(&mut self.inner).poll_next(cx) {
-            Poll::Ready(Some(Ok(batch))) => {
-                let doc_col = batch.column(0);
-                let batch = match self.field_type {
-                    DataType::List(_) => flatten_string_list::<i32>(&batch, doc_col).map_err(|e| {
-                        datafusion_common::error::DataFusionError::Execution(format!(
-                            "flatten string list error: {}",
-                            e
-                        ))
-                    }),
-                    DataType::LargeList(_) => {
-                        flatten_string_list::<i64>(&batch, doc_col).map_err(|e| {
-                            datafusion_common::error::DataFusionError::Execution(format!(
-                                "flatten string list error: {}",
-                                e
-                            ))
-                        })
-                    }
-                    _ => unreachable!(
-                        "expect data type List or LargeList but got {:?}",
-                        self.field_type
-                    ),
-                };
-                Poll::Ready(Some(batch))
-            }
-            Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
-            Poll::Ready(None) => Poll::Ready(None),
-            Poll::Pending => Poll::Pending,
-        }
-    }
-}
-
-impl RecordBatchStream for FlattenStream {
-    fn schema(&self) -> SchemaRef {
-        let schema = Schema::new(vec![
-            Field::new(
-                self.inner.schema().field(0).name(),
-                self.data_type.clone(),
-                true,
-            ),
-            ROW_ID_FIELD.clone(),
-        ]);
-
-        Arc::new(schema)
-    }
-}
-
-fn flatten_string_list<Offset: arrow::array::OffsetSizeTrait>(
-    batch: &RecordBatch,
-    doc_col: &Arc<dyn Array>,
-) -> Result<RecordBatch> {
-    let docs = doc_col.as_list::<Offset>();
-    let row_ids = batch[ROW_ID].as_primitive::<datatypes::UInt64Type>();
-
-    let row_ids = row_ids
-        .values()
-        .iter()
-        .zip(docs.iter())
-        .flat_map(|(row_id, doc)| std::iter::repeat_n(*row_id, doc.map(|d| d.len()).unwrap_or(0)));
-
-    let row_ids = Arc::new(UInt64Array::from_iter_values(row_ids));
-    let docs = match docs.value_type() {
-        datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => docs.values().clone(),
-        _ => {
-            return Err(Error::index(format!(
-                "expect data type String or LargeString but got {}",
-                docs.value_type()
-            )));
-        }
-    };
-
-    let schema = Schema::new(vec![
-        Field::new(
-            batch.schema().field(0).name(),
-            docs.data_type().clone(),
-            true,
-        ),
-        ROW_ID_FIELD.clone(),
-    ]);
-    let batch = RecordBatch::try_new(Arc::new(schema), vec![docs, row_ids])?;
-    Ok(batch)
-}
-
 pub(crate) fn token_file_path(partition_id: u64) -> String {
     format!("part_{}_{}", partition_id, TOKENS_FILE)
 }
@@ -1761,14 +1998,23 @@ pub(crate) fn doc_file_path(partition_id: u64) -> String {
 }
 
 pub(crate) fn part_metadata_file_path(partition_id: u64) -> String {
-    format!("part_{}_{}", partition_id, METADATA_FILE)
+    staged_partition_file_path(partition_id, METADATA_FILE)
 }
 
 const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE];
-// Each remapped file is renamed twice: first to a temp path (phase 1), then to
-// its final path (phase 2). Keep in sync with the two rename loops below in
-// `merge_metadata_files`.
-const PARTITION_FILE_RENAME_PHASES: u64 = 2;
+const STAGED_PARTITION_DIR: &str = "staging";
+
+fn partition_file_path(partition_id: u64, suffix: &str) -> String {
+    format!("part_{}_{}", partition_id, suffix)
+}
+
+fn staged_partition_file_path(partition_id: u64, suffix: &str) -> String {
+    format!(
+        "{}/{}",
+        STAGED_PARTITION_DIR,
+        partition_file_path(partition_id, suffix)
+    )
+}
 
 pub async fn merge_index_files(
     object_store: &ObjectStore,
@@ -1776,33 +2022,65 @@ pub async fn merge_index_files(
     store: Arc<dyn IndexStore>,
     progress: Arc<dyn IndexBuildProgress>,
 ) -> Result<()> {
-    // List all partition metadata files in the index directory
-    let part_metadata_files = list_metadata_files(object_store, index_dir).await?;
+    let metadata_path = index_dir.clone().join(METADATA_FILE);
+    if object_store.exists(&metadata_path).await? {
+        return Ok(());
+    }
+
+    // List all staged partition metadata files in the index directory
+    let index_files = list_index_files(object_store, index_dir).await?;
+    let part_metadata_files = metadata_files(&index_files);
+    if part_metadata_files.is_empty() {
+        return Err(Error::invalid_input_source(
+            format!(
+                "No partition metadata files found in index directory: {}",
+                index_dir
+            )
+            .into(),
+        ));
+    }
 
     // Call merge_metadata_files function for inverted index
     merge_metadata_files(store, &part_metadata_files, progress).await
 }
 
-/// List and filter metadata files from the index directory
-/// Returns partition metadata files
-async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result<Vec<String>> {
-    // List all partition metadata files in the index directory
-    let mut part_metadata_files = Vec::new();
-    let mut list_stream = object_store.list(Some(index_dir.clone()));
+async fn list_index_files(object_store: &ObjectStore, index_dir: &Path) -> Result<Vec<String>> {
+    let mut index_files = Vec::new();
+    let mut list_stream = object_store.read_dir_all(index_dir, None);
 
     while let Some(item) = list_stream.next().await {
         match item {
             Ok(meta) => {
-                let file_name = meta.location.filename().unwrap_or_default();
-                // Filter files matching the pattern part_*_metadata.lance
-                if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") {
-                    part_metadata_files.push(file_name.to_string());
-                }
+                let location = meta.location.as_ref().trim_start_matches('/');
+                let index_dir = index_dir.as_ref().trim_start_matches('/');
+                let relative_path = location
+                    .strip_prefix(index_dir)
+                    .map(|s| s.trim_start_matches('/').to_string())
+                    .unwrap_or_else(|| meta.location.filename().unwrap_or("").to_string());
+                index_files.push(relative_path);
             }
             Err(err) => return Err(err),
         }
     }
 
+    Ok(index_files)
+}
+
+fn metadata_files(index_files: &[String]) -> Vec<String> {
+    index_files
+        .iter()
+        .filter(|file_name| {
+            file_name.starts_with(&format!("{}/part_", STAGED_PARTITION_DIR))
+                && file_name.ends_with("_metadata.lance")
+        })
+        .cloned()
+        .collect()
+}
+
+#[cfg(test)]
+async fn list_metadata_files(object_store: &ObjectStore, index_dir: &Path) -> Result<Vec<String>> {
+    let index_files = list_index_files(object_store, index_dir).await?;
+    let part_metadata_files = metadata_files(&index_files);
     if part_metadata_files.is_empty() {
         return Err(Error::invalid_input_source(
             format!(
@@ -1893,89 +2171,35 @@ async fn merge_metadata_files(
     progress.stage_complete("read_partition_metadata").await?;
 
     // Create ID mapping: sorted original IDs -> 0,1,2...
-    let mut sorted_ids = all_partitions.clone();
+    let mut sorted_ids = all_partitions;
     sorted_ids.sort();
     sorted_ids.dedup();
 
-    let id_mapping: HashMap<u64, u64> = sorted_ids
+    let id_mapping: Vec<(u64, u64)> = sorted_ids
         .iter()
         .enumerate()
         .map(|(new_id, &old_id)| (old_id, new_id as u64))
         .collect();
 
-    // Safe rename partition files using temporary files to avoid overwrite
-    let timestamp = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap()
-        .as_secs();
-
-    let changed_partition_count = id_mapping
-        .iter()
-        .filter(|(old_id, new_id)| old_id != new_id)
-        .count() as u64;
-    let total_renames = changed_partition_count
-        * PARTITION_FILE_SUFFIXES.len() as u64
-        * PARTITION_FILE_RENAME_PHASES;
+    let total_copies = id_mapping.len() as u64 * PARTITION_FILE_SUFFIXES.len() as u64;
     progress
-        .stage_start("remap_partition_files", Some(total_renames), "files")
+        .stage_start("remap_partition_files", Some(total_copies), "files")
         .await?;
 
-    // Phase 1: Move files to temporary locations
-    let mut temp_files: Vec<(String, String, String)> = Vec::new(); // (temp_path, old_path, final_path)
-    let mut renamed_files = 0u64;
+    let mut copied_files = 0u64;
 
-    for (&old_id, &new_id) in &id_mapping {
-        if old_id != new_id {
-            for suffix in PARTITION_FILE_SUFFIXES {
-                let old_path = format!("part_{}_{}", old_id, suffix);
-                let new_path = format!("part_{}_{}", new_id, suffix);
-                let temp_path = format!("temp_{}_{}", timestamp, old_path);
-
-                // Move to temporary location first to avoid overwrite
-                if let Err(e) = store.rename_index_file(&old_path, &temp_path).await {
-                    // Rollback phase 1: restore files from temp locations
-                    for (temp_name, old_name, _) in temp_files.iter().rev() {
-                        let _ = store.rename_index_file(temp_name, old_name).await;
-                    }
-                    return Err(Error::index(format!(
-                        "Failed to move {} to temp {}: {}",
-                        old_path, temp_path, e
-                    )));
-                }
-                temp_files.push((temp_path, old_path, new_path));
-                renamed_files += 1;
-                progress
-                    .stage_progress("remap_partition_files", renamed_files)
-                    .await?;
-            }
-        }
-    }
-
-    // Phase 2: Move from temporary to final locations
-    let mut completed_renames: Vec<(String, String)> = Vec::new(); // (final_path, temp_path)
-
-    for (temp_path, _old_path, final_path) in &temp_files {
-        if let Err(e) = store.rename_index_file(temp_path, final_path).await {
-            // Rollback phase 2: restore completed renames and remaining temps
-            for (final_name, temp_name) in completed_renames.iter().rev() {
-                let _ = store.rename_index_file(final_name, temp_name).await;
-            }
-            // Restore remaining temp files to original locations
-            for (temp_name, orig_name, _) in temp_files.iter() {
-                if !completed_renames.iter().any(|(_, t)| t == temp_name) {
-                    let _ = store.rename_index_file(temp_name, orig_name).await;
-                }
-            }
-            return Err(Error::index(format!(
-                "Failed to rename {} to {}: {}",
-                temp_path, final_path, e
-            )));
+    for &(old_id, new_id) in &id_mapping {
+        for suffix in PARTITION_FILE_SUFFIXES {
+            let staged_path = staged_partition_file_path(old_id, suffix);
+            let final_path = partition_file_path(new_id, suffix);
+            store
+                .copy_index_file_to(&staged_path, &final_path, store.as_ref())
+                .await?;
+            copied_files += 1;
+            progress
+                .stage_progress("remap_partition_files", copied_files)
+                .await?;
         }
-        completed_renames.push((final_path.clone(), temp_path.clone()));
-        renamed_files += 1;
-        progress
-            .stage_progress("remap_partition_files", renamed_files)
-            .await?;
     }
     progress.stage_complete("remap_partition_files").await?;
 
@@ -2002,10 +2226,15 @@ async fn merge_metadata_files(
     progress.stage_progress("write_merged_metadata", 1).await?;
     progress.stage_complete("write_merged_metadata").await?;
 
-    // Cleanup partition metadata files
+    // Cleanup staged partition metadata files
     for file_name in part_metadata_files {
-        if file_name.starts_with("part_") && file_name.ends_with("_metadata.lance") {
-            let _ = store.delete_index_file(file_name).await;
+        let _ = store.delete_index_file(file_name).await;
+    }
+    for &(old_id, _) in &id_mapping {
+        for suffix in PARTITION_FILE_SUFFIXES {
+            let _ = store
+                .delete_index_file(&staged_partition_file_path(old_id, suffix))
+                .await;
         }
     }
 
@@ -2029,7 +2258,7 @@ pub fn document_input(
         DataType::List(field) | DataType::LargeList(field)
             if matches!(field.data_type(), DataType::Utf8 | DataType::LargeUtf8) =>
         {
-            Ok(Box::pin(FlattenStream::new(input)))
+            Ok(input)
         }
         DataType::LargeBinary => match field.metadata().get(ARROW_EXT_NAME_KEY) {
             Some(name) if name.as_str() == JSON_EXT_NAME => {
@@ -2220,13 +2449,250 @@ mod tests {
     }
 
     impl DeepSizeOf for CountingStore {
-        fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
             0
         }
     }
 
+    #[derive(Debug, Clone)]
+    struct NoRenameStore {
+        inner: Arc<dyn IndexStore>,
+        final_delete_count: Option<Arc<AtomicUsize>>,
+    }
+
+    impl NoRenameStore {
+        fn new(inner: Arc<dyn IndexStore>) -> Self {
+            Self {
+                inner,
+                final_delete_count: None,
+            }
+        }
+
+        fn with_final_delete_tracking(inner: Arc<dyn IndexStore>) -> Self {
+            Self {
+                inner,
+                final_delete_count: Some(Arc::new(AtomicUsize::new(0))),
+            }
+        }
+
+        fn final_delete_count(&self) -> usize {
+            self.final_delete_count
+                .as_ref()
+                .map(|count| count.load(Ordering::SeqCst))
+                .unwrap_or_default()
+        }
+
+        fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore {
+            dest_store
+                .as_any()
+                .downcast_ref::<Self>()
+                .map(|store| store.inner.as_ref())
+                .unwrap_or(dest_store)
+        }
+    }
+
+    impl DeepSizeOf for NoRenameStore {
+        fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+            self.inner.deep_size_of_children(context)
+        }
+    }
+
+    #[async_trait]
+    impl IndexStore for NoRenameStore {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(self.clone())
+        }
+
+        fn io_parallelism(&self) -> usize {
+            self.inner.io_parallelism()
+        }
+
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            self.inner.with_io_priority(io_priority)
+        }
+
+        async fn new_index_file(
+            &self,
+            name: &str,
+            schema: Arc<Schema>,
+        ) -> Result<Box<dyn IndexWriter>> {
+            self.inner.new_index_file(name, schema).await
+        }
+
+        async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
+            self.inner.open_index_file(name).await
+        }
+
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            self.inner
+                .copy_index_file(name, Self::unwrap_dest_store(dest_store))
+                .await
+        }
+
+        async fn copy_index_file_to(
+            &self,
+            name: &str,
+            new_name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            self.inner
+                .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store))
+                .await
+        }
+
+        async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile> {
+            Err(Error::internal(format!(
+                "merge_index_files should not rename partition file {name} to {new_name}"
+            )))
+        }
+
+        async fn delete_index_file(&self, name: &str) -> Result<()> {
+            if name.starts_with("part_")
+                && let Some(count) = &self.final_delete_count
+            {
+                count.fetch_add(1, Ordering::SeqCst);
+            }
+            self.inner.delete_index_file(name).await
+        }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+            self.inner.list_files_with_sizes().await
+        }
+    }
+
+    #[derive(Debug)]
+    struct FailMetadataStore {
+        inner: Arc<dyn IndexStore>,
+    }
+
+    impl FailMetadataStore {
+        fn new(inner: Arc<dyn IndexStore>) -> Self {
+            Self { inner }
+        }
+
+        fn unwrap_dest_store(dest_store: &dyn IndexStore) -> &dyn IndexStore {
+            dest_store
+                .as_any()
+                .downcast_ref::<Self>()
+                .map(|store| store.inner.as_ref())
+                .unwrap_or(dest_store)
+        }
+    }
+
+    impl DeepSizeOf for FailMetadataStore {
+        fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+            self.inner.deep_size_of_children(context)
+        }
+    }
+
+    #[async_trait]
+    impl IndexStore for FailMetadataStore {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn clone_arc(&self) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.clone(),
+            })
+        }
+
+        fn io_parallelism(&self) -> usize {
+            self.inner.io_parallelism()
+        }
+
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            self.inner.with_io_priority(io_priority)
+        }
+
+        async fn new_index_file(
+            &self,
+            name: &str,
+            schema: Arc<Schema>,
+        ) -> Result<Box<dyn IndexWriter>> {
+            let writer = self.inner.new_index_file(name, schema).await?;
+            if name == METADATA_FILE {
+                Ok(Box::new(FailFinishWriter { inner: writer }))
+            } else {
+                Ok(writer)
+            }
+        }
+
+        async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
+            self.inner.open_index_file(name).await
+        }
+
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            self.inner
+                .copy_index_file(name, Self::unwrap_dest_store(dest_store))
+                .await
+        }
+
+        async fn copy_index_file_to(
+            &self,
+            name: &str,
+            new_name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            self.inner
+                .copy_index_file_to(name, new_name, Self::unwrap_dest_store(dest_store))
+                .await
+        }
+
+        async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile> {
+            self.inner.rename_index_file(name, new_name).await
+        }
+
+        async fn delete_index_file(&self, name: &str) -> Result<()> {
+            self.inner.delete_index_file(name).await
+        }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+            self.inner.list_files_with_sizes().await
+        }
+    }
+
+    struct FailFinishWriter {
+        inner: Box<dyn IndexWriter>,
+    }
+
+    #[async_trait]
+    impl IndexWriter for FailFinishWriter {
+        async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64> {
+            self.inner.write_record_batch(batch).await
+        }
+
+        async fn add_global_buffer(&mut self, data: Bytes) -> Result<u32> {
+            self.inner.add_global_buffer(data).await
+        }
+
+        async fn finish(&mut self) -> Result<IndexFile> {
+            Err(Error::internal("injected metadata write failure"))
+        }
+
+        async fn finish_with_metadata(
+            &mut self,
+            _metadata: HashMap<String, String>,
+        ) -> Result<IndexFile> {
+            Err(Error::internal("injected metadata write failure"))
+        }
+    }
+
     #[derive(Debug)]
     struct CountingWriter {
+        path: String,
         write_count: Arc<AtomicUsize>,
     }
 
@@ -2237,17 +2703,25 @@ mod tests {
         }
 
         async fn add_global_buffer(&mut self, _data: Bytes) -> Result<u32> {
-            // The posting-list writer stores the group offsets as a global
-            // buffer; mirror the real writer's 1-indexed return value.
+            // Mirror the real writer's 1-indexed return value.
             Ok(1)
         }
 
-        async fn finish(&mut self) -> Result<()> {
-            Ok(())
+        async fn finish(&mut self) -> Result<IndexFile> {
+            Ok(IndexFile {
+                path: self.path.clone(),
+                size_bytes: 0,
+            })
         }
 
-        async fn finish_with_metadata(&mut self, _metadata: HashMap<String, String>) -> Result<()> {
-            Ok(())
+        async fn finish_with_metadata(
+            &mut self,
+            _metadata: HashMap<String, String>,
+        ) -> Result<IndexFile> {
+            Ok(IndexFile {
+                path: self.path.clone(),
+                size_bytes: 0,
+            })
         }
     }
 
@@ -2265,120 +2739,507 @@ mod tests {
             1
         }
 
-        async fn new_index_file(
-            &self,
-            _name: &str,
-            _schema: Arc<Schema>,
-        ) -> Result<Box<dyn IndexWriter>> {
-            Ok(Box::new(CountingWriter {
-                write_count: self.write_count.clone(),
-            }))
-        }
+        fn with_io_priority(&self, _io_priority: u64) -> Arc<dyn IndexStore> {
+            // No backing scheduler, so priority is meaningless here.
+            self.clone_arc()
+        }
+
+        async fn new_index_file(
+            &self,
+            name: &str,
+            _schema: Arc<Schema>,
+        ) -> Result<Box<dyn IndexWriter>> {
+            Ok(Box::new(CountingWriter {
+                path: name.to_string(),
+                write_count: self.write_count.clone(),
+            }))
+        }
+
+        async fn open_index_file(&self, _name: &str) -> Result<Arc<dyn IndexReader>> {
+            Err(Error::not_supported(
+                "CountingStore does not support reading",
+            ))
+        }
+
+        async fn copy_index_file(
+            &self,
+            _name: &str,
+            _dest_store: &dyn IndexStore,
+        ) -> Result<IndexFile> {
+            Err(Error::not_supported(
+                "CountingStore does not support copying",
+            ))
+        }
+
+        async fn rename_index_file(&self, _name: &str, _new_name: &str) -> Result<IndexFile> {
+            Err(Error::not_supported(
+                "CountingStore does not support renaming",
+            ))
+        }
+
+        async fn delete_index_file(&self, _name: &str) -> Result<()> {
+            Err(Error::not_supported(
+                "CountingStore does not support deleting",
+            ))
+        }
+
+        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
+            Ok(vec![])
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> {
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for doc_id in 0..3u64 {
+            builder.docs.append(doc_id, 1);
+        }
+
+        for doc_id in 0..3u32 {
+            let mut posting_list = PostingListBuilder::new(false);
+            posting_list.add(doc_id, PositionRecorder::Count(1));
+            builder.posting_lists.push(posting_list);
+        }
+
+        let store = CountingStore::new();
+        let docs = Arc::new(std::mem::take(&mut builder.docs));
+        builder
+            .write_posting_lists(&store, docs, &posting_file_path(0))
+            .await?;
+
+        assert_eq!(store.write_count(), 1);
+        Ok(())
+    }
+
+    async fn write_partition_file_marker(
+        store: &dyn IndexStore,
+        path: &str,
+        partition_id: u64,
+    ) -> Result<()> {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "partition_id",
+            DataType::UInt64,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(UInt64Array::from(vec![partition_id]))],
+        )?;
+        let mut writer = store.new_index_file(path, schema).await?;
+        writer.write_record_batch(batch).await?;
+        writer.finish().await?;
+        Ok(())
+    }
+
+    async fn write_partition_files(
+        store: &dyn IndexStore,
+        partition_id: u64,
+        target: PartitionWriteTarget,
+    ) -> Result<()> {
+        write_partition_file_marker(store, &target.token_path(partition_id), partition_id).await?;
+        write_partition_file_marker(store, &target.posting_path(partition_id), partition_id)
+            .await?;
+        write_partition_file_marker(store, &target.doc_path(partition_id), partition_id).await?;
+        Ok(())
+    }
+
+    async fn read_partition_file_marker(store: &dyn IndexStore, path: &str) -> Result<u64> {
+        let reader = store.open_index_file(path).await?;
+        let batch = reader.read_range(0..1, None).await?;
+        let partition_ids = batch.column(0).as_primitive::<datatypes::UInt64Type>();
+        Ok(partition_ids.value(0))
+    }
+
+    async fn assert_partition_file_markers(
+        store: &dyn IndexStore,
+        partition_id: u64,
+        expected_marker: u64,
+    ) -> Result<()> {
+        assert_eq!(
+            read_partition_file_marker(store, &token_file_path(partition_id)).await?,
+            expected_marker
+        );
+        assert_eq!(
+            read_partition_file_marker(store, &posting_file_path(partition_id)).await?,
+            expected_marker
+        );
+        assert_eq!(
+            read_partition_file_marker(store, &doc_file_path(partition_id)).await?,
+            expected_marker
+        );
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_merge_index_files_remaps_staged_partitions_without_rename() -> Result<()> {
+        let index_dir = TempDir::default();
+        let object_store = Arc::new(ObjectStore::local());
+        let base_store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let store = Arc::new(NoRenameStore::new(base_store.clone()));
+        let partitions = vec![5_u64, 1_u64, (17_u64 << 32) | 2];
+        let metadata_builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            Vec::new(),
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        );
+
+        for partition_id in &partitions {
+            write_partition_files(
+                base_store.as_ref(),
+                *partition_id,
+                PartitionWriteTarget::Staged,
+            )
+            .await?;
+            metadata_builder
+                .write_part_metadata(base_store.as_ref(), *partition_id)
+                .await?;
+        }
+
+        merge_index_files(
+            object_store.as_ref(),
+            &index_dir.obj_path(),
+            store,
+            noop_progress(),
+        )
+        .await?;
+
+        let metadata_reader = base_store.open_index_file(METADATA_FILE).await?;
+        let metadata = &metadata_reader.schema().metadata;
+        let written_partitions: Vec<u64> = serde_json::from_str(
+            metadata
+                .get("partitions")
+                .expect("partitions missing from metadata"),
+        )?;
+        let mut expected_partitions = partitions.clone();
+        expected_partitions.sort_unstable();
+        expected_partitions.dedup();
+        let remapped_partitions = (0..expected_partitions.len() as u64).collect::<Vec<_>>();
+        assert_eq!(written_partitions, remapped_partitions);
+
+        for (new_id, old_id) in expected_partitions.iter().enumerate() {
+            assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?;
+            assert!(
+                base_store
+                    .open_index_file(&part_metadata_file_path(*old_id))
+                    .await
+                    .is_err(),
+                "partition metadata should be cleaned up after final metadata is written"
+            );
+            for suffix in PARTITION_FILE_SUFFIXES {
+                assert!(
+                    base_store
+                        .open_index_file(&staged_partition_file_path(*old_id, suffix))
+                        .await
+                        .is_err(),
+                    "staged partition files should be cleaned up after final metadata is written"
+                );
+            }
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_merge_index_files_rewrites_partial_final_files_from_staging() -> Result<()> {
+        let index_dir = TempDir::default();
+        let object_store = Arc::new(ObjectStore::local());
+        let base_store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let store = Arc::new(NoRenameStore::with_final_delete_tracking(
+            base_store.clone(),
+        ));
+        let partitions = vec![1_u64, 5_u64];
+        let metadata_builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            Vec::new(),
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        );
+
+        for partition_id in &partitions {
+            write_partition_files(
+                base_store.as_ref(),
+                *partition_id,
+                PartitionWriteTarget::Staged,
+            )
+            .await?;
+            metadata_builder
+                .write_part_metadata(base_store.as_ref(), *partition_id)
+                .await?;
+        }
+
+        for suffix in PARTITION_FILE_SUFFIXES {
+            write_partition_file_marker(base_store.as_ref(), &partition_file_path(1, suffix), 999)
+                .await?;
+        }
+
+        merge_index_files(
+            object_store.as_ref(),
+            &index_dir.obj_path(),
+            store.clone(),
+            noop_progress(),
+        )
+        .await?;
+
+        assert_partition_file_markers(base_store.as_ref(), 0, 1).await?;
+        assert_partition_file_markers(base_store.as_ref(), 1, 5).await?;
+        assert_eq!(
+            store.final_delete_count(),
+            0,
+            "merge should overwrite final partition files without deleting them first"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_distributed_from_existing_copies_existing_partitions_to_staging_and_finalizes()
+    -> Result<()> {
+        let object_store = Arc::new(ObjectStore::local());
+        let source_dir = TempDir::default();
+        let dest_dir = TempDir::default();
+        let source_store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            source_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            dest_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let merge_store = Arc::new(NoRenameStore::new(dest_store.clone()));
+        let fragment_mask = 7_u64 << 32;
+        let partitions = vec![fragment_mask | 5, fragment_mask | 1];
+
+        for partition_id in &partitions {
+            write_partition_files(
+                source_store.as_ref(),
+                *partition_id,
+                PartitionWriteTarget::Final,
+            )
+            .await?;
+        }
+
+        let builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            Some(source_store.clone()),
+            partitions.clone(),
+            TokenSetFormat::default(),
+            Some(fragment_mask),
+            RoaringBitmap::new(),
+        );
+        builder.write(dest_store.as_ref()).await?;
 
-        async fn open_index_file(&self, _name: &str) -> Result<Arc<dyn IndexReader>> {
-            Err(Error::not_supported(
-                "CountingStore does not support reading",
-            ))
+        for partition_id in &partitions {
+            assert_partition_file_markers(source_store.as_ref(), *partition_id, *partition_id)
+                .await?;
+            for suffix in PARTITION_FILE_SUFFIXES {
+                let staged_path = staged_partition_file_path(*partition_id, suffix);
+                assert_eq!(
+                    read_partition_file_marker(dest_store.as_ref(), &staged_path).await?,
+                    *partition_id
+                );
+                assert!(
+                    dest_store
+                        .open_index_file(&partition_file_path(*partition_id, suffix))
+                        .await
+                        .is_err(),
+                    "distributed existing partition should be staged instead of copied to root"
+                );
+            }
+            dest_store
+                .open_index_file(&part_metadata_file_path(*partition_id))
+                .await?;
         }
 
-        async fn copy_index_file(&self, _name: &str, _dest_store: &dyn IndexStore) -> Result<()> {
-            Err(Error::not_supported(
-                "CountingStore does not support copying",
-            ))
-        }
+        merge_index_files(
+            object_store.as_ref(),
+            &dest_dir.obj_path(),
+            merge_store,
+            noop_progress(),
+        )
+        .await?;
 
-        async fn rename_index_file(&self, _name: &str, _new_name: &str) -> Result<()> {
-            Err(Error::not_supported(
-                "CountingStore does not support renaming",
-            ))
+        let mut expected_partitions = partitions.clone();
+        expected_partitions.sort_unstable();
+        for (new_id, old_id) in expected_partitions.iter().enumerate() {
+            assert_partition_file_markers(dest_store.as_ref(), new_id as u64, *old_id).await?;
+            for suffix in PARTITION_FILE_SUFFIXES {
+                assert!(
+                    dest_store
+                        .open_index_file(&staged_partition_file_path(*old_id, suffix))
+                        .await
+                        .is_err(),
+                    "staged partition files should be cleaned after final metadata is written"
+                );
+            }
         }
 
-        async fn delete_index_file(&self, _name: &str) -> Result<()> {
-            Err(Error::not_supported(
-                "CountingStore does not support deleting",
-            ))
-        }
+        Ok(())
+    }
 
-        async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
-            Ok(vec![])
+    #[tokio::test]
+    async fn test_merge_index_files_keeps_staging_when_final_metadata_write_fails() -> Result<()> {
+        let index_dir = TempDir::default();
+        let object_store = Arc::new(ObjectStore::local());
+        let base_store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let failing_store = Arc::new(FailMetadataStore::new(base_store.clone()));
+        let partitions = vec![1_u64, 5_u64];
+        let metadata_builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            Vec::new(),
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        );
+
+        for partition_id in &partitions {
+            write_partition_files(
+                base_store.as_ref(),
+                *partition_id,
+                PartitionWriteTarget::Staged,
+            )
+            .await?;
+            metadata_builder
+                .write_part_metadata(base_store.as_ref(), *partition_id)
+                .await?;
         }
-    }
 
-    fn collect_group_starts(config: PostingGroupConfig, sizes: &[usize]) -> Vec<u32> {
-        let mut acc = PostingGroupAccumulator::new(config);
-        for &size in sizes {
-            acc.push(size);
+        let err = merge_index_files(
+            object_store.as_ref(),
+            &index_dir.obj_path(),
+            failing_store,
+            noop_progress(),
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            err.to_string().contains("metadata write failure"),
+            "expected injected metadata failure, got: {err}"
+        );
+
+        for partition_id in &partitions {
+            base_store
+                .open_index_file(&part_metadata_file_path(*partition_id))
+                .await?;
+            for suffix in PARTITION_FILE_SUFFIXES {
+                let staged_path = staged_partition_file_path(*partition_id, suffix);
+                assert_eq!(
+                    read_partition_file_marker(base_store.as_ref(), &staged_path).await?,
+                    *partition_id
+                );
+            }
         }
-        acc.into_starts()
-    }
 
-    #[test]
-    fn test_group_accumulator_cuts_on_target_bytes() {
-        let config = PostingGroupConfig {
-            target_bytes: 100,
-            max_tokens: 1000,
-        };
-        // 40+40 -> cut at 80? no, 80 < 100; third 40 reaches 120 >= 100 -> cut.
-        // So group 0 = tokens [0,3), then a new group starts at token 3.
-        let starts = collect_group_starts(config, &[40, 40, 40, 10, 10]);
-        assert_eq!(starts, vec![0, 3]);
+        Ok(())
     }
 
-    #[test]
-    fn test_group_accumulator_cuts_on_max_tokens() {
-        let config = PostingGroupConfig {
-            target_bytes: 1_000_000,
-            max_tokens: 2,
-        };
-        // Byte target never reached; cap of 2 forces a cut every 2 tokens.
-        let starts = collect_group_starts(config, &[1, 1, 1, 1, 1]);
-        assert_eq!(starts, vec![0, 2, 4]);
-    }
+    #[tokio::test]
+    async fn test_distributed_build_writes_partition_data_to_staging() -> Result<()> {
+        let index_dir = TempDir::default();
+        let object_store = ObjectStore::local();
+        let store = Arc::new(LanceIndexStore::new(
+            object_store.into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
 
-    #[test]
-    fn test_group_accumulator_clamps_oversized_term() {
-        let config = PostingGroupConfig {
-            target_bytes: 100,
-            max_tokens: 64,
-        };
-        // A term larger than the target that *starts* a group occupies that
-        // group alone ([1, 2) here), so a single huge posting list is never
-        // forced to share a cache entry. Token 0 (==100) closes its own group
-        // first; the trailing small terms regroup after the big one.
-        let starts = collect_group_starts(config, &[100, 5000, 10, 10]);
-        assert_eq!(starts, vec![0, 1, 2]);
+        let fragment_mask = 7_u64 << 32;
+        let batch = make_doc_batch("hello world", fragment_mask);
+        let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)]));
+        let stream = Box::pin(stream);
+        let mut builder = InvertedIndexBuilder::new_with_fragment_mask(
+            InvertedIndexParams::default(),
+            Some(fragment_mask),
+        );
+        builder.update(stream, store.as_ref(), None).await?;
 
-        // A huge term encountered mid-group is absorbed and closes that group;
-        // we never split one term across groups.
-        let starts = collect_group_starts(config, &[10, 10, 5000, 10, 10]);
-        assert_eq!(starts, vec![0, 3]);
-    }
+        let part_metadata_files =
+            list_metadata_files(&ObjectStore::local(), &index_dir.obj_path()).await?;
+        assert_eq!(part_metadata_files.len(), 1);
+        assert!(
+            part_metadata_files[0].starts_with("staging/part_"),
+            "partition metadata should be written to staging"
+        );
+        let reader = store.open_index_file(&part_metadata_files[0]).await?;
+        let partition_ids: Vec<u64> = serde_json::from_str(
+            reader
+                .schema()
+                .metadata
+                .get("partitions")
+                .expect("partitions missing from metadata"),
+        )?;
+        assert_eq!(partition_ids.len(), 1);
+        let partition_id = partition_ids[0];
+
+        store
+            .open_index_file(&staged_partition_file_path(partition_id, TOKENS_FILE))
+            .await?;
+        assert!(
+            store
+                .open_index_file(&partition_file_path(partition_id, METADATA_FILE))
+                .await
+                .is_err(),
+            "distributed build-only metadata should not be written to root partition metadata paths"
+        );
+        assert!(
+            store
+                .open_index_file(&token_file_path(partition_id))
+                .await
+                .is_err(),
+            "distributed build-only data should not be written to final partition paths"
+        );
 
-    #[test]
-    fn test_group_accumulator_empty_and_single() {
-        let config = PostingGroupConfig::default();
-        assert_eq!(collect_group_starts(config, &[]), Vec::<u32>::new());
-        assert_eq!(collect_group_starts(config, &[10]), vec![0]);
+        Ok(())
     }
 
     #[tokio::test]
-    async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> {
-        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        for doc_id in 0..3u64 {
-            builder.docs.append(doc_id, 1);
-        }
-
-        for doc_id in 0..3u32 {
-            let mut posting_list = PostingListBuilder::new(false);
-            posting_list.add(doc_id, PositionRecorder::Count(1));
-            builder.posting_lists.push(posting_list);
-        }
+    async fn test_merge_index_files_is_noop_when_metadata_exists() -> Result<()> {
+        let index_dir = TempDir::default();
+        let object_store = Arc::new(ObjectStore::local());
+        let store: Arc<dyn IndexStore> = Arc::new(LanceIndexStore::new(
+            object_store.clone(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let metadata_builder = InvertedIndexBuilder::from_existing_index(
+            InvertedIndexParams::default(),
+            None,
+            vec![42],
+            TokenSetFormat::default(),
+            None,
+            RoaringBitmap::new(),
+        );
+        metadata_builder
+            .write_metadata(store.as_ref(), &[42])
+            .await?;
 
-        let store = CountingStore::new();
-        let docs = Arc::new(std::mem::take(&mut builder.docs));
-        builder.write_posting_lists(&store, docs).await?;
+        merge_index_files(
+            object_store.as_ref(),
+            &index_dir.obj_path(),
+            store,
+            noop_progress(),
+        )
+        .await?;
 
-        assert_eq!(store.write_count(), 1);
         Ok(())
     }
 
@@ -2412,6 +3273,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format,
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: params.block_size,
             },
         )
         .await?;
@@ -2435,6 +3297,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format,
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: params.block_size,
             },
         )
         .await?;
@@ -2458,7 +3321,8 @@ mod tests {
             token_set_format,
             None,
             RoaringBitmap::new(),
-        );
+        )
+        .with_format_version(InvertedListFormatVersion::V1);
         builder.write(dest_store.as_ref()).await?;
 
         let metadata_reader = dest_store.open_index_file(METADATA_FILE).await?;
@@ -2522,6 +3386,14 @@ mod tests {
             .await?;
 
         let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
+        let derived_params = index.derive_index_params()?;
+        let derived_params: InvertedIndexParams =
+            serde_json::from_str(derived_params.params.as_deref().unwrap())?;
+        assert_eq!(
+            derived_params.format_version,
+            Some(InvertedListFormatVersion::V1)
+        );
+
         let schema = Arc::new(Schema::new(vec![
             Field::new("doc", DataType::Utf8, true),
             Field::new(ROW_ID, DataType::UInt64, false),
@@ -2820,7 +3692,6 @@ mod tests {
                 }
             })
             .collect::<Vec<_>>();
-
         let read_start = tags
             .iter()
             .position(|e| e == "start:read_partition_metadata")
@@ -2858,8 +3729,8 @@ mod tests {
         );
         assert_eq!(
             remap_progress.last().copied().unwrap_or_default(),
-            12,
-            "expected remap_partition_files progress to cover both rename phases"
+            6,
+            "expected remap_partition_files progress to cover staged-to-final copies"
         );
         assert!(
             tags.iter().any(|e| e == "progress:write_merged_metadata"),
@@ -2909,6 +3780,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
@@ -2940,6 +3812,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
@@ -2978,6 +3851,7 @@ mod tests {
                 fragment_mask: None,
                 token_set_format: TokenSetFormat::default(),
                 worker_memory_limit_bytes: u64::MAX,
+                block_size: InvertedIndexParams::default().block_size,
             },
         )
         .await?;
@@ -3026,27 +3900,208 @@ mod tests {
         assert_eq!(resolve_worker_memory_limit_bytes(&params, 16), 256 << 20);
     }
 
+    fn tail_with_docs(id: u64, num_docs: u64) -> TailPartition {
+        let mut builder = InnerBuilder::new(id, false, TokenSetFormat::default());
+        let token = builder.tokens.add(format!("token{}", id));
+        builder
+            .posting_lists
+            .resize_with(builder.tokens.len(), || PostingListBuilder::new(false));
+        for row in 0..num_docs {
+            let doc = builder.docs.append(row, 1);
+            builder.posting_lists[token as usize].add(doc, PositionRecorder::Count(1));
+        }
+        TailPartition { builder }
+    }
+
     #[test]
-    fn test_merge_all_tail_partitions_combines_everything() -> Result<()> {
-        let merged = merge_all_tail_partitions(vec![
-            TailPartition {
-                builder: InnerBuilder::new(0, false, TokenSetFormat::default()),
-            },
-            TailPartition {
-                builder: InnerBuilder::new(1, false, TokenSetFormat::default()),
-            },
-            TailPartition {
-                builder: InnerBuilder::new(2, false, TokenSetFormat::default()),
-            },
-        ])?;
+    fn test_merge_all_tail_partitions_combines_under_budget() -> Result<()> {
+        let merged = merge_all_tail_partitions(
+            vec![
+                tail_with_docs(0, 4),
+                tail_with_docs(1, 4),
+                tail_with_docs(2, 4),
+            ],
+            u64::MAX,
+        )?;
+        assert_eq!(merged.len(), 1);
+        assert_eq!(merged[0].id(), 0);
+        assert_eq!(merged[0].docs.len(), 12);
+        Ok(())
+    }
 
-        assert_eq!(merged.expect("merged builder should exist").id(), 0);
+    #[test]
+    fn test_merge_all_tail_partitions_splits_on_memory_budget() -> Result<()> {
+        let tails = vec![
+            tail_with_docs(0, 64),
+            tail_with_docs(1, 64),
+            tail_with_docs(2, 64),
+            tail_with_docs(3, 64),
+        ];
+        let single = tails[0].builder.memory_size();
+        // A budget below two builders' footprint must keep them separate.
+        let merged = merge_all_tail_partitions(tails, single + 1)?;
+        assert_eq!(merged.len(), 4);
+        assert!(merged.iter().all(|builder| builder.docs.len() == 64));
         Ok(())
     }
 
     #[test]
     fn test_merge_all_tail_partitions_returns_none_for_empty_input() -> Result<()> {
-        assert!(merge_all_tail_partitions(Vec::new())?.is_none());
+        assert!(merge_all_tail_partitions(Vec::new(), u64::MAX)?.is_empty());
+        Ok(())
+    }
+
+    /// Build an inverted index over `batches` with an explicit worker/memory
+    /// layout and return it loaded, so tests can compare query behavior
+    /// across partition shapes of the same corpus.
+    async fn build_fuzzy_corpus_index(
+        batches: Vec<RecordBatch>,
+        num_workers: usize,
+        memory_limit_mb: u64,
+    ) -> Result<(TempDir, Arc<InvertedIndex>)> {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let schema = batches[0].schema();
+        let stream =
+            RecordBatchStreamAdapter::new(schema, stream::iter(batches.into_iter().map(Ok)));
+        let params =
+            InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English)
+                .with_position(false)
+                .remove_stop_words(false)
+                .stem(false)
+                .max_token_length(None)
+                .num_workers(num_workers)
+                .memory_limit_mb(memory_limit_mb);
+        let mut builder = InvertedIndexBuilder::new(params);
+        builder
+            .update(Box::pin(stream), store.as_ref(), None)
+            .await?;
+        let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?;
+        // The caller keeps the TempDir alive for as long as it queries the
+        // loaded index.
+        Ok((index_dir, index))
+    }
+
+    /// Regression test for tail-partition splitting vs fuzzy queries
+    /// (https://github.com/lance-format/lance/pull/7601#pullrequestreview):
+    /// splitting the leftover tails into budget-sized partitions must not
+    /// change fuzzy results while `max_expansions` is not the binding
+    /// constraint. Every doc carries one member of two dense fuzzy families
+    /// plus unique filler tokens, so a small worker memory budget forces
+    /// flushes and a tail split, while a fuzzy query matches every doc.
+    #[tokio::test]
+    async fn test_tail_partition_split_preserves_fuzzy_results() -> Result<()> {
+        use std::collections::HashMap;
+
+        use crate::prefilter::NoFilter;
+        use crate::scalar::inverted::document_tokenizer::DocType;
+        use crate::scalar::inverted::query::{FtsSearchParams, Operator, Tokens};
+
+        const NUM_DOCS: usize = 4000;
+        const DOCS_PER_BATCH: usize = 20;
+        let alpha_variants = ["alpha", "alphb", "alphc", "alphd", "alphe"];
+        let beta_variants = ["beta", "betb", "betc", "betd", "bete"];
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("doc", DataType::Utf8, true),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let batches = (0..NUM_DOCS / DOCS_PER_BATCH)
+            .map(|batch_idx| {
+                let mut docs = Vec::with_capacity(DOCS_PER_BATCH);
+                let mut row_ids = Vec::with_capacity(DOCS_PER_BATCH);
+                for row in 0..DOCS_PER_BATCH {
+                    let i = batch_idx * DOCS_PER_BATCH + row;
+                    // 8 unique filler tokens per doc grow the vocabulary so
+                    // the corpus comfortably exceeds the small worker budget.
+                    docs.push(format!(
+                        "{} {} f{i}a f{i}b f{i}c f{i}d f{i}e f{i}f f{i}g f{i}h",
+                        alpha_variants[i % alpha_variants.len()],
+                        beta_variants[i % beta_variants.len()],
+                    ));
+                    row_ids.push(i as u64);
+                }
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(StringArray::from(docs)),
+                        Arc::new(UInt64Array::from(row_ids)),
+                    ],
+                )
+                .unwrap()
+            })
+            .collect::<Vec<_>>();
+
+        // Reference shape: everything in one partition.
+        let (_ref_dir, ref_index) = build_fuzzy_corpus_index(batches.clone(), 1, 1024).await?;
+        assert_eq!(
+            ref_index.partitions.len(),
+            1,
+            "reference build should stay in a single partition"
+        );
+
+        // Tail-heavy shape: a 1MB budget across 2 workers forces flushes and
+        // splits the leftover tails by the same budget.
+        let (_split_dir, split_index) = build_fuzzy_corpus_index(batches, 2, 1).await?;
+        assert!(
+            split_index.partitions.len() > 1,
+            "small worker budget should produce multiple partitions, got {}",
+            split_index.partitions.len()
+        );
+
+        async fn fuzzy_search(
+            index: &InvertedIndex,
+            tokens: &[&str],
+            operator: Operator,
+        ) -> HashMap<u64, f32> {
+            let tokens = Arc::new(Tokens::new(
+                tokens.iter().map(|t| t.to_string()).collect(),
+                DocType::Text,
+            ));
+            // max_expansions=50 is far above the 10 family variants, so the
+            // per-partition vs global cap distinction cannot bind here.
+            let params = Arc::new(
+                FtsSearchParams::new()
+                    .with_limit(Some(NUM_DOCS))
+                    .with_fuzziness(Some(1))
+                    .with_max_expansions(50),
+            );
+            let (row_ids, scores) = index
+                .bm25_search(
+                    tokens,
+                    params,
+                    operator,
+                    Arc::new(NoFilter),
+                    Arc::new(NoOpMetricsCollector),
+                    None,
+                )
+                .await
+                .unwrap();
+            row_ids.into_iter().zip(scores).collect()
+        }
+
+        for (tokens, operator) in [
+            (vec!["alphx"], Operator::Or),
+            (vec!["alphx", "betx"], Operator::Or),
+            (vec!["alphx", "betx"], Operator::And),
+        ] {
+            let reference = fuzzy_search(&ref_index, &tokens, operator).await;
+            let split = fuzzy_search(&split_index, &tokens, operator).await;
+            assert_eq!(
+                reference.len(),
+                NUM_DOCS,
+                "every doc carries a family variant, {tokens:?} {operator:?} should match all"
+            );
+            assert_eq!(
+                reference, split,
+                "fuzzy {operator:?} results must not depend on the tail partition shape for {tokens:?}"
+            );
+        }
+
         Ok(())
     }
 
@@ -3068,10 +4123,15 @@ mod tests {
         let second_doc = second.docs.append(20, 2);
         second.posting_lists[world as usize].add(second_doc, PositionRecorder::Count(2));
 
-        let merged = merge_tail_partition_group(vec![
-            TailPartition { builder: first },
-            TailPartition { builder: second },
-        ])?;
+        let merged = merge_all_tail_partitions(
+            vec![
+                TailPartition { builder: first },
+                TailPartition { builder: second },
+            ],
+            u64::MAX,
+        )?;
+        assert_eq!(merged.len(), 1);
+        let merged = &merged[0];
 
         assert_eq!(merged.id(), 0);
         assert_eq!(merged.docs.len(), 2);
@@ -3088,6 +4148,118 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_merge_from_after_remap_does_not_panic() {
+        // `first` is the merge accumulator. Give it three tokens, then remap away the
+        // middle one, mirroring filter_old_data dropping a token whose postings emptied.
+        let mut first = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token in ["a", "b", "c"] {
+            first.tokens.add(token.to_owned());
+        }
+        first
+            .posting_lists
+            .resize_with(first.tokens.len(), || PostingListBuilder::new(false));
+        let first_doc = first.docs.append(10, 1);
+        first.posting_lists[0].add(first_doc, PositionRecorder::Count(1)); // "a"
+        first.posting_lists[2].add(first_doc, PositionRecorder::Count(1)); // "c"
+
+        // Remove token "b" (id 1) and compact its (empty) posting list to match.
+        first.tokens.remap(&[1]);
+        first.posting_lists.remove(1);
+        assert_eq!(first.tokens.len(), first.posting_lists.len());
+
+        // `second` contributes a brand-new token absent from `first`. Before the fix,
+        // get_or_add returned the stale next_id, indexing past posting_lists.
+        let mut second = InnerBuilder::new(1, false, TokenSetFormat::default());
+        let zeta = second.tokens.add("zeta".to_owned());
+        second
+            .posting_lists
+            .resize_with(second.tokens.len(), || PostingListBuilder::new(false));
+        let second_doc = second.docs.append(20, 1);
+        second.posting_lists[zeta as usize].add(second_doc, PositionRecorder::Count(1));
+
+        first.merge_from(second).unwrap();
+
+        assert_eq!(first.tokens.len(), 3);
+        assert_eq!(first.posting_lists.len(), 3);
+        let zeta_id = first.tokens.get("zeta").expect("zeta should be merged in");
+        assert!((zeta_id as usize) < first.posting_lists.len());
+    }
+
+    // FST token file with a stale next_id (above the token count), as a pre-#7115 writer left.
+    async fn write_stale_next_id_token_file(store: &dyn IndexStore, partition_id: u64) {
+        let mut tokens = TokenSet::default();
+        tokens.add("alpha".to_owned());
+        tokens.add("gamma".to_owned());
+        assert_eq!(tokens.len(), 2);
+        tokens.next_id = 9;
+        let batch = tokens.to_batch(TokenSetFormat::Fst).unwrap();
+        let mut writer = store
+            .new_index_file(&token_file_path(partition_id), batch.schema())
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    // load_fst recomputes next_id from the token count rather than trusting the persisted value.
+    #[tokio::test]
+    async fn test_load_fst_recomputes_stale_next_id() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_stale_next_id_token_file(store.as_ref(), 0).await;
+        let reader = store.open_index_file(&token_file_path(0)).await.unwrap();
+        let tokens = TokenSet::load(reader, TokenSetFormat::Fst).await.unwrap();
+        assert_eq!(tokens.len(), 2);
+        assert_eq!(tokens.next_id(), 2);
+    }
+
+    // A stale next_id loaded from disk must not leak an out-of-range token id into a merge.
+    #[tokio::test]
+    async fn test_merge_with_stale_next_id_token_file_does_not_panic() {
+        let index_dir = TempDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            index_dir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_stale_next_id_token_file(store.as_ref(), 0).await;
+        let reader = store.open_index_file(&token_file_path(0)).await.unwrap();
+        let tokens = TokenSet::load(reader, TokenSetFormat::Fst)
+            .await
+            .unwrap()
+            .into_mutable();
+
+        let mut first = InnerBuilder::new(0, false, TokenSetFormat::Fst);
+        first.set_tokens(tokens);
+        first
+            .posting_lists
+            .resize_with(first.tokens.len(), || PostingListBuilder::new(false));
+        let doc = first.docs.append(10, 1);
+        first.posting_lists[0].add(doc, PositionRecorder::Count(1));
+        first.posting_lists[1].add(doc, PositionRecorder::Count(1));
+
+        let mut second = InnerBuilder::new(1, false, TokenSetFormat::Fst);
+        let zeta = second.tokens.add("zeta".to_owned());
+        second
+            .posting_lists
+            .resize_with(second.tokens.len(), || PostingListBuilder::new(false));
+        let second_doc = second.docs.append(20, 1);
+        second.posting_lists[zeta as usize].add(second_doc, PositionRecorder::Count(1));
+
+        first.merge_from(second).unwrap();
+        assert_eq!(first.tokens.len(), 3);
+        assert_eq!(first.posting_lists.len(), 3);
+        let zeta_id = first.tokens.get("zeta").expect("zeta should be merged in");
+        assert!((zeta_id as usize) < first.posting_lists.len());
+    }
+
     #[tokio::test]
     async fn test_update_index_returns_worker_error_when_workers_exit_during_dispatch() {
         let num_batches = (*LANCE_FTS_NUM_SHARDS * 2 + 1) as u64;
@@ -3190,7 +4362,10 @@ mod tests {
         // Remap the index via the ScalarIndex trait method
         use crate::scalar::ScalarIndex;
         let mapping = HashMap::from([(0u64, Some(50 << 32))]);
-        index.remap(&mapping, dest_store.as_ref()).await.unwrap();
+        index
+            .remap(&RowAddrRemap::direct(mapping), dest_store.as_ref())
+            .await
+            .unwrap();
 
         // Reload from dest and verify deleted fragments are preserved
         let remapped_index = InvertedIndex::load(dest_store.clone(), None, &LanceCache::no_cache())
diff --git a/rust/lance-index/src/scalar/inverted/cache_codec.rs b/rust/lance-index/src/scalar/inverted/cache_codec.rs
index 74cfc98ef7b..e59b22dc501 100644
--- a/rust/lance-index/src/scalar/inverted/cache_codec.rs
+++ b/rust/lance-index/src/scalar/inverted/cache_codec.rs
@@ -4,16 +4,28 @@
 //! Cache codec impls for FTS index entries.
 //!
 //! Serializes [`PostingList`] and [`Positions`] cache values for persistent
-//! cache backends. The format is a small variant tag plus a JSON header for
-//! scalar metadata, with Arrow-backed payload sections written as zero-copy
-//! Arrow IPC streams via [`lance_arrow::ipc`]. The raw byte buffer inside
-//! [`SharedPositionStream`] is written via [`write_len_prefixed_bytes`] and
-//! read back via [`read_len_prefixed_bytes_at`] -- both zero-copy slices into
-//! the input `Bytes` allocation.
+//! cache backends, behind the stabilized envelope written by
+//! [`CacheCodec`](lance_core::cache::CacheCodec).
 //!
-//! This is the FTS counterpart of `partition_serde.rs` for vector indices.
+//! Every variant uses a protobuf header (see `protos-cache/cache.proto`, with the
+//! tail/position codecs and position-storage kind as proto enums) followed by
+//! 64-byte-aligned Arrow IPC sections and, where applicable, raw blobs:
+//!
+//! - the compressed posting list: an IPC section for `blocks`, then the
+//!   position sections (legacy IPC, or shared block-offsets IPC + a raw blob of
+//!   the [`SharedPositionStream`] byte buffer, which has its own portable
+//!   encoding), then an optional impact IPC section;
+//! - the plain posting list: an IPC section of `(row_ids, frequencies)`, then
+//!   an optional legacy position IPC section;
+//! - a packed posting-list group: one IPC section containing the original
+//!   `List<LargeBinary>` posting rows and optional impact rows; prewarmed groups
+//!   omit score/length metadata and inject it from the posting reader into
+//!   query-local views;
+//! - the standalone [`Positions`] codec: the position sections alone.
+//!
+//! All sections read back zero-copy via [`lance_arrow::ipc`]. This is the FTS
+//! counterpart of `partition_serde.rs` for vector indices.
 
-use std::io::Write;
 use std::sync::Arc;
 
 use arrow_array::cast::AsArray;
@@ -22,19 +34,22 @@ use arrow_array::{
     Array, Float32Array, LargeBinaryArray, ListArray, RecordBatch, UInt32Array, UInt64Array,
 };
 use arrow_schema::{DataType, Field, Schema};
-use bytes::Bytes;
-use lance_arrow::ipc::{
-    read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream,
-    write_len_prefixed_bytes,
-};
-use lance_core::cache::CacheCodecImpl;
+use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
 use lance_core::{Error, Result};
-use serde::{Deserialize, Serialize};
 
+use crate::cache_pb::{
+    CompressedPostingHeader, PlainPostingHeader, PositionStorage as PbPositionStorage,
+    PositionStreamCodec as PbPositionStreamCodec, PositionsHeader, PostingListGroupHeader,
+    PostingTailCodec as PbPostingTailCodec,
+};
+
+use super::impact::ImpactSkipData;
 use super::index::{
     CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec,
-    Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream,
+    Positions, PostingList, PostingListGroup, PostingListGroupStorage, PostingTailCodec,
+    SharedPositionStream,
 };
+use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size};
 
 // ---------------------------------------------------------------------------
 // Tags
@@ -42,87 +57,63 @@ use super::index::{
 
 const POSTING_VARIANT_PLAIN: u8 = 0;
 const POSTING_VARIANT_COMPRESSED: u8 = 1;
-
-const POSITIONS_TAG_NONE: u8 = 0;
-const POSITIONS_TAG_LEGACY: u8 = 1;
-const POSITIONS_TAG_SHARED: u8 = 2;
-
-const POSTING_TAIL_CODEC_FIXED32: u8 = 0;
-const POSTING_TAIL_CODEC_VARINT_DELTA: u8 = 1;
-
-const POSITION_STREAM_CODEC_VARINT_DOC_DELTA: u8 = 0;
-const POSITION_STREAM_CODEC_PACKED_DELTA: u8 = 1;
+const GROUP_VARIANT_MATERIALIZED: u8 = 0;
+const GROUP_VARIANT_PACKED: u8 = 1;
 
 // ---------------------------------------------------------------------------
-// Codec enum byte mappings
+// Codec enum mappings
 // ---------------------------------------------------------------------------
 
-fn posting_tail_codec_to_u8(c: PostingTailCodec) -> u8 {
+// Posting lists carry their discriminants as protobuf enums in the header;
+// these map to/from the in-memory Rust enums.
+
+fn posting_tail_codec_to_proto(c: PostingTailCodec) -> PbPostingTailCodec {
     match c {
-        PostingTailCodec::Fixed32 => POSTING_TAIL_CODEC_FIXED32,
-        PostingTailCodec::VarintDelta => POSTING_TAIL_CODEC_VARINT_DELTA,
+        PostingTailCodec::Fixed32 => PbPostingTailCodec::Fixed32,
+        PostingTailCodec::VarintDelta => PbPostingTailCodec::VarintDelta,
     }
 }
 
-fn u8_to_posting_tail_codec(v: u8) -> Result<PostingTailCodec> {
-    match v {
-        POSTING_TAIL_CODEC_FIXED32 => Ok(PostingTailCodec::Fixed32),
-        POSTING_TAIL_CODEC_VARINT_DELTA => Ok(PostingTailCodec::VarintDelta),
-        _ => Err(Error::io(format!("unknown posting tail codec: {v}"))),
+fn proto_to_posting_tail_codec(c: PbPostingTailCodec) -> PostingTailCodec {
+    match c {
+        PbPostingTailCodec::Fixed32 => PostingTailCodec::Fixed32,
+        PbPostingTailCodec::VarintDelta => PostingTailCodec::VarintDelta,
     }
 }
 
-fn position_stream_codec_to_u8(c: PositionStreamCodec) -> u8 {
+fn posting_tail_codec_to_tag(c: PostingTailCodec) -> u8 {
     match c {
-        PositionStreamCodec::VarintDocDelta => POSITION_STREAM_CODEC_VARINT_DOC_DELTA,
-        PositionStreamCodec::PackedDelta => POSITION_STREAM_CODEC_PACKED_DELTA,
+        PostingTailCodec::Fixed32 => 0,
+        PostingTailCodec::VarintDelta => 1,
     }
 }
 
-fn u8_to_position_stream_codec(v: u8) -> Result<PositionStreamCodec> {
-    match v {
-        POSITION_STREAM_CODEC_VARINT_DOC_DELTA => Ok(PositionStreamCodec::VarintDocDelta),
-        POSITION_STREAM_CODEC_PACKED_DELTA => Ok(PositionStreamCodec::PackedDelta),
-        _ => Err(Error::io(format!("unknown position stream codec: {v}"))),
+fn posting_tail_codec_from_tag(tag: u8) -> Result<PostingTailCodec> {
+    match tag {
+        0 => Ok(PostingTailCodec::Fixed32),
+        1 => Ok(PostingTailCodec::VarintDelta),
+        other => Err(Error::io(format!(
+            "unknown packed posting tail codec: {other}"
+        ))),
     }
 }
 
-// ---------------------------------------------------------------------------
-// Header / tag I/O helpers (mirrors partition_serde.rs)
-// ---------------------------------------------------------------------------
-
-fn write_json_header(writer: &mut dyn Write, header: &impl Serialize) -> Result<()> {
-    let bytes = serde_json::to_vec(header)?;
-    write_len_prefixed_bytes(writer, &bytes)?;
-    Ok(())
-}
-
-fn read_json_header<T: serde::de::DeserializeOwned>(data: &Bytes, offset: &mut usize) -> Result<T> {
-    let bytes = read_len_prefixed_bytes_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
-    serde_json::from_slice(&bytes)
-        .map_err(|e| Error::io(format!("failed to deserialize cache header: {e}")))
-}
-
-fn write_u8(writer: &mut dyn Write, value: u8) -> Result<()> {
-    writer
-        .write_all(&[value])
-        .map_err(|e| Error::io(format!("failed to write tag byte: {e}")))
+fn position_stream_codec_to_proto(c: PositionStreamCodec) -> PbPositionStreamCodec {
+    match c {
+        PositionStreamCodec::VarintDocDelta => PbPositionStreamCodec::VarintDocDelta,
+        PositionStreamCodec::PackedDelta => PbPositionStreamCodec::PackedDelta,
+    }
 }
 
-fn read_u8(data: &Bytes, offset: &mut usize) -> Result<u8> {
-    let bytes = data.as_ref();
-    if *offset >= bytes.len() {
-        return Err(Error::io(
-            "truncated cache entry: missing tag byte".to_string(),
-        ));
+fn proto_to_position_stream_codec(c: PbPositionStreamCodec) -> PositionStreamCodec {
+    match c {
+        PbPositionStreamCodec::VarintDocDelta => PositionStreamCodec::VarintDocDelta,
+        PbPositionStreamCodec::PackedDelta => PositionStreamCodec::PackedDelta,
     }
-    let v = bytes[*offset];
-    *offset += 1;
-    Ok(v)
 }
 
 // ---------------------------------------------------------------------------
-// Position storage serde (shared by PostingList variants and Positions)
+// Position storage sections (shared by PostingList variants and Positions)
 // ---------------------------------------------------------------------------
 
 const POSITION_LIST_COLUMN: &str = "position_list";
@@ -130,34 +121,39 @@ const BLOCK_OFFSETS_COLUMN: &str = "block_offsets";
 const ROW_IDS_COLUMN: &str = "row_ids";
 const FREQUENCIES_COLUMN: &str = "frequencies";
 const BLOCKS_COLUMN: &str = "blocks";
+const IMPACTS_COLUMN: &str = "impacts";
+
+fn legacy_positions_batch(list: &ListArray) -> Result<RecordBatch> {
+    let schema = Arc::new(Schema::new(vec![Field::new(
+        POSITION_LIST_COLUMN,
+        list.data_type().clone(),
+        list.is_nullable(),
+    )]));
+    Ok(RecordBatch::try_new(schema, vec![Arc::new(list.clone())])?)
+}
 
-#[derive(Serialize, Deserialize)]
-struct SharedPositionsHeader {
-    codec: u8,
+fn read_legacy_positions(r: &mut CacheEntryReader<'_>) -> Result<ListArray> {
+    let batch = r.read_ipc()?;
+    Ok(batch
+        .column_by_name(POSITION_LIST_COLUMN)
+        .ok_or_else(|| Error::io("legacy position column is missing".to_string()))?
+        .as_any()
+        .downcast_ref::<ListArray>()
+        .ok_or_else(|| Error::io("legacy position column is not a ListArray".to_string()))?
+        .clone())
 }
 
-fn write_position_storage(
-    writer: &mut dyn Write,
+/// Write the position sections (the bytes after the header) for `storage`. The
+/// caller's header proto carries the storage kind and shared-stream codec.
+fn write_position_sections(
+    w: &mut CacheEntryWriter<'_>,
     storage: &CompressedPositionStorage,
 ) -> Result<()> {
     match storage {
         CompressedPositionStorage::LegacyPerDoc(list) => {
-            write_u8(writer, POSITIONS_TAG_LEGACY)?;
-            let schema = Arc::new(Schema::new(vec![Field::new(
-                POSITION_LIST_COLUMN,
-                list.data_type().clone(),
-                list.is_nullable(),
-            )]));
-            let batch = RecordBatch::try_new(schema, vec![Arc::new(list.clone())])?;
-            write_ipc_stream(&batch, writer)?;
+            w.write_ipc(&legacy_positions_batch(list)?)?;
         }
         CompressedPositionStorage::SharedStream(stream) => {
-            write_u8(writer, POSITIONS_TAG_SHARED)?;
-            let header = SharedPositionsHeader {
-                codec: position_stream_codec_to_u8(stream.codec()),
-            };
-            write_json_header(writer, &header)?;
-
             let offsets = UInt32Array::from(stream.block_offsets().to_vec());
             let schema = Arc::new(Schema::new(vec![Field::new(
                 BLOCK_OFFSETS_COLUMN,
@@ -165,55 +161,43 @@ fn write_position_storage(
                 false,
             )]));
             let batch = RecordBatch::try_new(schema, vec![Arc::new(offsets)])?;
-            write_ipc_stream(&batch, writer)?;
-
-            write_len_prefixed_bytes(writer, stream.bytes())?;
+            w.write_ipc(&batch)?;
+            w.write_raw(stream.bytes())?;
         }
     }
     Ok(())
 }
 
-fn read_position_storage(
-    data: &Bytes,
-    offset: &mut usize,
-    tag: u8,
-) -> Result<CompressedPositionStorage> {
-    match tag {
-        POSITIONS_TAG_LEGACY => {
-            let batch =
-                read_ipc_stream_single_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
-            let list = batch
-                .column(0)
-                .as_any()
-                .downcast_ref::<ListArray>()
-                .ok_or_else(|| Error::io("legacy position column is not a ListArray".to_string()))?
-                .clone();
-            Ok(CompressedPositionStorage::LegacyPerDoc(list))
-        }
-        POSITIONS_TAG_SHARED => {
-            let header: SharedPositionsHeader = read_json_header(data, offset)?;
-            let codec = u8_to_position_stream_codec(header.codec)?;
-
-            let batch =
-                read_ipc_stream_single_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
+/// Read the position sections for the given `storage` kind and (for shared
+/// streams) `stream_codec`. Returns `None` only when `storage` is
+/// [`PbPositionStorage::None`].
+fn read_position_sections(
+    r: &mut CacheEntryReader<'_>,
+    storage: PbPositionStorage,
+    stream_codec: PositionStreamCodec,
+) -> Result<Option<CompressedPositionStorage>> {
+    match storage {
+        PbPositionStorage::None => Ok(None),
+        PbPositionStorage::Legacy => Ok(Some(CompressedPositionStorage::LegacyPerDoc(
+            read_legacy_positions(r)?,
+        ))),
+        PbPositionStorage::Shared => {
+            let batch = r.read_ipc()?;
             let block_offsets = batch
-                .column(0)
+                .column_by_name(BLOCK_OFFSETS_COLUMN)
+                .ok_or_else(|| Error::io("block_offsets column is missing".to_string()))?
                 .as_primitive_opt::<UInt32Type>()
                 .ok_or_else(|| Error::io("block_offsets column is not UInt32".to_string()))?
                 .values()
                 .to_vec();
-
-            // Zero copy: read_len_prefixed_bytes_at returns a Bytes slice
-            // backed by the same allocation as `data`, and SharedPositionStream
-            // now stores its byte buffer as Bytes -- no copy on read.
-            let bytes =
-                read_len_prefixed_bytes_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
-
-            Ok(CompressedPositionStorage::SharedStream(
-                SharedPositionStream::new(codec, block_offsets, bytes),
-            ))
+            // Zero copy: read_raw returns a Bytes slice backed by the same
+            // allocation as the input, and SharedPositionStream stores its byte
+            // buffer as Bytes -- no copy on read.
+            let bytes = r.read_raw()?;
+            Ok(Some(CompressedPositionStorage::SharedStream(
+                SharedPositionStream::new(stream_codec, block_offsets, bytes),
+            )))
         }
-        other => Err(Error::io(format!("unknown positions tag: {other}"))),
     }
 }
 
@@ -221,50 +205,58 @@ fn read_position_storage(
 // PostingList codec
 // ---------------------------------------------------------------------------
 
-#[derive(Serialize, Deserialize)]
-struct PlainPostingHeader {
-    max_score: Option<f32>,
-}
-
-#[derive(Serialize, Deserialize)]
-struct CompressedPostingHeader {
-    max_score: f32,
-    length: u32,
-    posting_tail_codec: u8,
-}
-
 impl CacheCodecImpl for PostingList {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+    const TYPE_ID: &'static str = "lance.fts.PostingList";
+    // Version 3 adds the optional impact IPC section. Main already used v2 for
+    // configurable posting block sizes, so impact data needs a distinct
+    // version to keep older readers from accepting a body with an extra
+    // section they cannot consume.
+    const CURRENT_VERSION: u32 = 3;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         match self {
             Self::Plain(plain) => {
-                write_u8(writer, POSTING_VARIANT_PLAIN)?;
-                serialize_plain(writer, plain)
+                w.write_u8(POSTING_VARIANT_PLAIN)?;
+                serialize_plain(w, plain)
             }
             Self::Compressed(compressed) => {
-                write_u8(writer, POSTING_VARIANT_COMPRESSED)?;
-                serialize_compressed(writer, compressed)
+                w.write_u8(POSTING_VARIANT_COMPRESSED)?;
+                serialize_compressed(w, compressed)
             }
         }
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let variant = read_u8(data, &mut offset)?;
-        match variant {
-            POSTING_VARIANT_PLAIN => Ok(Self::Plain(deserialize_plain(data, &mut offset)?)),
-            POSTING_VARIANT_COMPRESSED => {
-                Ok(Self::Compressed(deserialize_compressed(data, &mut offset)?))
-            }
-            other => Err(Error::io(format!("unknown PostingList variant: {other}"))),
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        match r.version() {
+            1 | 2 | Self::CURRENT_VERSION => deserialize_posting_list_body(r),
+            other => Err(Error::io(format!(
+                "unsupported PostingList cache version: {other}"
+            ))),
         }
     }
 }
 
-fn serialize_plain(writer: &mut dyn Write, plain: &PlainPostingList) -> Result<()> {
+fn deserialize_posting_list_body(r: &mut CacheEntryReader<'_>) -> Result<PostingList> {
+    let variant = r.read_u8()?;
+    match variant {
+        POSTING_VARIANT_PLAIN => Ok(PostingList::Plain(deserialize_plain(r)?)),
+        POSTING_VARIANT_COMPRESSED => Ok(PostingList::Compressed(deserialize_compressed(r)?)),
+        other => Err(Error::io(format!("unknown PostingList variant: {other}"))),
+    }
+}
+
+fn serialize_plain(w: &mut CacheEntryWriter<'_>, plain: &PlainPostingList) -> Result<()> {
+    // Plain postings carry only per-doc legacy positions (or none).
+    let position_storage = if plain.positions.is_some() {
+        PbPositionStorage::Legacy
+    } else {
+        PbPositionStorage::None
+    };
     let header = PlainPostingHeader {
         max_score: plain.max_score,
+        position_storage: position_storage as i32,
     };
-    write_json_header(writer, &header)?;
+    w.write_header(&header)?;
 
     let row_ids = UInt64Array::new(plain.row_ids.clone(), None);
     let frequencies = Float32Array::new(plain.frequencies.clone(), None);
@@ -273,52 +265,40 @@ fn serialize_plain(writer: &mut dyn Write, plain: &PlainPostingList) -> Result<(
         Field::new(FREQUENCIES_COLUMN, DataType::Float32, false),
     ]));
     let batch = RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(frequencies)])?;
-    write_ipc_stream(&batch, writer)?;
+    w.write_ipc(&batch)?;
 
-    match &plain.positions {
-        Some(list) => {
-            // Plain postings can only carry per-doc legacy positions; reuse
-            // the shared encoder.
-            write_position_storage(
-                writer,
-                &CompressedPositionStorage::LegacyPerDoc(list.clone()),
-            )?;
-        }
-        None => write_u8(writer, POSITIONS_TAG_NONE)?,
+    if let Some(list) = &plain.positions {
+        w.write_ipc(&legacy_positions_batch(list)?)?;
     }
     Ok(())
 }
 
-fn deserialize_plain(data: &Bytes, offset: &mut usize) -> Result<PlainPostingList> {
-    let header: PlainPostingHeader = read_json_header(data, offset)?;
+fn deserialize_plain(r: &mut CacheEntryReader<'_>) -> Result<PlainPostingList> {
+    let header: PlainPostingHeader = r.read_header()?;
 
-    let batch = read_ipc_stream_single_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
+    let batch = r.read_ipc()?;
     let row_ids = batch
-        .column(0)
+        .column_by_name(ROW_IDS_COLUMN)
+        .ok_or_else(|| Error::io("row_ids column is missing".to_string()))?
         .as_primitive_opt::<UInt64Type>()
         .ok_or_else(|| Error::io("row_ids column is not UInt64".to_string()))?
         .values()
         .clone();
     let frequencies = batch
-        .column(1)
+        .column_by_name(FREQUENCIES_COLUMN)
+        .ok_or_else(|| Error::io("frequencies column is missing".to_string()))?
         .as_primitive_opt::<Float32Type>()
         .ok_or_else(|| Error::io("frequencies column is not Float32".to_string()))?
         .values()
         .clone();
 
-    let positions_tag = read_u8(data, offset)?;
-    let positions = match positions_tag {
-        POSITIONS_TAG_NONE => None,
-        POSITIONS_TAG_LEGACY => match read_position_storage(data, offset, positions_tag)? {
-            CompressedPositionStorage::LegacyPerDoc(list) => Some(list),
-            CompressedPositionStorage::SharedStream(_) => {
-                unreachable!("shared stream tag was read as legacy variant (this is a bug)")
-            }
-        },
-        other => {
-            return Err(Error::io(format!(
-                "Plain posting list cannot have positions tag {other}"
-            )));
+    let positions = match header.position_storage() {
+        PbPositionStorage::None => None,
+        PbPositionStorage::Legacy => Some(read_legacy_positions(r)?),
+        PbPositionStorage::Shared => {
+            return Err(Error::io(
+                "Plain posting list cannot have a shared position stream".to_string(),
+            ));
         }
     };
 
@@ -330,13 +310,35 @@ fn deserialize_plain(data: &Bytes, offset: &mut usize) -> Result<PlainPostingLis
     ))
 }
 
-fn serialize_compressed(writer: &mut dyn Write, posting: &CompressedPostingList) -> Result<()> {
+/// The compressed posting list is serialized with a protobuf header followed
+/// by 64-byte-aligned Arrow IPC sections (for the `blocks`, and for shared
+/// position block-offsets) and a raw blob (for the shared position byte
+/// stream, which already has its own portable encoding).
+fn serialize_compressed(
+    w: &mut CacheEntryWriter<'_>,
+    posting: &CompressedPostingList,
+) -> Result<()> {
+    let (position_storage, position_stream_codec) = match &posting.positions {
+        None => (PbPositionStorage::None, PbPositionStreamCodec::default()),
+        Some(CompressedPositionStorage::LegacyPerDoc(_)) => {
+            (PbPositionStorage::Legacy, PbPositionStreamCodec::default())
+        }
+        Some(CompressedPositionStorage::SharedStream(stream)) => (
+            PbPositionStorage::Shared,
+            position_stream_codec_to_proto(stream.codec()),
+        ),
+    };
+
     let header = CompressedPostingHeader {
         max_score: posting.max_score,
         length: posting.length,
-        posting_tail_codec: posting_tail_codec_to_u8(posting.posting_tail_codec),
+        posting_tail_codec: posting_tail_codec_to_proto(posting.posting_tail_codec) as i32,
+        position_storage: position_storage as i32,
+        position_stream_codec: position_stream_codec as i32,
+        block_size: posting.block_size as u32,
+        has_impacts: posting.impacts.is_some(),
     };
-    write_json_header(writer, &header)?;
+    w.write_header(&header)?;
 
     let schema = Arc::new(Schema::new(vec![Field::new(
         BLOCKS_COLUMN,
@@ -344,32 +346,55 @@ fn serialize_compressed(writer: &mut dyn Write, posting: &CompressedPostingList)
         false,
     )]));
     let batch = RecordBatch::try_new(schema, vec![Arc::new(posting.blocks.clone())])?;
-    write_ipc_stream(&batch, writer)?;
+    w.write_ipc(&batch)?;
 
-    match &posting.positions {
-        Some(storage) => write_position_storage(writer, storage)?,
-        None => write_u8(writer, POSITIONS_TAG_NONE)?,
+    if let Some(storage) = &posting.positions {
+        write_position_sections(w, storage)?;
+    }
+    if let Some(impacts) = &posting.impacts {
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            IMPACTS_COLUMN,
+            DataType::LargeBinary,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(impacts.entries().clone())])?;
+        w.write_ipc(&batch)?;
     }
     Ok(())
 }
 
-fn deserialize_compressed(data: &Bytes, offset: &mut usize) -> Result<CompressedPostingList> {
-    let header: CompressedPostingHeader = read_json_header(data, offset)?;
-    let posting_tail_codec = u8_to_posting_tail_codec(header.posting_tail_codec)?;
+fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result<CompressedPostingList> {
+    let header: CompressedPostingHeader = r.read_header()?;
+    let posting_tail_codec = proto_to_posting_tail_codec(header.posting_tail_codec());
 
-    let batch = read_ipc_stream_single_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
+    let batch = r.read_ipc()?;
     let blocks = batch
-        .column(0)
+        .column_by_name(BLOCKS_COLUMN)
+        .ok_or_else(|| Error::io("blocks column is missing".to_string()))?
         .as_any()
         .downcast_ref::<LargeBinaryArray>()
         .ok_or_else(|| Error::io("blocks column is not a LargeBinaryArray".to_string()))?
         .clone();
 
-    let positions_tag = read_u8(data, offset)?;
-    let positions = if positions_tag == POSITIONS_TAG_NONE {
-        None
+    let stream_codec = proto_to_position_stream_codec(header.position_stream_codec());
+    let positions = read_position_sections(r, header.position_storage(), stream_codec)?;
+    let block_size = if header.block_size == 0 {
+        LEGACY_BLOCK_SIZE
     } else {
-        Some(read_position_storage(data, offset, positions_tag)?)
+        validate_block_size(header.block_size as usize)?
+    };
+    let impacts = if r.version() >= 3 && header.has_impacts {
+        let batch = r.read_ipc()?;
+        let entries = batch
+            .column_by_name(IMPACTS_COLUMN)
+            .ok_or_else(|| Error::io("impacts column is missing".to_string()))?
+            .as_any()
+            .downcast_ref::<LargeBinaryArray>()
+            .ok_or_else(|| Error::io("impacts column is not a LargeBinaryArray".to_string()))?
+            .clone();
+        Some(ImpactSkipData::new(entries, blocks.len())?)
+    } else {
+        None
     };
 
     Ok(CompressedPostingList::new(
@@ -377,7 +402,9 @@ fn deserialize_compressed(data: &Bytes, offset: &mut usize) -> Result<Compressed
         header.max_score,
         header.length,
         posting_tail_codec,
+        block_size,
         positions,
+        impacts,
     ))
 }
 
@@ -385,63 +412,115 @@ fn deserialize_compressed(data: &Bytes, offset: &mut usize) -> Result<Compressed
 // PostingListGroup codec
 // ---------------------------------------------------------------------------
 
-/// Serializes a [`PostingListGroup`] as a count followed by each member
-/// posting list, length-prefixed so the existing [`PostingList`] codec can be
-/// reused per entry (and its byte buffers read back zero-copy). See issue
-/// #7040.
+/// Version 2 distinguishes packed groups from the materialized fallback. A
+/// packed group writes one IPC batch; materialized groups retain the v1 inline
+/// member framing used by legacy and position-bearing prewarm paths. Version 3
+/// marks packed groups whose IPC schema can carry configurable posting block
+/// sizes while retaining the version-2 body framing. Version 4 adds optional
+/// impact data, either as an inline section for materialized members or as a
+/// column in the packed IPC batch.
 impl CacheCodecImpl for PostingListGroup {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        let count = u32::try_from(self.posting_lists.len())
+    const TYPE_ID: &'static str = "lance.fts.PostingListGroup";
+    const CURRENT_VERSION: u32 = 4;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let count = u32::try_from(self.len())
             .map_err(|_| Error::io("posting list group too large to serialize".to_string()))?;
-        writer
-            .write_all(&count.to_le_bytes())
-            .map_err(|e| Error::io(format!("failed to write group count: {e}")))?;
-        for posting in &self.posting_lists {
-            let mut buf = Vec::new();
-            posting.serialize(&mut buf)?;
-            write_len_prefixed_bytes(writer, &buf)?;
+        match &self.storage {
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                w.write_u8(GROUP_VARIANT_MATERIALIZED)?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                for posting in posting_lists {
+                    posting.serialize(w)?;
+                }
+            }
+            PostingListGroupStorage::Packed(group) => {
+                w.write_u8(GROUP_VARIANT_PACKED)?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                w.write_u8(posting_tail_codec_to_tag(group.posting_tail_codec))?;
+                w.write_ipc(&group.batch)?;
+            }
         }
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        if data.len() < 4 {
-            return Err(Error::io(
-                "truncated posting list group: missing count".to_string(),
-            ));
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        match r.version() {
+            1 => return deserialize_materialized_group(r),
+            2 | 3 | Self::CURRENT_VERSION => {}
+            other => {
+                return Err(Error::io(format!(
+                    "unsupported PostingListGroup cache version: {other}"
+                )));
+            }
         }
-        let count = u32::from_le_bytes(data[0..4].try_into().unwrap()) as usize;
-        offset += 4;
-        let mut posting_lists = Vec::with_capacity(count);
-        for _ in 0..count {
-            let entry = read_len_prefixed_bytes_at(data, &mut offset)
-                .map_err(|e| Error::io(e.to_string()))?;
-            posting_lists.push(PostingList::deserialize(&entry)?);
+
+        let variant = r.read_u8()?;
+        match variant {
+            GROUP_VARIANT_MATERIALIZED => deserialize_materialized_group(r),
+            GROUP_VARIANT_PACKED => {
+                let header: PostingListGroupHeader = r.read_header()?;
+                let posting_tail_codec = posting_tail_codec_from_tag(r.read_u8()?)?;
+                let batch = r.read_ipc()?;
+                if batch.num_rows() != header.count as usize {
+                    return Err(Error::io(format!(
+                        "packed posting group row count {} does not match header count {}",
+                        batch.num_rows(),
+                        header.count
+                    )));
+                }
+                Self::new_packed(batch, posting_tail_codec)
+            }
+            other => Err(Error::io(format!(
+                "unknown PostingListGroup variant: {other}"
+            ))),
         }
-        Ok(Self::new(posting_lists))
     }
 }
 
+fn deserialize_materialized_group(r: &mut CacheEntryReader<'_>) -> Result<PostingListGroup> {
+    let header: PostingListGroupHeader = r.read_header()?;
+    let mut posting_lists = Vec::with_capacity(header.count as usize);
+    for _ in 0..header.count {
+        posting_lists.push(deserialize_posting_list_body(r)?);
+    }
+    Ok(PostingListGroup::new(posting_lists))
+}
+
 // ---------------------------------------------------------------------------
 // Positions codec
 // ---------------------------------------------------------------------------
 
 impl CacheCodecImpl for Positions {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        write_position_storage(writer, &self.0)
+    const TYPE_ID: &'static str = "lance.fts.Positions";
+    const CURRENT_VERSION: u32 = 1;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let (position_storage, position_stream_codec) = match &self.0 {
+            CompressedPositionStorage::LegacyPerDoc(_) => {
+                (PbPositionStorage::Legacy, PbPositionStreamCodec::default())
+            }
+            CompressedPositionStorage::SharedStream(stream) => (
+                PbPositionStorage::Shared,
+                position_stream_codec_to_proto(stream.codec()),
+            ),
+        };
+        let header = PositionsHeader {
+            position_storage: position_storage as i32,
+            position_stream_codec: position_stream_codec as i32,
+        };
+        w.write_header(&header)?;
+        write_position_sections(w, &self.0)
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let tag = read_u8(data, &mut offset)?;
-        if tag == POSITIONS_TAG_NONE {
-            return Err(Error::io(
-                "Positions cache entry cannot encode the None variant".to_string(),
-            ));
-        }
-        let storage = read_position_storage(data, &mut offset, tag)?;
-        Ok(Self(storage))
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: PositionsHeader = r.read_header()?;
+        let stream_codec = proto_to_position_stream_codec(header.position_stream_codec());
+        read_position_sections(r, header.position_storage(), stream_codec)?
+            .map(Self)
+            .ok_or_else(|| {
+                Error::io("Positions cache entry cannot encode the None variant".to_string())
+            })
     }
 }
 
@@ -451,16 +530,26 @@ impl CacheCodecImpl for Positions {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
     use arrow::buffer::ScalarBuffer;
-    use arrow_array::LargeBinaryArray;
-    use arrow_array::builder::{Int32Builder, ListBuilder};
+    use arrow_array::builder::{Int32Builder, LargeBinaryBuilder, ListBuilder};
+    use arrow_array::{Array, LargeBinaryArray, RecordBatch};
+    use arrow_schema::{Field, Schema};
     use bytes::Bytes;
-    use lance_core::cache::CacheCodecImpl;
+    use lance_core::Result;
+    use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
 
+    use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec};
+
+    use super::super::impact::{ImpactSkipData, ImpactSkipDataBuilder};
     use super::super::index::{
-        CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec,
-        Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream,
+        CompressedPositionStorage, CompressedPostingList, IMPACT_COL, POSTING_BLOCK_SIZE_KEY,
+        POSTING_COL, PlainPostingList, PositionStreamCodec, Positions, PostingList,
+        PostingListGroup, PostingTailCodec, SharedPositionStream,
     };
+    use super::super::tokenizer::LEGACY_BLOCK_SIZE;
 
     fn legacy_positions(rows: &[&[i32]]) -> arrow_array::ListArray {
         let mut builder = ListBuilder::new(Int32Builder::new());
@@ -473,6 +562,73 @@ mod tests {
         builder.finish()
     }
 
+    fn packed_batch(postings: &[Vec<Vec<u8>>], block_size: Option<usize>) -> RecordBatch {
+        let mut builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for posting in postings {
+            for block in posting {
+                builder.values().append_value(block);
+            }
+            builder.append(true);
+        }
+        let postings = builder.finish();
+        let fields = vec![Field::new(POSTING_COL, postings.data_type().clone(), false)];
+        let schema = Arc::new(match block_size {
+            Some(block_size) => Schema::new_with_metadata(
+                fields,
+                HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string())]),
+            ),
+            None => Schema::new(fields),
+        });
+        RecordBatch::try_new(schema, vec![Arc::new(postings)]).unwrap()
+    }
+
+    fn packed_group(
+        postings: &[Vec<Vec<u8>>],
+        posting_tail_codec: PostingTailCodec,
+        block_size: Option<usize>,
+    ) -> PostingListGroup {
+        PostingListGroup::new_packed(packed_batch(postings, block_size), posting_tail_codec)
+            .unwrap()
+    }
+
+    fn packed_group_with_impacts(
+        postings: &[Vec<Vec<u8>>],
+        impacts: &[ImpactSkipData],
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> PostingListGroup {
+        assert_eq!(postings.len(), impacts.len());
+        let posting_batch = packed_batch(postings, Some(block_size));
+        let mut impacts_builder = ListBuilder::new(LargeBinaryBuilder::new());
+        for impacts in impacts {
+            for entry_idx in 0..impacts.entries().len() {
+                impacts_builder
+                    .values()
+                    .append_value(impacts.entries().value(entry_idx));
+            }
+            impacts_builder.append(true);
+        }
+        let impacts = impacts_builder.finish();
+        let fields = vec![
+            Field::new(
+                POSTING_COL,
+                posting_batch.column(0).data_type().clone(),
+                false,
+            ),
+            Field::new(IMPACT_COL, impacts.data_type().clone(), false),
+        ];
+        let schema = Arc::new(Schema::new_with_metadata(
+            fields,
+            posting_batch.schema_ref().metadata().clone(),
+        ));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![posting_batch.column(0).clone(), Arc::new(impacts)],
+        )
+        .unwrap();
+        PostingListGroup::new_packed(batch, posting_tail_codec).unwrap()
+    }
+
     fn assert_plain_eq(a: &PlainPostingList, b: &PlainPostingList) {
         assert_eq!(a.row_ids.as_ref(), b.row_ids.as_ref());
         assert_eq!(a.frequencies.as_ref(), b.frequencies.as_ref());
@@ -502,16 +658,68 @@ mod tests {
         }
     }
 
-    fn roundtrip_posting_list(entry: &PostingList) -> PostingList {
+    fn impact_skip_data(level0_len: usize, block_size: usize) -> ImpactSkipData {
+        let mut builder = ImpactSkipDataBuilder::with_capacity(level0_len, block_size);
+        for block_idx in 0..level0_len {
+            let doc_base = block_idx as u32 * 10;
+            builder
+                .append_block(&[
+                    (doc_base + 1, block_idx as u32 + 1, 10),
+                    (doc_base + 9, block_idx as u32 + 2, 8),
+                ])
+                .unwrap();
+        }
+        builder.finish().unwrap()
+    }
+
+    /// Serialize a codec body (no envelope) into a standalone buffer.
+    fn body_bytes<T: CacheCodecImpl>(entry: &T) -> Bytes {
         let mut buf = Vec::new();
-        entry.serialize(&mut buf).unwrap();
-        PostingList::deserialize(&Bytes::from(buf)).unwrap()
+        let mut w = CacheEntryWriter::new(&mut buf);
+        entry.serialize(&mut w).unwrap();
+        Bytes::from(buf)
     }
 
-    fn roundtrip_positions(entry: &Positions) -> Positions {
+    /// Deserialize a codec body (no envelope) at the current build's version.
+    fn from_body<T: CacheCodecImpl>(data: &Bytes) -> Result<T> {
+        let mut r = CacheEntryReader::new(data, 0, T::CURRENT_VERSION);
+        T::deserialize(&mut r)
+    }
+
+    fn from_body_version<T: CacheCodecImpl>(data: &Bytes, version: u32) -> Result<T> {
+        let mut r = CacheEntryReader::new(data, 0, version);
+        T::deserialize(&mut r)
+    }
+
+    fn compressed_body_with_ipc_sections(
+        blocks: &RecordBatch,
+        impacts: Option<&RecordBatch>,
+    ) -> Bytes {
         let mut buf = Vec::new();
-        entry.serialize(&mut buf).unwrap();
-        Positions::deserialize(&Bytes::from(buf)).unwrap()
+        let mut w = CacheEntryWriter::new(&mut buf);
+        w.write_u8(super::POSTING_VARIANT_COMPRESSED).unwrap();
+        w.write_header(&CompressedPostingHeader {
+            max_score: 1.0,
+            length: 1,
+            posting_tail_codec: PbPostingTailCodec::VarintDelta as i32,
+            block_size: 256,
+            has_impacts: impacts.is_some(),
+            ..Default::default()
+        })
+        .unwrap();
+        w.write_ipc(blocks).unwrap();
+        if let Some(impacts) = impacts {
+            w.write_ipc(impacts).unwrap();
+        }
+        Bytes::from(buf)
+    }
+
+    fn roundtrip_posting_list(entry: &PostingList) -> PostingList {
+        from_body::<PostingList>(&body_bytes(entry)).unwrap()
+    }
+
+    fn roundtrip_positions(entry: &Positions) -> Positions {
+        from_body::<Positions>(&body_bytes(entry)).unwrap()
     }
 
     fn assert_slice_points_into_bytes(slice: &[u8], bytes: &Bytes) {
@@ -563,14 +771,22 @@ mod tests {
             Some(&[1u8, 2, 3, 4, 5][..]),
             Some(&[6, 7, 8, 9, 10][..]),
         ]);
-        let posting =
-            CompressedPostingList::new(blocks, 3.5, 42, PostingTailCodec::VarintDelta, None);
+        let posting = CompressedPostingList::new(
+            blocks,
+            3.5,
+            42,
+            PostingTailCodec::VarintDelta,
+            256,
+            None,
+            None,
+        );
         let entry = PostingList::Compressed(posting.clone());
         match roundtrip_posting_list(&entry) {
             PostingList::Compressed(restored) => {
                 assert_eq!(restored.max_score, posting.max_score);
                 assert_eq!(restored.length, posting.length);
                 assert_eq!(restored.posting_tail_codec, posting.posting_tail_codec);
+                assert_eq!(restored.block_size, posting.block_size);
                 assert_eq!(restored.blocks, posting.blocks);
                 assert!(restored.positions.is_none());
             }
@@ -578,6 +794,73 @@ mod tests {
         }
     }
 
+    #[test]
+    fn compressed_posting_list_impacts_roundtrip() {
+        let blocks = LargeBinaryArray::from_opt_vec(vec![
+            Some(&[1u8, 2, 3, 4, 5][..]),
+            Some(&[6, 7, 8, 9, 10][..]),
+        ]);
+        let impacts = impact_skip_data(blocks.len(), 256);
+        let posting = CompressedPostingList::new(
+            blocks,
+            3.5,
+            42,
+            PostingTailCodec::VarintDelta,
+            256,
+            None,
+            Some(impacts.clone()),
+        );
+        let entry = PostingList::Compressed(posting);
+        match roundtrip_posting_list(&entry) {
+            PostingList::Compressed(restored) => {
+                let restored = restored.impacts.expect("impacts should roundtrip");
+                assert_eq!(restored.level0_len(), impacts.level0_len());
+                assert_eq!(restored.level1_len(), impacts.level1_len());
+                assert_eq!(restored.entries(), impacts.entries());
+            }
+            PostingList::Plain(_) => panic!("expected Compressed variant"),
+        }
+    }
+
+    #[test]
+    fn compressed_posting_list_missing_ipc_columns_returns_error() {
+        let empty = RecordBatch::new_empty(Arc::new(Schema::empty()));
+        assert!(
+            from_body::<PostingList>(&compressed_body_with_ipc_sections(&empty, None)).is_err()
+        );
+
+        let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1_u8, 2, 3][..])]);
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            super::BLOCKS_COLUMN,
+            blocks.data_type().clone(),
+            false,
+        )]));
+        let blocks = RecordBatch::try_new(schema, vec![Arc::new(blocks)]).unwrap();
+        assert!(
+            from_body::<PostingList>(&compressed_body_with_ipc_sections(&blocks, Some(&empty)))
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn compressed_posting_list_v1_cache_without_impacts_decodes() {
+        let posting = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]),
+            1.25,
+            5,
+            PostingTailCodec::Fixed32,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+            None,
+            None,
+        );
+        let data = body_bytes(&PostingList::Compressed(posting));
+        let restored = from_body_version::<PostingList>(&data, 1).unwrap();
+        let PostingList::Compressed(restored) = restored else {
+            panic!("expected Compressed variant");
+        };
+        assert!(restored.impacts.is_none());
+    }
+
     #[test]
     fn compressed_posting_list_legacy_positions_roundtrip() {
         let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]);
@@ -586,9 +869,11 @@ mod tests {
             1.25,
             5,
             PostingTailCodec::Fixed32,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
             Some(CompressedPositionStorage::LegacyPerDoc(legacy_positions(
                 &[&[0, 4, 8]],
             ))),
+            None,
         );
         let entry = PostingList::Compressed(posting.clone());
         match roundtrip_posting_list(&entry) {
@@ -620,7 +905,9 @@ mod tests {
                 7.0,
                 3,
                 PostingTailCodec::VarintDelta,
+                256,
                 Some(CompressedPositionStorage::SharedStream(stream)),
+                None,
             );
             let entry = PostingList::Compressed(posting.clone());
             match roundtrip_posting_list(&entry) {
@@ -648,17 +935,15 @@ mod tests {
             7.0,
             3,
             PostingTailCodec::VarintDelta,
+            256,
             Some(CompressedPositionStorage::SharedStream(
                 expected_stream.clone(),
             )),
+            None,
         );
-        let mut buf = Vec::new();
-        PostingList::Compressed(posting)
-            .serialize(&mut buf)
-            .unwrap();
-        let serialized = Bytes::from(buf);
+        let serialized = body_bytes(&PostingList::Compressed(posting));
 
-        let restored = PostingList::deserialize(&serialized).unwrap();
+        let restored = from_body::<PostingList>(&serialized).unwrap();
         let PostingList::Compressed(restored) = restored else {
             panic!("expected Compressed variant");
         };
@@ -686,6 +971,8 @@ mod tests {
             2.5,
             7,
             PostingTailCodec::VarintDelta,
+            256,
+            None,
             None,
         ));
 
@@ -695,12 +982,12 @@ mod tests {
             vec![plain.clone(), compressed, plain],
         ] {
             let group = PostingListGroup::new(members.clone());
-            let mut buf = Vec::new();
-            group.serialize(&mut buf).unwrap();
-            let restored = PostingListGroup::deserialize(&Bytes::from(buf)).unwrap();
-            assert_eq!(restored.posting_lists.len(), members.len());
-            for (a, b) in members.iter().zip(restored.posting_lists.iter()) {
-                match (a, b) {
+            let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+            assert!(!restored.is_packed());
+            assert_eq!(restored.len(), members.len());
+            for (index, a) in members.iter().enumerate() {
+                let b = restored.posting_list(index, None, None).unwrap().unwrap();
+                match (a, &b) {
                     (PostingList::Plain(x), PostingList::Plain(y)) => assert_plain_eq(x, y),
                     (PostingList::Compressed(x), PostingList::Compressed(y)) => {
                         assert_eq!(x.blocks, y.blocks);
@@ -713,6 +1000,167 @@ mod tests {
         }
     }
 
+    #[test]
+    fn packed_posting_list_group_roundtrip_and_v1_fallback() {
+        let group = packed_group(
+            &[vec![vec![1, 2, 3], vec![4, 5]], vec![vec![7; 16 * 1024]]],
+            PostingTailCodec::VarintDelta,
+            Some(256),
+        );
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(restored.is_packed());
+        assert_eq!(restored.len(), 2);
+        for slot in 0..2 {
+            let max_score = [1.5, 3.25][slot];
+            let length = [3, 4096][slot];
+            let expected = group
+                .posting_list(slot, Some(max_score), Some(length))
+                .unwrap()
+                .unwrap();
+            let actual = restored
+                .posting_list(slot, Some(max_score), Some(length))
+                .unwrap()
+                .unwrap();
+            let (PostingList::Compressed(expected), PostingList::Compressed(actual)) =
+                (expected, actual)
+            else {
+                panic!("expected compressed packed posting views");
+            };
+            assert_eq!(actual.blocks, expected.blocks);
+            assert_eq!(actual.max_score, expected.max_score);
+            assert_eq!(actual.length, expected.length);
+            assert_eq!(actual.posting_tail_codec, expected.posting_tail_codec);
+            assert_eq!(actual.block_size, 256);
+        }
+
+        let legacy_packed =
+            packed_group(&[vec![vec![9, 8, 7]]], PostingTailCodec::VarintDelta, None);
+        let restored = from_body::<PostingListGroup>(&body_bytes(&legacy_packed)).unwrap();
+        let PostingList::Compressed(posting) = restored
+            .posting_list(0, Some(2.0), Some(3))
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed legacy packed posting");
+        };
+        assert_eq!(posting.block_size, LEGACY_BLOCK_SIZE);
+
+        let legacy_member = PostingList::Compressed(CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]),
+            2.0,
+            3,
+            PostingTailCodec::VarintDelta,
+            LEGACY_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let mut legacy_body = Vec::new();
+        let mut writer = CacheEntryWriter::new(&mut legacy_body);
+        writer
+            .write_header(&crate::cache_pb::PostingListGroupHeader { count: 1 })
+            .unwrap();
+        legacy_member.serialize(&mut writer).unwrap();
+        let legacy_body = Bytes::from(legacy_body);
+        let mut reader = CacheEntryReader::new(&legacy_body, 0, 1);
+        let restored = PostingListGroup::deserialize(&mut reader).unwrap();
+        assert!(!restored.is_packed());
+        assert_eq!(restored.len(), 1);
+    }
+
+    #[test]
+    fn posting_list_group_impacted_compressed_members_roundtrip() {
+        let first = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..]), Some(&[4u8, 5, 6][..])]),
+            3.0,
+            256,
+            PostingTailCodec::VarintDelta,
+            LEGACY_BLOCK_SIZE,
+            None,
+            Some(impact_skip_data(2, LEGACY_BLOCK_SIZE)),
+        );
+        let second = CompressedPostingList::new(
+            LargeBinaryArray::from_opt_vec(vec![Some(&[7u8, 8, 9][..])]),
+            5.0,
+            128,
+            PostingTailCodec::Fixed32,
+            256,
+            Some(CompressedPositionStorage::SharedStream(
+                SharedPositionStream::new(
+                    PositionStreamCodec::PackedDelta,
+                    vec![0u32, 12],
+                    Bytes::from(vec![0xABu8; 32]),
+                ),
+            )),
+            Some(impact_skip_data(1, 256)),
+        );
+        let members = vec![
+            PostingList::Compressed(first.clone()),
+            PostingList::Compressed(second.clone()),
+        ];
+        let group = PostingListGroup::new(members);
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(!restored.is_packed());
+        assert_eq!(restored.len(), 2);
+
+        let expected = [&first, &second];
+        for (slot, expected) in expected.iter().enumerate() {
+            let restored = restored.posting_list(slot, None, None).unwrap().unwrap();
+            let PostingList::Compressed(restored) = restored else {
+                panic!("expected compressed member");
+            };
+            assert_eq!(restored.blocks, expected.blocks);
+            assert_eq!(restored.length, expected.length);
+            assert_eq!(restored.max_score, expected.max_score);
+            assert_eq!(restored.posting_tail_codec, expected.posting_tail_codec);
+            assert_eq!(restored.block_size, expected.block_size);
+            assert_eq!(
+                restored.impacts.as_ref().unwrap().entries(),
+                expected.impacts.as_ref().unwrap().entries()
+            );
+            match (&expected.positions, &restored.positions) {
+                (Some(expected), Some(restored)) => {
+                    assert_position_storage_eq(expected, restored);
+                }
+                (None, None) => {}
+                _ => panic!("position storage mismatch"),
+            }
+        }
+    }
+
+    #[test]
+    fn packed_posting_list_group_impacts_roundtrip() {
+        let postings = vec![vec![vec![1, 2, 3], vec![4, 5, 6]], vec![vec![7, 8, 9]]];
+        let expected_impacts = vec![impact_skip_data(2, 256), impact_skip_data(1, 256)];
+        let group = packed_group_with_impacts(
+            &postings,
+            &expected_impacts,
+            PostingTailCodec::VarintDelta,
+            256,
+        );
+
+        let restored = from_body::<PostingListGroup>(&body_bytes(&group)).unwrap();
+        assert!(restored.is_packed());
+        assert_eq!(restored.len(), expected_impacts.len());
+        for (slot, expected) in expected_impacts.iter().enumerate() {
+            let posting = restored
+                .posting_list(slot, Some(3.0), Some(256))
+                .unwrap()
+                .unwrap();
+            let PostingList::Compressed(posting) = posting else {
+                panic!("expected compressed packed posting");
+            };
+            let actual = posting.impacts.as_ref().expect("impacts should roundtrip");
+            assert_eq!(actual.entries(), expected.entries());
+            assert_eq!(actual.level0_len(), expected.level0_len());
+            assert_eq!(
+                actual.level1_doc_up_to(0),
+                expected.level1_doc_up_to(0),
+                "impact entries should remain decodable with the packed block size",
+            );
+            assert!(actual.level1_doc_up_to(0).is_some());
+        }
+    }
+
     #[test]
     fn positions_legacy_roundtrip() {
         let positions = Positions(CompressedPositionStorage::LegacyPerDoc(legacy_positions(
@@ -743,9 +1191,458 @@ mod tests {
             None,
         );
         let entry = PostingList::Plain(plain);
-        let mut buf = Vec::new();
-        entry.serialize(&mut buf).unwrap();
+        let mut buf = body_bytes(&entry).to_vec();
         buf.truncate(buf.len() / 2);
-        assert!(PostingList::deserialize(&Bytes::from(buf)).is_err());
+        assert!(from_body::<PostingList>(&Bytes::from(buf)).is_err());
+    }
+
+    /// Tests covering the stabilized envelope + compressed proto format,
+    /// exercised through the full type-erased [`CacheCodec`] (envelope + body).
+    mod stable_format {
+        use std::sync::Arc;
+
+        use arrow_array::Array;
+        use arrow_schema::DataType;
+        use lance_core::cache::{
+            CacheCodec, CacheCodecImpl, CacheDecode, CacheEntryReader, CacheEntryWriter,
+            CacheMissReason,
+        };
+        use lance_core::{Error, Result};
+        use prost::Message;
+
+        use super::super::{
+            BLOCKS_COLUMN, GROUP_VARIANT_PACKED, POSTING_VARIANT_COMPRESSED,
+            posting_tail_codec_to_tag,
+        };
+        use super::*;
+        use crate::cache_pb::{
+            CompressedPostingHeader, PostingListGroupHeader, PostingTailCodec as PbPostingTailCodec,
+        };
+
+        type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
+
+        struct PostingListV2Codec(PostingList);
+
+        impl CacheCodecImpl for PostingListV2Codec {
+            const TYPE_ID: &'static str = <PostingList as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 2;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                self.0.serialize(w)
+            }
+
+            fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                PostingList::deserialize(r).map(Self)
+            }
+        }
+
+        struct PostingListGroupV3Codec(PostingListGroup);
+
+        impl CacheCodecImpl for PostingListGroupV3Codec {
+            const TYPE_ID: &'static str = <PostingListGroup as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 3;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                self.0.serialize(w)
+            }
+
+            fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                PostingListGroup::deserialize(r).map(Self)
+            }
+        }
+
+        struct LegacyCompressedPostingV1 {
+            blocks: LargeBinaryArray,
+        }
+
+        impl CacheCodecImpl for LegacyCompressedPostingV1 {
+            const TYPE_ID: &'static str = <PostingList as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 1;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                w.write_u8(POSTING_VARIANT_COMPRESSED)?;
+                w.write_header(&CompressedPostingHeader {
+                    max_score: 2.0,
+                    length: 3,
+                    posting_tail_codec: PbPostingTailCodec::VarintDelta as i32,
+                    ..Default::default()
+                })?;
+                let schema = Arc::new(Schema::new(vec![Field::new(
+                    BLOCKS_COLUMN,
+                    DataType::LargeBinary,
+                    false,
+                )]));
+                let batch = RecordBatch::try_new(schema, vec![Arc::new(self.blocks.clone())])?;
+                w.write_ipc(&batch)
+            }
+
+            fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                Err(Error::io(
+                    "LegacyCompressedPostingV1 is a writer-only test codec".to_string(),
+                ))
+            }
+        }
+
+        struct LegacyPackedGroupV2 {
+            batch: RecordBatch,
+            posting_tail_codec: PostingTailCodec,
+        }
+
+        impl CacheCodecImpl for LegacyPackedGroupV2 {
+            const TYPE_ID: &'static str = <PostingListGroup as CacheCodecImpl>::TYPE_ID;
+            const CURRENT_VERSION: u32 = 2;
+
+            fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+                w.write_u8(GROUP_VARIANT_PACKED)?;
+                let count = u32::try_from(self.batch.num_rows())
+                    .map_err(|_| Error::io("legacy packed group is too large".to_string()))?;
+                w.write_header(&PostingListGroupHeader { count })?;
+                w.write_u8(posting_tail_codec_to_tag(self.posting_tail_codec))?;
+                w.write_ipc(&self.batch)
+            }
+
+            fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result<Self> {
+                Err(Error::io(
+                    "LegacyPackedGroupV2 is a writer-only test codec".to_string(),
+                ))
+            }
+        }
+
+        fn codec() -> CacheCodec {
+            CacheCodec::from_impl::<PostingList>()
+        }
+
+        fn serialize_typed_entry<T: CacheCodecImpl + 'static>(entry: T) -> Vec<u8> {
+            let any: ArcAny = Arc::new(entry);
+            let mut buf = Vec::new();
+            CacheCodec::from_impl::<T>()
+                .serialize(&any, &mut buf)
+                .unwrap();
+            buf
+        }
+
+        /// Serialize an entry through the full codec (envelope + body).
+        fn serialize_entry(entry: PostingList) -> Vec<u8> {
+            serialize_typed_entry(entry)
+        }
+
+        /// A `Bytes` whose base address is 64-byte aligned, modelling a backend
+        /// that reads cache entries into an aligned buffer.
+        fn aligned_bytes(payload: &[u8]) -> Bytes {
+            const ALIGN: usize = 64;
+            let mut v = vec![0u8; payload.len() + ALIGN];
+            let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+            v[pad..pad + payload.len()].copy_from_slice(payload);
+            Bytes::from(v).slice(pad..pad + payload.len())
+        }
+
+        fn compressed_with_shared_positions() -> PostingList {
+            let blocks =
+                LargeBinaryArray::from_opt_vec(vec![Some(&[9u8; 48][..]), Some(&[1u8; 48])]);
+            let stream = SharedPositionStream::new(
+                PositionStreamCodec::PackedDelta,
+                vec![0u32, 4, 11],
+                Bytes::from((0u8..64).collect::<Vec<_>>()),
+            );
+            PostingList::Compressed(CompressedPostingList::new(
+                blocks,
+                7.0,
+                3,
+                PostingTailCodec::VarintDelta,
+                256,
+                Some(CompressedPositionStorage::SharedStream(stream)),
+                None,
+            ))
+        }
+
+        /// The compressed `blocks` (an aligned IPC section) and the shared
+        /// position blob (a raw section) must both be borrowed zero-copy from
+        /// the input even though the envelope pushes them to a non-zero,
+        /// non-aligned starting offset.
+        #[test]
+        fn compressed_sections_are_zero_copy_through_envelope() {
+            let serialized = aligned_bytes(&serialize_entry(compressed_with_shared_positions()));
+            let restored = codec().deserialize(&serialized).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Compressed(restored) = restored.as_ref() else {
+                panic!("expected Compressed");
+            };
+
+            let base = serialized.as_ptr() as usize;
+            let end = base + serialized.len();
+            let points_in = |ptr: usize| ptr >= base && ptr < end;
+
+            // blocks IPC section decoded in place (no realigning memcpy).
+            for buf in restored.blocks.to_data().buffers() {
+                assert!(
+                    points_in(buf.as_ptr() as usize),
+                    "blocks buffer was realigned out of the input — misaligned IPC section",
+                );
+            }
+            // shared position raw blob borrowed in place.
+            let Some(CompressedPositionStorage::SharedStream(stream)) = &restored.positions else {
+                panic!("expected shared stream");
+            };
+            assert!(points_in(stream.bytes().as_ptr() as usize));
+        }
+
+        /// A packed group's single IPC batch and all posting views decoded from
+        /// it must borrow the cache entry's aligned input buffer.
+        #[test]
+        fn packed_group_sections_are_zero_copy_through_envelope() {
+            let postings = vec![
+                vec![vec![9; 48], vec![9; 48]],
+                vec![vec![1; 48], vec![1; 48]],
+            ];
+            let impacts = vec![impact_skip_data(2, 256), impact_skip_data(2, 256)];
+            let group =
+                packed_group_with_impacts(&postings, &impacts, PostingTailCodec::VarintDelta, 256);
+
+            let group_codec = CacheCodec::from_impl::<PostingListGroup>();
+            let any: ArcAny = Arc::new(group);
+            let mut buf = Vec::new();
+            group_codec.serialize(&any, &mut buf).unwrap();
+            let serialized = aligned_bytes(&buf);
+
+            let restored = group_codec.deserialize(&serialized).hit().unwrap();
+            let restored = restored.downcast::<PostingListGroup>().unwrap();
+
+            let base = serialized.as_ptr() as usize;
+            let end = base + serialized.len();
+            let points_in = |ptr: usize| ptr >= base && ptr < end;
+
+            assert!(restored.is_packed());
+            assert_eq!(restored.len(), 2);
+            for slot in 0..restored.len() {
+                let member = restored
+                    .posting_list(slot, Some(7.0), Some(3))
+                    .unwrap()
+                    .unwrap();
+                let PostingList::Compressed(member) = member else {
+                    panic!("expected Compressed member");
+                };
+                for buf in member.blocks.to_data().buffers() {
+                    assert!(
+                        points_in(buf.as_ptr() as usize),
+                        "group member blocks buffer was realigned out of the input — \
+                        misaligned IPC section",
+                    );
+                }
+                let impacts = member
+                    .impacts
+                    .as_ref()
+                    .expect("packed impacts should decode");
+                for buf in impacts.entries().to_data().buffers() {
+                    assert!(
+                        points_in(buf.as_ptr() as usize),
+                        "group member impact buffer was realigned out of the input",
+                    );
+                }
+            }
+        }
+
+        /// The plain posting's row-id/frequency IPC section must also decode
+        /// zero-copy through the envelope + proto header.
+        #[test]
+        fn plain_sections_are_zero_copy_through_envelope() {
+            let plain = PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from((0u64..64).collect::<Vec<_>>()),
+                ScalarBuffer::from(vec![1.0f32; 64]),
+                Some(2.0),
+                None,
+            ));
+            let serialized = aligned_bytes(&serialize_entry(plain));
+            let restored = codec().deserialize(&serialized).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Plain(restored) = restored.as_ref() else {
+                panic!("expected Plain");
+            };
+
+            let base = serialized.as_ptr() as usize;
+            let end = base + serialized.len();
+            // The row_ids ScalarBuffer must borrow from the input allocation.
+            let ptr = restored.row_ids.as_ptr() as usize;
+            assert!(
+                ptr >= base && ptr < end,
+                "row_ids buffer was realigned out of the input — misaligned IPC section",
+            );
+        }
+
+        /// Additive proto fields (lever #1) must not break decoding: an unknown
+        /// field number appended to the header is ignored.
+        #[test]
+        fn header_proto_ignores_unknown_fields() {
+            let header = CompressedPostingHeader {
+                max_score: 1.5,
+                length: 9,
+                posting_tail_codec: PbPostingTailCodec::VarintDelta as i32,
+                ..Default::default()
+            };
+            let mut bytes = header.encode_to_vec();
+            // Append an unknown field #15, varint wire type (0), value 7.
+            bytes.push(15 << 3);
+            bytes.push(7);
+            let decoded = CompressedPostingHeader::decode(bytes.as_slice()).unwrap();
+            assert_eq!(decoded.length, 9);
+            assert_eq!(decoded.max_score, 1.5);
+        }
+
+        /// An entry written by a different codec (foreign TYPE_ID) misses.
+        #[test]
+        fn foreign_type_id_is_miss() {
+            // A PostingListGroup entry carries a different TYPE_ID in its
+            // envelope; reading it as a PostingList must miss, not misread it.
+            let group = PostingListGroup::new(vec![]);
+            let any: ArcAny = Arc::new(group);
+            let mut buf = Vec::new();
+            CacheCodec::from_impl::<PostingListGroup>()
+                .serialize(&any, &mut buf)
+                .unwrap();
+            assert!(codec().deserialize(&Bytes::from(buf)).hit().is_none());
+        }
+
+        /// An entry written by a newer build (higher type_version) misses.
+        #[test]
+        fn future_type_version_is_miss() {
+            let mut buf = serialize_entry(compressed_with_shared_positions());
+            // Patch the envelope's type_version (magic[4] + ver[1] + len[2] +
+            // type_id[N]) to a value beyond what this build understands.
+            let type_id_len = u16::from_le_bytes([buf[5], buf[6]]) as usize;
+            let version_off = 4 + 1 + 2 + type_id_len;
+            buf[version_off..version_off + 4].copy_from_slice(&u32::MAX.to_le_bytes());
+            assert!(codec().deserialize(&Bytes::from(buf)).hit().is_none());
+        }
+
+        #[test]
+        fn old_codecs_reject_new_impact_envelopes_as_version_too_new() {
+            let posting = Bytes::from(serialize_entry(compressed_with_shared_positions()));
+            match CacheCodec::from_impl::<PostingListV2Codec>().deserialize(&posting) {
+                CacheDecode::Miss(reason) => {
+                    assert_eq!(reason, CacheMissReason::VersionTooNew)
+                }
+                CacheDecode::Hit(_) => panic!("v2 PostingList codec accepted a v3 envelope"),
+            }
+
+            let group = packed_group(
+                &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]],
+                PostingTailCodec::VarintDelta,
+                Some(256),
+            );
+            let group = Bytes::from(serialize_typed_entry(group));
+            match CacheCodec::from_impl::<PostingListGroupV3Codec>().deserialize(&group) {
+                CacheDecode::Miss(reason) => {
+                    assert_eq!(reason, CacheMissReason::VersionTooNew)
+                }
+                CacheDecode::Hit(_) => {
+                    panic!("v3 PostingListGroup codec accepted a v4 envelope")
+                }
+            }
+        }
+
+        #[test]
+        fn current_codecs_read_previous_main_versions() {
+            let previous_posting = PostingListV2Codec(compressed_with_shared_positions());
+            let previous_posting = Bytes::from(serialize_typed_entry(previous_posting));
+            let restored = codec().deserialize(&previous_posting).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Compressed(restored) = restored.as_ref() else {
+                panic!("expected compressed posting");
+            };
+            assert_eq!(restored.block_size, 256);
+            assert!(restored.impacts.is_none());
+
+            let previous_group = PostingListGroupV3Codec(packed_group(
+                &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]],
+                PostingTailCodec::VarintDelta,
+                Some(256),
+            ));
+            let previous_group = Bytes::from(serialize_typed_entry(previous_group));
+            let restored = CacheCodec::from_impl::<PostingListGroup>()
+                .deserialize(&previous_group)
+                .hit()
+                .unwrap()
+                .downcast::<PostingListGroup>()
+                .unwrap();
+            assert!(restored.is_packed());
+            assert_eq!(restored.len(), 2);
+            let PostingList::Compressed(restored) = restored
+                .posting_list(0, Some(2.0), Some(3))
+                .unwrap()
+                .unwrap()
+            else {
+                panic!("expected compressed packed posting");
+            };
+            assert_eq!(restored.block_size, 256);
+            assert!(restored.impacts.is_none());
+        }
+
+        #[test]
+        fn current_codecs_read_legacy_payloads_without_block_size() {
+            let legacy_posting = LegacyCompressedPostingV1 {
+                blocks: LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]),
+            };
+            let legacy_posting = Bytes::from(serialize_typed_entry(legacy_posting));
+            let restored = codec().deserialize(&legacy_posting).hit().unwrap();
+            let restored = restored.downcast::<PostingList>().unwrap();
+            let PostingList::Compressed(restored) = restored.as_ref() else {
+                panic!("expected a compressed legacy posting");
+            };
+            assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE);
+
+            let legacy_batch = packed_batch(&[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], None);
+            assert!(
+                !legacy_batch
+                    .schema_ref()
+                    .metadata()
+                    .contains_key(POSTING_BLOCK_SIZE_KEY)
+            );
+            let legacy_group = LegacyPackedGroupV2 {
+                batch: legacy_batch,
+                posting_tail_codec: PostingTailCodec::VarintDelta,
+            };
+            let legacy_group = Bytes::from(serialize_typed_entry(legacy_group));
+            let restored = CacheCodec::from_impl::<PostingListGroup>()
+                .deserialize(&legacy_group)
+                .hit()
+                .unwrap()
+                .downcast::<PostingListGroup>()
+                .unwrap();
+            let PostingList::Compressed(restored) = restored
+                .posting_list(0, Some(2.0), Some(3))
+                .unwrap()
+                .unwrap()
+            else {
+                panic!("expected a compressed legacy packed posting");
+            };
+            assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE);
+        }
+
+        /// A pre-stabilization blob (no magic) self-heals to a miss.
+        #[test]
+        fn pre_stabilization_blob_is_miss() {
+            // Old format led with a u64 LE length prefix, never our magic.
+            let mut blob = (30u64).to_le_bytes().to_vec();
+            blob.extend_from_slice(&[0u8; 30]);
+            assert!(codec().deserialize(&Bytes::from(blob)).hit().is_none());
+        }
+
+        /// A structurally-valid envelope whose body leads with an out-of-range
+        /// variant tag self-heals to a `BodyError` miss rather than panicking or
+        /// misreading the remaining bytes.
+        #[test]
+        fn unknown_posting_variant_is_miss() {
+            use lance_core::cache::{CacheDecode, CacheMissReason};
+
+            let mut buf = serialize_entry(compressed_with_shared_positions());
+            // The variant tag is the first body byte, right after the envelope
+            // (magic[4] + ver[1] + type_id_len[2] + type_id[N] + type_version[4]).
+            let type_id_len = u16::from_le_bytes([buf[5], buf[6]]) as usize;
+            let variant_off = 4 + 1 + 2 + type_id_len + 4;
+            buf[variant_off] = 2; // neither PLAIN (0) nor COMPRESSED (1)
+            match codec().deserialize(&Bytes::from(buf)) {
+                CacheDecode::Miss(reason) => assert_eq!(reason, CacheMissReason::BodyError),
+                CacheDecode::Hit(_) => panic!("expected a BodyError miss, got a hit"),
+            }
+        }
     }
 }
diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs
index fabc4ac9261..6efa6c0b7b3 100644
--- a/rust/lance-index/src/scalar/inverted/encoding.rs
+++ b/rust/lance-index/src/scalar/inverted/encoding.rs
@@ -5,10 +5,15 @@ use std::io::Write;
 
 use super::builder::BLOCK_SIZE;
 use super::index::{PositionStreamCodec, PostingTailCodec};
+#[cfg(test)]
+use super::tokenizer::LEGACY_BLOCK_SIZE;
+use super::tokenizer::validate_block_size;
 use arrow::array::LargeBinaryBuilder;
-use bitpacking::{BitPacker, BitPacker4x};
+use lance_bitpacking::{BitPacker, BitPacker4x, BitPacker8x};
 use lance_core::{Error, Result};
 
+pub const MAX_POSTING_BLOCK_SIZE: usize = BitPacker8x::BLOCK_LEN;
+
 // we compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE),
 // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies)
 // each block is:
@@ -41,18 +46,40 @@ pub fn compress_posting_list<'a>(
 
 #[cfg(test)]
 pub fn compress_posting_list_with_tail_codec<'a>(
+    length: usize,
+    doc_ids: impl Iterator<Item = &'a u32>,
+    frequencies: impl Iterator<Item = &'a u32>,
+    block_max_scores: impl Iterator<Item = f32>,
+    tail_codec: PostingTailCodec,
+) -> Result<arrow::array::LargeBinaryArray> {
+    compress_posting_list_with_tail_codec_and_block_size(
+        length,
+        doc_ids,
+        frequencies,
+        block_max_scores,
+        tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+#[cfg(test)]
+pub fn compress_posting_list_with_tail_codec_and_block_size<'a>(
     length: usize,
     doc_ids: impl Iterator<Item = &'a u32>,
     frequencies: impl Iterator<Item = &'a u32>,
     mut block_max_scores: impl Iterator<Item = f32>,
     tail_codec: PostingTailCodec,
+    block_size: usize,
 ) -> Result<arrow::array::LargeBinaryArray> {
-    if length < BLOCK_SIZE {
+    let block_size = validate_block_size(block_size)?;
+    if length < block_size {
         // directly do remainder compression to avoid overhead of creating buffer
         let mut builder = LargeBinaryBuilder::with_capacity(1, length * 4 * 2 + 1);
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
         compress_posting_remainder(
             doc_ids.copied().collect::<Vec<_>>().as_slice(),
             frequencies.copied().collect::<Vec<_>>().as_slice(),
@@ -63,27 +90,25 @@ pub fn compress_posting_list_with_tail_codec<'a>(
         return Ok(builder.finish());
     }
 
-    let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(BLOCK_SIZE), length * 3);
-    let mut buffer = [0u8; BLOCK_SIZE * 4 + 5];
-    let mut doc_id_buffer = Vec::with_capacity(BLOCK_SIZE);
-    let mut freq_buffer = Vec::with_capacity(BLOCK_SIZE);
+    let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(block_size), length * 3);
+    let mut doc_id_buffer = Vec::with_capacity(block_size);
+    let mut freq_buffer = Vec::with_capacity(block_size);
     for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) {
         doc_id_buffer.push(*doc_id);
         freq_buffer.push(*freq);
 
-        if doc_id_buffer.len() < BLOCK_SIZE {
+        if doc_id_buffer.len() < block_size {
             continue;
         }
 
-        assert_eq!(doc_id_buffer.len(), BLOCK_SIZE);
+        assert_eq!(doc_id_buffer.len(), block_size);
 
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
-        // delta encoding + bitpacking for doc ids
-        compress_sorted_block(&doc_id_buffer, &mut buffer, &mut builder)?;
-        // bitpacking for frequencies
-        compress_block(&freq_buffer, &mut buffer, &mut builder)?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
+        encode_posting_block_payload(&doc_id_buffer, &freq_buffer, &mut builder)?;
         builder.append_value("");
         doc_id_buffer.clear();
         freq_buffer.clear();
@@ -91,44 +116,187 @@ pub fn compress_posting_list_with_tail_codec<'a>(
 
     // we don't compress the last block if it is not full
     if !doc_id_buffer.is_empty() {
-        // write the max score of the block
-        let max_score = block_max_scores.next().unwrap();
-        let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        // write the max score of the block (128-doc blocks only)
+        if posting_block_score_prefix_len(block_size) > 0 {
+            let max_score = block_max_scores.next().unwrap();
+            let _ = builder.write(max_score.to_le_bytes().as_ref())?;
+        }
         compress_posting_remainder(&doc_id_buffer, &freq_buffer, tail_codec, &mut builder)?;
         builder.append_value("");
     }
     Ok(builder.finish())
 }
 
+/// Byte length of the block-max-score prefix on posting blocks. 128-doc
+/// blocks store a per-block max score, patched in at build time; 256-doc
+/// (V3) blocks always carry impact skip data, which supersedes it, so they
+/// store none.
+#[inline]
+pub fn posting_block_score_prefix_len(block_size: usize) -> usize {
+    if block_size == MAX_POSTING_BLOCK_SIZE {
+        0
+    } else {
+        4
+    }
+}
+
 pub fn encode_full_posting_block_into(
     doc_ids: &[u32],
     frequencies: &[u32],
     block: &mut Vec<u8>,
 ) -> Result<()> {
-    debug_assert_eq!(doc_ids.len(), BLOCK_SIZE);
-    debug_assert_eq!(frequencies.len(), BLOCK_SIZE);
-    block.extend_from_slice(&0f32.to_le_bytes());
-    let mut buffer = [0u8; BLOCK_SIZE * 4 + 5];
-    compress_sorted_block(doc_ids, &mut buffer, block)?;
-    compress_block(frequencies, &mut buffer, block)?;
+    validate_block_size(doc_ids.len())?;
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    if posting_block_score_prefix_len(doc_ids.len()) > 0 {
+        block.extend_from_slice(&0f32.to_le_bytes());
+    }
+    encode_posting_block_payload(doc_ids, frequencies, block)?;
     Ok(())
 }
 
+fn encode_posting_block_payload(
+    doc_ids: &[u32],
+    frequencies: &[u32],
+    block: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(doc_ids.len(), frequencies.len());
+    validate_block_size(doc_ids.len())?;
+    let mut buffer = [0u8; MAX_POSTING_BLOCK_SIZE * 4 + 5];
+    match doc_ids.len() {
+        BitPacker4x::BLOCK_LEN => {
+            compress_sorted_block_with::<BitPacker4x>(doc_ids, &mut buffer, block)?;
+            compress_block_with::<BitPacker4x>(frequencies, &mut buffer, block)?;
+        }
+        // 256-doc blocks (format V3) store frequencies with patched FOR:
+        // outliers no longer widen the whole block, which matters because one
+        // large tf per block otherwise doubles the frequency payload.
+        BitPacker8x::BLOCK_LEN => {
+            compress_sorted_block_with::<BitPacker8x>(doc_ids, &mut buffer, block)?;
+            compress_pfor_block_with::<BitPacker8x>(frequencies, &mut buffer, block)?;
+        }
+        _ => unreachable!("validated posting block size should be supported"),
+    }
+    Ok(())
+}
+
+/// Patched FOR (Lucene PForUtil style): pick the body bit width that
+/// minimizes total bytes, pack all values masked to that width, and append
+/// up to [`PFOR_MAX_EXCEPTIONS`] exceptions as (index u8, high-bits varint).
+const PFOR_MAX_EXCEPTIONS: usize = 31;
+
+#[inline]
+fn u32_bits(value: u32) -> usize {
+    (32 - value.leading_zeros()) as usize
+}
+
+#[inline]
+fn varint_u32_len(value: u32) -> usize {
+    u32_bits(value).max(1).div_ceil(7)
+}
+
+fn compress_pfor_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let max_bits = data.iter().map(|&v| u32_bits(v)).max().unwrap_or(0);
+    let mut best_width = max_bits;
+    let mut best_cost = P::BLOCK_LEN * max_bits / 8;
+    for width in (0..max_bits).rev() {
+        let mut exceptions = 0usize;
+        let mut exception_bytes = 0usize;
+        for &value in data {
+            if u32_bits(value) > width {
+                exceptions += 1;
+                exception_bytes += 1 + varint_u32_len(value >> width);
+            }
+        }
+        if exceptions > PFOR_MAX_EXCEPTIONS {
+            break;
+        }
+        let cost = P::BLOCK_LEN * width / 8 + exception_bytes;
+        if cost < best_cost {
+            best_cost = cost;
+            best_width = width;
+        }
+    }
+
+    let mask = if best_width >= 32 {
+        u32::MAX
+    } else {
+        (1u32 << best_width) - 1
+    };
+    let mut body = [0u32; MAX_POSTING_BLOCK_SIZE];
+    let mut exception_buf = Vec::new();
+    let mut exception_count = 0u8;
+    for (index, &value) in data.iter().enumerate() {
+        body[index] = value & mask;
+        if u32_bits(value) > best_width {
+            exception_buf.push(index as u8);
+            encode_varint_u32(&mut exception_buf, value >> best_width);
+            exception_count += 1;
+        }
+    }
+    let compressor = P::new();
+    let num_bytes = compressor.compress(&body[..P::BLOCK_LEN], buffer, best_width as u8);
+    let _ = builder.write(&[best_width as u8, exception_count])?;
+    let _ = builder.write(&buffer[..num_bytes])?;
+    let _ = builder.write(&exception_buf)?;
+    Ok(())
+}
+
+fn decompress_pfor_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let width = block[0];
+    let exception_count = block[1] as usize;
+    let compressor = P::new();
+    let num_bytes = compressor.decompress(&block[2..], buffer, width);
+    let mut offset = 2 + num_bytes;
+    for _ in 0..exception_count {
+        let index = block[offset] as usize;
+        offset += 1;
+        let high = decode_varint_u32(block, &mut offset)
+            .expect("pfor exception high bits should be a valid varint");
+        buffer[index] |= high << width;
+    }
+    res.extend_from_slice(buffer);
+    offset
+}
+
 pub fn encode_remainder_posting_block_into(
     doc_ids: &[u32],
     frequencies: &[u32],
     codec: PostingTailCodec,
+    block_size: usize,
     block: &mut Vec<u8>,
 ) -> Result<()> {
     debug_assert_eq!(doc_ids.len(), frequencies.len());
-    block.extend_from_slice(&0f32.to_le_bytes());
+    if posting_block_score_prefix_len(block_size) > 0 {
+        block.extend_from_slice(&0f32.to_le_bytes());
+    }
     compress_posting_remainder(doc_ids, frequencies, codec, block)?;
     Ok(())
 }
 
 #[inline]
 fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
-    let compressor = BitPacker4x::new();
+    compress_sorted_block_with::<BitPacker4x>(data, buffer, builder)
+}
+
+#[inline]
+fn compress_sorted_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let compressor = P::new();
     let num_bits = compressor.num_bits_sorted(data[0], data);
     let num_bytes = compressor.compress_sorted(data[0], data, buffer, num_bits);
     let _ = builder.write(data[0].to_le_bytes().as_ref())?;
@@ -139,7 +307,17 @@ fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Wri
 
 #[inline]
 fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> {
-    let compressor = BitPacker4x::new();
+    compress_block_with::<BitPacker4x>(data, buffer, builder)
+}
+
+#[inline]
+fn compress_block_with<P: BitPacker>(
+    data: &[u32],
+    buffer: &mut [u8],
+    builder: &mut impl Write,
+) -> Result<()> {
+    debug_assert_eq!(data.len(), P::BLOCK_LEN);
+    let compressor = P::new();
     let num_bits = compressor.num_bits(data);
     let num_bytes = compressor.compress(data, buffer, num_bits);
     let _ = builder.write(&[num_bits])?;
@@ -236,7 +414,7 @@ pub fn compress_positions(positions: &[u32]) -> Result<arrow::array::LargeBinary
 }
 
 #[inline]
-fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
+pub fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
     while value >= 0x80 {
         dst.push((value as u8) | 0x80);
         value >>= 7;
@@ -244,39 +422,6 @@ fn encode_varint_u32(dst: &mut Vec<u8>, mut value: u32) {
     dst.push(value as u8);
 }
 
-/// Encode a monotonically increasing sequence of `group_starts` (the first
-/// row of each posting-list cache group) as varint-encoded deltas. The first
-/// value is stored as-is and each subsequent value as its delta from the
-/// previous one; since deltas are group sizes (1..=cap) they fit in ~1 byte,
-/// keeping the buffer tiny even for indexes with millions of tokens. See
-/// issue #7040.
-pub(super) fn encode_group_starts(group_starts: &[u32]) -> Vec<u8> {
-    let mut dst = Vec::with_capacity(group_starts.len());
-    let mut previous = 0u32;
-    for &start in group_starts {
-        debug_assert!(start >= previous, "group_starts must be monotonic");
-        encode_varint_u32(&mut dst, start - previous);
-        previous = start;
-    }
-    dst
-}
-
-/// Decode the buffer produced by [`encode_group_starts`] back into the
-/// absolute `group_starts` values.
-pub(super) fn decode_group_starts(src: &[u8]) -> Result<Vec<u32>> {
-    let mut group_starts = Vec::new();
-    let mut offset = 0;
-    let mut previous = 0u32;
-    while offset < src.len() {
-        let delta = decode_varint_u32(src, &mut offset)?;
-        previous = previous
-            .checked_add(delta)
-            .ok_or_else(|| Error::index("group_starts delta decode overflowed u32".to_owned()))?;
-        group_starts.push(previous);
-    }
-    Ok(group_starts)
-}
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub(super) struct PositionBlockBuilder {
     codec: PositionStreamCodec,
@@ -379,7 +524,7 @@ impl PositionBlockBuilder {
 }
 
 #[inline]
-fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result<u32> {
+pub fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result<u32> {
     let mut value = 0u32;
     let mut shift = 0u32;
     while *offset < src.len() {
@@ -557,15 +702,10 @@ fn decode_position_stream_packed_block(
     let mut deltas = Vec::with_capacity(total_positions);
 
     for _ in 0..full_delta_blocks {
-        if packed_offset >= src.len() {
-            return Err(Error::index(
-                "unexpected EOF while decoding packed position stream".to_owned(),
-            ));
-        }
-        let num_bits = src[packed_offset];
-        packed_offset += 1;
-        let consumed = compressor.decompress(&src[packed_offset..], &mut packed_values, num_bits);
-        packed_offset += consumed;
+        let (num_bits, payload, next_offset) = packed_position_group(src, packed_offset)?;
+        let consumed = compressor.decompress(payload, &mut packed_values, num_bits);
+        debug_assert_eq!(consumed, payload.len());
+        packed_offset = next_offset;
         deltas.extend_from_slice(&packed_values);
     }
 
@@ -601,6 +741,144 @@ fn decode_position_stream_packed_block(
     Ok(())
 }
 
+fn packed_position_group(src: &[u8], offset: usize) -> Result<(u8, &[u8], usize)> {
+    let num_bits = *src.get(offset).ok_or_else(|| {
+        Error::index(format!(
+            "unexpected EOF reading packed position group header at byte offset {offset}; \
+             stream length is {}",
+            src.len()
+        ))
+    })?;
+    if num_bits > u32::BITS as u8 {
+        return Err(Error::index(format!(
+            "invalid packed position group bit width {num_bits} at byte offset {offset}; \
+             expected at most {}",
+            u32::BITS
+        )));
+    }
+
+    let payload_start = offset
+        .checked_add(1)
+        .ok_or_else(|| Error::index("packed position group offset overflow".to_owned()))?;
+    let payload_len = usize::from(num_bits) * BLOCK_SIZE / 8;
+    let payload_end = payload_start
+        .checked_add(payload_len)
+        .ok_or_else(|| Error::index("packed position group length overflow".to_owned()))?;
+    let payload = src.get(payload_start..payload_end).ok_or_else(|| {
+        Error::index(format!(
+            "unexpected EOF reading packed position group payload at byte offset {offset}; \
+             need {payload_len} bytes after the header but stream length is {}",
+            src.len()
+        ))
+    })?;
+    Ok((num_bits, payload, payload_end))
+}
+
+/// Decode one document's positions out of a PackedDelta position block
+/// without decoding the rest of the block. Full 128-delta groups are
+/// self-describing (`[num_bits u8][16 * num_bits packed bytes]`), so group
+/// byte offsets are recovered by hopping headers — no format change is
+/// involved. `delta_range` is the doc's range in the block-wide delta stream
+/// (from the frequency prefix sums); per-doc deltas reset at document
+/// boundaries, so decoding starts cleanly at `delta_range.start`.
+///
+/// The caller passes per-block scratch state that this function maintains:
+/// `group_offsets` (lazily extended header index, seeded with `[0]`),
+/// `unpacked_group`/`unpacked_group_idx` (the last unpacked group), and
+/// `tail_cache` (the varint tail, decoded in full on first touch). All are
+/// reset by the caller when the block cursor moves.
+#[allow(clippy::too_many_arguments)]
+pub(super) fn seek_packed_doc_positions(
+    src: &[u8],
+    total_deltas: usize,
+    delta_range: std::ops::Range<usize>,
+    group_offsets: &mut Vec<usize>,
+    unpacked_group: &mut [u32; BLOCK_SIZE],
+    unpacked_group_idx: &mut Option<usize>,
+    tail_cache: &mut Vec<u32>,
+    dst: &mut Vec<u32>,
+) -> Result<()> {
+    dst.clear();
+    if delta_range.start > delta_range.end || delta_range.end > total_deltas {
+        return Err(Error::index(format!(
+            "invalid packed position delta range {}..{} for {total_deltas} total deltas",
+            delta_range.start, delta_range.end
+        )));
+    }
+    if delta_range.is_empty() {
+        return Ok(());
+    }
+    let num_full_groups = total_deltas / BLOCK_SIZE;
+    let packed_deltas_end = num_full_groups * BLOCK_SIZE;
+
+    // Extend the header index far enough for this range (tail needs the
+    // offset one past the last full group).
+    let last_needed_group = if delta_range.end > packed_deltas_end {
+        num_full_groups
+    } else {
+        (delta_range.end - 1) / BLOCK_SIZE
+    };
+    while group_offsets.len() <= last_needed_group {
+        let last = *group_offsets
+            .last()
+            .ok_or_else(|| Error::index("packed position group offsets are empty".to_owned()))?;
+        let (_, _, next_offset) = packed_position_group(src, last)?;
+        group_offsets.push(next_offset);
+    }
+
+    let mut previous = 0u32;
+    let mut first = true;
+    let mut push_delta = |delta: u32, dst: &mut Vec<u32>| -> Result<()> {
+        let position = if first {
+            first = false;
+            delta
+        } else {
+            previous
+                .checked_add(delta)
+                .ok_or_else(|| Error::index("position stream overflow while decoding".to_owned()))?
+        };
+        dst.push(position);
+        previous = position;
+        Ok(())
+    };
+
+    for index in delta_range.start..delta_range.end.min(packed_deltas_end) {
+        let group = index / BLOCK_SIZE;
+        if *unpacked_group_idx != Some(group) {
+            let offset = *group_offsets.get(group).ok_or_else(|| {
+                Error::index(format!(
+                    "missing packed position group offset for group {group}; have {} offsets",
+                    group_offsets.len()
+                ))
+            })?;
+            let (num_bits, payload, _) = packed_position_group(src, offset)?;
+            BitPacker4x::new().decompress(payload, unpacked_group, num_bits);
+            *unpacked_group_idx = Some(group);
+        }
+        push_delta(unpacked_group[index % BLOCK_SIZE], dst)?;
+    }
+
+    if delta_range.end > packed_deltas_end {
+        let tail_len = total_deltas - packed_deltas_end;
+        if tail_cache.len() != tail_len {
+            tail_cache.clear();
+            tail_cache.reserve(tail_len);
+            let mut offset = *group_offsets.get(num_full_groups).ok_or_else(|| {
+                Error::index(format!(
+                    "missing packed position tail offset after {num_full_groups} full groups"
+                ))
+            })?;
+            for _ in 0..tail_len {
+                tail_cache.push(decode_varint_u32(src, &mut offset)?);
+            }
+        }
+        for index in delta_range.start.max(packed_deltas_end)..delta_range.end {
+            push_delta(tail_cache[index - packed_deltas_end], dst)?;
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 pub fn encode_position_stream_block_into(
     positions: &[u32],
@@ -650,23 +928,46 @@ pub fn decompress_posting_list_with_tail_codec(
     posting_list: &arrow::array::LargeBinaryArray,
     tail_codec: PostingTailCodec,
 ) -> Result<(Vec<u32>, Vec<u32>)> {
+    decompress_posting_list_with_tail_codec_and_block_size(
+        num_docs,
+        posting_list,
+        tail_codec,
+        LEGACY_BLOCK_SIZE,
+    )
+}
+
+#[cfg(test)]
+pub fn decompress_posting_list_with_tail_codec_and_block_size(
+    num_docs: u32,
+    posting_list: &arrow::array::LargeBinaryArray,
+    tail_codec: PostingTailCodec,
+    block_size: usize,
+) -> Result<(Vec<u32>, Vec<u32>)> {
+    let block_size = validate_block_size(block_size)?;
     let mut doc_ids: Vec<u32> = Vec::with_capacity(num_docs as usize);
     let mut frequencies: Vec<u32> = Vec::with_capacity(num_docs as usize);
 
-    let mut buffer = [0u32; BLOCK_SIZE];
-    let bitpacking_blocks = num_docs as usize / BLOCK_SIZE;
+    let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE];
+    let bitpacking_blocks = num_docs as usize / block_size;
     for compressed in posting_list.iter().take(bitpacking_blocks) {
         let compressed = compressed.unwrap();
-        decompress_posting_block(compressed, &mut buffer, &mut doc_ids, &mut frequencies);
+        decompress_posting_block(
+            compressed,
+            &mut buffer,
+            &mut doc_ids,
+            &mut frequencies,
+            block_size,
+        );
     }
 
-    let remainder = num_docs as usize % BLOCK_SIZE;
+    let remainder = num_docs as usize % block_size;
     if remainder > 0 {
         let compressed = posting_list.value(bitpacking_blocks);
         decompress_posting_remainder(
             compressed,
             remainder,
             tail_codec,
+            block_size,
             &mut doc_ids,
             &mut frequencies,
         );
@@ -701,24 +1002,46 @@ pub fn read_num_positions(compressed: &arrow::array::LargeBinaryArray) -> u32 {
 
 pub fn decompress_posting_block(
     block: &[u8],
-    buffer: &mut [u32; BLOCK_SIZE],
+    buffer: &mut [u32],
     doc_ids: &mut Vec<u32>,
     frequencies: &mut Vec<u32>,
+    block_size: usize,
 ) {
-    // skip the first 4 bytes for the max block score
-    let block = &block[4..];
-    let num_bytes = decompress_sorted_block(block, buffer, doc_ids);
-    decompress_block(&block[num_bytes..], buffer, frequencies);
+    debug_assert!(validate_block_size(block_size).is_ok());
+    debug_assert!(buffer.len() >= block_size);
+    // skip the block max score prefix (128-doc blocks only)
+    let mut block = &block[posting_block_score_prefix_len(block_size)..];
+    match block_size {
+        BitPacker4x::BLOCK_LEN => {
+            let num_bytes = decompress_sorted_block_with::<BitPacker4x>(block, buffer, doc_ids);
+            block = &block[num_bytes..];
+            let num_bytes = decompress_block_with::<BitPacker4x>(block, buffer, frequencies);
+            block = &block[num_bytes..];
+        }
+        BitPacker8x::BLOCK_LEN => {
+            let num_bytes = decompress_sorted_block_with::<BitPacker8x>(block, buffer, doc_ids);
+            block = &block[num_bytes..];
+            let num_bytes = decompress_pfor_block_with::<BitPacker8x>(block, buffer, frequencies);
+            block = &block[num_bytes..];
+        }
+        _ => unreachable!("validated posting block size should be supported"),
+    }
+    debug_assert!(
+        block.is_empty(),
+        "posting block has {} trailing bytes after decoding",
+        block.len()
+    );
 }
 
 pub fn decompress_posting_remainder(
     block: &[u8],
     n: usize,
     codec: PostingTailCodec,
+    block_size: usize,
     doc_ids: &mut Vec<u32>,
     frequencies: &mut Vec<u32>,
 ) {
-    let block = &block[4..];
+    let block = &block[posting_block_score_prefix_len(block_size)..];
     match codec {
         PostingTailCodec::Fixed32 => {
             decompress_raw_remainder(block, n, doc_ids);
@@ -755,9 +1078,14 @@ pub fn decompress_posting_remainder(
     }
 }
 
-pub fn decode_full_posting_block(block: &[u8], doc_ids: &mut Vec<u32>, frequencies: &mut Vec<u32>) {
-    let mut buffer = [0u32; BLOCK_SIZE];
-    decompress_posting_block(block, &mut buffer, doc_ids, frequencies);
+pub fn decode_full_posting_block(
+    block: &[u8],
+    doc_ids: &mut Vec<u32>,
+    frequencies: &mut Vec<u32>,
+    block_size: usize,
+) {
+    let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE];
+    decompress_posting_block(block, &mut buffer, doc_ids, frequencies, block_size);
 }
 
 pub fn decompress_sorted_block(
@@ -765,7 +1093,17 @@ pub fn decompress_sorted_block(
     buffer: &mut [u32; BLOCK_SIZE],
     res: &mut Vec<u32>,
 ) -> usize {
-    let compressor = BitPacker4x::new();
+    decompress_sorted_block_with::<BitPacker4x>(block, buffer, res)
+}
+
+fn decompress_sorted_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let compressor = P::new();
     let initial = u32::from_le_bytes(block[0..4].try_into().unwrap());
     let num_bits = block[4];
     let num_bytes = compressor.decompress_sorted(initial, &block[5..], buffer, num_bits);
@@ -773,11 +1111,18 @@ pub fn decompress_sorted_block(
     5 + num_bytes
 }
 
-fn decompress_block(block: &[u8], buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec<u32>) {
-    let compressor = BitPacker4x::new();
+fn decompress_block_with<P: BitPacker>(
+    block: &[u8],
+    buffer: &mut [u32],
+    res: &mut Vec<u32>,
+) -> usize {
+    debug_assert!(buffer.len() >= P::BLOCK_LEN);
+    let buffer = &mut buffer[..P::BLOCK_LEN];
+    let compressor = P::new();
     let num_bits = block[0];
-    compressor.decompress(&block[1..], buffer, num_bits);
+    let num_bytes = compressor.decompress(&block[1..], buffer, num_bits);
     res.extend_from_slice(&buffer[..]);
+    1 + num_bytes
 }
 
 pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>) {
@@ -787,11 +1132,18 @@ pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec<u32>
     }
 }
 
-pub fn read_posting_tail_first_doc(block: &[u8], codec: PostingTailCodec) -> u32 {
+pub fn read_posting_tail_first_doc(
+    block: &[u8],
+    codec: PostingTailCodec,
+    block_size: usize,
+) -> u32 {
+    let prefix = posting_block_score_prefix_len(block_size);
     match codec {
-        PostingTailCodec::Fixed32 => u32::from_le_bytes(block[4..8].try_into().unwrap()),
+        PostingTailCodec::Fixed32 => {
+            u32::from_le_bytes(block[prefix..prefix + 4].try_into().unwrap())
+        }
         PostingTailCodec::VarintDelta => {
-            let mut offset = 4usize;
+            let mut offset = prefix;
             decode_varint_u32(block, &mut offset)
                 .expect("posting tail block should contain a valid first doc id")
         }
@@ -805,31 +1157,6 @@ mod tests {
     use itertools::Itertools;
     use rand::Rng;
 
-    #[test]
-    fn test_group_starts_codec_roundtrip() {
-        for case in [
-            vec![],
-            vec![0u32],
-            vec![0, 1, 2, 3],
-            // realistic: monotonic with varied group sizes and a large jump
-            vec![0, 64, 128, 129, 4096, 1_000_000],
-        ] {
-            let encoded = encode_group_starts(&case);
-            let decoded = decode_group_starts(&encoded).unwrap();
-            assert_eq!(decoded, case, "roundtrip mismatch for {case:?}");
-        }
-    }
-
-    #[test]
-    fn test_decode_group_starts_rejects_overflow() {
-        // A crafted buffer whose deltas sum past u32::MAX must error rather
-        // than wrap. Encodes u32::MAX followed by a +1 delta.
-        let mut buf = Vec::new();
-        encode_varint_u32(&mut buf, u32::MAX);
-        encode_varint_u32(&mut buf, 1);
-        assert!(decode_group_starts(&buf).is_err());
-    }
-
     #[test]
     fn test_compress_posting_list() -> Result<()> {
         let num_rows: usize = BLOCK_SIZE * 1024 - 7;
@@ -867,6 +1194,84 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_compress_posting_list_supported_block_sizes() -> Result<()> {
+        for block_size in [128, 256] {
+            let num_rows: usize = block_size * 2 + 7;
+            let doc_ids = (0..num_rows as u32).collect::<Vec<_>>();
+            let frequencies = (0..num_rows as u32)
+                .map(|value| value % 7 + 1)
+                .collect::<Vec<_>>();
+            let block_max_scores =
+                (0..num_rows.div_ceil(block_size)).map(|value| value as f32 + 1.0);
+
+            let posting_list = compress_posting_list_with_tail_codec_and_block_size(
+                doc_ids.len(),
+                doc_ids.iter(),
+                frequencies.iter(),
+                block_max_scores,
+                PostingTailCodec::VarintDelta,
+                block_size,
+            )?;
+            assert_eq!(posting_list.len(), num_rows.div_ceil(block_size));
+
+            let (decoded_doc_ids, decoded_frequencies) =
+                decompress_posting_list_with_tail_codec_and_block_size(
+                    num_rows as u32,
+                    &posting_list,
+                    PostingTailCodec::VarintDelta,
+                    block_size,
+                )?;
+            assert_eq!(decoded_doc_ids, doc_ids);
+            assert_eq!(decoded_frequencies, frequencies);
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_256_posting_block_uses_single_physical_bitpack_chunk() -> Result<()> {
+        let block_size = BitPacker8x::BLOCK_LEN;
+        let doc_ids = (0..block_size as u32).collect::<Vec<_>>();
+        let frequencies = (0..block_size as u32)
+            .map(|value| value % 13 + 1)
+            .collect::<Vec<_>>();
+
+        let posting_list = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(1.0),
+            PostingTailCodec::VarintDelta,
+            block_size,
+        )?;
+        assert_eq!(posting_list.len(), 1);
+
+        let block = posting_list.value(0);
+        // 256-doc blocks carry no block-max-score prefix (impacts supply the
+        // per-block bound): [first_doc u32][doc num_bits u8][doc payload]...
+        let doc_num_bits = block[4];
+        let doc_bytes = BitPacker8x::compressed_block_size(doc_num_bits);
+        let freq_header_offset = 5 + doc_bytes;
+        // 256-doc blocks use patched FOR for frequencies:
+        // [width u8][exception_count u8][body][exceptions...]
+        let freq_num_bits = block[freq_header_offset];
+        let exception_count = block[freq_header_offset + 1] as usize;
+        assert_eq!(exception_count, 0, "uniform freqs need no exceptions");
+        let freq_bytes = BitPacker8x::compressed_block_size(freq_num_bits);
+        assert_eq!(block.len(), freq_header_offset + 2 + freq_bytes);
+
+        let (decoded_doc_ids, decoded_frequencies) =
+            decompress_posting_list_with_tail_codec_and_block_size(
+                doc_ids.len() as u32,
+                &posting_list,
+                PostingTailCodec::VarintDelta,
+                block_size,
+            )?;
+        assert_eq!(decoded_doc_ids, doc_ids);
+        assert_eq!(decoded_frequencies, frequencies);
+        Ok(())
+    }
+
     #[test]
     fn test_compress_posting_list_fixed32_tail_still_roundtrips() -> Result<()> {
         let doc_ids = vec![3_u32, 10_u32, 24_u32];
@@ -913,6 +1318,116 @@ mod tests {
         Ok(())
     }
 
+    /// Per-doc seek decoding of a PackedDelta position block must return
+    /// exactly the same positions as decoding the whole block, for every doc,
+    /// across group-boundary-straddling docs and varint tails.
+    #[test]
+    fn test_packed_position_doc_seek_matches_block_decode() -> Result<()> {
+        let mut rng = rand::rng();
+        // Frequency shapes: tiny blocks (tail only), exactly one group, a doc
+        // straddling group boundaries, and a large multi-group block.
+        let freq_shapes: Vec<Vec<u32>> = vec![
+            vec![1],
+            vec![3, 1, 5],
+            vec![64, 64],
+            vec![100, 60, 40],
+            (0..256u32).map(|i| (i % 7) + 1).collect(),
+            vec![300, 2, 129, 1, 77],
+        ];
+        for frequencies in freq_shapes {
+            let total: usize = frequencies.iter().map(|&f| f as usize).sum();
+            // Positions ascend within each doc; docs are independent.
+            let mut positions = Vec::with_capacity(total);
+            for &freq in &frequencies {
+                let mut current = rng.random_range(0..1000u32);
+                for _ in 0..freq {
+                    positions.push(current);
+                    current += rng.random_range(1..50u32);
+                }
+            }
+
+            let mut encoded = Vec::new();
+            encode_position_stream_block_into(
+                &positions,
+                &frequencies,
+                PositionStreamCodec::PackedDelta,
+                &mut encoded,
+            )?;
+
+            let mut whole = Vec::new();
+            decode_position_stream_block(
+                &encoded,
+                &frequencies,
+                PositionStreamCodec::PackedDelta,
+                &mut whole,
+            )?;
+            assert_eq!(whole, positions);
+
+            let mut group_offsets = vec![0usize];
+            let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]);
+            let mut unpacked_group_idx = None;
+            let mut tail_cache = Vec::new();
+            let mut scratch = Vec::new();
+            let mut delta_start = 0usize;
+            for &freq in &frequencies {
+                let delta_end = delta_start + freq as usize;
+                seek_packed_doc_positions(
+                    &encoded,
+                    total,
+                    delta_start..delta_end,
+                    &mut group_offsets,
+                    &mut unpacked_group,
+                    &mut unpacked_group_idx,
+                    &mut tail_cache,
+                    &mut scratch,
+                )?;
+                assert_eq!(
+                    scratch,
+                    whole[delta_start..delta_end],
+                    "doc positions mismatch for range {delta_start}..{delta_end} freqs={frequencies:?}"
+                );
+                delta_start = delta_end;
+            }
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_packed_position_decoders_reject_malformed_groups() {
+        let frequencies = [BLOCK_SIZE as u32];
+        for encoded in [&[1_u8][..], &[33_u8][..]] {
+            let mut decoded = Vec::new();
+            assert!(
+                decode_position_stream_block(
+                    encoded,
+                    &frequencies,
+                    PositionStreamCodec::PackedDelta,
+                    &mut decoded,
+                )
+                .is_err()
+            );
+
+            let mut group_offsets = vec![0usize];
+            let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]);
+            let mut unpacked_group_idx = None;
+            let mut tail_cache = Vec::new();
+            let mut scratch = Vec::new();
+            assert!(
+                seek_packed_doc_positions(
+                    encoded,
+                    BLOCK_SIZE,
+                    0..BLOCK_SIZE,
+                    &mut group_offsets,
+                    &mut unpacked_group,
+                    &mut unpacked_group_idx,
+                    &mut tail_cache,
+                    &mut scratch,
+                )
+                .is_err()
+            );
+        }
+    }
+
     #[test]
     fn test_encode_position_stream_block_roundtrip() -> Result<()> {
         let frequencies = vec![1, 3, 2, 4];
diff --git a/rust/lance-index/src/scalar/inverted/impact.rs b/rust/lance-index/src/scalar/inverted/impact.rs
new file mode 100644
index 00000000000..505d9f613d9
--- /dev/null
+++ b/rust/lance-index/src/scalar/inverted/impact.rs
@@ -0,0 +1,1055 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::mem::size_of;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use arrow_array::builder::LargeBinaryBuilder;
+use arrow_array::{Array, LargeBinaryArray};
+use lance_core::{Error, Result};
+
+use super::scorer::Scorer;
+
+pub const IMPACT_LEVEL1_BLOCKS: usize = 32;
+const SMALL_FRONTIER_FREQ_LIMIT: usize = 256;
+
+/// On-disk encoding of one impact entry, shared by every posting block size.
+///
+/// Entries contain `[doc_up_to varint][pair_count varint][pairs...]`. Each
+/// pair stores a varint whose high bits are `freq_delta - 1` and whose low bit
+/// reports whether a one-byte norm delta follows. The norm itself is the
+/// quantized `u8` document-length code; the common `norm_delta == 1` case needs
+/// no norm byte.
+#[derive(Debug, Clone)]
+pub struct ImpactSkipData {
+    entries: LargeBinaryArray,
+    level0_len: usize,
+    // Last doc id covered by each entry (level0 entries then level1 entries),
+    // decoded once at construction. Level1 markers are fully validated because
+    // WAND may use them to skip a group; u32::MAX marks malformed entries.
+    entry_doc_up_tos: Arc<[u32]>,
+    // The most recently baked bounds with a stable scorer key. Each query holds
+    // its own Arc in ImpactScoreCache, so replacing this slot for another scorer
+    // cannot change bounds already in use. Scorers without a key never enter the
+    // shared slot. Malformed entries bake to INFINITY so pruning stays safe.
+    last_keyed_bounds: Arc<Mutex<LastKeyedImpactBounds>>,
+}
+
+impl PartialEq for ImpactSkipData {
+    fn eq(&self, other: &Self) -> bool {
+        self.entries == other.entries && self.level0_len == other.level0_len
+    }
+}
+
+#[cfg(test)]
+#[derive(Debug, Clone, Copy)]
+pub struct ImpactScore {
+    pub score: f32,
+    pub entries_scanned: usize,
+}
+
+#[derive(Debug)]
+struct ImpactBounds {
+    per_entry: Box<[f32]>,
+    global: f32,
+}
+
+type LastKeyedImpactBounds = Option<(u64, Arc<ImpactBounds>)>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ImpactBoundsCacheKey {
+    Keyed(u64),
+    QueryLocal,
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct ImpactScoreCache {
+    key: Option<ImpactBoundsCacheKey>,
+    bounds: Option<Arc<ImpactBounds>>,
+}
+
+impl ImpactScoreCache {
+    fn bounds<'a, S: Scorer + ?Sized>(
+        &'a mut self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> &'a ImpactBounds {
+        let scorer_key = scorer.doc_weight_cache_key();
+        let cache_key = scorer_key
+            .map(ImpactBoundsCacheKey::Keyed)
+            .unwrap_or(ImpactBoundsCacheKey::QueryLocal);
+        if self.key != Some(cache_key) {
+            self.key = Some(cache_key);
+            self.bounds = None;
+        }
+
+        self.bounds
+            .get_or_insert_with(|| impacts.bounds_for_scorer(scorer, scorer_key))
+    }
+
+    fn entry_score<S: Scorer + ?Sized>(
+        &mut self,
+        impacts: &ImpactSkipData,
+        entry_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+    ) -> f32 {
+        if query_weight <= 0.0 {
+            return 0.0;
+        }
+        query_weight * self.bounds(impacts, scorer).per_entry[entry_idx]
+    }
+}
+
+impl ImpactSkipData {
+    pub fn new(entries: LargeBinaryArray, level0_len: usize) -> Result<Self> {
+        let expected_len = level0_len + level1_len(level0_len);
+        if entries.len() != expected_len {
+            return Err(Error::index(format!(
+                "impact entry count mismatch: got {}, expected {} for {} level0 blocks",
+                entries.len(),
+                expected_len,
+                level0_len
+            )));
+        }
+        let entry_doc_up_tos = (0..entries.len())
+            .map(|entry_idx| {
+                if entries.is_null(entry_idx) {
+                    return u32::MAX;
+                }
+                let bytes = entries.value(entry_idx);
+                let doc_up_to = if entry_idx < level0_len {
+                    decode_level0_entry_doc_up_to(bytes)
+                } else {
+                    decode_entry_doc_up_to(bytes)
+                };
+                doc_up_to.unwrap_or(u32::MAX)
+            })
+            .collect::<Arc<[u32]>>();
+        Ok(Self {
+            entries,
+            level0_len,
+            entry_doc_up_tos,
+            last_keyed_bounds: Arc::new(Mutex::new(None)),
+        })
+    }
+
+    fn keyed_bounds_guard(&self) -> MutexGuard<'_, LastKeyedImpactBounds> {
+        match self.last_keyed_bounds.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        }
+    }
+
+    fn bounds_for_scorer<S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        scorer_key: Option<u64>,
+    ) -> Arc<ImpactBounds> {
+        let Some(scorer_key) = scorer_key else {
+            return Arc::new(self.compute_bounds(scorer));
+        };
+
+        {
+            let cached = self.keyed_bounds_guard();
+            if let Some((cached_key, bounds)) = cached.as_ref()
+                && *cached_key == scorer_key
+            {
+                return bounds.clone();
+            }
+        }
+
+        // Compute outside the mutex. Concurrent misses may duplicate this work,
+        // but only the short publication/check below holds the shared lock.
+        let computed = Arc::new(self.compute_bounds(scorer));
+        let mut cached = self.keyed_bounds_guard();
+        if let Some((cached_key, bounds)) = cached.as_ref()
+            && *cached_key == scorer_key
+        {
+            return bounds.clone();
+        }
+        *cached = Some((scorer_key, computed.clone()));
+        computed
+    }
+
+    fn compute_bounds<S: Scorer + ?Sized>(&self, scorer: &S) -> ImpactBounds {
+        let per_entry = (0..self.entries.len())
+            .map(|entry_idx| {
+                if self.entries.is_null(entry_idx) {
+                    return f32::INFINITY;
+                }
+                let bytes = self.entries.value(entry_idx);
+                let mut max_doc_weight = 0.0_f32;
+                match for_each_entry_pair(bytes, |freq, doc_len| {
+                    max_doc_weight = max_doc_weight.max(scorer.doc_weight(freq, doc_len));
+                }) {
+                    Ok(()) => max_doc_weight,
+                    Err(_) => f32::INFINITY,
+                }
+            })
+            .collect::<Box<[f32]>>();
+        // The level1 entries cover every block, so their max is the list-wide
+        // max doc weight; zero-entry lists fall back to the empty level0 slab.
+        let global = if per_entry.len() > self.level0_len {
+            per_entry[self.level0_len..]
+                .iter()
+                .copied()
+                .fold(0.0_f32, f32::max)
+        } else {
+            per_entry.iter().copied().fold(0.0_f32, f32::max)
+        };
+        ImpactBounds { per_entry, global }
+    }
+
+    /// List-wide max doc weight, from the scorer-specific cached bounds. The
+    /// tightest valid global score bound is `query_weight * this`, matching what
+    /// the non-impact format stores as `max_score` at build time.
+    pub fn global_max_doc_weight_cached<S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        cache.bounds(self, scorer).global
+    }
+
+    /// Cached per-block max doc weights (level0 entries only), for bulk skip
+    /// scans over dead ranges without per-block window bookkeeping.
+    pub(crate) fn level0_doc_weight_bounds_cached<'a, S: Scorer + ?Sized>(
+        &self,
+        scorer: &S,
+        cache: &'a mut ImpactScoreCache,
+    ) -> &'a [f32] {
+        &cache.bounds(self, scorer).per_entry[..self.level0_len]
+    }
+
+    pub fn entries(&self) -> &LargeBinaryArray {
+        &self.entries
+    }
+
+    /// Conservative heap charge for query-independent derived state and one
+    /// shared keyed-bound slab, whether or not that slab has been initialized
+    /// yet. The Arrow impact entries are owned by the enclosing batch and are
+    /// deliberately excluded so packed-group cache accounting counts them once.
+    pub(crate) fn derived_cache_bytes(&self) -> usize {
+        Self::derived_cache_bytes_for_entries(self.entries.len())
+    }
+
+    pub(crate) fn derived_cache_bytes_for_entries(entry_count: usize) -> usize {
+        entry_count * size_of::<u32>()
+            + size_of::<Mutex<LastKeyedImpactBounds>>()
+            + size_of::<ImpactBounds>()
+            + entry_count * size_of::<f32>()
+    }
+
+    #[cfg(test)]
+    pub(crate) fn shares_derived_state_with(&self, other: &Self) -> bool {
+        Arc::ptr_eq(&self.entry_doc_up_tos, &other.entry_doc_up_tos)
+            && Arc::ptr_eq(&self.last_keyed_bounds, &other.last_keyed_bounds)
+    }
+
+    #[cfg(test)]
+    pub fn level0_len(&self) -> usize {
+        self.level0_len
+    }
+
+    #[cfg(test)]
+    pub fn level1_len(&self) -> usize {
+        level1_len(self.level0_len)
+    }
+
+    pub(crate) fn level1_doc_up_to(&self, group_idx: usize) -> Option<u32> {
+        if group_idx >= level1_len(self.level0_len) {
+            return None;
+        }
+        match self.entry_doc_up_tos[self.level0_len + group_idx] {
+            u32::MAX => None,
+            doc_up_to => Some(doc_up_to),
+        }
+    }
+
+    /// Last doc id covered by the level0 entry of `block_idx`, or `None` when
+    /// the entry is missing or malformed.
+    pub(crate) fn level0_doc_up_to(&self, block_idx: usize) -> Option<u32> {
+        if block_idx >= self.level0_len {
+            return None;
+        }
+        match self.entry_doc_up_tos[block_idx] {
+            u32::MAX => None,
+            doc_up_to => Some(doc_up_to),
+        }
+    }
+
+    pub fn level0_score_cached<S: Scorer + ?Sized>(
+        &self,
+        block_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        if block_idx >= self.level0_len {
+            return 0.0;
+        }
+        cache.entry_score(self, block_idx, query_weight, scorer)
+    }
+
+    /// Max score of the docs covered by the level1 entry of `group_idx`,
+    /// answered from the scorer-specific cached bounds slab.
+    pub(crate) fn level1_score_cached<S: Scorer + ?Sized>(
+        &self,
+        group_idx: usize,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> f32 {
+        if group_idx >= level1_len(self.level0_len) {
+            return 0.0;
+        }
+        cache.entry_score(self, self.level0_len + group_idx, query_weight, scorer)
+    }
+
+    #[cfg(test)]
+    pub fn max_score_up_to_cached<S>(
+        &self,
+        start_block_idx: usize,
+        up_to: u64,
+        query_weight: f32,
+        scorer: &S,
+        cache: &mut ImpactScoreCache,
+    ) -> ImpactScore
+    where
+        S: Scorer + ?Sized,
+    {
+        let mut block_idx = start_block_idx;
+        let mut max_score = 0.0_f32;
+        let mut entries_scanned = 0usize;
+
+        while block_idx < self.level0_len {
+            let group_idx = block_idx / IMPACT_LEVEL1_BLOCKS;
+            let group_start = group_idx * IMPACT_LEVEL1_BLOCKS;
+            let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(self.level0_len);
+            if block_idx == group_start {
+                let level1_entry_idx = self.level0_len + group_idx;
+                match self.entry_doc_up_tos[level1_entry_idx] {
+                    u32::MAX => {
+                        return ImpactScore {
+                            score: f32::INFINITY,
+                            entries_scanned: entries_scanned + 1,
+                        };
+                    }
+                    doc_up_to if u64::from(doc_up_to) <= up_to => {
+                        max_score = max_score.max(cache.entry_score(
+                            self,
+                            level1_entry_idx,
+                            query_weight,
+                            scorer,
+                        ));
+                        entries_scanned += 1;
+                        block_idx = group_end;
+                        continue;
+                    }
+                    _ => {}
+                }
+            }
+
+            max_score = max_score.max(cache.entry_score(self, block_idx, query_weight, scorer));
+            entries_scanned += 1;
+            match self.entry_doc_up_tos[block_idx] {
+                u32::MAX => {
+                    return ImpactScore {
+                        score: f32::INFINITY,
+                        entries_scanned,
+                    };
+                }
+                doc_up_to if u64::from(doc_up_to) >= up_to => break,
+                _ => {}
+            }
+            block_idx += 1;
+        }
+
+        ImpactScore {
+            score: max_score,
+            entries_scanned,
+        }
+    }
+}
+
+pub struct ImpactSkipDataBuilder {
+    entries: LargeBinaryBuilder,
+    level0_len: usize,
+    level1_entries: Vec<Vec<u8>>,
+    level1_docs: Vec<(u32, u32, u32)>,
+}
+
+impl ImpactSkipDataBuilder {
+    pub fn with_capacity(level0_blocks: usize, block_size: usize) -> Self {
+        Self {
+            entries: LargeBinaryBuilder::with_capacity(
+                level0_blocks + level1_len(level0_blocks),
+                0,
+            ),
+            level0_len: 0,
+            level1_entries: Vec::with_capacity(level1_len(level0_blocks)),
+            level1_docs: Vec::with_capacity(IMPACT_LEVEL1_BLOCKS * block_size),
+        }
+    }
+
+    pub fn append_block(&mut self, docs: &[(u32, u32, u32)]) -> Result<()> {
+        let bytes = encode_impact_entry(docs)?;
+        self.entries.append_value(bytes.as_slice());
+        self.level0_len += 1;
+        self.level1_docs.extend_from_slice(docs);
+        if self.level0_len.is_multiple_of(IMPACT_LEVEL1_BLOCKS) {
+            self.flush_level1()?;
+        }
+        Ok(())
+    }
+
+    pub fn finish(mut self) -> Result<ImpactSkipData> {
+        if !self.level1_docs.is_empty() {
+            self.flush_level1()?;
+        }
+        for entry in self.level1_entries {
+            self.entries.append_value(entry.as_slice());
+        }
+        ImpactSkipData::new(self.entries.finish(), self.level0_len)
+    }
+
+    fn flush_level1(&mut self) -> Result<()> {
+        let bytes = encode_impact_entry(self.level1_docs.as_slice())?;
+        self.level1_entries.push(bytes);
+        self.level1_docs.clear();
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+pub fn build_impact_skip_data(blocks: &[Vec<(u32, u32, u32)>]) -> Result<ImpactSkipData> {
+    let block_size = blocks.iter().map(Vec::len).max().unwrap_or(0).max(1);
+    let mut builder = ImpactSkipDataBuilder::with_capacity(blocks.len(), block_size);
+    for block in blocks {
+        builder.append_block(block)?;
+    }
+    builder.finish()
+}
+
+fn encode_impact_entry(docs: &[(u32, u32, u32)]) -> Result<Vec<u8>> {
+    if docs.is_empty() {
+        return Err(Error::index(
+            "cannot encode an empty impact entry".to_owned(),
+        ));
+    }
+    let doc_up_to = docs
+        .last()
+        .map(|(doc_id, _, _)| *doc_id)
+        .expect("non-empty impact entry was validated above");
+    let frontier = quantized_impact_frontier(docs);
+    let pair_count = u32::try_from(frontier.len()).map_err(|_| {
+        Error::index("impact frontier too large to encode as u32 pair count".to_string())
+    })?;
+    let mut bytes = Vec::with_capacity(5 + frontier.len() * 2);
+    super::encoding::encode_varint_u32(&mut bytes, doc_up_to);
+    super::encoding::encode_varint_u32(&mut bytes, pair_count);
+    let mut previous_freq = 0u32;
+    let mut previous_norm = 0u8;
+    for (pair_idx, (freq, norm)) in frontier.into_iter().enumerate() {
+        let freq_delta_minus_one = freq
+            .checked_sub(previous_freq)
+            .and_then(|delta| delta.checked_sub(1))
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "impact frequencies must be positive and strictly increasing: previous={previous_freq}, current={freq}"
+                ))
+            })?;
+        let norm_delta = norm.checked_sub(previous_norm).ok_or_else(|| {
+            Error::index(format!(
+                "impact norms must be non-decreasing: previous={previous_norm}, current={norm}"
+            ))
+        })?;
+        if pair_idx > 0 && norm_delta == 0 {
+            return Err(Error::index(format!(
+                "impact norms must be strictly increasing after quantization: norm={norm}"
+            )));
+        }
+
+        let has_explicit_norm_delta = norm_delta != 1;
+        let packed_freq_delta =
+            (u64::from(freq_delta_minus_one) << 1) | u64::from(has_explicit_norm_delta);
+        encode_varint_u64(&mut bytes, packed_freq_delta);
+        if has_explicit_norm_delta {
+            bytes.push(norm_delta);
+        }
+        previous_freq = freq;
+        previous_norm = norm;
+    }
+    Ok(bytes)
+}
+
+fn decode_entry_doc_up_to(bytes: &[u8]) -> Result<u32> {
+    let mut offset = 0usize;
+    let doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    // Level-1 doc ids drive whole-group skips, so only publish a doc id after
+    // validating the complete entry. A truncated entry may still have a valid
+    // first varint.
+    for_each_entry_pair(bytes, |_, _| {})?;
+    Ok(doc_up_to)
+}
+
+fn decode_level0_entry_doc_up_to(bytes: &[u8]) -> Result<u32> {
+    // A malformed level0 frontier bakes an INFINITY score before this marker
+    // can terminate a range scan, so avoid parsing every frontier twice on the
+    // query-load path. Level1 entries are fully validated.
+    let mut offset = 0usize;
+    super::encoding::decode_varint_u32(bytes, &mut offset)
+}
+
+/// Walk an entry's (freq, doc_len) frontier pairs, validating the layout.
+fn for_each_entry_pair(bytes: &[u8], mut visit: impl FnMut(u32, u32)) -> Result<()> {
+    let mut offset = 0usize;
+    let _doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    let pair_count = super::encoding::decode_varint_u32(bytes, &mut offset)?;
+    if pair_count == 0 {
+        return Err(Error::index(
+            "impact entry must contain at least one frontier pair".to_owned(),
+        ));
+    }
+
+    let mut previous_freq = 0u32;
+    let mut previous_norm = 0u16;
+    for pair_idx in 0..pair_count {
+        let packed_freq_delta = decode_varint_u64(bytes, &mut offset)?;
+        let freq_delta_minus_one = u32::try_from(packed_freq_delta >> 1)
+            .map_err(|_| Error::index("impact freq delta exceeds u32".to_owned()))?;
+        let freq_delta = freq_delta_minus_one
+            .checked_add(1)
+            .ok_or_else(|| Error::index("impact freq delta overflow".to_owned()))?;
+        let freq = previous_freq
+            .checked_add(freq_delta)
+            .ok_or_else(|| Error::index("impact frequency overflow".to_owned()))?;
+
+        let has_explicit_norm_delta = packed_freq_delta & 1 != 0;
+        let norm_delta = if has_explicit_norm_delta {
+            let norm_delta = bytes.get(offset).copied().ok_or_else(|| {
+                Error::index("unexpected EOF while decoding impact norm delta".to_owned())
+            })?;
+            offset += 1;
+            norm_delta
+        } else {
+            1
+        };
+        if pair_idx > 0 && norm_delta == 0 {
+            return Err(Error::index(
+                "impact norms must be strictly increasing".to_owned(),
+            ));
+        }
+
+        let norm = previous_norm
+            .checked_add(u16::from(norm_delta))
+            .filter(|norm| *norm <= u16::from(u8::MAX))
+            .ok_or_else(|| Error::index("impact norm delta overflow".to_owned()))?;
+        let norm = norm as u8;
+        visit(freq, super::index::dequantize_doc_length(norm));
+        previous_freq = freq;
+        previous_norm = u16::from(norm);
+    }
+    if offset != bytes.len() {
+        return Err(Error::index(format!(
+            "impact entry has {} trailing bytes",
+            bytes.len() - offset
+        )));
+    }
+    Ok(())
+}
+
+#[inline]
+fn encode_varint_u64(dst: &mut Vec<u8>, mut value: u64) {
+    while value >= 0x80 {
+        dst.push((value as u8) | 0x80);
+        value >>= 7;
+    }
+    dst.push(value as u8);
+}
+
+#[inline]
+fn decode_varint_u64(src: &[u8], offset: &mut usize) -> Result<u64> {
+    let mut value = 0u64;
+    let mut shift = 0u32;
+    while *offset < src.len() {
+        let byte = src[*offset];
+        *offset += 1;
+        if shift == 63 && byte & 0xFE != 0 {
+            return Err(Error::index(
+                "invalid u64 varint in impact entry".to_owned(),
+            ));
+        }
+        value |= u64::from(byte & 0x7F) << shift;
+        if byte & 0x80 == 0 {
+            return Ok(value);
+        }
+        shift += 7;
+        if shift > 63 {
+            return Err(Error::index(
+                "invalid u64 varint in impact entry".to_owned(),
+            ));
+        }
+    }
+    Err(Error::index(
+        "unexpected EOF while decoding impact entry".to_owned(),
+    ))
+}
+
+fn quantized_impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u8)> {
+    let raw_frontier = impact_frontier(docs);
+    let mut frontier: Vec<(u32, u8)> = Vec::with_capacity(raw_frontier.len());
+    for (freq, doc_len) in raw_frontier {
+        let norm = super::index::quantize_doc_length(doc_len);
+        match frontier.last_mut() {
+            Some((last_freq, last_norm)) if *last_norm == norm => {
+                // At the same quantized norm, the larger frequency dominates.
+                *last_freq = freq;
+            }
+            Some((_, last_norm)) => {
+                debug_assert!(
+                    *last_norm < norm,
+                    "raw impact frontier document lengths must be increasing"
+                );
+                frontier.push((freq, norm));
+            }
+            None => frontier.push((freq, norm)),
+        }
+    }
+    frontier
+}
+
+fn impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> {
+    let max_freq = docs.iter().map(|(_, freq, _)| *freq).max().unwrap_or(0) as usize;
+    if max_freq <= SMALL_FRONTIER_FREQ_LIMIT {
+        return impact_frontier_small_freq(docs, max_freq);
+    }
+
+    impact_frontier_sparse_freq(docs)
+}
+
+fn impact_frontier_small_freq(docs: &[(u32, u32, u32)], max_freq: usize) -> Vec<(u32, u32)> {
+    let mut min_doc_len_by_freq = [u32::MAX; SMALL_FRONTIER_FREQ_LIMIT + 1];
+    for (_, freq, doc_len) in docs {
+        min_doc_len_by_freq[*freq as usize] = min_doc_len_by_freq[*freq as usize].min(*doc_len);
+    }
+
+    let min_doc_lens = min_doc_len_by_freq[..=max_freq]
+        .iter()
+        .enumerate()
+        .filter_map(|(freq, doc_len)| (*doc_len != u32::MAX).then_some((freq as u32, *doc_len)))
+        .collect::<Vec<_>>();
+    frontier_from_min_doc_lens(min_doc_lens)
+}
+
+fn impact_frontier_sparse_freq(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> {
+    let mut pairs = docs
+        .iter()
+        .map(|(_, freq, doc_len)| (*freq, *doc_len))
+        .collect::<Vec<_>>();
+    pairs.sort_unstable_by_key(|(freq, _)| *freq);
+
+    let mut min_doc_lens: Vec<(u32, u32)> = Vec::with_capacity(pairs.len());
+    for (freq, doc_len) in pairs {
+        match min_doc_lens.last_mut() {
+            Some((last_freq, last_doc_len)) if *last_freq == freq => {
+                *last_doc_len = (*last_doc_len).min(doc_len);
+            }
+            _ => min_doc_lens.push((freq, doc_len)),
+        }
+    }
+
+    frontier_from_min_doc_lens(min_doc_lens)
+}
+
+fn frontier_from_min_doc_lens(min_doc_lens: Vec<(u32, u32)>) -> Vec<(u32, u32)> {
+    let mut best_doc_len = u32::MAX;
+    let mut frontier = Vec::with_capacity(min_doc_lens.len());
+    for (freq, doc_len) in min_doc_lens.into_iter().rev() {
+        if doc_len < best_doc_len {
+            frontier.push((freq, doc_len));
+            best_doc_len = doc_len;
+        }
+    }
+    frontier.reverse();
+    frontier
+}
+
+fn level1_len(level0_len: usize) -> usize {
+    level0_len.div_ceil(IMPACT_LEVEL1_BLOCKS)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer};
+
+    use super::*;
+    use crate::scalar::inverted::scorer::{MemBM25Scorer, Scorer};
+
+    struct KeyedCountingScorer {
+        key: u64,
+        calls: Arc<AtomicUsize>,
+    }
+
+    impl Scorer for KeyedCountingScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+            self.calls.fetch_add(1, Ordering::Relaxed);
+            freq as f32 / doc_tokens as f32
+        }
+
+        fn doc_weight_cache_key(&self) -> Option<u64> {
+            Some(self.key)
+        }
+    }
+
+    #[test]
+    fn impact_entry_frontier_drops_dominated_pairs() {
+        let docs = vec![(0, 1, 10), (1, 1, 8), (2, 2, 9), (3, 3, 20)];
+        assert_eq!(impact_frontier(&docs), vec![(1, 8), (2, 9), (3, 20)]);
+    }
+
+    #[test]
+    fn impact_entry_frontier_handles_sparse_large_frequencies() {
+        let docs = vec![
+            (0, 1, 100),
+            (1, 1, 80),
+            (2, 512, 90),
+            (3, 1_000, 120),
+            (4, 1_000, 110),
+        ];
+        assert_eq!(
+            impact_frontier(&docs),
+            vec![(1, 80), (512, 90), (1_000, 110)]
+        );
+    }
+
+    #[test]
+    fn quantized_impact_frontier_drops_equal_norms() {
+        let docs = vec![(0, 1, 16), (1, 2, 17), (2, 3, 24)];
+        assert_eq!(
+            quantized_impact_frontier(&docs),
+            vec![
+                (2, super::super::index::quantize_doc_length(17)),
+                (3, super::super::index::quantize_doc_length(24)),
+            ]
+        );
+    }
+
+    #[test]
+    fn impact_max_score_can_use_level1_entry() {
+        let blocks = (0..40)
+            .map(|block| vec![(block as u32, 1 + block as u32 % 3, 10)])
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        assert_eq!(impacts.level0_len(), 40);
+        assert_eq!(impacts.level1_len(), 2);
+        let scorer = MemBM25Scorer::new(400, 40, HashMap::from([(String::from("token"), 40usize)]));
+        let mut cache = ImpactScoreCache::default();
+        let score = impacts.max_score_up_to_cached(0, 31, 1.0, &scorer, &mut cache);
+        assert!(score.entries_scanned < IMPACT_LEVEL1_BLOCKS);
+        assert!(score.score > 0.0);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_reports_full_and_partial_groups() {
+        let blocks = (0..40)
+            .map(|block| vec![(block as u32, 1, 10)])
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+
+        assert_eq!(
+            impacts.level1_doc_up_to(0),
+            Some((IMPACT_LEVEL1_BLOCKS - 1) as u32)
+        );
+        assert_eq!(impacts.level1_doc_up_to(1), Some(39));
+        assert_eq!(impacts.level1_doc_up_to(2), None);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_returns_none_for_malformed_entry() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let malformed_level1 = vec![1, 2, 3];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(malformed_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+    }
+
+    #[test]
+    fn impact_level1_doc_up_to_validates_complete_entry() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        // A complete first varint is not enough: the pair count and frontier
+        // are required before this doc id can safely drive a group skip.
+        let truncated_level1 = [31_u8];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(truncated_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+    }
+
+    #[test]
+    fn empty_impact_frontiers_are_malformed_bounds() {
+        let scorer = MemBM25Scorer::new(10, 1, HashMap::new());
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let empty_level1 = [0, 0];
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0.as_slice()),
+            Some(empty_level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+        let mut cache = ImpactScoreCache::default();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+        assert!(
+            impacts
+                .global_max_doc_weight_cached(&scorer, &mut cache)
+                .is_infinite()
+        );
+        assert!(
+            ImpactSkipDataBuilder::with_capacity(1, 128)
+                .append_block(&[])
+                .is_err()
+        );
+    }
+
+    #[test]
+    fn null_impact_entry_is_an_infinite_bound_even_with_hidden_bytes() {
+        let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let level1 = encode_impact_entry(&[(0, 2, 8)]).unwrap();
+        let mut values = level0.clone();
+        values.extend_from_slice(&level1);
+        let entries = LargeBinaryArray::new(
+            OffsetBuffer::new(ScalarBuffer::from(vec![
+                0_i64,
+                level0.len() as i64,
+                values.len() as i64,
+            ])),
+            Buffer::from_vec(values),
+            Some(NullBuffer::from(vec![true, false])),
+        );
+        let impacts = ImpactSkipData::new(entries, 1).unwrap();
+        let scorer = MemBM25Scorer::new(10, 1, HashMap::new());
+        let mut cache = ImpactScoreCache::default();
+
+        assert_eq!(impacts.level1_doc_up_to(0), None);
+        assert!(
+            impacts
+                .global_max_doc_weight_cached(&scorer, &mut cache)
+                .is_infinite()
+        );
+    }
+
+    #[test]
+    fn impact_bounds_follow_changed_bm25_average_doc_length() {
+        let impacts = build_impact_skip_data(&[vec![(0, 1, 100)]]).unwrap();
+        let low_avgdl = MemBM25Scorer::new(1, 1, HashMap::new());
+        let high_avgdl = MemBM25Scorer::new(100, 1, HashMap::new());
+        let mut low_cache = ImpactScoreCache::default();
+        let mut high_cache = ImpactScoreCache::default();
+
+        let low_bound = impacts.global_max_doc_weight_cached(&low_avgdl, &mut low_cache);
+        let high_bound = impacts.global_max_doc_weight_cached(&high_avgdl, &mut high_cache);
+        let quantized_doc_length = super::super::index::dequantize_doc_length(
+            super::super::index::quantize_doc_length(100),
+        );
+
+        assert!((low_bound - low_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6);
+        assert!((high_bound - high_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6);
+        assert!(low_bound >= low_avgdl.doc_weight(1, 100));
+        assert!(high_bound >= high_avgdl.doc_weight(1, 100));
+        assert!(
+            high_bound > low_bound,
+            "larger avgdl must recompute a larger bound: low={low_bound}, high={high_bound}"
+        );
+    }
+
+    #[test]
+    fn impact_bounds_reuse_same_scorer_key_across_queries() {
+        let impacts = build_impact_skip_data(&[vec![(0, 2, 10)]]).unwrap();
+        let cloned = impacts.clone();
+        assert!(impacts.shares_derived_state_with(&cloned));
+        let calls = Arc::new(AtomicUsize::new(0));
+        let scorer = KeyedCountingScorer {
+            key: 7,
+            calls: calls.clone(),
+        };
+        let mut first_query = ImpactScoreCache::default();
+        let mut second_query = ImpactScoreCache::default();
+
+        let first = impacts.global_max_doc_weight_cached(&scorer, &mut first_query);
+        let baked_calls = calls.load(Ordering::Relaxed);
+        assert!(baked_calls > 0);
+        let second = cloned.global_max_doc_weight_cached(&scorer, &mut second_query);
+
+        assert_eq!(second, first);
+        assert_eq!(
+            calls.load(Ordering::Relaxed),
+            baked_calls,
+            "the same keyed bounds should be shared across query caches"
+        );
+    }
+
+    #[test]
+    fn malformed_unscanned_entry_does_not_poison_range_score() {
+        let level0_0 = encode_impact_entry(&[(0, 1, 10)]).unwrap();
+        let malformed_level0_1 = vec![1, 2, 3];
+        let level1 = encode_impact_entry(&[(0, 1, 10), (1, 1, 10)]).unwrap();
+        let entries = LargeBinaryArray::from_opt_vec(vec![
+            Some(level0_0.as_slice()),
+            Some(malformed_level0_1.as_slice()),
+            Some(level1.as_slice()),
+        ]);
+        let impacts = ImpactSkipData::new(entries, 2).unwrap();
+        let scorer = MemBM25Scorer::new(10, 10, HashMap::from([(String::from("token"), 2usize)]));
+        let mut cache = ImpactScoreCache::default();
+
+        let score = impacts.max_score_up_to_cached(0, 0, 1.0, &scorer, &mut cache);
+        assert!(score.score.is_finite());
+        assert_eq!(score.entries_scanned, 1);
+
+        assert_eq!(
+            impacts.level0_score_cached(1, 1.0, &scorer, &mut cache),
+            f32::INFINITY
+        );
+    }
+
+    #[test]
+    fn impact_entries_store_quantized_norm_deltas() {
+        let docs = vec![(7, 1, 1), (9, 2, 2), (12, 3, 5)];
+        let encoded = encode_impact_entry(&docs).unwrap();
+
+        // doc_up_to=12, pair_count=3, two implicit +1 norm deltas, then an
+        // explicit +3 norm delta folded behind the frequency varint's low bit.
+        assert_eq!(encoded, vec![12, 3, 0, 0, 1, 3]);
+
+        let mut pairs = Vec::new();
+        for_each_entry_pair(&encoded, |freq, doc_len| pairs.push((freq, doc_len))).unwrap();
+        assert_eq!(pairs, vec![(1, 1), (2, 2), (3, 5)]);
+    }
+
+    #[test]
+    fn malformed_norm_deltas_are_rejected() {
+        // doc_up_to=0, pair_count=1, and the pair flag promises a norm byte
+        // that is not present.
+        let truncated = [0, 1, 1];
+        let error = for_each_entry_pair(&truncated, |_, _| {}).unwrap_err();
+        assert!(matches!(&error, Error::Index { .. }));
+        assert!(error.to_string().contains("impact norm delta"));
+
+        // The first pair reaches norm 255, so an implicit +1 on the second
+        // pair must fail instead of wrapping back to zero.
+        let overflowing = [0, 2, 1, 255, 0];
+        let error = for_each_entry_pair(&overflowing, |_, _| {}).unwrap_err();
+        assert!(matches!(&error, Error::Index { .. }));
+        assert!(error.to_string().contains("impact norm delta overflow"));
+    }
+
+    #[test]
+    fn impact_entries_roundtrip_quantized_frontier() {
+        let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80), (4095, 130, 900)];
+        let encoded = encode_impact_entry(&docs).unwrap();
+        assert_eq!(decode_entry_doc_up_to(&encoded).unwrap(), 4095);
+        let mut decoded_pairs = Vec::new();
+        for_each_entry_pair(&encoded, |freq, doc_len| {
+            decoded_pairs.push((freq, doc_len))
+        })
+        .unwrap();
+        let expected_pairs = quantized_impact_frontier(&docs)
+            .into_iter()
+            .map(|(freq, norm)| (freq, super::super::index::dequantize_doc_length(norm)))
+            .collect::<Vec<_>>();
+        assert_eq!(decoded_pairs, expected_pairs);
+        assert!(!decoded_pairs.is_empty());
+
+        // A 256-doc-block skip data goes through the shared codec end to end.
+        let blocks: Vec<Vec<(u32, u32, u32)>> = (0..3)
+            .map(|b| (0..256).map(|i| (b * 256 + i, 1 + i % 5, 10)).collect())
+            .collect();
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        assert_eq!(impacts.level1_doc_up_to(0), Some(767));
+        let scorer = MemBM25Scorer::new(400, 768, HashMap::from([(String::from("t"), 768usize)]));
+        let mut cache = ImpactScoreCache::default();
+        assert!(
+            impacts
+                .level0_score_cached(0, 1.0, &scorer, &mut cache)
+                .is_finite()
+        );
+        let level1 = impacts.max_score_up_to_cached(0, 767, 1.0, &scorer, &mut cache);
+        assert!(level1.score.is_finite() && level1.score > 0.0);
+    }
+
+    #[test]
+    fn v2_and_v3_impacts_use_identical_encoding() {
+        let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80)];
+        let mut v2_builder = ImpactSkipDataBuilder::with_capacity(1, 128);
+        v2_builder.append_block(&docs).unwrap();
+        let v2 = v2_builder.finish().unwrap();
+        let mut v3_builder = ImpactSkipDataBuilder::with_capacity(1, 256);
+        v3_builder.append_block(&docs).unwrap();
+        let v3 = v3_builder.finish().unwrap();
+
+        assert_eq!(v2.entries(), v3.entries());
+    }
+
+    #[test]
+    fn impact_upper_bound_covers_real_scores() {
+        let blocks = vec![
+            vec![(0, 1, 100), (3, 2, 40), (7, 4, 80)],
+            vec![(9, 3, 15), (10, 1, 5), (12, 5, 30)],
+            vec![(16, 2, 10), (18, 6, 70), (21, 3, 12)],
+            vec![(24, 1, 4), (28, 7, 100), (30, 2, 8)],
+        ];
+        let impacts = build_impact_skip_data(&blocks).unwrap();
+        let scorer = MemBM25Scorer::new(474, 31, HashMap::from([(String::from("token"), 4usize)]));
+        let query_weight = scorer.query_weight("token");
+        let mut cache = ImpactScoreCache::default();
+
+        for start_block_idx in 0..blocks.len() {
+            let up_to = blocks
+                .iter()
+                .skip(start_block_idx)
+                .take(2)
+                .flatten()
+                .map(|(doc_id, _, _)| *doc_id)
+                .max()
+                .unwrap();
+            let upper_bound = impacts.max_score_up_to_cached(
+                start_block_idx,
+                u64::from(up_to),
+                query_weight,
+                &scorer,
+                &mut cache,
+            );
+            let exact_max = blocks
+                .iter()
+                .skip(start_block_idx)
+                .flatten()
+                .take_while(|(doc_id, _, _)| *doc_id <= up_to)
+                .map(|(_, freq, doc_len)| query_weight * scorer.doc_weight(*freq, *doc_len))
+                .fold(0.0_f32, f32::max);
+            assert!(
+                upper_bound.score + 1e-6 >= exact_max,
+                "upper bound {} should cover exact max {} from block {} up to doc {}",
+                upper_bound.score,
+                exact_max,
+                start_block_idx,
+                up_to
+            );
+        }
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs
index 5b67ba1da9b..c1740545226 100644
--- a/rust/lance-index/src/scalar/inverted/index.rs
+++ b/rust/lance-index/src/scalar/inverted/index.rs
@@ -1,15 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::fmt::{Debug, Display};
-use std::sync::Arc;
-use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock};
 use std::{
     cmp::{Reverse, min},
     collections::BinaryHeap,
 };
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     ops::Range,
     time::Instant,
 };
@@ -35,29 +36,34 @@ use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::metrics::Time;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use deepsize::DeepSizeOf;
 use fst::{Automaton, IntoStreamer, Streamer};
 use futures::{FutureExt, Stream, StreamExt, TryStreamExt, stream};
-use itertools::Itertools;
+use itertools::{Either, Itertools};
 use lance_arrow::{RecordBatchExt, iter_str_array};
 use lance_core::cache::{CacheCodec, CacheKey, LanceCache, WeakLanceCache};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::{DataFusionResult, LanceOptionExt};
+use lance_core::utils::address::RowAddress;
 use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_core::utils::tracing::{IO_TYPE_LOAD_SCALAR_PART, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result};
 use lance_select::{RowAddrMask, RowAddrTreeMap};
 use roaring::RoaringBitmap;
 use std::sync::LazyLock;
-use tokio::task::spawn_blocking;
-use tracing::{info, instrument};
+use tokio::{sync::OnceCell, task::spawn_blocking};
+use tracing::{info, instrument, warn};
 
-use super::encoding::{PositionBlockBuilder, decode_group_starts};
+use super::encoding::{MAX_POSTING_BLOCK_SIZE, PositionBlockBuilder};
+use super::impact::{IMPACT_LEVEL1_BLOCKS, ImpactSkipData, ImpactSkipDataBuilder};
 use super::iter::PostingListIterator;
+use super::lazy_docset::LazyDocSet;
+use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size};
 use super::{InvertedIndexBuilder, InvertedIndexParams, wand::*};
 use super::{
     builder::{
-        BLOCK_SIZE, PostingGroupAccumulator, PostingGroupConfig, ScoredDoc, doc_file_path,
-        inverted_list_schema_for_version, posting_file_path, token_file_path,
+        BLOCK_SIZE, ScoredDoc, doc_file_path,
+        inverted_list_schema_for_version_with_block_size_and_impacts, posting_file_path,
+        token_file_path,
     },
     iter::PlainPostingListIterator,
     query::*,
@@ -67,14 +73,14 @@ use super::{
     builder::{InnerBuilder, PositionRecorder},
     iter::CompressedPostingListIterator,
 };
-use crate::frag_reuse::FragReuseIndex;
 use crate::pbold;
 use crate::progress::IndexBuildProgress;
 use crate::scalar::inverted::scorer::MemBM25Scorer;
 use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer;
 use crate::scalar::{
     AnyQuery, BuiltinIndexType, CreatedIndex, IndexReader, IndexStore, MetricsCollector,
-    OldIndexDataFilter, ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery, UpdateCriteria,
+    OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery,
+    UpdateCriteria,
 };
 use crate::{FtsPrewarmOptions, Index};
 use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys};
@@ -83,8 +89,10 @@ use std::str::FromStr;
 // Version 0: Arrow TokenSetFormat (legacy)
 // Version 1: Fst TokenSetFormat with per-doc compressed positions
 // Version 2: Fst TokenSetFormat with shared posting-list position streams.
+// Version 3: Version 2 layout with 256-document physical posting blocks.
 pub const INVERTED_INDEX_VERSION_V1: u32 = 1;
 pub const INVERTED_INDEX_VERSION_V2: u32 = 2;
+pub const INVERTED_INDEX_VERSION_V3: u32 = 3;
 pub const TOKENS_FILE: &str = "tokens.lance";
 pub const INVERT_LIST_FILE: &str = "invert.lance";
 pub const DOCS_FILE: &str = "docs.lance";
@@ -100,6 +108,7 @@ pub const POSITION_COL: &str = "_position";
 pub const COMPRESSED_POSITION_COL: &str = "_compressed_position";
 pub const POSITION_BLOCK_OFFSET_COL: &str = "_position_block_offset";
 pub const POSTING_COL: &str = "_posting";
+pub const IMPACT_COL: &str = "_impacts";
 pub const MAX_SCORE_COL: &str = "_max_score";
 pub const LENGTH_COL: &str = "_length";
 pub const BLOCK_MAX_SCORE_COL: &str = "_block_max_score";
@@ -107,13 +116,10 @@ pub const NUM_TOKEN_COL: &str = "_num_tokens";
 pub const SCORE_COL: &str = "_score";
 pub const TOKEN_SET_FORMAT_KEY: &str = "token_set_format";
 pub const POSTING_TAIL_CODEC_KEY: &str = "posting_tail_codec";
+pub const FTS_FORMAT_VERSION_KEY: &str = "format_version";
 pub const POSITIONS_LAYOUT_KEY: &str = "positions_layout";
 pub const POSITIONS_CODEC_KEY: &str = "positions_codec";
-/// Schema-metadata key holding the 1-indexed global-buffer id of the
-/// varint-delta-encoded posting-list cache-group boundaries (issue #7040).
-/// Absent on indexes written before grouping was introduced, which fall back
-/// to the per-token cache path.
-pub const POSTING_GROUP_OFFSETS_BUF_KEY: &str = "posting_group_offsets_buf";
+pub const POSTING_BLOCK_SIZE_KEY: &str = "posting_block_size";
 pub const POSTING_TAIL_CODEC_FIXED32_V1: &str = "fixed32_v1";
 pub const POSTING_TAIL_CODEC_VARINT_DELTA_V1: &str = "varint_delta_v1";
 pub const POSITIONS_LAYOUT_SHARED_STREAM_V2: &str = "shared_stream_v2";
@@ -131,26 +137,33 @@ pub static FTS_SCHEMA: LazyLock<SchemaRef> =
 static ROW_ID_SCHEMA: LazyLock<SchemaRef> =
     LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone()])));
 
-fn resolve_fts_format_version(
+pub fn resolve_fts_format_version(
     value: Option<&str>,
 ) -> std::result::Result<InvertedListFormatVersion, Error> {
-    value.unwrap_or("1").parse()
+    match value {
+        Some(value) => value.parse(),
+        None => Ok(default_fts_format_version()),
+    }
+}
+
+pub fn default_fts_format_version() -> InvertedListFormatVersion {
+    InvertedListFormatVersion::V2
 }
 
 pub fn current_fts_format_version() -> InvertedListFormatVersion {
-    resolve_fts_format_version(std::env::var("LANCE_FTS_FORMAT_VERSION").ok().as_deref())
-        .expect("failed to parse LANCE_FTS_FORMAT_VERSION")
+    default_fts_format_version()
 }
 
 pub fn max_supported_fts_format_version() -> InvertedListFormatVersion {
-    InvertedListFormatVersion::V2
+    InvertedListFormatVersion::V3
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
 pub enum InvertedListFormatVersion {
-    #[default]
     V1,
+    #[default]
     V2,
+    V3,
 }
 
 impl InvertedListFormatVersion {
@@ -161,29 +174,50 @@ impl InvertedListFormatVersion {
         }
     }
 
+    pub fn from_posting_tail_codec_and_block_size(
+        codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Result<Self> {
+        validate_block_size(block_size)?;
+        let format_version = match (codec, block_size) {
+            (PostingTailCodec::Fixed32, LEGACY_BLOCK_SIZE) => Self::V1,
+            (PostingTailCodec::VarintDelta, LEGACY_BLOCK_SIZE) => Self::V2,
+            (PostingTailCodec::VarintDelta, 256) => Self::V3,
+            (PostingTailCodec::Fixed32, 256) => {
+                return Err(Error::invalid_input(
+                    "FTS format_version=3 requires the varint-delta posting tail codec".to_string(),
+                ));
+            }
+            _ => unreachable!("validate_block_size limits supported block sizes"),
+        };
+        validate_format_version_block_size(format_version, block_size)?;
+        Ok(format_version)
+    }
+
     pub fn index_version(self) -> u32 {
         match self {
             Self::V1 => INVERTED_INDEX_VERSION_V1,
             Self::V2 => INVERTED_INDEX_VERSION_V2,
+            Self::V3 => INVERTED_INDEX_VERSION_V3,
         }
     }
 
     pub fn posting_tail_codec(self) -> PostingTailCodec {
         match self {
             Self::V1 => PostingTailCodec::Fixed32,
-            Self::V2 => PostingTailCodec::VarintDelta,
+            Self::V2 | Self::V3 => PostingTailCodec::VarintDelta,
         }
     }
 
     pub fn position_codec(self) -> Option<PositionStreamCodec> {
         match self {
             Self::V1 => None,
-            Self::V2 => Some(PositionStreamCodec::PackedDelta),
+            Self::V2 | Self::V3 => Some(PositionStreamCodec::PackedDelta),
         }
     }
 
     pub fn uses_shared_position_stream(self) -> bool {
-        matches!(self, Self::V2)
+        matches!(self, Self::V2 | Self::V3)
     }
 }
 
@@ -194,17 +228,51 @@ impl FromStr for InvertedListFormatVersion {
         match s.trim() {
             "1" | "v1" | "V1" => Ok(Self::V1),
             "2" | "v2" | "V2" => Ok(Self::V2),
+            "3" | "v3" | "V3" => Ok(Self::V3),
             other => Err(Error::index(format!(
-                "unsupported FTS format version {}, expected 1 or 2",
+                "unsupported FTS format version {}, expected 1, 2, or 3",
                 other
             ))),
         }
     }
 }
 
+pub fn default_fts_format_version_for_block_size(
+    block_size: usize,
+) -> Result<InvertedListFormatVersion> {
+    validate_block_size(block_size)?;
+    match block_size {
+        LEGACY_BLOCK_SIZE => Ok(InvertedListFormatVersion::V2),
+        256 => Ok(InvertedListFormatVersion::V3),
+        _ => unreachable!("validate_block_size limits supported block sizes"),
+    }
+}
+
+pub fn validate_format_version_block_size(
+    format_version: InvertedListFormatVersion,
+    block_size: usize,
+) -> Result<()> {
+    validate_block_size(block_size)?;
+    match (format_version, block_size) {
+        (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)
+        | (InvertedListFormatVersion::V3, 256) => Ok(()),
+        (InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2, 256) => {
+            Err(Error::invalid_input(format!(
+                "FTS format_version={} is incompatible with block_size=256; use format_version=3",
+                format_version.index_version()
+            )))
+        }
+        (InvertedListFormatVersion::V3, other) => Err(Error::invalid_input(format!(
+            "FTS format_version=3 requires block_size=256, got {other}"
+        ))),
+        _ => unreachable!("validate_block_size limits supported block sizes"),
+    }
+}
+
 #[derive(Debug)]
 struct PartitionCandidates {
     tokens_by_position: Vec<String>,
+    grouped_expansions: Vec<GroupedExpansionTerms>,
     candidates: Vec<DocCandidate>,
 }
 
@@ -212,11 +280,76 @@ impl PartitionCandidates {
     fn empty() -> Self {
         Self {
             tokens_by_position: Vec::new(),
+            grouped_expansions: Vec::new(),
             candidates: Vec::new(),
         }
     }
 }
 
+#[derive(Debug)]
+struct LoadedPostings {
+    postings: Vec<PostingIterator>,
+    grouped_expansions: Vec<GroupedExpansionTerms>,
+    impact_safe: bool,
+}
+
+impl LoadedPostings {
+    fn empty() -> Self {
+        Self {
+            postings: Vec::new(),
+            grouped_expansions: Vec::new(),
+            impact_safe: false,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct GroupedExpansionTerms {
+    position: u32,
+    terms: Vec<ExpansionTermFreqs>,
+}
+
+fn grouped_rescore_wand_limit(
+    limit: Option<usize>,
+    grouped_expansions: &[GroupedExpansionTerms],
+) -> Option<usize> {
+    let limit = limit?;
+    // Grouped fuzzy AND rescoring needs a small candidate cushion because WAND
+    // ranks by the unioned group posting first and the exact expansion IDF later.
+    let expansion_terms = grouped_expansions
+        .iter()
+        .map(|group| group.terms.len())
+        .sum::<usize>()
+        .max(1);
+    Some(limit.saturating_mul(expansion_terms))
+}
+
+#[derive(Debug)]
+struct ExpansionTermFreqs {
+    token: String,
+    freqs_by_posting_doc_id: Vec<(u64, u32)>,
+}
+
+impl ExpansionTermFreqs {
+    fn new(token: String, posting: &PostingList) -> Self {
+        let freqs_by_posting_doc_id = posting
+            .iter()
+            .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq))
+            .collect();
+        Self {
+            token,
+            freqs_by_posting_doc_id,
+        }
+    }
+
+    fn frequency(&self, posting_doc_id: u64) -> Option<u32> {
+        self.freqs_by_posting_doc_id
+            .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id)
+            .ok()
+            .map(|idx| self.freqs_by_posting_doc_id[idx].1)
+    }
+}
+
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Default)]
 pub enum TokenSetFormat {
     Arrow,
@@ -250,7 +383,7 @@ impl FromStr for TokenSetFormat {
 }
 
 impl DeepSizeOf for TokenSetFormat {
-    fn deep_size_of_children(&self, _: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize {
         0
     }
 }
@@ -299,6 +432,21 @@ pub(super) fn parse_posting_tail_codec(
         .unwrap_or(PostingTailCodec::Fixed32))
 }
 
+pub(super) fn parse_posting_block_size(metadata: &HashMap<String, String>) -> Result<usize> {
+    metadata
+        .get(POSTING_BLOCK_SIZE_KEY)
+        .map(|value| {
+            let block_size = value.parse::<usize>().map_err(|err| {
+                Error::index(format!(
+                    "invalid {POSTING_BLOCK_SIZE_KEY} metadata value {value:?}: {err}"
+                ))
+            })?;
+            validate_block_size(block_size)
+        })
+        .transpose()
+        .map(|block_size| block_size.unwrap_or(LEGACY_BLOCK_SIZE))
+}
+
 impl PositionStreamCodec {
     pub fn as_str(self) -> &'static str {
         match self {
@@ -336,6 +484,25 @@ fn parse_shared_position_codec(metadata: &HashMap<String, String>) -> Result<Pos
 pub(super) fn parse_format_version_from_metadata(
     metadata: &HashMap<String, String>,
 ) -> Result<InvertedListFormatVersion> {
+    if let Some(value) = metadata.get(FTS_FORMAT_VERSION_KEY) {
+        let format_version = InvertedListFormatVersion::from_str(value)?;
+        validate_format_version_block_size(format_version, parse_posting_block_size(metadata)?)?;
+        return Ok(format_version);
+    }
+    let block_size = parse_posting_block_size(metadata)?;
+    if block_size == 256 {
+        if metadata
+            .get(POSTING_TAIL_CODEC_KEY)
+            .map(|_| parse_posting_tail_codec(metadata))
+            .transpose()?
+            .is_some_and(|posting_tail_codec| posting_tail_codec != PostingTailCodec::VarintDelta)
+        {
+            return Err(Error::index(
+                "FTS block_size=256 requires the varint-delta posting tail codec".to_string(),
+            ));
+        }
+        return Ok(InvertedListFormatVersion::V3);
+    }
     if metadata.contains_key(POSITIONS_CODEC_KEY) || metadata.contains_key(POSITIONS_LAYOUT_KEY) {
         return Ok(InvertedListFormatVersion::V2);
     }
@@ -352,7 +519,9 @@ pub struct InvertedIndex {
     store: Arc<dyn IndexStore>,
     tokenizer: Box<dyn LanceTokenizer>,
     token_set_format: TokenSetFormat,
+    format_version: InvertedListFormatVersion,
     pub(crate) partitions: Vec<Arc<InvertedPartition>>,
+    corpus_stats: Arc<OnceCell<(u64, usize)>>,
     // Fragments which are contained in the index, but no longer in the dataset.
     // These should be pruned at search time since we don't prune them at update time.
     deleted_fragments: RoaringBitmap,
@@ -363,6 +532,7 @@ impl Debug for InvertedIndex {
         f.debug_struct("InvertedIndex")
             .field("params", &self.params)
             .field("token_set_format", &self.token_set_format)
+            .field("format_version", &self.format_version)
             .field("partitions", &self.partitions)
             .field("deleted_fragments", &self.deleted_fragments)
             .finish()
@@ -370,27 +540,52 @@ impl Debug for InvertedIndex {
 }
 
 impl DeepSizeOf for InvertedIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.partitions.deep_size_of_children(context)
     }
 }
 
+/// Resolve any `Pending` candidates that wand emitted via the
+/// deferred-row_id path. After this returns, every entry in
+/// `candidates` carries a real row_id.
+async fn resolve_deferred_candidates(
+    docs: &LazyDocSet,
+    candidates: &mut [DocCandidate],
+) -> Result<()> {
+    let pending: Vec<u32> = candidates
+        .iter()
+        .filter_map(|c| match c.addr {
+            CandidateAddr::Pending(d) => Some(d),
+            CandidateAddr::RowId(_) => None,
+        })
+        .collect();
+    if pending.is_empty() {
+        return Ok(());
+    }
+    let mut iter = docs.resolve_row_ids(&pending).await?.into_iter();
+    for c in candidates {
+        if matches!(c.addr, CandidateAddr::Pending(_)) {
+            let r = iter.next().ok_or_else(|| {
+                Error::internal("resolve_row_ids returned fewer items than requested")
+            })?;
+            c.addr = CandidateAddr::RowId(r);
+        }
+    }
+    Ok(())
+}
+
 impl InvertedIndex {
     fn format_version(&self) -> InvertedListFormatVersion {
-        self.partitions
-            .first()
-            .map(|partition| {
-                InvertedListFormatVersion::from_posting_tail_codec(
-                    partition.inverted_list.posting_tail_codec(),
-                )
-            })
-            .unwrap_or_else(current_fts_format_version)
+        self.format_version
     }
 
     fn index_version(&self) -> u32 {
-        match self.token_set_format {
-            TokenSetFormat::Arrow => 0,
-            TokenSetFormat::Fst => self.format_version().index_version(),
+        match (self.token_set_format, self.format_version()) {
+            (
+                TokenSetFormat::Arrow,
+                InvertedListFormatVersion::V1 | InvertedListFormatVersion::V2,
+            ) => 0,
+            (_, format_version) => format_version.index_version(),
         }
     }
 
@@ -508,7 +703,7 @@ impl InvertedIndex {
             .with_token_set_format(first.token_set_format)
             .with_format_version(first.format_version())
             .with_posting_tail_codec(first.posting_tail_codec());
-        builder
+        let files = builder
             .update_from_segments(new_data, dest_store, segments, old_data_filter)
             .await?;
 
@@ -517,7 +712,7 @@ impl InvertedIndex {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
             index_version: first.index_version(),
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -532,22 +727,25 @@ impl InvertedIndex {
         query_tokens: &Tokens,
         params: &FtsSearchParams,
     ) -> Result<MemBM25Scorer> {
-        let scorer = IndexBM25Scorer::new(self.partitions.iter().map(|part| part.as_ref()));
-        let mut terms: Vec<String> = Vec::new();
-        let mut seen = HashSet::new();
         if matches!(params.fuzziness, Some(n) if n != 0) {
             let expanded = self.expand_fuzzy_tokens(query_tokens, params)?;
-            for idx in 0..expanded.len() {
-                let token = expanded.get_token(idx);
-                if seen.insert(token.to_string()) {
-                    terms.push(token.to_string());
-                }
-            }
+            self.bm25_scorer_for_final_tokens(&expanded).await
         } else {
-            for token in query_tokens {
-                if seen.insert(token.to_string()) {
-                    terms.push(token.to_string());
-                }
+            self.bm25_scorer_for_final_tokens(query_tokens).await
+        }
+    }
+
+    /// Scorer for a token list that needs no further fuzzy expansion: dedup
+    /// the terms and pull their document frequencies. `bm25_search` calls
+    /// this with the tokens it already expanded, so the expansion runs once
+    /// per query rather than once for the scorer and once per partition.
+    async fn bm25_scorer_for_final_tokens(&self, tokens: &Tokens) -> Result<MemBM25Scorer> {
+        let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?;
+        let mut terms: Vec<String> = Vec::new();
+        let mut seen = HashSet::new();
+        for token in tokens {
+            if seen.insert(token.to_string()) {
+                terms.push(token.to_string());
             }
         }
         let mut token_docs = HashMap::with_capacity(terms.len());
@@ -555,18 +753,42 @@ impl InvertedIndex {
             let df = self.df_for_term(term).await?;
             token_docs.insert(term.clone(), df);
         }
-        Ok(MemBM25Scorer::new(
-            scorer.total_tokens(),
-            scorer.num_docs(),
-            token_docs,
-        ))
+        Ok(MemBM25Scorer::new(total_tokens, num_docs, token_docs))
     }
 
     pub async fn bm25_stats_for_terms(&self, terms: &[String]) -> Result<(u64, usize, Vec<usize>)> {
-        let scorer = IndexBM25Scorer::new(self.partitions.iter().map(|part| part.as_ref()));
+        let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?;
         let token_docs =
             futures::future::try_join_all(terms.iter().map(|term| self.df_for_term(term))).await?;
-        Ok((scorer.total_tokens(), scorer.num_docs(), token_docs))
+        Ok((total_tokens, num_docs, token_docs))
+    }
+
+    /// Aggregate per-partition `total_tokens` and `num_docs` across the
+    /// index. `len` is cheap (no IO); `total_tokens_num` reads only the
+    /// num_tokens column the first time per partition and caches it on
+    /// `LazyDocSet`. Avoids materializing the full DocSet just to get
+    /// these two scalars.
+    async fn aggregate_corpus_stats(&self) -> Result<(u64, usize)> {
+        self.corpus_stats
+            .get_or_try_init(|| async {
+                let io_parallelism = self.store.io_parallelism();
+                let num_docs: usize = self.partitions.iter().map(|p| p.docs.len()).sum();
+                let futures = self
+                    .partitions
+                    .iter()
+                    .map(|p| {
+                        let docs = p.docs.clone();
+                        async move { docs.total_tokens_num().await }
+                    })
+                    .collect::<Vec<_>>();
+                let totals: Vec<u64> = stream::iter(futures)
+                    .buffer_unordered(io_parallelism)
+                    .try_collect()
+                    .await?;
+                Ok((totals.into_iter().sum(), num_docs))
+            })
+            .await
+            .copied()
     }
 
     /// Sum the posting-list length for `term` across this index's partitions
@@ -595,17 +817,47 @@ impl InvertedIndex {
     }
 
     /// Expand fuzzy query tokens against all partitions in this segment.
+    ///
+    /// `params.max_expansions` caps the whole query's expansion, not any
+    /// single partition's: for each query token the per-partition candidates
+    /// (each streamed in FST key order) merge into one lexicographically
+    /// ordered set, and the remaining budget takes a prefix of it. The
+    /// selected terms are a pure function of the segment's vocabulary, so
+    /// splitting the same corpus into more partitions cannot change which
+    /// terms a fuzzy query matches.
     pub fn expand_fuzzy_tokens(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result<Tokens> {
         let mut expanded_tokens = Vec::new();
         let mut expanded_positions = Vec::new();
         let mut seen = HashSet::new();
-        for partition in &self.partitions {
-            let expanded = partition.expand_fuzzy(tokens, params)?;
-            for idx in 0..expanded.len() {
-                let token = expanded.get_token(idx);
-                if seen.insert(token.to_string()) {
-                    expanded_tokens.push(token.to_string());
-                    expanded_positions.push(expanded.position(idx));
+        for token_idx in 0..tokens.len() {
+            let remaining = params.max_expansions.saturating_sub(expanded_tokens.len());
+            if remaining == 0 {
+                break;
+            }
+            let token = tokens.get_token(token_idx);
+            let position = tokens.position(token_idx);
+            // Each partition contributes at most its `remaining`
+            // lexicographically smallest candidates, so the global
+            // lex-smallest `remaining` selection below is unaffected by the
+            // per-partition truncation.
+            let mut candidates = BTreeSet::new();
+            let base_prefix_len = tokens.token_type().prefix_len(token) as u32;
+            for partition in &self.partitions {
+                partition.collect_fuzzy_candidates(
+                    token,
+                    base_prefix_len,
+                    params,
+                    remaining,
+                    &mut candidates,
+                )?;
+            }
+            for candidate in candidates {
+                if expanded_tokens.len() >= params.max_expansions {
+                    break;
+                }
+                if seen.insert((candidate.clone(), position)) {
+                    expanded_tokens.push(candidate);
+                    expanded_positions.push(position);
                 }
             }
         }
@@ -630,27 +882,81 @@ impl InvertedIndex {
         metrics: Arc<dyn MetricsCollector>,
         base_scorer: Option<&MemBM25Scorer>,
     ) -> Result<(Vec<u64>, Vec<f32>)> {
+        // Fuzzy expansion runs once here, with the global `max_expansions`
+        // budget, instead of once per partition: partitions receive the
+        // final token list, so the matched terms cannot depend on how the
+        // corpus happens to be partitioned.
+        let tokens = if matches!(params.fuzziness, Some(n) if n != 0) {
+            let expanded = Arc::new(self.expand_fuzzy_tokens(tokens.as_ref(), params.as_ref())?);
+            if operator == Operator::And || params.phrase_slop.is_some() {
+                // AND/phrase semantics require every original token position
+                // to keep at least one expansion; a position that expands to
+                // nothing anywhere in the segment can never be matched.
+                let surviving = (0..expanded.len())
+                    .map(|idx| expanded.position(idx))
+                    .collect::<HashSet<_>>();
+                if (0..tokens.len()).any(|idx| !surviving.contains(&tokens.position(idx))) {
+                    return Ok((Vec::new(), Vec::new()));
+                }
+            }
+            expanded
+        } else {
+            tokens
+        };
+
         // The wand only consults `scorer.doc_weight`, which is metadata-free.
         // The outer aggregation below consults `scorer.query_weight`, which
         // hits per-token `posting_len`; building a `MemBM25Scorer` with
         // precomputed per-term IDFs avoids the v2 bulk metadata pull.
         let local_scorer;
-        let scorer: &dyn Scorer = if let Some(base_scorer) = base_scorer {
+        let scorer: &MemBM25Scorer = if let Some(base_scorer) = base_scorer {
             base_scorer
         } else {
-            local_scorer = self
-                .bm25_base_scorer(tokens.as_ref(), params.as_ref())
-                .await?;
+            local_scorer = self.bm25_scorer_for_final_tokens(tokens.as_ref()).await?;
             &local_scorer
         };
+        let impact_scorer = Arc::new(scorer.clone());
 
         let limit = params.limit.unwrap_or(usize::MAX);
         if limit == 0 {
             return Ok((Vec::new(), Vec::new()));
         }
+
+        fn push_scored_candidate(
+            candidates: &mut BinaryHeap<Reverse<ScoredDoc>>,
+            limit: usize,
+            addr: CandidateAddr,
+            score: f32,
+        ) -> Result<()> {
+            // resolve_deferred_candidates ran upstream, so every candidate
+            // carries a real row_id at this point.
+            let row_id = match addr {
+                CandidateAddr::RowId(r) => r,
+                CandidateAddr::Pending(_) => {
+                    return Err(Error::internal(
+                        "bm25_search post-condition: deferred candidate left unresolved",
+                    ));
+                }
+            };
+
+            if candidates.len() < limit {
+                candidates.push(Reverse(ScoredDoc::new(row_id, score)));
+            } else if candidates.peek().unwrap().0.score.0 < score {
+                candidates.pop();
+                candidates.push(Reverse(ScoredDoc::new(row_id, score)));
+            }
+            Ok(())
+        }
+
         let mask = prefilter.mask();
 
         let mut candidates = BinaryHeap::new();
+        // Shared top-k floor across this query's partitions. Seeded to -inf so
+        // the first real score wins; each partition publishes its local k-th
+        // and prunes against the running global k-th (a lower bound on the true
+        // global k-th - see `Wand::shared_threshold`).
+        let impact_shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+        let legacy_shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
         let parts = self
             .partitions
             .iter()
@@ -660,13 +966,31 @@ impl InvertedIndex {
                 let params = params.clone();
                 let mask = mask.clone();
                 let metrics = metrics.clone();
+                let impact_scorer = impact_scorer.clone();
+                let impact_shared_threshold = impact_shared_threshold.clone();
+                let legacy_shared_threshold = legacy_shared_threshold.clone();
                 async move {
-                    let postings = part
-                        .load_posting_lists(tokens.as_ref(), params.as_ref(), metrics.as_ref())
+                    let loaded_postings = part
+                        .load_posting_lists(
+                            tokens.as_ref(),
+                            params.as_ref(),
+                            operator,
+                            impact_scorer.as_ref(),
+                            metrics.as_ref(),
+                        )
                         .await?;
+                    let LoadedPostings {
+                        postings,
+                        grouped_expansions,
+                        impact_safe,
+                    } = loaded_postings;
                     if postings.is_empty() {
+                        // No hits in this partition; its DocSet stays
+                        // unloaded, so we never pay the per-doc
+                        // row_id/num_tokens download for it.
                         return Result::Ok(PartitionCandidates::empty());
                     }
+                    let docs_for_wand = part.docs.docs_for_wand(mask.as_ref()).await?;
                     let max_position = postings
                         .iter()
                         .map(|posting| posting.term_index() as usize)
@@ -680,20 +1004,47 @@ impl InvertedIndex {
                     let params = params.clone();
                     let mask = mask.clone();
                     let metrics = metrics.clone();
-                    spawn_cpu(move || {
-                        let candidates = part.bm25_search(
-                            params.as_ref(),
+                    let part_for_wand = part.clone();
+                    let has_grouped_expansions = !grouped_expansions.is_empty();
+                    let use_impact_path = impact_safe && !has_grouped_expansions;
+                    let wand_params = if has_grouped_expansions {
+                        let mut rescoring_params = params.as_ref().clone();
+                        rescoring_params.limit =
+                            grouped_rescore_wand_limit(params.limit, &grouped_expansions);
+                        Arc::new(rescoring_params)
+                    } else {
+                        params.clone()
+                    };
+                    let partition_threshold = if has_grouped_expansions {
+                        Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()))
+                    } else if use_impact_path {
+                        impact_shared_threshold
+                    } else {
+                        legacy_shared_threshold
+                    };
+                    let wand_scorer = use_impact_path.then(|| impact_scorer.clone());
+                    let candidates = spawn_cpu(move || {
+                        let candidates = part_for_wand.bm25_search(
+                            docs_for_wand.as_ref(),
+                            wand_params.as_ref(),
                             operator,
                             mask,
                             postings,
+                            wand_scorer,
                             metrics.as_ref(),
+                            partition_threshold,
                         )?;
-                        Ok(PartitionCandidates {
-                            tokens_by_position,
-                            candidates,
-                        })
+                        std::result::Result::<_, Error>::Ok(candidates)
                     })
-                    .await
+                    .await?;
+                    let mut partition_result = PartitionCandidates {
+                        tokens_by_position,
+                        grouped_expansions,
+                        candidates,
+                    };
+                    resolve_deferred_candidates(&part.docs, &mut partition_result.candidates)
+                        .await?;
+                    Result::Ok(partition_result)
                 }
             })
             .collect::<Vec<_>>();
@@ -703,8 +1054,13 @@ impl InvertedIndex {
             if res.candidates.is_empty() {
                 continue;
             }
-            let mut idf_by_position = Vec::with_capacity(res.tokens_by_position.len());
-            for token in &res.tokens_by_position {
+            let PartitionCandidates {
+                tokens_by_position,
+                grouped_expansions,
+                candidates: part_candidates,
+            } = res;
+            let mut idf_by_position = Vec::with_capacity(tokens_by_position.len());
+            for token in &tokens_by_position {
                 let idf_weight = match idf_cache.get(token) {
                     Some(weight) => *weight,
                     None => {
@@ -715,23 +1071,61 @@ impl InvertedIndex {
                 };
                 idf_by_position.push(idf_weight);
             }
-            for DocCandidate {
-                row_id,
-                freqs,
-                doc_length,
-            } in res.candidates
-            {
-                let mut score = 0.0;
-                for (term_index, freq) in freqs.into_iter() {
-                    debug_assert!((term_index as usize) < idf_by_position.len());
-                    score +=
-                        idf_by_position[term_index as usize] * scorer.doc_weight(freq, doc_length);
+
+            if grouped_expansions.is_empty() {
+                for DocCandidate {
+                    addr,
+                    freqs,
+                    doc_length,
+                    ..
+                } in part_candidates
+                {
+                    let mut score = 0.0;
+                    for (term_index, freq) in freqs.into_iter() {
+                        debug_assert!((term_index as usize) < idf_by_position.len());
+                        score += idf_by_position[term_index as usize]
+                            * scorer.doc_weight(freq, doc_length);
+                    }
+                    push_scored_candidate(&mut candidates, limit, addr, score)?;
                 }
-                if candidates.len() < limit {
-                    candidates.push(Reverse(ScoredDoc::new(row_id, score)));
-                } else if candidates.peek().unwrap().0.score.0 < score {
-                    candidates.pop();
-                    candidates.push(Reverse(ScoredDoc::new(row_id, score)));
+            } else {
+                let grouped_positions = grouped_expansions
+                    .iter()
+                    .map(|group| group.position)
+                    .collect::<HashSet<_>>();
+                for DocCandidate {
+                    addr,
+                    posting_doc_id,
+                    freqs,
+                    doc_length,
+                } in part_candidates
+                {
+                    let mut score = 0.0;
+                    for (term_index, freq) in freqs.into_iter() {
+                        if grouped_positions.contains(&term_index) {
+                            continue;
+                        }
+                        debug_assert!((term_index as usize) < idf_by_position.len());
+                        score += idf_by_position[term_index as usize]
+                            * scorer.doc_weight(freq, doc_length);
+                    }
+                    for group in &grouped_expansions {
+                        for term in &group.terms {
+                            let Some(freq) = term.frequency(posting_doc_id) else {
+                                continue;
+                            };
+                            let idf_weight = match idf_cache.get(&term.token) {
+                                Some(weight) => *weight,
+                                None => {
+                                    let weight = scorer.query_weight(&term.token);
+                                    idf_cache.insert(term.token.clone(), weight);
+                                    weight
+                                }
+                            };
+                            score += idf_weight * scorer.doc_weight(freq, doc_length);
+                        }
+                    }
+                    push_scored_candidate(&mut candidates, limit, addr, score)?;
                 }
             }
         }
@@ -745,7 +1139,7 @@ impl InvertedIndex {
 
     async fn load_legacy_index(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         log::warn!("loading legacy FTS index");
@@ -794,14 +1188,16 @@ impl InvertedIndex {
             store: store.clone(),
             tokenizer,
             token_set_format: TokenSetFormat::Arrow,
+            format_version: InvertedListFormatVersion::V1,
             partitions: vec![Arc::new(InvertedPartition {
                 id: 0,
                 store,
                 tokens,
                 inverted_list,
-                docs,
+                docs: Arc::new(LazyDocSet::from_loaded(docs)),
                 token_set_format: TokenSetFormat::Arrow,
             })],
+            corpus_stats: Arc::new(OnceCell::new()),
             deleted_fragments: RoaringBitmap::new(),
         }))
     }
@@ -810,9 +1206,36 @@ impl InvertedIndex {
         self.partitions.len() == 1 && self.partitions[0].is_legacy()
     }
 
+    /// Read only the index's [`InvertedIndexParams`],
+    /// Contains more complete info than manifest's lossy `InvertedIndexDetails`.
+    pub async fn load_params(store: &dyn IndexStore) -> Result<InvertedIndexParams> {
+        match store.open_index_file(METADATA_FILE).await {
+            Ok(reader) => {
+                let params = reader
+                    .schema()
+                    .metadata
+                    .get("params")
+                    .ok_or(Error::index("params not found in metadata".to_owned()))?;
+                Ok(serde_json::from_str::<InvertedIndexParams>(params)?)
+            }
+            Err(_) => {
+                // Legacy format: params live in the tokens file (see
+                // `load_legacy_index`).
+                let reader = store.open_index_file(TOKENS_FILE).await?;
+                Ok(reader
+                    .schema()
+                    .metadata
+                    .get("tokenizer")
+                    .map(|s| serde_json::from_str::<InvertedIndexParams>(s))
+                    .transpose()?
+                    .unwrap_or_default())
+            }
+        }
+    }
+
     pub async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>>
     where
@@ -843,6 +1266,7 @@ impl InvertedIndex {
                     .map(|name| TokenSetFormat::from_str(name))
                     .transpose()?
                     .unwrap_or(TokenSetFormat::Arrow);
+                let format_version = parse_format_version_from_metadata(&reader.schema().metadata)?;
 
                 // Load deleted_fragments if present (optional for backward compatibility)
                 let deleted_fragments = if reader.num_rows() > 0 {
@@ -858,8 +1282,8 @@ impl InvertedIndex {
                 };
 
                 let format = token_set_format;
-                let partitions = partitions.into_iter().map(|id| {
-                    let store = store.clone();
+                let partitions = partitions.into_iter().enumerate().map(|(priority, id)| {
+                    let store = store.with_io_priority(priority as u64);
                     let frag_reuse_index_clone = frag_reuse_index.clone();
                     let index_cache_for_part =
                         index_cache.with_key_prefix(format!("part-{}", id).as_str());
@@ -888,7 +1312,9 @@ impl InvertedIndex {
                     store,
                     tokenizer,
                     token_set_format,
+                    format_version,
                     partitions,
+                    corpus_stats: Arc::new(OnceCell::new()),
                     deleted_fragments,
                 }))
             }
@@ -910,12 +1336,6 @@ impl Index for InvertedIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::invalid_input(
-            "inverted index cannot be cast to vector index",
-        ))
-    }
-
     fn statistics(&self) -> Result<serde_json::Value> {
         let num_tokens = self
             .partitions
@@ -948,24 +1368,232 @@ impl Index for InvertedIndex {
     }
 }
 
+/// Target on-disk size of one prewarm chunk. Keep this large enough that cloud
+/// stores do not spend prewarm time on thousands of tiny range reads, but still
+/// bounded so one large partition is not materialized all at once.
+const PREWARM_CHUNK_TARGET_BYTES: u64 = 128 << 20;
+
+/// Cap on token rows per chunk, bounding the built `Vec` when posting lists are tiny.
+const PREWARM_MAX_CHUNK_TOKENS: usize = 256 * 1024;
+
+/// Floor on token rows per chunk, so a partition always makes progress.
+const PREWARM_MIN_CHUNK_TOKENS: usize = 1;
+
+/// Maximum number of posting lists in a runtime synthetic cache group. This is
+/// deliberately token-count based so grouping works for old v2 indexes without
+/// scanning posting lengths or requiring index rebuilds.
+static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock<usize> = LazyLock::new(|| {
+    std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
+        .unwrap_or_else(|_| "128".to_string())
+        .parse()
+        .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS")
+});
+
+fn runtime_posting_group_tokens() -> usize {
+    (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1)
+}
+
+/// Runtime posting-list cache grouping. Non-empty v2 indexes synthesize fixed
+/// groups at read time so prewarm and queries share group cache entries without
+/// persisted grouping metadata or index rebuilds.
+#[derive(Debug, Clone, DeepSizeOf)]
+enum PostingGrouping {
+    /// Leaves legacy or empty partitions ungrouped.
+    None,
+    /// Uses a fixed runtime cache group size measured in token rows, not posting bytes.
+    SyntheticFixed { group_size: u32 },
+}
+
+impl PostingGrouping {
+    fn for_reader(is_legacy_layout: bool, token_count: usize) -> Self {
+        if is_legacy_layout || token_count == 0 {
+            return Self::None;
+        }
+
+        let group_size = u32::try_from(runtime_posting_group_tokens())
+            .unwrap_or(u32::MAX)
+            .max(1);
+        Self::SyntheticFixed { group_size }
+    }
+
+    fn is_grouped(&self) -> bool {
+        !matches!(self, Self::None)
+    }
+
+    fn range_for_token(&self, token_id: u32, token_count: usize) -> Option<(u32, u32)> {
+        match self {
+            Self::None => None,
+            Self::SyntheticFixed { group_size } => {
+                let token_count = u32::try_from(token_count).unwrap_or(u32::MAX);
+                let start = (token_id / *group_size) * *group_size;
+                let end = start.saturating_add(*group_size).min(token_count);
+                Some((start, end))
+            }
+        }
+    }
+
+    fn aligned_chunk_end(&self, token_count: usize, tok_start: usize, desired_end: usize) -> usize {
+        match self {
+            Self::None => desired_end,
+            Self::SyntheticFixed { group_size } => synthetic_group_aligned_chunk_end(
+                usize::try_from(*group_size).unwrap_or(usize::MAX).max(1),
+                token_count,
+                tok_start,
+                desired_end,
+            ),
+        }
+    }
+
+    fn ranges_for_chunk(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        token_count: usize,
+    ) -> Vec<(u32, u32)> {
+        match self {
+            Self::None => Vec::new(),
+            Self::SyntheticFixed { group_size } => synthetic_group_ranges_for_chunk(
+                usize::try_from(*group_size).unwrap_or(usize::MAX).max(1),
+                tok_start,
+                tok_end,
+                token_count,
+            ),
+        }
+    }
+}
+
+/// Token rows per chunk: byte target / average bytes-per-token, clamped to `[MIN, MAX]`.
+fn prewarm_chunk_tokens(token_count: usize, file_size_bytes: u64) -> usize {
+    if token_count == 0 {
+        return PREWARM_MIN_CHUNK_TOKENS;
+    }
+    let bytes_per_token = (file_size_bytes / token_count as u64).max(1); // >= 1: no div-by-zero
+    let by_bytes = (PREWARM_CHUNK_TARGET_BYTES / bytes_per_token) as usize;
+    by_bytes.clamp(PREWARM_MIN_CHUNK_TOKENS, PREWARM_MAX_CHUNK_TOKENS)
+}
+
+fn synthetic_group_aligned_chunk_end(
+    group_size: usize,
+    token_count: usize,
+    tok_start: usize,
+    desired_end: usize,
+) -> usize {
+    if desired_end >= token_count {
+        return token_count;
+    }
+
+    let boundary = desired_end - (desired_end % group_size);
+    if boundary > tok_start {
+        boundary
+    } else {
+        tok_start.saturating_add(group_size).min(token_count)
+    }
+}
+
+fn synthetic_group_ranges_for_chunk(
+    group_size: usize,
+    tok_start: usize,
+    tok_end: usize,
+    token_count: usize,
+) -> Vec<(u32, u32)> {
+    let mut ranges = Vec::new();
+    let mut start = tok_start - (tok_start % group_size);
+    if start < tok_start {
+        start = start.saturating_add(group_size).min(token_count);
+    }
+    while start < tok_end {
+        let end = start.saturating_add(group_size).min(token_count);
+        ranges.push((
+            u32::try_from(start).unwrap_or(u32::MAX),
+            u32::try_from(end).unwrap_or(u32::MAX),
+        ));
+        start = end;
+    }
+    ranges
+}
+
+fn prewarm_chunk_ranges(
+    grouping: &PostingGrouping,
+    token_count: usize,
+    chunk_tokens: usize,
+) -> Vec<(usize, usize)> {
+    let mut ranges = Vec::new();
+    let mut tok_start = 0usize;
+    while tok_start < token_count {
+        let mut tok_end = (tok_start + chunk_tokens).min(token_count);
+        // `tok_start` is always a group boundary; snap `tok_end` back to one too.
+        if grouping.is_grouped() {
+            tok_end = grouping.aligned_chunk_end(token_count, tok_start, tok_end);
+        }
+        ranges.push((tok_start, tok_end));
+        tok_start = tok_end;
+    }
+    ranges
+}
+
 impl InvertedIndex {
     pub async fn prewarm_with_options(&self, options: &FtsPrewarmOptions) -> Result<()> {
         let with_position = options.with_position;
-        let io_parallelism = self.store.io_parallelism();
-        let prewarm_futures = self
-            .partitions
-            .iter()
-            .map(Arc::clone)
-            .map(|part| async move {
-                part.inverted_list
-                    .prewarm_posting_lists(with_position)
-                    .await?;
-                Result::Ok(())
-            });
-        stream::iter(prewarm_futures)
-            .buffer_unordered(io_parallelism)
-            .try_collect::<Vec<_>>()
-            .await?;
+        let chunk_concurrency = self.store.io_parallelism().max(1);
+        let prewarm_started = Instant::now();
+        info!(
+            partition_count = self.partitions.len(),
+            with_position, chunk_concurrency, "fts index prewarm started"
+        );
+        for part in &self.partitions {
+            let partition_started = Instant::now();
+            info!(
+                partition_id = part.id(),
+                token_count = part.tokens.len(),
+                with_position,
+                chunk_concurrency,
+                "fts partition prewarm started"
+            );
+            if let Err(err) = part
+                .inverted_list
+                .prewarm_posting_lists(with_position, chunk_concurrency)
+                .await
+            {
+                warn!(
+                    partition_id = part.id(),
+                    error = %err,
+                    elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                    "fts partition posting list prewarm failed"
+                );
+                return Err(err);
+            }
+            info!(
+                partition_id = part.id(),
+                elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                "fts partition posting lists prewarmed"
+            );
+            // Materialize the deferred DocSet too: prewarm's contract is
+            // that subsequent queries do no IO, so the per-doc row_ids /
+            // num_tokens must be resident, not lazily faulted in at query
+            // time. `ensure_loaded` opens, reads, and drops the reader.
+            let docs_started = Instant::now();
+            if let Err(err) = part.docs.ensure_loaded().await {
+                warn!(
+                    partition_id = part.id(),
+                    error = %err,
+                    elapsed_ms = docs_started.elapsed().as_millis() as u64,
+                    total_elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                    "fts partition docset prewarm failed"
+                );
+                return Err(err);
+            }
+            info!(
+                partition_id = part.id(),
+                docset_elapsed_ms = docs_started.elapsed().as_millis() as u64,
+                elapsed_ms = partition_started.elapsed().as_millis() as u64,
+                "fts partition prewarm finished"
+            );
+        }
+        info!(
+            partition_count = self.partitions.len(),
+            elapsed_ms = prewarm_started.elapsed().as_millis() as u64,
+            "fts index prewarm finished"
+        );
         Ok(())
     }
     /// Search docs match the input text.
@@ -1024,10 +1652,11 @@ impl ScalarIndex for InvertedIndex {
 
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
-        self.to_builder()
+        let files = self
+            .to_builder()
             .remap(mapping, self.store.clone(), dest_store)
             .await?;
 
@@ -1036,7 +1665,7 @@ impl ScalarIndex for InvertedIndex {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
             index_version: self.index_version(),
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -1046,7 +1675,8 @@ impl ScalarIndex for InvertedIndex {
         dest_store: &dyn IndexStore,
         old_data_filter: Option<crate::scalar::OldIndexDataFilter>,
     ) -> Result<CreatedIndex> {
-        self.to_builder()
+        let files = self
+            .to_builder()
             .update(new_data, dest_store, old_data_filter)
             .await?;
 
@@ -1055,7 +1685,7 @@ impl ScalarIndex for InvertedIndex {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&details).unwrap(),
             index_version: self.index_version(),
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -1074,8 +1704,9 @@ impl ScalarIndex for InvertedIndex {
             // Empty tokenizer metadata only appears in legacy simple-tokenizer indexes.
             params.base_tokenizer = "simple".to_string();
         }
+        params = params.format_version(self.format_version());
 
-        let params_json = serde_json::to_string(&params)?;
+        let params_json = params.to_training_json()?.to_string();
 
         Ok(ScalarIndexParams {
             index_type: BuiltinIndexType::Inverted.as_str().to_string(),
@@ -1091,7 +1722,10 @@ pub struct InvertedPartition {
     store: Arc<dyn IndexStore>,
     pub(crate) tokens: TokenSet,
     pub(crate) inverted_list: Arc<PostingListReader>,
-    pub(crate) docs: DocSet,
+    /// Per-doc row_id + num_tokens. Wrapped in `LazyDocSet` so partitions
+    /// that don't contribute hits to a query never pay the full-array
+    /// download. Scoring paths call `ensure_loaded` before walking wand.
+    pub(crate) docs: Arc<LazyDocSet>,
     token_set_format: TokenSetFormat,
 }
 
@@ -1125,7 +1759,7 @@ impl InvertedPartition {
     pub async fn load(
         store: Arc<dyn IndexStore>,
         id: u64,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
         token_set_format: TokenSetFormat,
     ) -> Result<Self> {
@@ -1133,8 +1767,23 @@ impl InvertedPartition {
         let tokens = TokenSet::load(token_file, token_set_format).await?;
         let invert_list_file = store.open_index_file(&posting_file_path(id)).await?;
         let inverted_list = PostingListReader::try_new(invert_list_file, index_cache).await?;
-        let docs_file = store.open_index_file(&doc_file_path(id)).await?;
-        let docs = DocSet::load(docs_file, false, frag_reuse_index).await?;
+        // Defer the per-doc row_id/num_tokens read. Construction reads only
+        // the doc count (one footer read) and then drops the reader; the bulk
+        // load happens on first scoring use, re-opening the docs file on
+        // demand, and partitions that never score skip it entirely. Storing
+        // the store + path instead of an open reader keeps a cached partition
+        // from pinning a docs-file handle for its whole lifetime.
+        let docs_path = doc_file_path(id);
+        let num_docs = store.open_index_file(&docs_path).await?.num_rows();
+        let docs = Arc::new(LazyDocSet::new(
+            store.clone(),
+            docs_path,
+            num_docs,
+            false,
+            frag_reuse_index,
+            // V3 (256-doc block) partitions score with quantized doc lengths.
+            inverted_list.block_size() == MAX_POSTING_BLOCK_SIZE,
+        ));
 
         Ok(Self {
             id,
@@ -1152,128 +1801,414 @@ impl InvertedPartition {
 
     pub fn expand_fuzzy(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result<Tokens> {
         let mut new_tokens = Vec::with_capacity(min(tokens.len(), params.max_expansions));
-        for token in tokens {
-            let fuzziness = match params.fuzziness {
-                Some(fuzziness) => fuzziness,
-                None => MatchQuery::auto_fuzziness(token),
-            };
-            let lev = fst::automaton::Levenshtein::new(token, fuzziness)
-                .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?;
-
-            let base_len = tokens.token_type().prefix_len(token) as u32;
-            if let TokenMap::Fst(ref map) = self.tokens.tokens {
-                match base_len + params.prefix_length {
-                    0 => take_fst_keys(map.search(lev), &mut new_tokens, params.max_expansions),
-                    prefix_length => {
-                        let prefix = &token[..min(prefix_length as usize, token.len())];
-                        let prefix = fst::automaton::Str::new(prefix).starts_with();
-                        take_fst_keys(
-                            map.search(lev.intersection(prefix)),
-                            &mut new_tokens,
-                            params.max_expansions,
-                        )
-                    }
+        let mut new_positions = Vec::with_capacity(new_tokens.capacity());
+        let mut seen = HashSet::new();
+        for token_idx in 0..tokens.len() {
+            let remaining = params.max_expansions.saturating_sub(new_tokens.len());
+            if remaining == 0 {
+                break;
+            }
+            let token = tokens.get_token(token_idx);
+            let position = tokens.position(token_idx);
+            let base_prefix_len = tokens.token_type().prefix_len(token) as u32;
+            let mut candidates = BTreeSet::new();
+            self.collect_fuzzy_candidates(
+                token,
+                base_prefix_len,
+                params,
+                remaining,
+                &mut candidates,
+            )?;
+            for candidate in candidates {
+                if new_tokens.len() >= params.max_expansions {
+                    break;
+                }
+                if seen.insert((candidate.clone(), position)) {
+                    new_tokens.push(candidate);
+                    new_positions.push(position);
                 }
-            } else {
-                return Err(Error::index(
-                    "tokens is not fst, which is not expected".to_owned(),
-                ));
             }
         }
-        Ok(Tokens::new(new_tokens, tokens.token_type().clone()))
+        Ok(Tokens::with_positions(
+            new_tokens,
+            new_positions,
+            tokens.token_type().clone(),
+        ))
     }
 
-    // search the documents that contain the query
-    // return the doc info and the doc length
-    // ref: https://en.wikipedia.org/wiki/Okapi_BM25
-    #[instrument(level = "debug", skip_all)]
-    pub async fn load_posting_lists(
+    /// Collect up to `limit` fuzzy candidates for one query token from this
+    /// partition's token FST, in key (lexicographic) order. Callers merge
+    /// candidates across partitions and apply the query-wide
+    /// `max_expansions` budget; truncating each partition at `limit` is
+    /// lossless for that selection because any term among the merged
+    /// lexicographically-smallest `limit` is also among its own partition's
+    /// smallest `limit`.
+    fn collect_fuzzy_candidates(
         &self,
-        tokens: &Tokens,
+        token: &str,
+        base_prefix_len: u32,
         params: &FtsSearchParams,
-        metrics: &dyn MetricsCollector,
-    ) -> Result<Vec<PostingIterator>> {
-        let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0);
-        let is_phrase_query = params.phrase_slop.is_some();
-        let tokens = match is_fuzzy {
-            true => self.expand_fuzzy(tokens, params)?,
-            false => tokens.clone(),
+        limit: usize,
+        candidates: &mut BTreeSet<String>,
+    ) -> Result<()> {
+        let fuzziness = match params.fuzziness {
+            Some(fuzziness) => fuzziness,
+            None => MatchQuery::auto_fuzziness(token),
         };
+        let lev = fst::automaton::Levenshtein::new(token, fuzziness)
+            .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?;
+
+        if let TokenMap::Fst(ref map) = self.tokens.tokens {
+            let mut expanded = Vec::new();
+            match base_prefix_len + params.prefix_length {
+                0 => take_fst_keys(map.search(lev), &mut expanded, limit),
+                prefix_length => {
+                    let prefix = &token[..min(prefix_length as usize, token.len())];
+                    let prefix = fst::automaton::Str::new(prefix).starts_with();
+                    take_fst_keys(map.search(lev.intersection(prefix)), &mut expanded, limit)
+                }
+            }
+            candidates.extend(expanded);
+            Ok(())
+        } else {
+            Err(Error::index(
+                "tokens is not fst, which is not expected".to_owned(),
+            ))
+        }
+    }
+
+    fn union_plain_posting_lists(postings: Vec<PostingList>) -> Result<PostingList> {
+        let mut freqs_by_row_id = BTreeMap::new();
+        for posting in postings {
+            for (row_id, freq, _) in posting.iter() {
+                let entry = freqs_by_row_id.entry(row_id).or_insert(0u32);
+                *entry = entry.checked_add(freq).ok_or_else(|| {
+                    Error::index(format!("posting frequency overflow for row id {}", row_id))
+                })?;
+            }
+        }
+        let mut row_ids = Vec::with_capacity(freqs_by_row_id.len());
+        let mut frequencies = Vec::with_capacity(freqs_by_row_id.len());
+        for (row_id, freq) in freqs_by_row_id {
+            row_ids.push(row_id);
+            frequencies.push(freq as f32);
+        }
+        Ok(PostingList::Plain(PlainPostingList::new(
+            ScalarBuffer::from(row_ids),
+            ScalarBuffer::from(frequencies),
+            None,
+            None,
+        )))
+    }
+
+    fn union_compressed_posting_lists(
+        postings: Vec<PostingList>,
+        docs: &DocSet,
+    ) -> Result<PostingList> {
+        let block_size = postings
+            .iter()
+            .find_map(|posting| match posting {
+                PostingList::Compressed(posting) => Some(posting.block_size),
+                PostingList::Plain(_) => None,
+            })
+            .unwrap_or(LEGACY_BLOCK_SIZE);
+        let mut freqs_by_doc_id = BTreeMap::new();
+        for posting in postings {
+            for (doc_id, freq, _) in posting.iter() {
+                let doc_id = u32::try_from(doc_id).map_err(|_| {
+                    Error::index(format!(
+                        "compressed posting doc id {} exceeds u32::MAX",
+                        doc_id
+                    ))
+                })?;
+                let entry = freqs_by_doc_id.entry(doc_id).or_insert(0u32);
+                *entry = entry.checked_add(freq).ok_or_else(|| {
+                    Error::index(format!("posting frequency overflow for doc id {}", doc_id))
+                })?;
+            }
+        }
+        if freqs_by_doc_id.is_empty() {
+            return Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            )));
+        }
+
+        let mut builder = PostingListBuilder::new_with_block_size(false, block_size);
+        let mut doc_ids = Vec::with_capacity(freqs_by_doc_id.len());
+        let mut frequencies = Vec::with_capacity(freqs_by_doc_id.len());
+        for (doc_id, freq) in freqs_by_doc_id {
+            builder.add(doc_id, PositionRecorder::Count(freq));
+            doc_ids.push(doc_id);
+            frequencies.push(freq);
+        }
+        let block_max_scores = docs.calculate_block_max_scores_with_block_size(
+            doc_ids.iter(),
+            frequencies.iter(),
+            block_size,
+        );
+        let batch = builder.to_batch(block_max_scores)?;
+        let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+        let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+        PostingList::from_batch(&batch, Some(max_score), Some(length))
+    }
+
+    fn union_posting_lists(postings: Vec<PostingList>, docs: &DocSet) -> Result<PostingList> {
+        let has_plain = postings
+            .iter()
+            .any(|posting| matches!(posting, PostingList::Plain(_)));
+        let has_compressed = postings
+            .iter()
+            .any(|posting| matches!(posting, PostingList::Compressed(_)));
+        match (has_plain, has_compressed) {
+            (true, true) => Err(Error::index(
+                "cannot union mixed plain and compressed posting lists".to_owned(),
+            )),
+            (true, false) => Self::union_plain_posting_lists(postings),
+            (false, true) => Self::union_compressed_posting_lists(postings, docs),
+            (false, false) => Ok(PostingList::Plain(PlainPostingList::new(
+                ScalarBuffer::from(Vec::<u64>::new()),
+                ScalarBuffer::from(Vec::<f32>::new()),
+                None,
+                None,
+            ))),
+        }
+    }
+
+    // search the documents that contain the query
+    // return the doc info and the doc length
+    // ref: https://en.wikipedia.org/wiki/Okapi_BM25
+    #[instrument(level = "debug", skip_all)]
+    async fn load_posting_lists(
+        &self,
+        tokens: &Tokens,
+        params: &FtsSearchParams,
+        operator: Operator,
+        impact_scorer: &MemBM25Scorer,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<LoadedPostings> {
+        let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0);
+        let is_phrase_query = params.phrase_slop.is_some();
+        let is_and_query = operator == Operator::And;
+        let required_positions = (is_and_query || is_phrase_query).then(|| {
+            (0..tokens.len())
+                .map(|index| tokens.position(index))
+                .collect::<HashSet<_>>()
+        });
+        // Fuzzy expansion already ran once at the index level (see
+        // `InvertedIndex::bm25_search`) under the global `max_expansions`
+        // budget; the incoming tokens are final and `is_fuzzy` only drives
+        // the grouped dedup/scoring semantics below.
+        let tokens = tokens.clone();
         let token_positions = (0..tokens.len())
             .map(|index| tokens.position(index))
             .collect::<Vec<_>>();
         let mut token_ids = Vec::with_capacity(tokens.len());
+        let mut matched_positions = required_positions.as_ref().map(|_| HashSet::new());
         for (index, token) in tokens.into_iter().enumerate() {
             let token_id = self.map(&token);
             if let Some(token_id) = token_id {
-                token_ids.push((token_id, token, token_positions[index]));
-            } else if is_phrase_query {
-                // if the token is not found, we can't do phrase query
-                return Ok(Vec::new());
+                let position = token_positions[index];
+                if let Some(matched_positions) = matched_positions.as_mut() {
+                    matched_positions.insert(position);
+                }
+                token_ids.push((token_id, token, position));
+            } else if is_phrase_query || is_and_query {
+                // if the token is not found, we can't do phrase or AND query
+                return Ok(LoadedPostings::empty());
             }
         }
         if token_ids.is_empty() {
-            return Ok(Vec::new());
+            return Ok(LoadedPostings::empty());
+        }
+        if let Some(required_positions) = required_positions.as_ref()
+            && let Some(matched_positions) = matched_positions.as_ref()
+            && !required_positions.is_subset(matched_positions)
+        {
+            return Ok(LoadedPostings::empty());
         }
+
+        let is_fuzzy_and_query = is_fuzzy && is_and_query && !is_phrase_query;
         if !is_phrase_query {
-            token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id);
-            token_ids.dedup_by_key(|(token_id, _, _)| *token_id);
+            if is_fuzzy_and_query {
+                token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id));
+                token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2);
+            } else {
+                token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id);
+                token_ids.dedup_by_key(|(token_id, _, _)| *token_id);
+            }
         }
 
         let num_docs = self.docs.len();
-        stream::iter(token_ids)
+        let loaded_postings = stream::iter(token_ids)
             .map(|(token_id, token, position)| async move {
                 let posting = self
                     .inverted_list
                     .posting_list(token_id, is_phrase_query, metrics)
                     .await?;
 
-                let query_weight = idf(posting.len(), num_docs);
-
-                Result::Ok(PostingIterator::with_query_weight(
-                    token,
-                    token_id,
-                    position,
-                    query_weight,
-                    posting,
-                    num_docs,
-                ))
+                Result::Ok((token_id, token, position, posting))
             })
             .buffered(self.store.io_parallelism())
             .try_collect::<Vec<_>>()
-            .await
+            .await?;
+
+        if (is_and_query || is_phrase_query)
+            && !is_fuzzy_and_query
+            && loaded_postings
+                .iter()
+                .any(|(_, _, _, posting)| posting.is_empty())
+        {
+            return Ok(LoadedPostings::empty());
+        }
+
+        if !is_fuzzy_and_query {
+            let impact_safe = loaded_postings
+                .iter()
+                .all(|(_, _, _, posting)| posting.has_impacts());
+            return Ok(LoadedPostings {
+                postings: loaded_postings
+                    .into_iter()
+                    .map(|(token_id, token, position, posting)| {
+                        let query_weight = if impact_safe {
+                            impact_scorer.query_weight(&token)
+                        } else {
+                            idf(posting.len(), num_docs)
+                        };
+                        PostingIterator::with_query_weight(
+                            token,
+                            token_id,
+                            position,
+                            query_weight,
+                            posting,
+                            num_docs,
+                        )
+                    })
+                    .collect(),
+                grouped_expansions: Vec::new(),
+                impact_safe,
+            });
+        }
+
+        let needs_union = loaded_postings
+            .windows(2)
+            .any(|window| window[0].2 == window[1].2);
+        let docs_for_union = if needs_union {
+            Some(self.docs.ensure_num_tokens_loaded().await?)
+        } else {
+            None
+        };
+
+        // WAND's AND mode treats every iterator as required, so expansions from
+        // one original query position must be merged before scoring.
+        let mut grouped_postings = Vec::new();
+        let mut grouped_expansions = Vec::new();
+        let mut iter = loaded_postings.into_iter().peekable();
+        while let Some((token_id, token, position, posting)) = iter.next() {
+            let mut group = vec![(token_id, token, posting)];
+            while matches!(iter.peek(), Some((_, _, next_position, _)) if *next_position == position)
+            {
+                let (token_id, token, _, posting) = iter.next().expect("peeked item must exist");
+                group.push((token_id, token, posting));
+            }
+
+            let (token_id, token, posting) = if group.len() == 1 {
+                group.pop().expect("single-item group must exist")
+            } else {
+                let token_id = group[0].0;
+                let token = group[0].1.clone();
+                grouped_expansions.push(GroupedExpansionTerms {
+                    position,
+                    terms: group
+                        .iter()
+                        .map(|(_, token, posting)| ExpansionTermFreqs::new(token.clone(), posting))
+                        .collect(),
+                });
+                let postings = group
+                    .into_iter()
+                    .map(|(_, _, posting)| posting)
+                    .collect::<Vec<_>>();
+                let posting = Self::union_posting_lists(
+                    postings,
+                    docs_for_union
+                        .as_deref()
+                        .expect("union docs must be loaded for grouped fuzzy AND"),
+                )?;
+                (token_id, token, posting)
+            };
+            if posting.is_empty() {
+                return Ok(LoadedPostings::empty());
+            }
+
+            let query_weight = idf(posting.len(), num_docs);
+            grouped_postings.push(PostingIterator::with_query_weight(
+                token,
+                token_id,
+                position,
+                query_weight,
+                posting,
+                num_docs,
+            ));
+        }
+
+        Ok(LoadedPostings {
+            postings: grouped_postings,
+            grouped_expansions,
+            impact_safe: false,
+        })
     }
 
     #[instrument(level = "debug", skip_all)]
+    // Deferred-DocSet adds the `docs` param (caller materializes it) on top of
+    // the cross-partition `shared_threshold`, tipping this hot-path search fn
+    // one over the limit. Bundling args isn't worth the churn here.
+    #[allow(clippy::too_many_arguments)]
     pub fn bm25_search(
         &self,
+        docs: &DocSet,
         params: &FtsSearchParams,
         operator: Operator,
         mask: Arc<RowAddrMask>,
         postings: Vec<PostingIterator>,
+        impact_scorer: Option<Arc<MemBM25Scorer>>,
         metrics: &dyn MetricsCollector,
+        shared_threshold: Arc<AtomicU32>,
     ) -> Result<Vec<DocCandidate>> {
         if postings.is_empty() {
             return Ok(Vec::new());
         }
 
-        // let local_metrics = LocalMetricsCollector::default();
-        let scorer = IndexBM25Scorer::new(std::iter::once(self));
-        let mut wand = Wand::new(operator, postings.into_iter(), &self.docs, scorer);
-        let hits = wand.search(params, mask, metrics)?;
-        // local_metrics.dump_into(metrics);
+        // Caller selects the DocSet shape via `LazyDocSet::docs_for_wand`
+        // and passes it in here; wand uses `docs.has_row_ids()` to
+        // handle the num_tokens-only case.
+        let hits = if let Some(scorer) = impact_scorer {
+            let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer)
+                .with_shared_threshold(shared_threshold);
+            wand.search(params, mask, metrics)?
+        } else {
+            let scorer = IndexBM25Scorer::new(std::iter::once(self));
+            let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer)
+                .with_shared_threshold(shared_threshold);
+            wand.search(params, mask, metrics)?
+        };
         Ok(hits)
     }
 
     pub async fn into_builder(self) -> Result<InnerBuilder> {
-        let mut builder = InnerBuilder::new_with_posting_tail_codec(
+        let mut builder = InnerBuilder::new_with_posting_tail_codec_and_block_size(
             self.id,
             self.inverted_list.has_positions(),
             self.token_set_format,
             self.inverted_list.posting_tail_codec(),
+            self.inverted_list.block_size(),
         );
         builder.tokens = self.tokens.into_mutable();
-        builder.docs = self.docs;
+        // into_builder rewrites every doc, so materialize the full
+        // DocSet now and clone it out of the Arc.
+        let docs_arc = self.docs.ensure_loaded().await?;
+        builder.docs = (*docs_arc).clone();
 
         builder
             .posting_lists
@@ -1307,7 +2242,7 @@ impl Default for TokenMap {
 }
 
 impl DeepSizeOf for TokenMap {
-    fn deep_size_of_children(&self, ctx: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, ctx: &mut lance_core::deepsize::Context) -> usize {
         match self {
             Self::HashMap(map) => map.deep_size_of_children(ctx),
             Self::Fst(map) => map.as_fst().size(),
@@ -1511,15 +2446,13 @@ impl TokenSet {
         let map = fst::Map::new(bytes.to_vec())
             .map_err(|e| Error::index(format!("failed to load fst tokens: {}", e)))?;
 
-        let next_id_col = batch[TOKEN_NEXT_ID_COL].as_primitive::<datatypes::UInt32Type>();
         let total_length_col =
             batch[TOKEN_TOTAL_LENGTH_COL].as_primitive::<datatypes::UInt64Type>();
 
-        let next_id = next_id_col
-            .values()
-            .first()
-            .copied()
-            .ok_or(Error::index("token next id column is empty".to_owned()))?;
+        // Token ids are dense `[0, len)`, so `next_id` must equal the token count. Recompute
+        // it instead of trusting the persisted value, which writers before #7115 could leave
+        // stale. Mirrors `load_arrow`.
+        let next_id = map.len() as u32;
 
         let total_length = total_length_col
             .values()
@@ -1629,17 +2562,26 @@ impl TokenSet {
             }
         };
 
+        let mut retained_length = 0;
         map.retain(
-            |_, token_id| match removed_token_ids.binary_search(token_id) {
+            |token, token_id| match removed_token_ids.binary_search(token_id) {
                 Ok(_) => false,
                 Err(index) => {
                     *token_id -= index as u32;
+                    retained_length += token.len();
                     true
                 }
             },
         );
 
         self.tokens = TokenMap::HashMap(map);
+
+        // The retain above compacts the surviving token ids into a dense `[0, len)`
+        // range, so `next_id` (handed to the next new token) must follow them down.
+        // `total_length` likewise must drop the removed tokens' bytes; it is persisted
+        // and feeds memory accounting, so a stale value drifts across remap/merge cycles.
+        self.next_id = self.tokens.len() as u32;
+        self.total_length = retained_length;
     }
 
     pub fn next_id(&self) -> u32 {
@@ -1669,14 +2611,15 @@ pub struct PostingListReader {
     metadata: PostingMetadata,
 
     has_position: bool,
+    has_impacts: bool,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     positions_layout: PositionsLayout,
 
-    /// First row of each posting-list cache group, decoded at open from the
-    /// global buffer named by [`POSTING_GROUP_OFFSETS_BUF_KEY`] (issue #7040).
-    /// `None` for indexes written before grouping; those use the per-token
-    /// cache path. Always present for grouped v2 indexes with `>0` rows.
-    group_starts: Option<Vec<u32>>,
+    /// Runtime posting-list cache grouping. Non-empty v2 indexes use synthetic
+    /// fixed groups so prewarm can improve cache density without rebuilding the
+    /// index or relying on persisted grouping metadata.
+    grouping: PostingGrouping,
 
     index_cache: WeakLanceCache,
 }
@@ -1698,7 +2641,7 @@ enum PostingMetadata {
     /// `ensure_metadata_loaded`, and the stats path can also fetch a single
     /// token via `posting_len_for_token` without forcing the bulk load.
     V2 {
-        metadata: tokio::sync::OnceCell<LoadedPostingMetadata>,
+        metadata: OnceCell<LoadedPostingMetadata>,
     },
 }
 
@@ -1737,7 +2680,7 @@ impl std::fmt::Debug for PostingListReader {
 }
 
 impl DeepSizeOf for PostingListReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let metadata_size = match &self.metadata {
             PostingMetadata::LegacyV1 {
                 offsets,
@@ -1751,7 +2694,7 @@ impl DeepSizeOf for PostingListReader {
                 })
                 .unwrap_or(0),
         };
-        metadata_size + self.group_starts.deep_size_of_children(context)
+        metadata_size + self.grouping.deep_size_of_children(context)
     }
 }
 
@@ -1768,7 +2711,9 @@ impl PostingListReader {
             PositionsLayout::None
         };
         let posting_tail_codec = parse_posting_tail_codec(&reader.schema().metadata)?;
+        let block_size = parse_posting_block_size(&reader.schema().metadata)?;
         let has_position = positions_layout != PositionsLayout::None;
+        let has_impacts = reader.schema().field(IMPACT_COL).is_some();
         let metadata = if reader.schema().field(POSTING_COL).is_none() {
             let (offsets, max_scores) = Self::load_metadata(reader.schema())?;
             PostingMetadata::LegacyV1 {
@@ -1777,40 +2722,26 @@ impl PostingListReader {
             }
         } else {
             PostingMetadata::V2 {
-                metadata: tokio::sync::OnceCell::new(),
+                metadata: OnceCell::new(),
             }
         };
 
-        let group_starts = Self::load_group_starts(reader.as_ref()).await?;
+        let is_legacy_layout = matches!(&metadata, PostingMetadata::LegacyV1 { .. });
+        let grouping = PostingGrouping::for_reader(is_legacy_layout, reader.num_rows());
 
         Ok(Self {
             reader,
             metadata,
             has_position,
+            has_impacts,
             posting_tail_codec,
+            block_size,
             positions_layout,
-            group_starts,
+            grouping,
             index_cache: WeakLanceCache::from(index_cache),
         })
     }
 
-    /// Decode the posting-list cache-group boundaries from the global buffer
-    /// recorded in schema metadata, if present (issue #7040). Returns `None`
-    /// for indexes written before grouping was introduced.
-    async fn load_group_starts(reader: &dyn IndexReader) -> Result<Option<Vec<u32>>> {
-        let Some(buf_id) = reader.schema().metadata.get(POSTING_GROUP_OFFSETS_BUF_KEY) else {
-            return Ok(None);
-        };
-        let buf_id: u32 = buf_id.parse().map_err(|e| {
-            Error::index(format!(
-                "invalid {POSTING_GROUP_OFFSETS_BUF_KEY} metadata value {buf_id:?}: {e}"
-            ))
-        })?;
-        let bytes = reader.read_global_buffer(buf_id).await?;
-        let group_starts = decode_group_starts(&bytes)?;
-        Ok(Some(group_starts))
-    }
-
     // for legacy format
     // returns the offsets and max scores
     fn load_metadata(
@@ -1850,6 +2781,10 @@ impl PostingListReader {
         self.posting_tail_codec
     }
 
+    pub(crate) fn block_size(&self) -> usize {
+        self.block_size
+    }
+
     fn is_legacy_layout(&self) -> bool {
         matches!(self.metadata, PostingMetadata::LegacyV1 { .. })
     }
@@ -1878,10 +2813,10 @@ impl PostingListReader {
     }
 
     /// Async access to a single token's posting list length. For v2
-    /// indexes this reads a single row from `LENGTH_COL` if the bulk metadata
-    /// has not been loaded yet, and never triggers the bulk load itself. The
-    /// stats path uses this so a single-term `df` lookup costs O(1) bytes
-    /// rather than O(num_unique_tokens).
+    /// indexes this reads one row of posting metadata if the bulk metadata has
+    /// not been loaded yet, and never triggers the bulk load itself. The stats
+    /// path uses this so a single-term `df` lookup costs O(1) bytes rather
+    /// than O(num_unique_tokens).
     pub(crate) async fn posting_len_for_token(&self, token_id: u32) -> Result<usize> {
         match &self.metadata {
             PostingMetadata::LegacyV1 { .. } => Ok(self.posting_len(token_id)),
@@ -1889,13 +2824,10 @@ impl PostingListReader {
                 if let Some(metadata) = metadata.get() {
                     return Ok(metadata.lengths[token_id as usize] as usize);
                 }
-                let token_id = token_id as usize;
-                let batch = self
-                    .reader
-                    .read_range(token_id..token_id + 1, Some(&[LENGTH_COL]))
-                    .await?;
-                let len = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
-                Ok(len as usize)
+                let (_, length) = self.posting_metadata_for_token(token_id).await?;
+                length
+                    .map(|len| len as usize)
+                    .ok_or_else(|| Error::index("posting length metadata missing".to_string()))
             }
         }
     }
@@ -1921,17 +2853,20 @@ impl PostingListReader {
                         Some(loaded.lengths[token_id as usize]),
                     ));
                 }
-                let token_id_usize = token_id as usize;
-                let batch = self
-                    .reader
-                    .read_range(
-                        token_id_usize..token_id_usize + 1,
-                        Some(&[MAX_SCORE_COL, LENGTH_COL]),
-                    )
+                let metadata = self
+                    .index_cache
+                    .get_or_insert_with_key(PostingMetadataKey { token_id }, || async move {
+                        let token_id = token_id as usize;
+                        let batch = self
+                            .reader
+                            .read_range(token_id..token_id + 1, Some(&[MAX_SCORE_COL, LENGTH_COL]))
+                            .await?;
+                        let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+                        let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+                        Ok(PostingMetadataValue { max_score, length })
+                    })
                     .await?;
-                let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
-                let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
-                Ok((Some(max_score), Some(length)))
+                Ok((Some(metadata.max_score), Some(metadata.length)))
             }
         }
     }
@@ -1978,7 +2913,7 @@ impl PostingListReader {
             self.posting_batch_legacy(token_id, with_position).await
         } else {
             let token_id = token_id as usize;
-            let columns = if with_position {
+            let mut columns = if with_position {
                 match self.positions_layout {
                     PositionsLayout::SharedStream(_) => {
                         vec![
@@ -1993,6 +2928,9 @@ impl PostingListReader {
             } else {
                 vec![POSTING_COL]
             };
+            if self.has_impacts {
+                columns.push(IMPACT_COL);
+            }
             let batch = self
                 .reader
                 .read_range(token_id..token_id + 1, Some(&columns))
@@ -2037,39 +2975,49 @@ impl PostingListReader {
             Some((start, end)) => {
                 let group = self
                     .index_cache
-                    .get_or_insert_with_key(PostingListGroupKey { start, end }, || async move {
-                        metrics.record_part_load();
-                        info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start);
-                        self.load_posting_list_group(start, end).await
-                    })
+                    .get_or_insert_with_key(
+                        posting_list_group_cache_key(start, end, self.has_impacts),
+                        || async move {
+                            metrics.record_part_load();
+                            info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start);
+                            self.load_posting_list_group(start, end).await
+                        },
+                    )
                     .await?;
+                let (max_score, length) = if group.needs_external_metadata() {
+                    self.posting_metadata_for_token(token_id).await?
+                } else {
+                    (None, None)
+                };
                 let slot = (token_id - start) as usize;
                 group
-                    .get(slot)
+                    .posting_list(slot, max_score, length)?
                     .ok_or_else(|| {
                         Error::index(format!(
                             "token {token_id} maps to slot {slot} outside posting group [{start}, {end})"
                         ))
                     })?
-                    .clone()
             }
-            // Fallback for indexes written before grouping: one cache entry
-            // per token.
+            // Fallback for layouts that cannot use row-based groups: one cache
+            // entry per token.
             None => self
                 .index_cache
-                .get_or_insert_with_key(PostingListKey { token_id }, || async move {
-                    metrics.record_part_load();
-                    info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id);
-                    // Fetch the posting batch and this token's (max_score,
-                    // length) in parallel; for cold v2 partitions this is one
-                    // single-row metadata read plus one posting-row read,
-                    // instead of pulling the full per-token metadata table.
-                    let (batch, (max_score, length)) = futures::try_join!(
-                        self.posting_batch(token_id, false),
-                        self.posting_metadata_for_token(token_id),
-                    )?;
-                    self.posting_list_from_batch(&batch, max_score, length)
-                })
+                .get_or_insert_with_key(
+                    posting_list_cache_key(token_id, self.has_impacts),
+                    || async move {
+                        metrics.record_part_load();
+                        info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id);
+                        // Fetch the posting batch and this token's (max_score,
+                        // length) in parallel; for cold v2 partitions this is one
+                        // single-row metadata read plus one posting-row read,
+                        // instead of pulling the full per-token metadata table.
+                        let (batch, (max_score, length)) = futures::try_join!(
+                            self.posting_batch(token_id, false),
+                            self.posting_metadata_for_token(token_id),
+                        )?;
+                        self.posting_list_from_batch(&batch, max_score, length)
+                    },
+                )
                 .await?
                 .as_ref()
                 .clone(),
@@ -2085,55 +3033,30 @@ impl PostingListReader {
     }
 
     /// Map a token id to its cache group's row range `[start, end)`, or `None`
-    /// when grouping is not available (pre-grouping indexes) so the caller
-    /// falls back to the per-token path. In v2 the token id is the row offset,
-    /// so the group range is also the physical row range.
+    /// when grouping is not available so the caller falls back to the per-token
+    /// path. In v2 the token id is the row offset, so the group range is also
+    /// the physical row range.
     fn group_range_for_token(&self, token_id: u32) -> Option<(u32, u32)> {
-        let starts = self.group_starts.as_ref()?;
-        // partition_point returns the count of group starts <= token_id, so the
-        // owning group begins at index k - 1 and the next start (if any) is its
-        // exclusive end.
-        let k = starts.partition_point(|&s| s <= token_id);
-        // k == 0 means token_id precedes the first group start, which cannot
-        // happen for a valid token in a grouped index (the first group starts
-        // at row 0); guard anyway and fall back to the per-token path.
-        if k == 0 {
-            return None;
-        }
-        let start = starts[k - 1];
-        // The last group runs to the final posting list. `self.len()` is the
-        // authoritative posting-list count (offsets length for v1, row count for
-        // v2), and prewarm derives the same `end` from it — so warm- and
-        // cold-cache group keys are identical by construction, not by the
-        // incidental v2 `num_rows == token_count` equality.
-        let end = starts.get(k).copied().unwrap_or(self.len() as u32);
-        Some((start, end))
+        self.grouping.range_for_token(token_id, self.len())
     }
 
-    /// Read rows `[start, end)` of the posting file and decode them into a
-    /// [`PostingListGroup`] cache value (issue #7040). Positions are excluded;
-    /// phrase queries load them on demand via [`Self::read_positions`].
+    /// Read rows `[start, end)` into one compact Arrow-backed cache value.
+    /// Positions are excluded; phrase queries load them on demand via
+    /// [`Self::read_positions`].
     async fn load_posting_list_group(&self, start: u32, end: u32) -> Result<PostingListGroup> {
+        let mut columns = vec![POSTING_COL, MAX_SCORE_COL, LENGTH_COL];
+        if self.has_impacts {
+            columns.push(IMPACT_COL);
+        }
         let batch = self
             .reader
-            .read_range(
-                start as usize..end as usize,
-                Some(&[POSTING_COL, MAX_SCORE_COL, LENGTH_COL]),
-            )
+            .read_range(start as usize..end as usize, Some(&columns))
             .await?;
-        let max_scores = batch[MAX_SCORE_COL].as_primitive::<Float32Type>();
-        let lengths = batch[LENGTH_COL].as_primitive::<UInt32Type>();
-        let mut posting_lists = Vec::with_capacity(batch.num_rows());
-        for i in 0..batch.num_rows() {
-            let row = batch.slice(i, 1);
-            let posting = self.posting_list_from_batch(
-                &row,
-                Some(max_scores.value(i)),
-                Some(lengths.value(i)),
-            )?;
-            posting_lists.push(posting);
-        }
-        Ok(PostingListGroup::new(posting_lists))
+        PostingListGroup::new_packed_with_block_size(
+            batch.shrink_to_fit()?,
+            self.posting_tail_codec,
+            self.block_size,
+        )
     }
 
     fn posting_list_from_batch_parts(
@@ -2141,6 +3064,7 @@ impl PostingListReader {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions_layout: PositionsLayout,
     ) -> Result<PostingList> {
         let posting_list = PostingList::from_batch_with_tail_codec_and_positions_layout(
@@ -2148,6 +3072,7 @@ impl PostingListReader {
             max_score,
             length,
             posting_tail_codec,
+            block_size,
             positions_layout,
         )?;
         Ok(posting_list)
@@ -2164,93 +3089,251 @@ impl PostingListReader {
             max_score,
             length,
             self.posting_tail_codec,
+            self.block_size,
             self.positions_layout,
         )
     }
 
-    fn build_prewarm_posting_lists(
-        batch: RecordBatch,
-        offsets: Option<Vec<usize>>,
-        max_scores: Option<Vec<f32>>,
-        lengths: Option<Vec<u32>>,
-        posting_tail_codec: PostingTailCodec,
-        positions_layout: PositionsLayout,
+    /// Build posting lists for one chunk's token range from `chunk_batch`, rebasing
+    /// global offsets to chunk-local rows. Returns `(global token_id, PostingList)`
+    /// pairs identical to the whole-file path, only bounded to one chunk.
+    fn build_prewarm_posting_lists_chunk(
+        chunk_batch: RecordBatch,
+        chunk: PrewarmChunk<'_>,
+        ctx: &PrewarmBuildCtx<'_>,
     ) -> Result<Vec<(u32, PostingList)>> {
-        let token_count = if let Some(offsets) = offsets.as_ref() {
-            offsets.len()
-        } else if let Some(lengths) = lengths.as_ref() {
-            lengths.len()
-        } else {
-            batch.num_rows()
-        };
-
-        let mut posting_lists = Vec::with_capacity(token_count);
-        for token_id in 0..token_count {
-            let batch = if let Some(offsets) = offsets.as_ref() {
-                let start = offsets[token_id];
-                let end = if token_id + 1 < offsets.len() {
-                    offsets[token_id + 1]
+        let mut posting_lists = Vec::with_capacity(chunk.token_count);
+        for local in 0..chunk.token_count {
+            let global = chunk.tok_start + local;
+            let row_batch = if let Some(chunk_offsets) = chunk.offsets {
+                // Legacy v1: rebase global offsets to chunk row 0; the last token
+                // ends at `chunk.end_row` (no trailing sentinel in chunk_offsets).
+                let base = chunk_offsets[0];
+                let start = chunk_offsets[local] - base;
+                let end = if local + 1 < chunk_offsets.len() {
+                    chunk_offsets[local + 1] - base
                 } else {
-                    batch.num_rows()
+                    chunk.end_row - base
                 };
-                batch.slice(start, end - start)
+                chunk_batch.slice(start, end - start)
             } else {
-                batch.slice(token_id, 1)
+                // V2: one posting row per token; row `local` within the chunk.
+                chunk_batch.slice(local, 1)
             };
-            let batch = batch.shrink_to_fit()?;
+            let row_batch = row_batch.shrink_to_fit()?;
             let posting_list = Self::posting_list_from_batch_parts(
-                &batch,
-                max_scores.as_ref().map(|scores| scores[token_id]),
-                lengths.as_ref().map(|lengths| lengths[token_id]),
-                posting_tail_codec,
-                positions_layout,
+                &row_batch,
+                ctx.max_scores.map(|scores| scores[global]),
+                ctx.lengths.map(|lengths| lengths[global]),
+                ctx.posting_tail_codec,
+                ctx.block_size,
+                ctx.positions_layout,
             )?;
-            posting_lists.push((token_id as u32, posting_list));
+            posting_lists.push((global as u32, posting_list));
         }
 
         Ok(posting_lists)
     }
 
-    async fn prewarm_posting_lists(&self, with_position: bool) -> Result<()> {
+    /// Read the posting rows for token ids `[tok_start, tok_end)` into one RecordBatch.
+    /// For v2 the token range is the row range; for v1 it's derived from the offsets.
+    async fn read_chunk_batch(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        with_position: bool,
+    ) -> Result<RecordBatch> {
+        let columns = self.posting_columns(with_position);
+        let row_range = match &self.metadata {
+            PostingMetadata::LegacyV1 { offsets, .. } => {
+                let start = offsets[tok_start];
+                let end = offsets
+                    .get(tok_end)
+                    .copied()
+                    .unwrap_or_else(|| self.reader.num_rows());
+                start..end
+            }
+            PostingMetadata::V2 { .. } => tok_start..tok_end,
+        };
+        let batch = self.reader.read_range(row_range, Some(&columns)).await?;
+        Ok(batch)
+    }
+
+    async fn prewarm_posting_lists(
+        &self,
+        with_position: bool,
+        chunk_concurrency: usize,
+    ) -> Result<()> {
+        self.prewarm_posting_lists_chunked(with_position, None, chunk_concurrency)
+            .await?;
+        Ok(())
+    }
+
+    /// Stream the partition's posting lists into the cache in bounded token-row chunks
+    /// (read -> build -> insert -> drop), so peak resident set is ~one chunk. Returns
+    /// the chunk count (tests assert it split). `chunk_tokens_override` is test-only.
+    async fn prewarm_posting_lists_chunked(
+        &self,
+        with_position: bool,
+        chunk_tokens_override: Option<usize>,
+        chunk_concurrency: usize,
+    ) -> Result<usize> {
         if with_position && !self.has_positions() {
             return Err(Error::invalid_input(
                 "cannot prewarm positions for an inverted index that was built without positions; recreate the index with with_position=true".to_owned(),
             ));
         }
 
-        // Make sure max_scores/lengths are populated before we clone them into
-        // the blocking task; otherwise the v2 branch would unwrap empty
-        // OnceCells.
+        // Make max_scores/lengths available for query-local packed views. The
+        // materialized fallback also clones them into its blocking build task.
         self.ensure_metadata_loaded().await?;
 
-        let read_batch_start = Instant::now();
-        let batch = self.read_batch(with_position).await?;
-        let read_batch_elapsed = read_batch_start.elapsed();
+        // With grouping the cache stores one entry per group, so a group's
+        // posting lists must all be resident at once: align chunk boundaries to
+        // whole groups. Without grouping, chunks are plain token ranges.
+        let grouping = self.grouping.clone();
+        let use_packed_groups = grouping.is_grouped() && !with_position;
+        // Packed groups reuse the reader's bulk metadata at query time, so they
+        // do not need the temporary full-partition metadata clones used by the
+        // materialized fallback.
+        let state = (!use_packed_groups).then(|| self.chunk_build_state());
+        let token_count = self.len();
+        let posting_data_size_bytes = self.posting_data_size_bytes();
+        let chunk_tokens = chunk_tokens_override
+            .unwrap_or_else(|| prewarm_chunk_tokens(token_count, posting_data_size_bytes))
+            .max(1);
+        let chunk_ranges = prewarm_chunk_ranges(&grouping, token_count, chunk_tokens);
+        let chunk_count = chunk_ranges.len();
+        let chunk_concurrency = chunk_concurrency.max(1);
+
+        let read_build_start = Instant::now();
+        stream::iter(chunk_ranges)
+            .map(|(tok_start, tok_end)| {
+                let state = state.as_ref();
+                let grouping = &grouping;
+                async move {
+                    if use_packed_groups {
+                        let groups = self
+                            .build_packed_chunk_groups(tok_start, tok_end, token_count, grouping)
+                            .await?;
+                        for (start, end, group) in groups {
+                            self.index_cache
+                                .insert_with_key(
+                                    &posting_list_group_cache_key(start, end, self.has_impacts),
+                                    Arc::new(group),
+                                )
+                                .await;
+                        }
+                    } else {
+                        let state = state.expect(
+                            "materialized prewarm must initialize posting-list build state",
+                        );
+                        let posting_lists = self
+                            .build_chunk_postings(tok_start, tok_end, with_position, state)
+                            .await?;
+                        self.publish_chunk_postings(
+                            posting_lists,
+                            grouping,
+                            tok_start,
+                            tok_end,
+                            token_count,
+                            with_position,
+                        )
+                        .await;
+                    }
+                    Result::Ok(())
+                }
+            })
+            .buffer_unordered(chunk_concurrency)
+            .try_collect::<()>()
+            .await?;
+        let read_build_elapsed = read_build_start.elapsed();
+
+        info!(
+            legacy_layout = self.is_legacy_layout(),
+            with_position,
+            token_count,
+            chunk_count,
+            chunk_tokens,
+            chunk_concurrency,
+            posting_data_size_bytes,
+            read_build_ms = read_build_elapsed.as_secs_f64() * 1000.0,
+            "posting list prewarm timing"
+        );
+
+        Ok(chunk_count)
+    }
 
-        let (legacy_layout, offsets, max_scores, lengths) = match &self.metadata {
+    /// Loop-invariant inputs shared by every chunk build: the metadata vecs
+    /// (`Arc`d so chunks share them without re-cloning) plus codec/layout.
+    fn chunk_build_state(&self) -> ChunkBuildState {
+        let (offsets, max_scores, lengths) = match &self.metadata {
             PostingMetadata::LegacyV1 {
                 offsets,
                 max_scores,
-            } => (true, Some(offsets.clone()), max_scores.clone(), None),
+            } => (Some(offsets.clone()), max_scores.clone(), None),
             PostingMetadata::V2 { metadata } => (
-                false,
                 None,
                 metadata.get().map(|loaded| loaded.max_scores.clone()),
                 metadata.get().map(|loaded| loaded.lengths.clone()),
             ),
         };
-        let posting_tail_codec = self.posting_tail_codec;
-        let positions_layout = self.positions_layout;
-        let populate_start = Instant::now();
+        ChunkBuildState {
+            offsets: offsets.map(Arc::new),
+            max_scores: max_scores.map(Arc::new),
+            lengths: lengths.map(Arc::new),
+            posting_tail_codec: self.posting_tail_codec,
+            block_size: self.block_size,
+            positions_layout: self.positions_layout,
+        }
+    }
+
+    /// Read one token-row chunk and build its posting lists off the runtime thread.
+    /// The large batch is dropped inside the blocking task once built, bounding
+    /// resident memory to one chunk.
+    async fn build_chunk_postings(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        with_position: bool,
+        state: &ChunkBuildState,
+    ) -> Result<Vec<(u32, PostingList)>> {
+        let chunk_token_count = tok_end - tok_start;
+        let chunk_batch = self
+            .read_chunk_batch(tok_start, tok_end, with_position)
+            .await?;
+
+        let (chunk_offsets, chunk_end_row) = match state.offsets.as_ref() {
+            Some(offsets) => {
+                let end_row = offsets
+                    .get(tok_end)
+                    .copied()
+                    .unwrap_or_else(|| self.reader.num_rows());
+                (Some(offsets[tok_start..tok_end].to_vec()), end_row)
+            }
+            // V2 doesn't use chunk_end_row (one row per token); pass tok_end.
+            None => (None, tok_end),
+        };
+        let max_scores = state.max_scores.clone();
+        let lengths = state.lengths.clone();
+        let posting_tail_codec = state.posting_tail_codec;
+        let block_size = state.block_size;
+        let positions_layout = state.positions_layout;
         let posting_lists = spawn_blocking(move || {
-            Self::build_prewarm_posting_lists(
-                batch,
-                offsets,
-                max_scores,
-                lengths,
+            let ctx = PrewarmBuildCtx {
+                max_scores: max_scores.as_deref().map(|v| v.as_slice()),
+                lengths: lengths.as_deref().map(|v| v.as_slice()),
                 posting_tail_codec,
+                block_size,
                 positions_layout,
-            )
+            };
+            let chunk = PrewarmChunk {
+                tok_start,
+                token_count: chunk_token_count,
+                offsets: chunk_offsets.as_deref(),
+                end_row: chunk_end_row,
+            };
+            Self::build_prewarm_posting_lists_chunk(chunk_batch, chunk, &ctx)
         })
         .await
         .map_err(|err| {
@@ -2258,60 +3341,143 @@ impl PostingListReader {
                 "Failed to build prewarm posting lists in blocking task: {err}"
             ))
         })??;
-        // Strip positions into their own per-token cache entries first
-        // (unchanged); the posting cache holds positions-free lists.
-        let mut postings_by_token = Vec::with_capacity(posting_lists.len());
-        for (token_id, mut posting_list) in posting_lists {
-            if with_position && let Some(positions) = posting_list.take_positions() {
-                self.index_cache
-                    .insert_with_key(&PositionKey { token_id }, Arc::new(Positions(positions)))
-                    .await;
+        // The chunk yields its token range as contiguous ascending ids from
+        // `tok_start`; the group publish path relies on this to index the lists.
+        debug_assert_eq!(posting_lists.len(), chunk_token_count);
+        debug_assert!(
+            posting_lists
+                .iter()
+                .enumerate()
+                .all(|(i, (token_id, _))| *token_id as usize == tok_start + i)
+        );
+        Ok(posting_lists)
+    }
+
+    /// Build compact v2 groups directly from one posting-row chunk. Each group
+    /// slice is deep-copied once, so it owns only its Arrow buffers without
+    /// materializing a `Vec<PostingList>` or retaining the full chunk.
+    async fn build_packed_chunk_groups(
+        &self,
+        tok_start: usize,
+        tok_end: usize,
+        token_count: usize,
+        grouping: &PostingGrouping,
+    ) -> Result<Vec<(u32, u32, PostingListGroup)>> {
+        debug_assert!(grouping.is_grouped());
+        debug_assert!(!self.is_legacy_layout());
+
+        let chunk_batch = self.read_chunk_batch(tok_start, tok_end, false).await?;
+        let ranges = grouping.ranges_for_chunk(tok_start, tok_end, token_count);
+        let posting_tail_codec = self.posting_tail_codec;
+        let block_size = self.block_size;
+
+        spawn_blocking(move || {
+            let mut groups = Vec::with_capacity(ranges.len());
+            for (start, end) in ranges {
+                let start_usize = start as usize;
+                let end_usize = end as usize;
+                let local_start = start_usize - tok_start;
+                let group_len = end_usize - start_usize;
+                let group_batch = chunk_batch.slice(local_start, group_len).shrink_to_fit()?;
+                groups.push((
+                    start,
+                    end,
+                    PostingListGroup::new_packed_with_block_size(
+                        group_batch,
+                        posting_tail_codec,
+                        block_size,
+                    )?,
+                ));
             }
-            debug_assert_eq!(token_id as usize, postings_by_token.len());
-            postings_by_token.push(posting_list);
-        }
-        // Populate the same cache keys the read path uses: grouped entries when
-        // grouping is active (issue #7040), per-token entries otherwise.
-        match self.group_starts.as_ref() {
-            Some(starts) => {
-                // The read path derives the last group's `end` from `self.len()`;
-                // match it here so both produce identical `PostingListGroupKey`s.
-                debug_assert_eq!(postings_by_token.len(), self.len());
-                for (k, &start) in starts.iter().enumerate() {
-                    let end = starts.get(k + 1).copied().unwrap_or(self.len() as u32);
-                    let group = PostingListGroup::new(
-                        postings_by_token[start as usize..end as usize].to_vec(),
-                    );
+            Result::Ok(groups)
+        })
+        .await
+        .map_err(|err| {
+            Error::internal(format!(
+                "Failed to build packed prewarm posting groups in blocking task: {err}"
+            ))
+        })?
+    }
+
+    /// Strip positions into their own per-token cache entries (the posting cache
+    /// holds positions-free lists), then populate the same cache keys the read
+    /// path uses: grouped entries when grouping is active, per-token entries
+    /// otherwise. Called once per chunk; the chunk's lists drop on return.
+    async fn publish_chunk_postings(
+        &self,
+        posting_lists: Vec<(u32, PostingList)>,
+        grouping: &PostingGrouping,
+        tok_start: usize,
+        tok_end: usize,
+        token_count: usize,
+        with_position: bool,
+    ) {
+        match grouping {
+            PostingGrouping::None => {
+                for (token_id, mut posting_list) in posting_lists {
+                    self.cache_positions(&mut posting_list, token_id, with_position)
+                        .await;
                     self.index_cache
-                        .insert_with_key(&PostingListGroupKey { start, end }, Arc::new(group))
+                        .insert_with_key(
+                            &posting_list_cache_key(token_id, self.has_impacts),
+                            Arc::new(posting_list),
+                        )
                         .await;
                 }
             }
-            None => {
-                for (token_id, posting_list) in postings_by_token.into_iter().enumerate() {
+            PostingGrouping::SyntheticFixed { .. } => {
+                let mut chunk_postings = Vec::with_capacity(posting_lists.len());
+                for (token_id, mut posting_list) in posting_lists {
+                    self.cache_positions(&mut posting_list, token_id, with_position)
+                        .await;
+                    chunk_postings.push(posting_list);
+                }
+                // Chunk is group-aligned, so every group starting in it also ends
+                // in it; `chunk_postings[i]` is token `tok_start + i`. The last
+                // group's `end` derives from `token_count`, matching the read path
+                // so both produce identical `PostingListGroupKey`s.
+                for (start, end) in grouping.ranges_for_chunk(tok_start, tok_end, token_count) {
+                    let start_usize = start as usize;
+                    let lo = start_usize - tok_start;
+                    let hi = end as usize - tok_start;
+                    let group = PostingListGroup::new(chunk_postings[lo..hi].to_vec());
                     self.index_cache
                         .insert_with_key(
-                            &PostingListKey {
-                                token_id: token_id as u32,
-                            },
-                            Arc::new(posting_list),
+                            &posting_list_group_cache_key(start, end, self.has_impacts),
+                            Arc::new(group),
                         )
                         .await;
                 }
             }
         }
-        let populate_elapsed = populate_start.elapsed();
+    }
 
-        info!(
-            legacy_layout,
-            with_position,
-            token_count = self.len(),
-            read_batch_ms = read_batch_elapsed.as_secs_f64() * 1000.0,
-            post_read_loop_ms = populate_elapsed.as_secs_f64() * 1000.0,
-            "posting list prewarm timing"
-        );
+    /// Move a posting list's positions (when present and requested) into the
+    /// dedicated per-token position cache, leaving the posting list positions-free.
+    async fn cache_positions(
+        &self,
+        posting_list: &mut PostingList,
+        token_id: u32,
+        with_position: bool,
+    ) {
+        if with_position && let Some(positions) = posting_list.take_positions() {
+            self.index_cache
+                .insert_with_key(&PositionKey { token_id }, Arc::new(Positions(positions)))
+                .await;
+        }
+    }
 
-        Ok(())
+    /// Cheap `invert.lance` size estimate (file length from object metadata, no
+    /// data read), used only to size prewarm chunks. Falls back to a row-count
+    /// proxy when the reader can't surface the length (legacy v1).
+    pub(crate) fn posting_data_size_bytes(&self) -> u64 {
+        if let Some(size) = self.reader.file_size_bytes() {
+            return size;
+        }
+        // Fallback proxy for readers that don't cache their file length: just needs
+        // to be monotonic in partition size.
+        const ESTIMATED_BYTES_PER_ROW: u64 = 16;
+        (self.reader.num_rows() as u64).saturating_mul(ESTIMATED_BYTES_PER_ROW)
     }
 
     pub(crate) async fn read_batch(&self, with_position: bool) -> Result<RecordBatch> {
@@ -2449,15 +3615,52 @@ impl PostingListReader {
                 }
             }
         }
+        if self.has_impacts {
+            base_columns.push(IMPACT_COL);
+        }
         base_columns
     }
 }
 
-/// New type just to allow Positions implement DeepSizeOf so it can be put
-/// in the cache.
-#[derive(Clone)]
-pub struct Positions(pub(super) CompressedPositionStorage);
-
+/// Loop-invariant state for [`InvertedPartition::build_chunk_postings`]. The
+/// metadata vecs are `Arc`d so each chunk's blocking build shares them cheaply.
+struct ChunkBuildState {
+    offsets: Option<Arc<Vec<usize>>>,
+    max_scores: Option<Arc<Vec<f32>>>,
+    lengths: Option<Arc<Vec<u32>>>,
+    posting_tail_codec: PostingTailCodec,
+    block_size: usize,
+    positions_layout: PositionsLayout,
+}
+
+/// Chunk-invariant inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]:
+/// the per-partition codec/layout and the (shared, whole-partition) metadata
+/// slices indexed by global token id. These don't change across chunks.
+struct PrewarmBuildCtx<'a> {
+    max_scores: Option<&'a [f32]>,
+    lengths: Option<&'a [u32]>,
+    posting_tail_codec: PostingTailCodec,
+    block_size: usize,
+    positions_layout: PositionsLayout,
+}
+
+/// Per-chunk inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]:
+/// the token sub-range `[tok_start, tok_start + token_count)` and, for legacy
+/// v1, the rebased offset slice plus the chunk's end row.
+struct PrewarmChunk<'a> {
+    tok_start: usize,
+    token_count: usize,
+    /// Legacy v1 only: `offsets[tok_start..tok_start+token_count]` (no sentinel).
+    offsets: Option<&'a [usize]>,
+    /// Legacy v1 only: global row at which this chunk's posting rows end.
+    end_row: usize,
+}
+
+/// New type just to allow Positions implement DeepSizeOf so it can be put
+/// in the cache.
+#[derive(Clone)]
+pub struct Positions(pub(super) CompressedPositionStorage);
+
 /// Slice-aware cache-size charge for the Arrow array shapes stored in posting
 /// caches. [`Array::get_buffer_memory_size`] reports the full capacity of shared
 /// backing buffers; cached posting lists often reference only a small slice of a
@@ -2502,7 +3705,7 @@ fn sliced_cache_bytes(array: &dyn Array) -> usize {
 }
 
 impl DeepSizeOf for Positions {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.0.deep_size_of_children(context)
     }
 }
@@ -2531,8 +3734,8 @@ impl CacheKey for PostingListKey {
 
 /// Cache key for a group of consecutive posting lists stored as a single
 /// entry, covering rows `[start, end)` (issue #7040). The range, not a token
-/// id, is the key so that a write-time config change that reshapes groups
-/// simply misses old entries instead of serving a differently-shaped group.
+/// id, is the key so a runtime group-size change simply misses old entries
+/// instead of serving a differently-shaped group.
 #[derive(Debug, Clone)]
 pub struct PostingListGroupKey {
     pub start: u32,
@@ -2555,6 +3758,75 @@ impl CacheKey for PostingListGroupKey {
     }
 }
 
+/// Internal cache-key decorator that isolates impact-bearing posting values
+/// without changing the source-compatible public posting key structs.
+#[derive(Debug, Clone)]
+struct ImpactAwareCacheKey<K> {
+    inner: K,
+    has_impacts: bool,
+}
+
+impl<K: CacheKey> CacheKey for ImpactAwareCacheKey<K> {
+    type ValueType = K::ValueType;
+
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        if self.has_impacts {
+            format!("{}-impacts", self.inner.key()).into()
+        } else {
+            self.inner.key()
+        }
+    }
+
+    fn type_name() -> &'static str {
+        K::type_name()
+    }
+
+    fn codec() -> Option<CacheCodec> {
+        K::codec()
+    }
+}
+
+fn posting_list_cache_key(token_id: u32, has_impacts: bool) -> ImpactAwareCacheKey<PostingListKey> {
+    ImpactAwareCacheKey {
+        inner: PostingListKey { token_id },
+        has_impacts,
+    }
+}
+
+fn posting_list_group_cache_key(
+    start: u32,
+    end: u32,
+    has_impacts: bool,
+) -> ImpactAwareCacheKey<PostingListGroupKey> {
+    ImpactAwareCacheKey {
+        inner: PostingListGroupKey { start, end },
+        has_impacts,
+    }
+}
+
+#[derive(Debug, Clone, DeepSizeOf)]
+struct PostingMetadataValue {
+    max_score: f32,
+    length: u32,
+}
+
+#[derive(Debug, Clone)]
+struct PostingMetadataKey {
+    token_id: u32,
+}
+
+impl CacheKey for PostingMetadataKey {
+    type ValueType = PostingMetadataValue;
+
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        format!("posting-metadata-{}", self.token_id).into()
+    }
+
+    fn type_name() -> &'static str {
+        "PostingMetadata"
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct PositionKey {
     pub token_id: u32,
@@ -2583,7 +3855,7 @@ pub enum CompressedPositionStorage {
 }
 
 impl DeepSizeOf for CompressedPositionStorage {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         match self {
             Self::LegacyPerDoc(positions) => sliced_cache_bytes(positions),
             Self::SharedStream(stream) => stream.size(),
@@ -2594,10 +3866,9 @@ impl DeepSizeOf for CompressedPositionStorage {
 #[derive(Debug, Clone, PartialEq, Eq, Default)]
 pub struct SharedPositionStream {
     codec: PositionStreamCodec,
-    block_offsets: Vec<u32>,
-    // Stored as `Bytes` so that the cache deserialization path can hand
-    // ownership of an IPC-decoded slice in without copying. Cloning the
-    // stream is then an `Arc` bump rather than an O(N) buffer copy.
+    block_offsets: Arc<[u32]>,
+    // Stored with shared ownership so cache hits can clone position streams
+    // without copying either offsets or bytes.
     bytes: bytes::Bytes,
 }
 
@@ -2605,7 +3876,7 @@ impl SharedPositionStream {
     pub fn new(codec: PositionStreamCodec, block_offsets: Vec<u32>, bytes: bytes::Bytes) -> Self {
         Self {
             codec,
-            block_offsets,
+            block_offsets: Arc::from(block_offsets.into_boxed_slice()),
             bytes,
         }
     }
@@ -2638,35 +3909,360 @@ impl SharedPositionStream {
     }
 
     pub fn block_offsets(&self) -> &[u32] {
-        &self.block_offsets
+        self.block_offsets.as_ref()
     }
 
     pub fn size(&self) -> usize {
-        self.block_offsets.capacity() * std::mem::size_of::<u32>() + self.bytes.len()
+        self.block_offsets.len() * std::mem::size_of::<u32>() + self.bytes.len()
     }
 }
 
 /// A group of consecutive posting lists held in a single cache entry, in row
-/// order (issue #7040). `posting_lists[i]` corresponds to row `start + i`,
-/// where `start` is the group's first row from [`PostingListGroupKey`].
-#[derive(Debug, Clone, DeepSizeOf)]
+/// order (issue #7040). Prewarmed modern groups without positions retain only
+/// the compact Arrow posting rows read from `invert.lance`; max-score/length
+/// metadata stays in the reader and is injected when a query creates a
+/// posting-list view. Cold-loaded groups may keep inline metadata to preserve
+/// one-read query loading. Legacy and position-bearing prewarm paths use the
+/// materialized fallback.
+#[derive(Debug, Clone)]
 pub struct PostingListGroup {
-    pub(super) posting_lists: Vec<PostingList>,
+    pub(super) storage: PostingListGroupStorage,
+}
+
+#[derive(Debug, Clone)]
+pub(super) enum PostingListGroupStorage {
+    Packed(PackedPostingListGroup),
+    Materialized(Vec<PostingList>),
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct PackedPostingListGroup {
+    pub(super) batch: RecordBatch,
+    pub(super) posting_tail_codec: PostingTailCodec,
+    pub(super) block_size: usize,
+    first_docs_states: Arc<[OnceLock<Box<[u32]>>]>,
+    first_docs_state_capacity_bytes: usize,
+    impact_states: Option<Arc<[OnceLock<Box<ImpactSkipData>>]>>,
+    impact_state_capacity_bytes: usize,
+}
+
+impl DeepSizeOf for PostingListGroup {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => group
+                .batch
+                .columns()
+                .iter()
+                .map(|column| sliced_cache_bytes(column.as_ref()))
+                .sum::<usize>()
+                .saturating_add(group.first_docs_state_capacity_bytes)
+                .saturating_add(group.impact_state_capacity_bytes),
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                posting_lists.deep_size_of_children(context)
+            }
+        }
+    }
 }
 
 impl PostingListGroup {
     pub(super) fn new(posting_lists: Vec<PostingList>) -> Self {
-        Self { posting_lists }
+        Self {
+            storage: PostingListGroupStorage::Materialized(posting_lists),
+        }
+    }
+
+    pub(super) fn new_packed(
+        batch: RecordBatch,
+        posting_tail_codec: PostingTailCodec,
+    ) -> Result<Self> {
+        let block_size = parse_posting_block_size(batch.schema_ref().metadata())?;
+        Self::new_packed_with_block_size(batch, posting_tail_codec, block_size)
+    }
+
+    fn new_packed_with_block_size(
+        batch: RecordBatch,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Result<Self> {
+        validate_block_size(block_size)?;
+        if let Some(encoded_block_size) = batch.schema_ref().metadata().get(POSTING_BLOCK_SIZE_KEY)
+        {
+            let encoded_block_size = encoded_block_size.parse::<usize>().map_err(|err| {
+                Error::index(format!(
+                    "invalid {POSTING_BLOCK_SIZE_KEY} metadata value {encoded_block_size:?}: {err}"
+                ))
+            })?;
+            if encoded_block_size != block_size {
+                return Err(Error::index(format!(
+                    "packed posting group {POSTING_BLOCK_SIZE_KEY}={encoded_block_size} does not match block_size={block_size}"
+                )));
+            }
+        }
+
+        // Projected reads may drop schema metadata. Restore the reader's
+        // validated block size before the batch enters the packed cache so IPC
+        // roundtrips remain self-describing. Older packed cache entries omit
+        // the key and enter through new_packed with the legacy 128-doc default.
+        let mut schema = batch.schema().as_ref().clone();
+        schema
+            .metadata
+            .insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string());
+        let batch = batch.with_schema(Arc::new(schema))?;
+        let postings = batch
+            .column_by_name(POSTING_COL)
+            .and_then(|column| column.as_list_opt::<i32>())
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "packed posting group column {POSTING_COL} must be List<LargeBinary>"
+                ))
+            })?;
+        if postings.values().data_type() != &DataType::LargeBinary {
+            return Err(Error::index(format!(
+                "packed posting group column {POSTING_COL} must contain LargeBinary values, got {}",
+                postings.values().data_type()
+            )));
+        }
+        if postings.null_count() != 0 {
+            return Err(Error::index(
+                "packed posting group column must not contain nulls".to_string(),
+            ));
+        }
+        let total_posting_blocks = (0..batch.num_rows())
+            .map(|slot| postings.value_length(slot) as usize)
+            .sum::<usize>();
+        let first_docs_states: Arc<[OnceLock<Box<[u32]>>]> = (0..batch.num_rows())
+            .map(|_| OnceLock::new())
+            .collect::<Vec<_>>()
+            .into();
+        // Reserve the compact per-slot state slab and the block-head arrays it
+        // can lazily retain, so warming these derived values cannot grow the
+        // cache beyond its admission charge.
+        let first_docs_state_capacity_bytes = first_docs_states
+            .len()
+            .saturating_mul(std::mem::size_of::<OnceLock<Box<[u32]>>>())
+            .saturating_add(total_posting_blocks.saturating_mul(std::mem::size_of::<u32>()));
+        let (impact_states, impact_state_capacity_bytes) = if let Some(impacts) =
+            batch.column_by_name(IMPACT_COL)
+        {
+            let impacts = impacts.as_list_opt::<i32>().ok_or_else(|| {
+                Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must be List<LargeBinary>"
+                ))
+            })?;
+            if impacts.values().data_type() != &DataType::LargeBinary {
+                return Err(Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must contain LargeBinary values, got {}",
+                    impacts.values().data_type()
+                )));
+            }
+            if impacts.null_count() != 0 {
+                return Err(Error::index(format!(
+                    "packed posting group column {IMPACT_COL} must not contain nulls"
+                )));
+            }
+            let mut derived_cache_bytes = 0usize;
+            for slot in 0..batch.num_rows() {
+                let posting_blocks = postings.value_length(slot) as usize;
+                let impact_entries = impacts.value_length(slot) as usize;
+                let expected_impact_entries =
+                    posting_blocks.saturating_add(posting_blocks.div_ceil(IMPACT_LEVEL1_BLOCKS));
+                if impact_entries != expected_impact_entries {
+                    return Err(Error::index(format!(
+                        "packed posting group impact slot {slot} has {impact_entries} entries, expected {expected_impact_entries} for {posting_blocks} posting blocks"
+                    )));
+                }
+                derived_cache_bytes = derived_cache_bytes.saturating_add(
+                    ImpactSkipData::derived_cache_bytes_for_entries(impact_entries),
+                );
+            }
+
+            let states: Arc<[OnceLock<Box<ImpactSkipData>>]> = (0..batch.num_rows())
+                .map(|_| OnceLock::new())
+                .collect::<Vec<_>>()
+                .into();
+            // Account up front for every allocation that the lazy states can
+            // eventually retain. The impact entry bytes themselves remain in
+            // `batch` and are already charged exactly once above.
+            let per_slot_bytes = std::mem::size_of::<OnceLock<Box<ImpactSkipData>>>()
+                .saturating_add(std::mem::size_of::<ImpactSkipData>());
+            let capacity_bytes = states
+                .len()
+                .saturating_mul(per_slot_bytes)
+                .saturating_add(derived_cache_bytes);
+            (Some(states), capacity_bytes)
+        } else {
+            (None, 0)
+        };
+        match (
+            batch.column_by_name(MAX_SCORE_COL),
+            batch.column_by_name(LENGTH_COL),
+        ) {
+            (None, None) => {}
+            (Some(max_scores), Some(lengths)) => {
+                let max_scores = max_scores
+                    .as_primitive_opt::<Float32Type>()
+                    .ok_or_else(|| {
+                        Error::index(format!(
+                            "packed posting group column {MAX_SCORE_COL} must be Float32"
+                        ))
+                    })?;
+                let lengths = lengths.as_primitive_opt::<UInt32Type>().ok_or_else(|| {
+                    Error::index(format!(
+                        "packed posting group column {LENGTH_COL} must be UInt32"
+                    ))
+                })?;
+                if max_scores.null_count() != 0 || lengths.null_count() != 0 {
+                    return Err(Error::index(
+                        "packed posting group metadata columns must not contain nulls".to_string(),
+                    ));
+                }
+            }
+            _ => {
+                return Err(Error::index(format!(
+                    "packed posting group must contain both {MAX_SCORE_COL} and {LENGTH_COL}, or neither"
+                )));
+            }
+        }
+
+        Ok(Self {
+            storage: PostingListGroupStorage::Packed(PackedPostingListGroup {
+                batch,
+                posting_tail_codec,
+                block_size,
+                first_docs_states,
+                first_docs_state_capacity_bytes,
+                impact_states,
+                impact_state_capacity_bytes,
+            }),
+        })
+    }
+
+    pub(super) fn len(&self) -> usize {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => group.batch.num_rows(),
+            PostingListGroupStorage::Materialized(posting_lists) => posting_lists.len(),
+        }
+    }
+
+    #[cfg(test)]
+    pub(super) fn is_packed(&self) -> bool {
+        matches!(&self.storage, PostingListGroupStorage::Packed(_))
+    }
+
+    fn needs_external_metadata(&self) -> bool {
+        match &self.storage {
+            PostingListGroupStorage::Packed(group) => {
+                group.batch.column_by_name(MAX_SCORE_COL).is_none()
+            }
+            PostingListGroupStorage::Materialized(_) => false,
+        }
     }
 
-    /// Borrow the posting list at offset `slot` within the group (i.e.
-    /// `token_id - start`).
-    pub(super) fn get(&self, slot: usize) -> Option<&PostingList> {
-        self.posting_lists.get(slot)
+    /// Build an owned posting-list view for `slot`. Packed groups clone only
+    /// Arrow array metadata; the compressed posting bytes remain shared with
+    /// the group's `List<LargeBinary>` child buffers.
+    pub(super) fn posting_list(
+        &self,
+        slot: usize,
+        max_score: Option<f32>,
+        length: Option<u32>,
+    ) -> Result<Option<PostingList>> {
+        match &self.storage {
+            PostingListGroupStorage::Materialized(posting_lists) => {
+                Ok(posting_lists.get(slot).cloned())
+            }
+            PostingListGroupStorage::Packed(group) => {
+                if slot >= group.batch.num_rows() {
+                    return Ok(None);
+                }
+                let postings = group
+                    .batch
+                    .column_by_name(POSTING_COL)
+                    .and_then(|column| column.as_list_opt::<i32>())
+                    .ok_or_else(|| {
+                        Error::index(format!(
+                            "packed posting group column {POSTING_COL} must be List<LargeBinary>"
+                        ))
+                    })?;
+                let blocks = postings.value(slot);
+                let blocks = blocks.as_binary_opt::<i64>().ok_or_else(|| {
+                    Error::index(format!(
+                        "packed posting group slot {slot} is not LargeBinary"
+                    ))
+                })?;
+                let max_score = match group.batch.column_by_name(MAX_SCORE_COL) {
+                    Some(column) => column
+                        .as_primitive_opt::<Float32Type>()
+                        .expect("packed group metadata was validated at construction")
+                        .value(slot),
+                    None => max_score.ok_or_else(|| {
+                        Error::index("packed posting group requires max-score metadata".to_string())
+                    })?,
+                };
+                let length = match group.batch.column_by_name(LENGTH_COL) {
+                    Some(column) => column
+                        .as_primitive_opt::<UInt32Type>()
+                        .expect("packed group metadata was validated at construction")
+                        .value(slot),
+                    None => length.ok_or_else(|| {
+                        Error::index("packed posting group requires length metadata".to_string())
+                    })?,
+                };
+                let impacts = match (
+                    group.impact_states.as_ref(),
+                    group.batch.column_by_name(IMPACT_COL),
+                ) {
+                    (Some(states), Some(column)) => {
+                        let state = states.get(slot).ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group impact state missing slot {slot}"
+                            ))
+                        })?;
+                        let impact_lists = column.as_list_opt::<i32>().ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group column {IMPACT_COL} must be List<LargeBinary>"
+                            ))
+                        })?;
+                        let entries = impact_lists.value(slot);
+                        let entries = entries.as_binary_opt::<i64>().ok_or_else(|| {
+                            Error::index(format!(
+                                "packed posting group impact slot {slot} is not LargeBinary"
+                            ))
+                        })?;
+                        let impacts =
+                            state.get_or_init(|| {
+                                Box::new(ImpactSkipData::new(entries.clone(), blocks.len()).expect(
+                                    "packed impact entry count was validated at construction",
+                                ))
+                            });
+                        Some(impacts.as_ref().clone())
+                    }
+                    (None, None) => None,
+                    _ => {
+                        return Err(Error::internal(
+                            "packed posting group impact column/state mismatch".to_string(),
+                        ));
+                    }
+                };
+                Ok(Some(PostingList::Compressed(
+                    CompressedPostingList::new(
+                        blocks.clone(),
+                        max_score,
+                        length,
+                        group.posting_tail_codec,
+                        group.block_size,
+                        None,
+                        impacts,
+                    )
+                    .with_packed_first_docs(group.first_docs_states.clone(), slot),
+                )))
+            }
+        }
     }
 }
 
 #[derive(Debug, Clone, DeepSizeOf)]
+#[allow(clippy::large_enum_variant)]
 pub enum PostingList {
     Plain(PlainPostingList),
     Compressed(CompressedPostingList),
@@ -2679,7 +4275,8 @@ impl PostingList {
         length: Option<u32>,
     ) -> Result<Self> {
         let posting_tail_codec = parse_posting_tail_codec(batch.schema_ref().metadata())?;
-        Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec)
+        let block_size = parse_posting_block_size(batch.schema_ref().metadata())?;
+        Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec, block_size)
     }
 
     pub fn from_batch_with_tail_codec(
@@ -2687,6 +4284,7 @@ impl PostingList {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
     ) -> Result<Self> {
         let positions_layout = if batch.column_by_name(COMPRESSED_POSITION_COL).is_some() {
             PositionsLayout::SharedStream(parse_shared_position_codec(
@@ -2702,6 +4300,7 @@ impl PostingList {
             max_score,
             length,
             posting_tail_codec,
+            block_size,
             positions_layout,
         )
     }
@@ -2711,6 +4310,7 @@ impl PostingList {
         max_score: Option<f32>,
         length: Option<u32>,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions_layout: PositionsLayout,
     ) -> Result<Self> {
         match batch.column_by_name(POSTING_COL) {
@@ -2725,8 +4325,9 @@ impl PostingList {
                     max_score.unwrap(),
                     length.unwrap(),
                     posting_tail_codec,
+                    block_size,
                     shared_position_codec,
-                );
+                )?;
                 Ok(Self::Compressed(posting))
             }
             None => {
@@ -2747,6 +4348,13 @@ impl PostingList {
         }
     }
 
+    pub fn has_impacts(&self) -> bool {
+        match self {
+            Self::Plain(_) => false,
+            Self::Compressed(posting) => posting.impacts.is_some(),
+        }
+    }
+
     pub fn set_positions(&mut self, positions: CompressedPositionStorage) {
         match self {
             Self::Plain(posting) => match positions {
@@ -2796,9 +4404,14 @@ impl PostingList {
             Self::Plain(_) => PostingTailCodec::Fixed32,
             Self::Compressed(posting) => posting.posting_tail_codec,
         };
-        let mut builder = PostingListBuilder::new_with_posting_tail_codec(
+        let block_size = match &self {
+            Self::Plain(_) => LEGACY_BLOCK_SIZE,
+            Self::Compressed(posting) => posting.block_size,
+        };
+        let mut builder = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
             self.has_position(),
             posting_tail_codec,
+            block_size,
         );
         match self {
             // legacy format
@@ -2860,7 +4473,7 @@ pub struct PlainPostingList {
 }
 
 impl DeepSizeOf for PlainPostingList {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         self.row_ids.len() * std::mem::size_of::<u64>()
             + self.frequencies.len() * std::mem::size_of::<f32>()
             + self
@@ -2951,52 +4564,149 @@ impl PlainPostingList {
     }
 }
 
-#[derive(Debug, PartialEq, Clone)]
+#[derive(Debug, Clone)]
+enum FirstDocsState {
+    Standalone(Arc<OnceLock<Box<[u32]>>>),
+    Packed {
+        states: Arc<[OnceLock<Box<[u32]>>]>,
+        slot: usize,
+    },
+}
+
+impl FirstDocsState {
+    fn standalone() -> Self {
+        Self::Standalone(Arc::new(OnceLock::new()))
+    }
+
+    fn state(&self) -> &OnceLock<Box<[u32]>> {
+        match self {
+            Self::Standalone(state) => state,
+            Self::Packed { states, slot } => &states[*slot],
+        }
+    }
+
+    fn get_or_init(&self, initialize: impl FnOnce() -> Box<[u32]>) -> &[u32] {
+        self.state().get_or_init(initialize)
+    }
+
+    fn capacity_bytes(
+        &self,
+        block_count: usize,
+        context: &mut lance_core::deepsize::Context,
+    ) -> usize {
+        if context.mark_seen(self.state() as *const _ as usize) {
+            std::mem::size_of::<OnceLock<Box<[u32]>>>()
+                .saturating_add(block_count.saturating_mul(std::mem::size_of::<u32>()))
+        } else {
+            0
+        }
+    }
+
+    #[cfg(test)]
+    fn shares_state_with(&self, other: &Self) -> bool {
+        std::ptr::eq(self.state(), other.state())
+    }
+}
+
+#[derive(Debug, Clone)]
 pub struct CompressedPostingList {
     pub max_score: f32,
     pub length: u32,
     // each binary is a block of compressed data
-    // that contains `BLOCK_SIZE` doc ids and then `BLOCK_SIZE` frequencies
+    // that contains `block_size` doc ids and then `block_size` frequencies,
+    // packed by the physical bitpacker matching that block size.
     pub blocks: LargeBinaryArray,
     pub posting_tail_codec: PostingTailCodec,
+    pub block_size: usize,
     pub positions: Option<CompressedPositionStorage>,
+    pub(crate) impacts: Option<ImpactSkipData>,
+    // First doc id per block, baked lazily and shared across per-query clones
+    // of the cached list. See `block_first_docs`.
+    first_docs: FirstDocsState,
+}
+
+impl PartialEq for CompressedPostingList {
+    fn eq(&self, other: &Self) -> bool {
+        self.max_score == other.max_score
+            && self.length == other.length
+            && self.blocks == other.blocks
+            && self.posting_tail_codec == other.posting_tail_codec
+            && self.block_size == other.block_size
+            && self.positions == other.positions
+            && self.impacts == other.impacts
+    }
 }
 
 impl DeepSizeOf for CompressedPostingList {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         sliced_cache_bytes(&self.blocks)
             + self
                 .positions
                 .as_ref()
                 .map(|positions| positions.deep_size_of_children(context))
                 .unwrap_or(0)
+            + self
+                .impacts
+                .as_ref()
+                .map(|impacts| {
+                    sliced_cache_bytes(impacts.entries())
+                        .saturating_add(impacts.derived_cache_bytes())
+                })
+                .unwrap_or(0)
+            + self.first_docs.capacity_bytes(self.blocks.len(), context)
     }
 }
 
 impl CompressedPostingList {
-    pub fn new(
+    pub(crate) fn new(
         blocks: LargeBinaryArray,
         max_score: f32,
         length: u32,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         positions: Option<CompressedPositionStorage>,
+        impacts: Option<ImpactSkipData>,
     ) -> Self {
+        debug_assert!(block_size.is_power_of_two());
         Self {
             max_score,
             length,
             blocks,
             posting_tail_codec,
+            block_size,
             positions,
+            impacts,
+            first_docs: FirstDocsState::standalone(),
         }
     }
 
+    fn with_packed_first_docs(mut self, states: Arc<[OnceLock<Box<[u32]>>]>, slot: usize) -> Self {
+        debug_assert!(slot < states.len());
+        self.first_docs = FirstDocsState::Packed { states, slot };
+        self
+    }
+
+    /// Block sizes are validated powers of two, so per-doc hot loops derive
+    /// block indices with shift/mask instead of runtime division, which is
+    /// measurably slower in the iterator advance path.
+    #[inline]
+    pub(crate) fn block_shift(&self) -> u32 {
+        self.block_size.trailing_zeros()
+    }
+
+    #[inline]
+    pub(crate) fn block_mask(&self) -> usize {
+        self.block_size - 1
+    }
+
     pub fn from_batch(
         batch: &RecordBatch,
         max_score: f32,
         length: u32,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         shared_position_codec: Option<PositionStreamCodec>,
-    ) -> Self {
+    ) -> Result<Self> {
         debug_assert_eq!(batch.num_rows(), 1);
         let blocks = batch[POSTING_COL]
             .as_list::<i32>()
@@ -3025,14 +4735,24 @@ impl CompressedPostingList {
                 )
             })
         };
+        let impacts = batch
+            .column_by_name(IMPACT_COL)
+            .map(|col| {
+                let entries = col.as_list::<i32>().value(0).as_binary::<i64>().clone();
+                ImpactSkipData::new(entries, blocks.len())
+            })
+            .transpose()?;
 
-        Self {
+        Ok(Self {
             max_score,
             length,
             blocks,
             posting_tail_codec,
+            block_size,
             positions,
-        }
+            impacts,
+            first_docs: FirstDocsState::standalone(),
+        })
     }
 
     pub fn iter(&self) -> CompressedPostingListIterator {
@@ -3041,23 +4761,58 @@ impl CompressedPostingList {
             self.blocks.clone(),
             self.posting_tail_codec,
             self.positions.clone(),
+            self.block_size,
         )
     }
 
     pub fn block_max_score(&self, block_idx: usize) -> f32 {
+        // 256-doc (V3) blocks store no per-block max score: their impact
+        // skip data supplies the tight per-block bound, so callers on that
+        // path never reach here. Fall back to the list-level max, which is
+        // still a valid (looser) bound for any block.
+        if super::encoding::posting_block_score_prefix_len(self.block_size) == 0 {
+            return self.max_score;
+        }
         let block = self.blocks.value(block_idx);
         block[0..4].try_into().map(f32::from_le_bytes).unwrap()
     }
 
+    #[inline]
     pub fn block_least_doc_id(&self, block_idx: usize) -> u32 {
-        let block = self.blocks.value(block_idx);
-        let remainder = self.length as usize % BLOCK_SIZE;
-        let is_remainder_block = remainder > 0 && block_idx + 1 == self.blocks.len();
-        if is_remainder_block {
-            super::encoding::read_posting_tail_first_doc(block, self.posting_tail_codec)
-        } else {
-            block[4..8].try_into().map(u32::from_le_bytes).unwrap()
-        }
+        self.block_first_docs()[block_idx]
+    }
+
+    /// First doc id of every block, decoded once per cached list and shared by
+    /// the per-query clones. Block boundary lookups (window bounds, block
+    /// binary searches) are hot enough that re-reading the block headers —
+    /// and re-decoding the tail block — shows up in profiles.
+    pub(crate) fn block_first_docs(&self) -> &[u32] {
+        self.first_docs.get_or_init(|| {
+            (0..self.blocks.len())
+                .map(|block_idx| {
+                    let block = self.blocks.value(block_idx);
+                    let remainder = self.length as usize % self.block_size;
+                    if block_idx + 1 == self.blocks.len() && remainder > 0 {
+                        return super::encoding::read_posting_tail_first_doc(
+                            block,
+                            self.posting_tail_codec,
+                            self.block_size,
+                        );
+                    }
+                    let prefix = super::encoding::posting_block_score_prefix_len(self.block_size);
+                    block[prefix..prefix + 4]
+                        .try_into()
+                        .map(u32::from_le_bytes)
+                        .unwrap()
+                })
+                .collect::<Vec<_>>()
+                .into_boxed_slice()
+        })
+    }
+
+    #[cfg(test)]
+    fn shares_first_docs_with(&self, other: &Self) -> bool {
+        self.first_docs.shares_state_with(&other.first_docs)
     }
 }
 
@@ -3108,12 +4863,14 @@ impl EncodedBlocks {
         doc_ids: &[u32],
         frequencies: &[u32],
         codec: PostingTailCodec,
+        block_size: usize,
     ) -> Result<()> {
         self.offsets.push(self.bytes.len() as u32);
         super::encoding::encode_remainder_posting_block_into(
             doc_ids,
             frequencies,
             codec,
+            block_size,
             &mut self.bytes,
         )
     }
@@ -3182,6 +4939,7 @@ pub struct PostingListBuilder {
     open_doc_id: Option<u32>,
     open_doc_frequency: u32,
     open_doc_last_position: Option<u32>,
+    block_size: usize,
     memory_size_bytes: u32,
     len: u32,
 }
@@ -3189,14 +4947,11 @@ pub struct PostingListBuilder {
 pub(super) struct PostingListBatchBuilder {
     schema: SchemaRef,
     postings: ListBuilder<LargeBinaryBuilder>,
+    impacts: Option<ListBuilder<LargeBinaryBuilder>>,
     max_scores: Float32Builder,
     lengths: UInt32Builder,
     positions: BatchPositionsBuilder,
     len: usize,
-    /// Tracks posting-list cache-group boundaries in row order across all
-    /// batches this builder produces (issue #7040). Outlives `finish`, which
-    /// only resets the per-batch column builders.
-    group_accumulator: PostingGroupAccumulator,
 }
 
 enum BatchPositionsBuilder {
@@ -3211,6 +4966,7 @@ enum BatchPositionsBuilder {
 struct PostingListParts<'a> {
     with_positions: bool,
     posting_tail_codec: PostingTailCodec,
+    block_size: usize,
     length: usize,
     encoded_blocks: EncodedBlocks,
     encoded_position_blocks: EncodedPositionBlocks,
@@ -3224,7 +4980,6 @@ impl PostingListBatchBuilder {
         with_positions: bool,
         format_version: InvertedListFormatVersion,
         capacity: usize,
-        group_config: PostingGroupConfig,
     ) -> Self {
         let positions = if !with_positions {
             BatchPositionsBuilder::None
@@ -3239,14 +4994,18 @@ impl PostingListBatchBuilder {
                 capacity,
             ))
         };
+        let impacts = schema
+            .field_with_name(IMPACT_COL)
+            .ok()
+            .map(|_| ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity));
         Self {
             schema,
             postings: ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity),
+            impacts,
             max_scores: Float32Builder::with_capacity(capacity),
             lengths: UInt32Builder::with_capacity(capacity),
             positions,
             len: 0,
-            group_accumulator: PostingGroupAccumulator::new(group_config),
         }
     }
 
@@ -3261,11 +5020,11 @@ impl PostingListBatchBuilder {
     fn append(
         &mut self,
         compressed: LargeBinaryArray,
+        impacts: Option<&ImpactSkipData>,
         max_score: f32,
         length: u32,
         positions: Option<&CompressedPositionStorage>,
     ) -> Result<()> {
-        let posting_bytes = compressed.value_data().len();
         {
             let values = self.postings.values();
             for index in 0..compressed.len() {
@@ -3273,7 +5032,19 @@ impl PostingListBatchBuilder {
             }
         }
         self.postings.append(true);
-        self.group_accumulator.push(posting_bytes);
+        if let Some(impacts_builder) = &mut self.impacts {
+            let impacts = impacts.ok_or_else(|| {
+                Error::index(format!(
+                    "impacts builder missing impact data for posting length {}",
+                    length
+                ))
+            })?;
+            let values = impacts_builder.values();
+            for index in 0..impacts.entries().len() {
+                values.append_value(impacts.entries().value(index));
+            }
+            impacts_builder.append(true);
+        }
         self.max_scores.append_value(max_score);
         self.lengths.append_value(length);
 
@@ -3338,6 +5109,9 @@ impl PostingListBatchBuilder {
             Arc::new(self.max_scores.finish()) as ArrayRef,
             Arc::new(self.lengths.finish()) as ArrayRef,
         ];
+        if let Some(impacts) = &mut self.impacts {
+            columns.push(Arc::new(impacts.finish()) as ArrayRef);
+        }
         match &mut self.positions {
             BatchPositionsBuilder::None => {}
             BatchPositionsBuilder::Legacy(position_lists) => {
@@ -3354,13 +5128,6 @@ impl PostingListBatchBuilder {
         self.len = 0;
         RecordBatch::try_new(self.schema.clone(), columns).map_err(Error::from)
     }
-
-    /// Consume the builder and return the posting-list cache-group boundaries
-    /// accumulated across all batches (issue #7040). Each entry is the first
-    /// row of a group; the sequence is monotonically increasing.
-    pub fn into_group_starts(self) -> Vec<u32> {
-        self.group_accumulator.into_starts()
-    }
 }
 
 impl PostingListBuilder {
@@ -3373,9 +5140,10 @@ impl PostingListBuilder {
     }
 
     pub fn new(with_position: bool) -> Self {
-        Self::new_with_posting_tail_codec(
+        Self::new_with_posting_tail_codec_and_block_size(
             with_position,
             current_fts_format_version().posting_tail_codec(),
+            LEGACY_BLOCK_SIZE,
         )
     }
 
@@ -3383,6 +5151,27 @@ impl PostingListBuilder {
         with_position: bool,
         posting_tail_codec: PostingTailCodec,
     ) -> Self {
+        Self::new_with_posting_tail_codec_and_block_size(
+            with_position,
+            posting_tail_codec,
+            LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    pub fn new_with_block_size(with_position: bool, block_size: usize) -> Self {
+        Self::new_with_posting_tail_codec_and_block_size(
+            with_position,
+            current_fts_format_version().posting_tail_codec(),
+            block_size,
+        )
+    }
+
+    pub fn new_with_posting_tail_codec_and_block_size(
+        with_position: bool,
+        posting_tail_codec: PostingTailCodec,
+        block_size: usize,
+    ) -> Self {
+        validate_block_size(block_size).expect("invalid posting list block size");
         Self {
             with_positions: with_position,
             posting_tail_codec,
@@ -3393,6 +5182,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             len: 0,
             memory_size_bytes: 0,
         }
@@ -3414,8 +5204,8 @@ impl PostingListBuilder {
         &self,
         mut visit: impl FnMut(u32, u32, Option<Vec<u32>>) -> std::result::Result<(), E>,
     ) -> std::result::Result<(), E> {
-        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
-        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        let mut doc_ids = Vec::with_capacity(self.block_size);
+        let mut frequencies = Vec::with_capacity(self.block_size);
         let mut decoded_positions = Vec::new();
         let mut position_block_index = 0usize;
 
@@ -3423,7 +5213,12 @@ impl PostingListBuilder {
             for block in encoded_blocks.iter() {
                 doc_ids.clear();
                 frequencies.clear();
-                super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
+                super::encoding::decode_full_posting_block(
+                    block,
+                    &mut doc_ids,
+                    &mut frequencies,
+                    self.block_size,
+                );
                 decoded_positions.clear();
                 if self.with_positions {
                     let position_blocks = self
@@ -3503,7 +5298,7 @@ impl PostingListBuilder {
         }
         self.len += 1;
 
-        if self.tail_entries.len() == BLOCK_SIZE {
+        if self.tail_entries.len() == self.block_size {
             self.flush_tail_block()
                 .expect("posting list block compression should succeed");
         }
@@ -3562,7 +5357,7 @@ impl PostingListBuilder {
                 self.open_doc_id = None;
                 self.open_doc_frequency = 0;
                 self.open_doc_last_position = None;
-                if self.tail_entries.len() == BLOCK_SIZE {
+                if self.tail_entries.len() == self.block_size {
                     self.flush_tail_block()?;
                 }
                 Ok(())
@@ -3613,13 +5408,17 @@ impl PostingListBuilder {
             self.open_doc_id.is_none(),
             "cannot flush a posting block while a document is still open"
         );
-        debug_assert_eq!(self.tail_entries.len(), BLOCK_SIZE);
-        let mut doc_ids = [0u32; BLOCK_SIZE];
-        let mut frequencies = [0u32; BLOCK_SIZE];
-        for (index, entry) in self.tail_entries.iter().enumerate() {
-            doc_ids[index] = entry.doc_id;
-            frequencies[index] = entry.frequency;
-        }
+        debug_assert_eq!(self.tail_entries.len(), self.block_size);
+        let doc_ids = self
+            .tail_entries
+            .iter()
+            .map(|entry| entry.doc_id)
+            .collect::<Vec<_>>();
+        let frequencies = self
+            .tail_entries
+            .iter()
+            .map(|entry| entry.frequency)
+            .collect::<Vec<_>>();
         let encoded_blocks_size_before = self
             .encoded_blocks
             .as_ref()
@@ -3723,6 +5522,7 @@ impl PostingListBuilder {
     fn build_batch(
         self,
         compressed: LargeBinaryArray,
+        impacts: Option<ImpactSkipData>,
         max_score: f32,
         schema: SchemaRef,
         positions: Option<CompressedPositionStorage>,
@@ -3741,6 +5541,22 @@ impl PostingListBuilder {
                 length as u32,
             ))) as ArrayRef,
         ];
+        if schema.field_with_name(IMPACT_COL).is_ok() {
+            let impacts = impacts.ok_or_else(|| {
+                Error::index(format!(
+                    "impact column requested without impact data for posting length {}",
+                    length
+                ))
+            })?;
+            let impact_offsets =
+                OffsetBuffer::new(ScalarBuffer::from(vec![0, impacts.entries().len() as i32]));
+            columns.push(Arc::new(ListArray::try_new(
+                Arc::new(Field::new("item", datatypes::DataType::LargeBinary, true)),
+                impact_offsets,
+                Arc::new(impacts.entries().clone()),
+                None,
+            )?) as ArrayRef);
+        }
         columns.extend(Self::build_position_columns(positions)?);
 
         let batch = RecordBatch::try_new(schema, columns)?;
@@ -3790,6 +5606,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -3799,6 +5616,7 @@ impl PostingListBuilder {
         let parts = PostingListParts {
             with_positions,
             posting_tail_codec,
+            block_size,
             length: len as usize,
             encoded_blocks: encoded_blocks
                 .map(|encoded_blocks| *encoded_blocks)
@@ -3809,14 +5627,20 @@ impl PostingListBuilder {
             tail_entries: tail_entries.as_slice(),
             tail_position_block: with_positions.then(|| tail_positions.finish()),
         };
-        let (compressed, shared_positions, max_score) =
+        let (compressed, shared_positions, max_score, impacts) =
             Self::build_compressed_with_scores_from_parts(parts, docs)?;
         let positions = match legacy_positions {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        batch_builder.append(compressed, max_score, len, positions.as_ref())
-    }
+        batch_builder.append(
+            compressed,
+            Some(&impacts),
+            max_score,
+            len,
+            positions.as_ref(),
+        )
+    }
 
     fn extend_tail_components(
         tail_entries: &[RawDocInfo],
@@ -3832,11 +5656,17 @@ impl PostingListBuilder {
     fn build_compressed_with_scores_from_parts(
         parts: PostingListParts<'_>,
         docs: &DocSet,
-    ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+    ) -> Result<(
+        LargeBinaryArray,
+        Option<SharedPositionStream>,
+        f32,
+        ImpactSkipData,
+    )> {
         let PostingListParts {
             with_positions,
             posting_tail_codec,
             length,
+            block_size,
             mut encoded_blocks,
             mut encoded_position_blocks,
             tail_entries,
@@ -3845,41 +5675,58 @@ impl PostingListBuilder {
         let avgdl = docs.average_length();
         let idf_scale = idf(length, docs.len()) * (K1 + 1.0);
         let mut max_score = f32::MIN;
-        let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
-        let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
+        let mut doc_ids = Vec::with_capacity(block_size);
+        let mut frequencies = Vec::with_capacity(block_size);
+        let mut impact_block = Vec::with_capacity(block_size);
+        let mut impact_builder =
+            ImpactSkipDataBuilder::with_capacity(length.div_ceil(block_size), block_size);
 
         for index in 0..encoded_blocks.len() {
             let block = encoded_blocks.block(index);
             doc_ids.clear();
             frequencies.clear();
-            super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies);
-            let block_score = compute_block_score(
+            super::encoding::decode_full_posting_block(
+                block,
+                &mut doc_ids,
+                &mut frequencies,
+                block_size,
+            );
+            let block_score = compute_block_score_and_impact_block(
                 docs,
                 avgdl,
                 idf_scale,
                 doc_ids.iter().copied(),
                 frequencies.iter().copied(),
+                &mut impact_block,
             );
+            impact_builder.append_block(impact_block.as_slice())?;
             max_score = max_score.max(block_score);
-            encoded_blocks.set_block_score(index, block_score);
+            if super::encoding::posting_block_score_prefix_len(block_size) > 0 {
+                encoded_blocks.set_block_score(index, block_score);
+            }
         }
 
         if !tail_entries.is_empty() {
             Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies);
-            let block_score = compute_block_score(
+            let block_score = compute_block_score_and_impact_block(
                 docs,
                 avgdl,
                 idf_scale,
                 doc_ids.iter().copied(),
                 frequencies.iter().copied(),
+                &mut impact_block,
             );
+            impact_builder.append_block(impact_block.as_slice())?;
             max_score = max_score.max(block_score);
             encoded_blocks.append_remainder_block_with_codec(
                 doc_ids.as_slice(),
                 frequencies.as_slice(),
                 posting_tail_codec,
+                block_size,
             )?;
-            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if super::encoding::posting_block_score_prefix_len(block_size) > 0 {
+                encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            }
             if with_positions {
                 encoded_position_blocks.push_encoded_block(
                     tail_position_block
@@ -3889,22 +5736,27 @@ impl PostingListBuilder {
             }
         }
 
+        let impacts = impact_builder.finish()?;
         Ok((
             encoded_blocks.into_array(),
             with_positions.then(|| encoded_position_blocks.into_stream()),
             max_score,
+            impacts,
         ))
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn build_compressed_with_block_scores_from_parts(
         with_positions: bool,
         posting_tail_codec: PostingTailCodec,
+        block_size: usize,
         mut encoded_blocks: EncodedBlocks,
         mut encoded_position_blocks: EncodedPositionBlocks,
         tail_entries: &[RawDocInfo],
         tail_position_block: Option<Vec<u8>>,
         mut block_max_scores: impl Iterator<Item = f32>,
     ) -> Result<(LargeBinaryArray, Option<SharedPositionStream>, f32)> {
+        let has_score_prefix = super::encoding::posting_block_score_prefix_len(block_size) > 0;
         let mut max_score = f32::MIN;
         let mut doc_ids = Vec::with_capacity(BLOCK_SIZE);
         let mut frequencies = Vec::with_capacity(BLOCK_SIZE);
@@ -3914,7 +5766,9 @@ impl PostingListBuilder {
                 .next()
                 .ok_or_else(|| Error::index("missing block max score".to_owned()))?;
             max_score = max_score.max(block_score);
-            encoded_blocks.set_block_score(index, block_score);
+            if has_score_prefix {
+                encoded_blocks.set_block_score(index, block_score);
+            }
         }
 
         if !tail_entries.is_empty() {
@@ -3927,8 +5781,11 @@ impl PostingListBuilder {
                 doc_ids.as_slice(),
                 frequencies.as_slice(),
                 posting_tail_codec,
+                block_size,
             )?;
-            encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            if has_score_prefix {
+                encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score);
+            }
             if with_positions {
                 encoded_position_blocks.push_encoded_block(
                     tail_position_block
@@ -3946,12 +5803,16 @@ impl PostingListBuilder {
     }
 
     pub fn to_batch(self, block_max_scores: Vec<f32>) -> Result<RecordBatch> {
-        let format_version = if self.posting_tail_codec == PostingTailCodec::Fixed32 {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
-        let schema = inverted_list_schema_for_version(self.has_positions(), format_version);
+        let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size(
+            self.posting_tail_codec,
+            self.block_size,
+        )?;
+        let schema = inverted_list_schema_for_version_with_block_size_and_impacts(
+            self.has_positions(),
+            format_version,
+            self.block_size,
+            false,
+        );
         let legacy_positions =
             if self.with_positions && !format_version.uses_shared_position_stream() {
                 Some(self.build_legacy_positions()?)
@@ -3968,6 +5829,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -3978,6 +5840,7 @@ impl PostingListBuilder {
             Self::build_compressed_with_block_scores_from_parts(
                 with_positions,
                 posting_tail_codec,
+                block_size,
                 encoded_blocks
                     .map(|encoded_blocks| *encoded_blocks)
                     .unwrap_or_default(),
@@ -3998,6 +5861,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             memory_size_bytes: 0,
             len,
         };
@@ -4005,17 +5869,11 @@ impl PostingListBuilder {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        builder.build_batch(compressed, max_score, schema, positions)
+        builder.build_batch(compressed, None, max_score, schema, positions)
     }
 
     pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result<RecordBatch> {
-        let format_version = if schema.column_with_name(POSITION_COL).is_some()
-            && schema.column_with_name(COMPRESSED_POSITION_COL).is_none()
-        {
-            InvertedListFormatVersion::V1
-        } else {
-            InvertedListFormatVersion::V2
-        };
+        let format_version = parse_format_version_from_metadata(schema.metadata())?;
         let legacy_positions =
             if self.with_positions && !format_version.uses_shared_position_stream() {
                 Some(self.build_legacy_positions()?)
@@ -4032,6 +5890,7 @@ impl PostingListBuilder {
             open_doc_id,
             open_doc_frequency,
             open_doc_last_position,
+            block_size,
             len,
             ..
         } = self;
@@ -4041,6 +5900,7 @@ impl PostingListBuilder {
         let parts = PostingListParts {
             with_positions,
             posting_tail_codec,
+            block_size,
             length: len as usize,
             encoded_blocks: encoded_blocks
                 .map(|encoded_blocks| *encoded_blocks)
@@ -4051,7 +5911,7 @@ impl PostingListBuilder {
             tail_entries: tail_entries.as_slice(),
             tail_position_block: with_positions.then(|| tail_positions.finish()),
         };
-        let (compressed, shared_positions, max_score) =
+        let (compressed, shared_positions, max_score, impacts) =
             Self::build_compressed_with_scores_from_parts(parts, docs)?;
         let builder = Self {
             with_positions,
@@ -4063,6 +5923,7 @@ impl PostingListBuilder {
             open_doc_id: None,
             open_doc_frequency: 0,
             open_doc_last_position: None,
+            block_size,
             memory_size_bytes: 0,
             len,
         };
@@ -4070,13 +5931,16 @@ impl PostingListBuilder {
             Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)),
             None => shared_positions.map(CompressedPositionStorage::SharedStream),
         };
-        builder.build_batch(compressed, max_score, schema, positions)
+        builder.build_batch(compressed, Some(impacts), max_score, schema, positions)
     }
 
     pub fn remap(&mut self, removed: &[u32]) {
         let mut cursor = 0;
-        let mut new_builder =
-            Self::new_with_posting_tail_codec(self.has_positions(), self.posting_tail_codec);
+        let mut new_builder = Self::new_with_posting_tail_codec_and_block_size(
+            self.has_positions(),
+            self.posting_tail_codec,
+            self.block_size,
+        );
         for (doc_id, freq, positions) in self.iter() {
             while cursor < removed.len() && removed[cursor] < doc_id {
                 cursor += 1;
@@ -4095,19 +5959,23 @@ impl PostingListBuilder {
     }
 }
 
-fn compute_block_score(
+fn compute_block_score_and_impact_block(
     docs: &DocSet,
     avgdl: f32,
     idf_scale: f32,
     doc_ids: impl Iterator<Item = u32>,
     frequencies: impl Iterator<Item = u32>,
+    impact_block: &mut Vec<(u32, u32, u32)>,
 ) -> f32 {
+    impact_block.clear();
     let mut block_max_score = f32::MIN;
     for (doc_id, freq) in doc_ids.zip(frequencies) {
-        let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl);
-        let freq = freq as f32;
-        let score = freq / (freq + doc_norm);
+        let doc_len = docs.num_tokens(doc_id);
+        let doc_norm = K1 * (1.0 - B + B * doc_len as f32 / avgdl);
+        let freq_f32 = freq as f32;
+        let score = freq_f32 / (freq_f32 + doc_norm);
         block_max_score = block_max_score.max(score);
+        impact_block.push((doc_id, freq, doc_len));
     }
     block_max_score * idf_scale
 }
@@ -4218,28 +6086,170 @@ impl Ord for RawDocInfo {
     }
 }
 
+/// Lucene SmallFloat-style doc-length quantization for V3 scoring and impact
+/// norms: a 4-mantissa-bit float-like byte code. Values 0-7 are exact; larger
+/// values keep their top four significand bits (relative error <= 6.25%) and
+/// decode to their bucket floor. The floor only ever shortens a doc, so impact
+/// bounds remain conservative for exact-scoring V2 as well as quantized V3.
+pub(super) fn quantize_doc_length(value: u32) -> u8 {
+    let num_bits = 32 - value.leading_zeros();
+    if num_bits < 4 {
+        value as u8
+    } else {
+        let shift = num_bits - 4;
+        (((value >> shift) as u8) & 0x07) | (((shift + 1) as u8) << 3)
+    }
+}
+
+#[inline]
+pub(super) fn dequantize_doc_length(code: u8) -> u32 {
+    DEQUANTIZED_DOC_LENGTHS[code as usize]
+}
+
+pub(super) static DEQUANTIZED_DOC_LENGTHS: [u32; 256] = build_dequantized_doc_lengths();
+
+const fn build_dequantized_doc_lengths() -> [u32; 256] {
+    let mut table = [0u32; 256];
+    let mut code = 0usize;
+    while code < 256 {
+        let bits = (code & 0x07) as u64;
+        let shift = (code >> 3) as i64 - 1;
+        let decoded = if shift < 0 {
+            bits
+        } else {
+            (bits | 0x08) << shift
+        };
+        // Codes past the largest u32 encoding are never produced; saturate so
+        // the table stays total.
+        table[code] = if decoded > u32::MAX as u64 {
+            u32::MAX
+        } else {
+            decoded as u32
+        };
+        code += 1;
+    }
+    table
+}
+
+#[derive(Debug, Clone)]
+enum NumTokens {
+    Owned(Vec<u32>),
+    Shared(ScalarBuffer<u32>),
+}
+
+impl Default for NumTokens {
+    fn default() -> Self {
+        Self::Owned(Vec::new())
+    }
+}
+
+impl std::ops::Deref for NumTokens {
+    type Target = [u32];
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values,
+        }
+    }
+}
+
+impl DeepSizeOf for NumTokens {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match self {
+            Self::Owned(values) => values.deep_size_of_children(context),
+            Self::Shared(values) => values.deep_size_of_children(context),
+        }
+    }
+}
+
+impl NumTokens {
+    fn with_capacity(capacity: usize) -> Self {
+        Self::Owned(Vec::with_capacity(capacity))
+    }
+
+    fn into_owned(self) -> Vec<u32> {
+        match self {
+            Self::Owned(values) => values,
+            Self::Shared(values) => values.to_vec(),
+        }
+    }
+
+    fn push(&mut self, value: u32) {
+        match self {
+            Self::Owned(values) => values.push(value),
+            Self::Shared(values) => {
+                let mut owned = values.to_vec();
+                owned.push(value);
+                *self = Self::Owned(owned);
+            }
+        }
+    }
+
+    fn memory_size(&self) -> usize {
+        match self {
+            Self::Owned(values) => values.capacity() * std::mem::size_of::<u32>(),
+            Self::Shared(values) => values.inner().capacity(),
+        }
+    }
+}
+
 // DocSet is a mapping from row ids to the number of tokens in the document
 // It's used to sort the documents by the bm25 score
-#[derive(Debug, Clone, Default, DeepSizeOf)]
+#[derive(Debug, Clone, Default)]
 pub struct DocSet {
     row_ids: Vec<u64>,
-    num_tokens: Vec<u32>,
+    num_tokens: NumTokens,
     // (row_id, doc_id) pairs sorted by row_id
     inv: Vec<(u64, u32)>,
 
     total_tokens: u64,
+
+    // V3 (256-doc block) partitions score with quantized doc lengths: the
+    // flag is set at partition load and the byte-norm slab bakes lazily on
+    // first scoring use (shared by clones of the loaded set). 128-block
+    // partitions never set the flag and keep exact scoring.
+    scoring_quantized: bool,
+    norms: Arc<std::sync::OnceLock<Box<[u8]>>>,
+}
+
+impl DeepSizeOf for DocSet {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.row_ids.deep_size_of_children(context)
+            + self.num_tokens.deep_size_of_children(context)
+            + self.inv.deep_size_of_children(context)
+            + self
+                .norms
+                .get()
+                .map(|slab| std::mem::size_of_val(slab.as_ref()))
+                .unwrap_or(0)
+    }
 }
 
 impl DocSet {
     #[inline]
     pub fn len(&self) -> usize {
-        self.row_ids.len()
+        // Use num_tokens instead of row_ids so the deferred-row_ids
+        // scoring path (which constructs a DocSet via
+        // [`Self::from_num_tokens_only`]) still reports the right doc
+        // count.
+        self.num_tokens.len()
     }
 
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 
+    /// True iff the per-doc `row_id` array is populated. The
+    /// deferred-row_id scoring path constructs DocSets with the array
+    /// left empty so wand can skip the load; callers that need to do
+    /// row_id lookups in the inner loop must check this and fall back
+    /// to async resolution otherwise.
+    #[inline]
+    pub fn has_row_ids(&self) -> bool {
+        !self.row_ids.is_empty()
+    }
+
     pub fn iter(&self) -> impl Iterator<Item = (&u64, &u32)> {
         self.row_ids.iter().zip(self.num_tokens.iter())
     }
@@ -4248,18 +6258,24 @@ impl DocSet {
         self.row_ids[doc_id as usize]
     }
 
-    pub fn doc_id(&self, row_id: u64) -> Option<u64> {
+    /// Resolve a `row_id` to every `doc_id` it owns.
+    ///
+    /// Modern indexes map each row to a single document. Older list indexes
+    /// may have indexed each list element as its own document, so a single
+    /// `row_id` can still own several `doc_id`s sharing that key in `inv`.
+    /// The prefilter path (`flat_search`) walks an allow-list of row_ids and
+    /// must evaluate all legacy documents for that row.
+    pub fn doc_ids(&self, row_id: u64) -> impl Iterator<Item = u64> + '_ {
         if self.inv.is_empty() {
-            // in legacy format, the row id is doc id
-            match self.row_ids.binary_search(&row_id) {
-                Ok(_) => Some(row_id),
-                Err(_) => None,
-            }
+            // in legacy format, the row id is doc id (one document per row)
+            let found = self.row_ids.binary_search(&row_id).is_ok();
+            Either::Left(found.then_some(row_id).into_iter())
         } else {
-            match self.inv.binary_search_by_key(&row_id, |x| x.0) {
-                Ok(idx) => Some(self.inv[idx].1 as u64),
-                Err(_) => None,
-            }
+            // `inv` is sorted by row_id, so the entries sharing this key form a
+            // contiguous run; yield the doc_id of each.
+            let lo = self.inv.partition_point(|entry| entry.0 < row_id);
+            let hi = self.inv.partition_point(|entry| entry.0 <= row_id);
+            Either::Right(self.inv[lo..hi].iter().map(|entry| entry.1 as u64))
         }
     }
     pub fn total_tokens_num(&self) -> u64 {
@@ -4276,9 +6292,19 @@ impl DocSet {
         doc_ids: impl Iterator<Item = &'a u32>,
         freqs: impl Iterator<Item = &'a u32>,
     ) -> Vec<f32> {
+        self.calculate_block_max_scores_with_block_size(doc_ids, freqs, LEGACY_BLOCK_SIZE)
+    }
+
+    pub fn calculate_block_max_scores_with_block_size<'a>(
+        &self,
+        doc_ids: impl Iterator<Item = &'a u32>,
+        freqs: impl Iterator<Item = &'a u32>,
+        block_size: usize,
+    ) -> Vec<f32> {
+        validate_block_size(block_size).expect("invalid posting list block size");
         let avgdl = self.average_length();
         let length = doc_ids.size_hint().0;
-        let num_blocks = length.div_ceil(BLOCK_SIZE);
+        let num_blocks = length.div_ceil(block_size);
         let mut block_max_scores = Vec::with_capacity(num_blocks);
         let idf_scale = idf(length, self.len()) * (K1 + 1.0);
         let mut max_score = f32::MIN;
@@ -4289,13 +6315,13 @@ impl DocSet {
             if score > max_score {
                 max_score = score;
             }
-            if (i + 1) % BLOCK_SIZE == 0 {
+            if (i + 1) % block_size == 0 {
                 max_score *= idf_scale;
                 block_max_scores.push(max_score);
                 max_score = f32::MIN;
             }
         }
-        if !length.is_multiple_of(BLOCK_SIZE) {
+        if !length.is_multiple_of(block_size) {
             max_score *= idf_scale;
             block_max_scores.push(max_score);
         }
@@ -4324,12 +6350,51 @@ impl DocSet {
     pub async fn load(
         reader: Arc<dyn IndexReader>,
         is_legacy: bool,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Self> {
         let batch = reader.read_range(0..reader.num_rows(), None).await?;
         let row_id_col = batch[ROW_ID].as_primitive::<datatypes::UInt64Type>();
         let num_tokens_col = batch[NUM_TOKEN_COL].as_primitive::<datatypes::UInt32Type>();
+        Self::from_columns(row_id_col, num_tokens_col, is_legacy, frag_reuse_index)
+    }
+
+    /// Build a `DocSet` carrying only the per-doc `num_tokens` array;
+    /// `row_ids` and `inv` are left empty. Used by the deferred-row_id
+    /// scoring path: wand checks `has_row_ids()` to skip `row_id` /
+    /// `num_tokens_by_row_id` calls, and the per-partition caller
+    /// resolves doc_id → row_id for the surviving top-K post-wand.
+    pub fn from_num_tokens_only(num_tokens_col: &arrow_array::UInt32Array) -> Self {
+        let total_tokens = num_tokens_col.values().iter().map(|&n| n as u64).sum();
+        Self::from_cached_num_tokens(num_tokens_col, total_tokens)
+    }
+
+    /// Build a zero-copy num-tokens-only view from an Arrow column and its
+    /// already-computed total. The caller must guarantee that `total_tokens`
+    /// is the sum of `num_tokens_col`.
+    pub(crate) fn from_cached_num_tokens(
+        num_tokens_col: &arrow_array::UInt32Array,
+        total_tokens: u64,
+    ) -> Self {
+        Self {
+            row_ids: Vec::new(),
+            num_tokens: NumTokens::Shared(num_tokens_col.values().clone()),
+            inv: Vec::new(),
+            total_tokens,
+            scoring_quantized: false,
+            norms: Arc::new(std::sync::OnceLock::new()),
+        }
+    }
 
+    /// Build a `DocSet` from already-loaded `row_id` and `num_tokens`
+    /// arrow columns. Lets callers that have one column already in hand
+    /// (e.g. `LazyDocSet` after `total_tokens_num` pre-fetched
+    /// `num_tokens`) skip re-reading that column.
+    pub fn from_columns(
+        row_id_col: &UInt64Array,
+        num_tokens_col: &arrow_array::UInt32Array,
+        is_legacy: bool,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+    ) -> Result<Self> {
         // for legacy format, the row id is doc id; sorting keeps binary search viable
         if is_legacy {
             let (row_ids, num_tokens): (Vec<_>, Vec<_>) = row_id_col
@@ -4349,37 +6414,54 @@ impl DocSet {
             let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
             return Ok(Self {
                 row_ids,
-                num_tokens,
+                num_tokens: NumTokens::Owned(num_tokens),
                 inv: Vec::new(),
                 total_tokens,
+                scoring_quantized: false,
+                norms: Arc::new(std::sync::OnceLock::new()),
             });
         }
 
-        // if frag reuse happened, we'll need to remap the row_ids. And after row_ids been
-        // remapped, we'll need resort to make sure binary_search works.
+        // If frag reuse happened, remap the row_ids through it. Crucially we
+        // must NOT drop the rows the reuse index deleted, because the posting
+        // lists reference doc_ids *positionally* (a doc_id is an index into
+        // these arrays, fixed at build time). Dropping deleted rows would
+        // renumber every later doc_id and desync the posting lists, so wand
+        // would index `num_tokens`/`row_ids` out of bounds or score the wrong
+        // doc. Instead we tombstone deleted rows in place: their slot survives
+        // (so doc_ids stay aligned with the posting lists) carrying
+        // `RowAddress::TOMBSTONE_ROW`, which wand skips, and they are left out
+        // of `inv` so a row_id lookup never resolves to a deleted doc. The
+        // heavyweight physical remap (`DocSet::remap`) is what actually
+        // renumbers and compacts; this load-time path only has to stay
+        // consistent until then.
         if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() {
             let mut row_ids = Vec::with_capacity(row_id_col.len());
-            let mut num_tokens = Vec::with_capacity(num_tokens_col.len());
-            for (row_id, num_token) in row_id_col.values().iter().zip(num_tokens_col.values()) {
-                if let Some(new_row_id) = frag_reuse_index_ref.remap_row_id(*row_id) {
-                    row_ids.push(new_row_id);
-                    num_tokens.push(*num_token);
+            let num_tokens = num_tokens_col.values().to_vec();
+            let mut inv = Vec::with_capacity(row_id_col.len());
+            for (doc_id, row_id) in row_id_col.values().iter().enumerate() {
+                match frag_reuse_index_ref.remap_row_id(*row_id) {
+                    Some(new_row_id) => {
+                        row_ids.push(new_row_id);
+                        inv.push((new_row_id, doc_id as u32));
+                    }
+                    None => {
+                        // Deleted: keep the slot (doc_ids must not shift) but
+                        // tombstone it and leave it out of `inv`.
+                        row_ids.push(RowAddress::TOMBSTONE_ROW);
+                    }
                 }
             }
-
-            let mut inv: Vec<(u64, u32)> = row_ids
-                .iter()
-                .enumerate()
-                .map(|(doc_id, row_id)| (*row_id, doc_id as u32))
-                .collect();
             inv.sort_unstable_by_key(|entry| entry.0);
 
             let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
             return Ok(Self {
                 row_ids,
-                num_tokens,
+                num_tokens: NumTokens::Owned(num_tokens),
                 inv,
                 total_tokens,
+                scoring_quantized: false,
+                norms: Arc::new(std::sync::OnceLock::new()),
             });
         }
 
@@ -4396,24 +6478,28 @@ impl DocSet {
         let total_tokens = num_tokens.iter().map(|&x| x as u64).sum();
         Ok(Self {
             row_ids,
-            num_tokens,
+            num_tokens: NumTokens::Owned(num_tokens),
             inv,
             total_tokens,
+            scoring_quantized: false,
+            norms: Arc::new(std::sync::OnceLock::new()),
         })
     }
 
     // remap the row ids to the new row ids
     // returns the removed doc ids
-    pub fn remap(&mut self, mapping: &HashMap<u64, Option<u64>>) -> Vec<u32> {
+    pub fn remap(&mut self, mapping: &RowAddrRemap) -> Vec<u32> {
         let mut removed = Vec::new();
         let len = self.len();
         let row_ids = std::mem::replace(&mut self.row_ids, Vec::with_capacity(len));
-        let num_tokens = std::mem::replace(&mut self.num_tokens, Vec::with_capacity(len));
+        let num_tokens =
+            std::mem::replace(&mut self.num_tokens, NumTokens::with_capacity(len)).into_owned();
+        self.invalidate_norms();
         self.total_tokens = 0;
         for (doc_id, (row_id, num_token)) in std::iter::zip(row_ids, num_tokens).enumerate() {
-            match mapping.get(&row_id) {
+            match mapping.get(row_id) {
                 Some(Some(new_row_id)) => {
-                    self.row_ids.push(*new_row_id);
+                    self.row_ids.push(new_row_id);
                     self.num_tokens.push(num_token);
                     self.total_tokens += num_token as u64;
                 }
@@ -4435,6 +6521,39 @@ impl DocSet {
         self.num_tokens[doc_id as usize]
     }
 
+    /// Enable quantized doc-length scoring (V3 / 256-doc block partitions).
+    pub fn set_quantized_scoring(&mut self, quantized: bool) {
+        self.scoring_quantized = quantized;
+    }
+
+    /// The quantized doc-length slab when this set scores quantized (V3
+    /// partitions), baked on first use; `None` for exact-scoring sets.
+    pub fn scoring_norms(&self) -> Option<&[u8]> {
+        if !self.scoring_quantized {
+            return None;
+        }
+        Some(
+            self.norms
+                .get_or_init(|| {
+                    self.num_tokens
+                        .iter()
+                        .map(|&n| quantize_doc_length(n))
+                        .collect()
+                })
+                .as_ref(),
+        )
+    }
+
+    /// Doc length as scoring sees it: the quantized bucket floor for V3
+    /// partitions, the exact value otherwise.
+    #[inline]
+    pub fn scoring_num_tokens(&self, doc_id: u32) -> u32 {
+        match self.scoring_norms() {
+            Some(norms) => dequantize_doc_length(norms[doc_id as usize]),
+            None => self.num_tokens[doc_id as usize],
+        }
+    }
+
     // this can be used only if it's a legacy format,
     // which store the sorted row ids so that we can use binary search
     #[inline]
@@ -4451,12 +6570,21 @@ impl DocSet {
         self.row_ids.push(row_id);
         self.num_tokens.push(num_tokens);
         self.total_tokens += num_tokens as u64;
+        self.invalidate_norms();
         self.row_ids.len() as u32 - 1
     }
 
+    // Drop the baked norm slab after a mutation; it re-bakes on the next
+    // scoring use.
+    fn invalidate_norms(&mut self) {
+        if self.norms.get().is_some() {
+            self.norms = Arc::new(std::sync::OnceLock::new());
+        }
+    }
+
     pub(crate) fn memory_size(&self) -> usize {
         self.row_ids.capacity() * std::mem::size_of::<u64>()
-            + self.num_tokens.capacity() * std::mem::size_of::<u32>()
+            + self.num_tokens.memory_size()
             + self.inv.capacity() * std::mem::size_of::<(u64, u32)>()
     }
 }
@@ -4480,6 +6608,12 @@ pub fn flat_full_text_search(
     match batches[0][doc_col].data_type() {
         DataType::Utf8 => do_flat_full_text_search::<i32>(batches, doc_col, query, tokenizer),
         DataType::LargeUtf8 => do_flat_full_text_search::<i64>(batches, doc_col, query, tokenizer),
+        DataType::List(_) => {
+            do_flat_full_text_search_list::<i32>(batches, doc_col, query, tokenizer)
+        }
+        DataType::LargeList(_) => {
+            do_flat_full_text_search_list::<i64>(batches, doc_col, query, tokenizer)
+        }
         data_type => Err(Error::invalid_input(format!(
             "unsupported data type {} for inverted index",
             data_type
@@ -4515,6 +6649,46 @@ fn do_flat_full_text_search<Offset: OffsetSizeTrait>(
     Ok(results)
 }
 
+fn do_flat_full_text_search_list<ListOffset: OffsetSizeTrait>(
+    batches: &[&RecordBatch],
+    doc_col: &str,
+    query: &str,
+    tokenizer: Option<Box<dyn LanceTokenizer>>,
+) -> Result<Vec<u64>> {
+    let mut results = Vec::new();
+    let mut tokenizer =
+        tokenizer.unwrap_or_else(|| InvertedIndexParams::default().build().unwrap());
+    let query_tokens = collect_query_tokens(query, &mut tokenizer);
+
+    for batch in batches {
+        let row_id_array = batch[ROW_ID].as_primitive::<UInt64Type>();
+        let doc_array = batch[doc_col].as_list::<ListOffset>();
+        match doc_array.value_type() {
+            DataType::Utf8 | DataType::LargeUtf8 => {}
+            data_type => {
+                return Err(Error::invalid_input(format!(
+                    "unsupported list item data type {} for inverted index",
+                    data_type
+                )));
+            }
+        }
+        for i in 0..row_id_array.len() {
+            if doc_array.is_null(i) {
+                continue;
+            }
+            let elements = doc_array.value(i);
+            if iter_str_array(elements.as_ref())
+                .flatten()
+                .any(|element| has_query_token(element, &mut tokenizer, &query_tokens))
+            {
+                results.push(row_id_array.value(i));
+            }
+        }
+    }
+
+    Ok(results)
+}
+
 const FLAT_ROW_ID_COL_IDX: usize = 0;
 const FLAT_ALL_TOKENS_COL_IDX: usize = 1;
 const FLAT_QUERY_TOKEN_COUNTS_COL_IDX: usize = 2;
@@ -4572,6 +6746,8 @@ async fn tokenize_and_count(
                 // thread is invisible to the caller's poll timer otherwise).
                 let start = std::time::Instant::now();
                 let batch = batch?;
+                let row_id_array = batch[ROW_ID].as_primitive::<UInt64Type>();
+                let mut row_ids = UInt64Builder::with_capacity(batch.num_rows());
                 let mut all_token_counts = UInt64Builder::with_capacity(batch.num_rows());
                 let mut query_token_counts = FixedSizeListBuilder::with_capacity(
                     UInt64Builder::with_capacity(batch.num_rows() * query_tokens.len()),
@@ -4579,20 +6755,7 @@ async fn tokenize_and_count(
                     batch.num_rows(),
                 );
                 let mut temp_query_token_counts = Vec::with_capacity(query_tokens.len());
-                let doc_iter = iter_str_array(batch.column(doc_col_idx));
-                for doc in doc_iter {
-                    let Some(doc) = doc else {
-                        all_token_counts.append_value(0);
-                        query_token_counts
-                            .values()
-                            .append_value_n(0, query_tokens.len());
-                        query_token_counts.append(true);
-                        continue;
-                    };
-
-                    temp_query_token_counts.clear();
-                    temp_query_token_counts.extend(std::iter::repeat_n(0, query_tokens.len()));
-
+                let mut count_text = |doc: &str, temp_query_token_counts: &mut Vec<u64>| -> u64 {
                     let mut stream = tokenizer.token_stream_for_doc(doc);
                     let mut all_tokens = 0;
                     while let Some(token) = stream.next() {
@@ -4601,20 +6764,67 @@ async fn tokenize_and_count(
                             temp_query_token_counts[token_index] += 1;
                         }
                     }
-                    all_token_counts.append_value(all_tokens);
-                    for count in temp_query_token_counts.iter().copied() {
-                        query_token_counts.values().append_value(count);
+                    all_tokens
+                };
+                let mut append_counts =
+                    |row_id: u64, all_tokens: u64, temp_query_token_counts: &[u64]| {
+                        row_ids.append_value(row_id);
+                        all_token_counts.append_value(all_tokens);
+                        for count in temp_query_token_counts.iter().copied() {
+                            query_token_counts.values().append_value(count);
+                        }
+                        query_token_counts.append(true);
+                    };
+                match batch.column(doc_col_idx).data_type() {
+                    DataType::Utf8 | DataType::LargeUtf8 => {
+                        let doc_iter = iter_str_array(batch.column(doc_col_idx));
+                        for (doc, row_id) in doc_iter.zip(row_id_array.values().iter()) {
+                            temp_query_token_counts.clear();
+                            temp_query_token_counts
+                                .extend(std::iter::repeat_n(0, query_tokens.len()));
+
+                            let Some(doc) = doc else {
+                                append_counts(*row_id, 0, &temp_query_token_counts);
+                                continue;
+                            };
+
+                            let all_tokens = count_text(doc, &mut temp_query_token_counts);
+                            append_counts(*row_id, all_tokens, &temp_query_token_counts);
+                        }
+                    }
+                    DataType::List(_) => {
+                        tokenize_and_count_list::<i32>(
+                            batch.column(doc_col_idx),
+                            row_id_array,
+                            &mut count_text,
+                            &mut append_counts,
+                            &mut temp_query_token_counts,
+                            query_tokens.len(),
+                        )?;
+                    }
+                    DataType::LargeList(_) => {
+                        tokenize_and_count_list::<i64>(
+                            batch.column(doc_col_idx),
+                            row_id_array,
+                            &mut count_text,
+                            &mut append_counts,
+                            &mut temp_query_token_counts,
+                            query_tokens.len(),
+                        )?;
+                    }
+                    data_type => {
+                        return DataFusionResult::Err(datafusion_common::DataFusionError::Execution(
+                            format!("unsupported data type {} for flat full text search", data_type),
+                        ));
                     }
-                    query_token_counts.append(true);
                 }
-                let row_ids = batch[ROW_ID].clone();
+                let row_ids = row_ids.finish();
                 let all_token_counts = all_token_counts.finish();
                 let query_token_counts = query_token_counts.finish();
                 let result_batch = RecordBatch::try_new(
-
                     output_schema,
                     vec![
-                        row_ids,
+                        Arc::new(row_ids) as ArrayRef,
                         Arc::new(all_token_counts) as ArrayRef,
                         Arc::new(query_token_counts) as ArrayRef,
                     ],
@@ -4640,6 +6850,47 @@ async fn tokenize_and_count(
     )?)
 }
 
+fn tokenize_and_count_list<ListOffset: OffsetSizeTrait>(
+    doc_col: &ArrayRef,
+    row_id_array: &arrow_array::PrimitiveArray<UInt64Type>,
+    count_text: &mut impl FnMut(&str, &mut Vec<u64>) -> u64,
+    append_counts: &mut impl FnMut(u64, u64, &[u64]),
+    temp_query_token_counts: &mut Vec<u64>,
+    query_tokens_len: usize,
+) -> DataFusionResult<()> {
+    let doc_array = doc_col.as_list::<ListOffset>();
+    match doc_array.value_type() {
+        DataType::Utf8 | DataType::LargeUtf8 => {}
+        data_type => {
+            return Err(datafusion_common::DataFusionError::Execution(format!(
+                "unsupported list item data type {} for flat full text search",
+                data_type
+            )));
+        }
+    }
+
+    for i in 0..row_id_array.len() {
+        if doc_array.is_null(i) {
+            continue;
+        }
+
+        temp_query_token_counts.clear();
+        temp_query_token_counts.extend(std::iter::repeat_n(0, query_tokens_len));
+
+        let elements = doc_array.value(i);
+        let mut all_tokens = 0;
+        for element in iter_str_array(elements.as_ref()).flatten() {
+            all_tokens += count_text(element, temp_query_token_counts);
+        }
+
+        if all_tokens > 0 {
+            append_counts(row_id_array.value(i), all_tokens, temp_query_token_counts);
+        }
+    }
+
+    Ok(())
+}
+
 /// Initialize the BM25 scorer
 ///
 /// In order to calculate BM25 scores we need to know token counts for the entire corpus.  We extract these from the
@@ -4680,7 +6931,9 @@ fn initialize_scorer(
 
     for _ in 0..counted_input.num_rows() {
         for token_count in all_token_counts.iter_mut() {
-            *token_count += input_token_counters.next().unwrap_or_default();
+            if input_token_counters.next().unwrap_or_default() > 0 {
+                *token_count += 1;
+            }
         }
     }
 
@@ -4795,8 +7048,22 @@ pub async fn flat_bm25_search_stream_with_metrics(
     elapsed_compute: Option<Time>,
 ) -> DataFusionResult<SendableRecordBatchStream> {
     let mut tokenizer = tokenizer;
+
+    // Pre-await synchronous work: query tokenization + chunk-stream setup.
+    let pre_await_start = std::time::Instant::now();
     let query_tokens = Arc::new(collect_query_tokens(&query, &mut tokenizer));
 
+    // A query that tokenizes to no terms (e.g. only stop words) has no
+    // searchable content and matches nothing. Return early rather than
+    // proceeding. This mirrors the indexed search path, which already
+    // short-circuits on empty query tokens.
+    if query_tokens.is_empty() {
+        return Ok(Box::pin(RecordBatchStreamAdapter::new(
+            FTS_SCHEMA.clone(),
+            stream::empty::<DataFusionResult<RecordBatch>>(),
+        )));
+    }
+
     let input_schema = input.schema();
     let doc_col_idx = input_schema.index_of(&doc_col)?;
 
@@ -4813,6 +7080,9 @@ pub async fn flat_bm25_search_stream_with_metrics(
         ACCUMULATE_BYTES,
         SLICE_BYTES,
     );
+    if let Some(t) = &elapsed_compute {
+        t.add_duration(pre_await_start.elapsed());
+    }
 
     // Phase 2 - For each row we need to know the total number of tokens and the count of each
     // of the query tokens.  For example, if the query is "book" and the row is "the book shop"
@@ -4828,13 +7098,10 @@ pub async fn flat_bm25_search_stream_with_metrics(
     .await?;
 
     // Phase 3 - Calculate final scores (this is fairly cheap, probably don't need to parallelize).
-    // Synchronous and single-threaded, so we time it from this thread.
-    let scoring_start = std::time::Instant::now();
+    // All post-await work is synchronous; time the scorer + score + slicing loop together.
+    let post_await_start = std::time::Instant::now();
     let scorer = initialize_scorer(base_scorer.as_ref(), query_tokens.as_ref(), &counted_input);
     let scores = flat_bm25_score(query_tokens.as_ref(), &counted_input, &scorer)?;
-    if let Some(t) = &elapsed_compute {
-        t.add_duration(scoring_start.elapsed());
-    }
 
     // Finally we emit batches according to the target batch size
     let num_out_batches = scores.num_rows().div_ceil(target_batch_size);
@@ -4844,6 +7111,9 @@ pub async fn flat_bm25_search_stream_with_metrics(
         let len = (scores.num_rows() - start).min(target_batch_size);
         batches.push(Ok(scores.slice(start, len)));
     }
+    if let Some(t) = &elapsed_compute {
+        t.add_duration(post_await_start.elapsed());
+    }
     Ok(Box::pin(RecordBatchStreamAdapter::new(
         FTS_SCHEMA.clone(),
         stream::iter(batches),
@@ -4863,11 +7133,14 @@ mod tests {
     use lance_core::utils::tempfile::TempObjDir;
     use lance_io::object_store::ObjectStore;
 
-    use crate::metrics::NoOpMetricsCollector;
+    use crate::metrics::{LocalMetricsCollector, NoOpMetricsCollector};
     use crate::prefilter::NoFilter;
     use crate::scalar::ScalarIndex;
     use crate::scalar::inverted::builder::{
-        InnerBuilder, InvertedIndexBuilder, PositionRecorder, inverted_list_schema,
+        InnerBuilder, InvertedIndexBuilder, PositionRecorder, doc_file_path, inverted_list_schema,
+        inverted_list_schema_for_version_with_block_size,
+        inverted_list_schema_for_version_with_block_size_and_impacts, posting_file_path,
+        token_file_path,
     };
     use crate::scalar::inverted::encoding::{
         compress_positions, compress_posting_list_with_tail_codec,
@@ -4875,12 +7148,19 @@ mod tests {
     };
     use crate::scalar::inverted::query::{FtsSearchParams, Operator};
     use crate::scalar::lance_format::LanceIndexStore;
-    use arrow::array::{AsArray, Int32Builder, LargeBinaryBuilder, ListBuilder, UInt32Builder};
+    use arrow::array::{
+        AsArray, GenericListBuilder, GenericStringBuilder, Int32Builder, LargeBinaryBuilder,
+        ListBuilder, UInt32Builder,
+    };
     use arrow::datatypes::{Float32Type, UInt32Type};
     use arrow_array::{ArrayRef, Float32Array, RecordBatch, StringArray, UInt32Array, UInt64Array};
     use arrow_schema::{DataType, Field, Schema};
     use std::collections::HashMap;
     use std::sync::Arc;
+    use std::sync::atomic::{AtomicU32, Ordering};
+
+    use crate::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer;
+    use lance_tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer};
 
     use super::*;
 
@@ -4891,15 +7171,21 @@ mod tests {
         token: &str,
         row_id: u64,
     ) -> Result<Arc<InvertedIndex>> {
-        let mut partition = InnerBuilder::new_with_format_version(
+        let block_size = params.posting_block_size();
+        let format_version = params.resolved_format_version();
+        let mut partition = InnerBuilder::new_with_format_version_and_block_size(
             0,
             false,
             token_set_format,
-            InvertedListFormatVersion::V1,
+            format_version,
+            block_size,
         );
         partition.tokens.add(token.to_owned());
-        let mut posting_list =
-            PostingListBuilder::new_with_posting_tail_codec(false, PostingTailCodec::Fixed32);
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            block_size,
+        );
         posting_list.add(0, PositionRecorder::Count(1));
         partition.posting_lists.push(posting_list);
         partition.docs.append(row_id, 1);
@@ -4915,6 +7201,15 @@ mod tests {
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 token_set_format.to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                format_version.posting_tail_codec().as_str().to_owned(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
         ]);
         let mut writer = store
             .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
@@ -4935,57 +7230,244 @@ mod tests {
         ))
     }
 
-    #[tokio::test]
-    async fn test_posting_builder_remap() {
-        let posting_tail_codec = PostingTailCodec::Fixed32;
-        let mut builder =
-            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
-        let n = BLOCK_SIZE + 3;
-        for i in 0..n {
-            builder.add(i as u32, PositionRecorder::Count(1));
-        }
-        let removed = vec![5, 7];
-        builder.remap(&removed);
+    #[test]
+    fn test_posting_block_size_schema_metadata() {
+        assert_eq!(parse_posting_block_size(&HashMap::new()).unwrap(), 128);
 
-        let mut expected =
-            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
-        for i in 0..n - removed.len() {
-            expected.add(i as u32, PositionRecorder::Count(1));
-        }
-        let expected_entries = expected.iter().collect::<Vec<_>>();
-        let actual_entries = builder.iter().collect::<Vec<_>>();
-        assert_eq!(actual_entries, expected_entries);
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "512".to_owned())]);
+        let err = parse_posting_block_size(&metadata).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
 
-        // BLOCK_SIZE + 3 elements should be reduced to BLOCK_SIZE + 1,
-        // there are still 2 blocks.
-        let batch = builder.to_batch(vec![1.0, 2.0]).unwrap();
-        let (doc_ids, freqs) = decompress_posting_list_with_tail_codec(
-            (n - removed.len()) as u32,
-            batch[POSTING_COL]
-                .as_list::<i32>()
-                .value(0)
-                .as_binary::<i64>(),
-            posting_tail_codec,
-        )
-        .unwrap();
-        assert!(
-            doc_ids
-                .iter()
-                .zip(expected_entries.iter().map(|(doc_id, _, _)| doc_id))
-                .all(|(a, b)| a == b)
-        );
-        assert!(
-            freqs
-                .iter()
-                .zip(expected_entries.iter().map(|(_, freq, _)| freq))
-                .all(|(a, b)| a == b)
-        );
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "129".to_owned())]);
+        let err = parse_posting_block_size(&metadata).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
     }
 
     #[test]
-    fn test_posting_builder_size_tracking_matches_structure() {
-        fn tracked_memory_size(builder: &PostingListBuilder) -> u64 {
-            let encoded_blocks_size = builder
+    fn test_num_tokens_only_reuses_sliced_arrow_storage() {
+        let docs = {
+            let source = UInt32Array::from(vec![999, 7, 16, 1024, 888]);
+            let sliced = source.slice(1, 3);
+            let mut docs = DocSet::from_num_tokens_only(&sliced);
+
+            let NumTokens::Shared(values) = &docs.num_tokens else {
+                panic!("num-tokens-only DocSet must retain shared Arrow storage");
+            };
+            assert!(values.ptr_eq(sliced.values()));
+            assert_eq!(values.as_ref(), &[7, 16, 1024]);
+            assert_eq!(docs.total_tokens_num(), 1047);
+            docs.set_quantized_scoring(true);
+            assert_eq!(docs.scoring_norms().unwrap().len(), 3);
+            assert_eq!(
+                docs.scoring_num_tokens(0),
+                dequantize_doc_length(quantize_doc_length(7))
+            );
+            assert_eq!(
+                docs.scoring_num_tokens(2),
+                dequantize_doc_length(quantize_doc_length(1024))
+            );
+            docs
+        };
+
+        assert_eq!(docs.len(), 3);
+        assert_eq!(docs.num_tokens(0), 7);
+        assert_eq!(docs.num_tokens(2), 1024);
+    }
+
+    #[test]
+    fn test_cached_num_tokens_uses_supplied_total_and_full_stays_owned() {
+        const CACHED_TOTAL_MARKER: u64 = 123_456;
+
+        let num_tokens = UInt32Array::from(vec![3, 5, 8]);
+        let docs = DocSet::from_cached_num_tokens(&num_tokens, CACHED_TOTAL_MARKER);
+        assert_eq!(docs.total_tokens_num(), CACHED_TOTAL_MARKER);
+        assert!(matches!(&docs.num_tokens, NumTokens::Shared(_)));
+
+        let row_ids = UInt64Array::from(vec![10, 20, 30]);
+        let full = DocSet::from_columns(&row_ids, &num_tokens, false, None).unwrap();
+        assert!(matches!(&full.num_tokens, NumTokens::Owned(_)));
+        assert_eq!(full.total_tokens_num(), 16);
+        assert_eq!(full.row_id(1), 20);
+    }
+
+    #[test]
+    fn test_posting_builder_writes_impacts_for_supported_block_sizes() {
+        for block_size in [128, 256] {
+            let format_version = default_fts_format_version_for_block_size(block_size).unwrap();
+            let num_docs = block_size * 33 + 1;
+            let mut docs = DocSet::default();
+            let mut posting = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                false,
+                format_version.posting_tail_codec(),
+                block_size,
+            );
+            for doc_id in 0..num_docs {
+                docs.append(doc_id as u64, (doc_id % 5 + 1) as u32);
+                posting.add(
+                    doc_id as u32,
+                    PositionRecorder::Count((doc_id % 3 + 1) as u32),
+                );
+            }
+            let schema =
+                inverted_list_schema_for_version_with_block_size(false, format_version, block_size);
+            let batch = posting.to_batch_with_docs(&docs, schema).unwrap();
+            assert!(batch.column_by_name(IMPACT_COL).is_some());
+            let max_score = batch[MAX_SCORE_COL].as_primitive::<Float32Type>().value(0);
+            let length = batch[LENGTH_COL].as_primitive::<UInt32Type>().value(0);
+            let posting = PostingList::from_batch(&batch, Some(max_score), Some(length)).unwrap();
+            let PostingList::Compressed(posting) = posting else {
+                panic!("expected compressed posting list");
+            };
+            let impacts = posting.impacts.expect("posting should include impacts");
+            assert_eq!(impacts.level0_len(), posting.blocks.len());
+            assert_eq!(impacts.level1_len(), posting.blocks.len().div_ceil(32));
+            assert_eq!(
+                impacts.entries().len(),
+                impacts.level0_len() + impacts.level1_len()
+            );
+        }
+    }
+
+    #[test]
+    fn test_posting_builder_without_impact_column_roundtrips_without_impacts() {
+        let mut posting = PostingListBuilder::new(false);
+        for doc_id in 0..BLOCK_SIZE + 3 {
+            posting.add(doc_id as u32, PositionRecorder::Count(1));
+        }
+        let batch = posting.to_batch(vec![1.0, 1.0]).unwrap();
+        assert!(batch.column_by_name(IMPACT_COL).is_none());
+        let posting =
+            PostingList::from_batch(&batch, Some(1.0), Some((BLOCK_SIZE + 3) as u32)).unwrap();
+        assert!(!posting.has_impacts());
+    }
+
+    #[tokio::test]
+    async fn test_build_search_uses_configured_posting_block_size() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let format_version = params.resolved_format_version();
+        let block_size = params.posting_block_size();
+        let num_docs = block_size + 7;
+
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+            block_size,
+        );
+        builder.tokens.add("needle".to_owned());
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            block_size,
+        );
+        for doc_id in 0..num_docs {
+            posting_list.add(doc_id as u32, PositionRecorder::Count(1));
+            builder.docs.append(1_000 + doc_id as u64, 1);
+        }
+        builder.posting_lists.push(posting_list);
+        builder.write(store.as_ref()).await.unwrap();
+        write_test_metadata(&store, vec![0], params).await;
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        assert_eq!(index.partitions[0].inverted_list.block_size(), block_size);
+
+        let posting = index.partitions[0]
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        let PostingList::Compressed(posting) = posting else {
+            panic!("expected compressed posting list");
+        };
+        assert_eq!(posting.block_size, block_size);
+        assert_eq!(posting.blocks.len(), num_docs.div_ceil(block_size));
+        let impacts = posting
+            .impacts
+            .as_ref()
+            .expect("newly written posting list should include impacts");
+        assert_eq!(impacts.level0_len(), posting.blocks.len());
+        assert_eq!(impacts.level1_len(), posting.blocks.len().div_ceil(32));
+        assert_eq!(
+            impacts.entries().len(),
+            impacts.level0_len() + impacts.level1_len()
+        );
+
+        let tokens = Arc::new(Tokens::new(vec!["needle".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+        let (row_ids, scores) = index
+            .bm25_search(tokens, params, Operator::Or, prefilter, metrics, None)
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids.len(), 10);
+        assert_eq!(scores.len(), 10);
+        assert!(row_ids.iter().all(|row_id| *row_id >= 1_000));
+    }
+
+    #[tokio::test]
+    async fn test_posting_builder_remap() {
+        let posting_tail_codec = PostingTailCodec::Fixed32;
+        let mut builder =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        let n = BLOCK_SIZE + 3;
+        for i in 0..n {
+            builder.add(i as u32, PositionRecorder::Count(1));
+        }
+        let removed = vec![5, 7];
+        builder.remap(&removed);
+
+        let mut expected =
+            PostingListBuilder::new_with_posting_tail_codec(false, posting_tail_codec);
+        for i in 0..n - removed.len() {
+            expected.add(i as u32, PositionRecorder::Count(1));
+        }
+        let expected_entries = expected.iter().collect::<Vec<_>>();
+        let actual_entries = builder.iter().collect::<Vec<_>>();
+        assert_eq!(actual_entries, expected_entries);
+
+        // BLOCK_SIZE + 3 elements should be reduced to BLOCK_SIZE + 1,
+        // there are still 2 blocks.
+        let batch = builder.to_batch(vec![1.0, 2.0]).unwrap();
+        let (doc_ids, freqs) = decompress_posting_list_with_tail_codec(
+            (n - removed.len()) as u32,
+            batch[POSTING_COL]
+                .as_list::<i32>()
+                .value(0)
+                .as_binary::<i64>(),
+            posting_tail_codec,
+        )
+        .unwrap();
+        assert!(
+            doc_ids
+                .iter()
+                .zip(expected_entries.iter().map(|(doc_id, _, _)| doc_id))
+                .all(|(a, b)| a == b)
+        );
+        assert!(
+            freqs
+                .iter()
+                .zip(expected_entries.iter().map(|(_, freq, _)| freq))
+                .all(|(a, b)| a == b)
+        );
+    }
+
+    #[test]
+    fn test_posting_builder_size_tracking_matches_structure() {
+        fn tracked_memory_size(builder: &PostingListBuilder) -> u64 {
+            let encoded_blocks_size = builder
                 .encoded_blocks
                 .iter()
                 .map(|encoded_blocks| std::mem::size_of::<EncodedBlocks>() + encoded_blocks.size())
@@ -5060,6 +7542,21 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_shared_position_stream_clone_shares_block_offsets() {
+        let stream = SharedPositionStream::new(
+            PositionStreamCodec::PackedDelta,
+            vec![0_u32, 4, 11],
+            bytes::Bytes::from_static(b"shared position bytes"),
+        );
+        let original_offsets = stream.block_offsets().as_ptr();
+
+        let cloned = stream.clone();
+
+        assert_eq!(cloned.block_offsets(), stream.block_offsets());
+        assert_eq!(cloned.block_offsets().as_ptr(), original_offsets);
+    }
+
     #[test]
     fn test_posting_builder_roundtrip_shared_positions() {
         let entries = vec![
@@ -5150,15 +7647,28 @@ mod tests {
     }
 
     #[test]
-    fn test_resolve_fts_format_version_defaults_to_v1() {
+    fn test_resolve_fts_format_version_defaults_to_v2() {
         assert_eq!(
             resolve_fts_format_version(None).unwrap(),
-            InvertedListFormatVersion::V1
+            InvertedListFormatVersion::V2
         );
         assert_eq!(
             resolve_fts_format_version(Some("2")).unwrap(),
             InvertedListFormatVersion::V2
         );
+        assert_eq!(
+            resolve_fts_format_version(Some("3")).unwrap(),
+            InvertedListFormatVersion::V3
+        );
+    }
+
+    #[test]
+    fn test_block_size_256_metadata_resolves_to_v3() {
+        let metadata = HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), "256".to_owned())]);
+        assert_eq!(
+            parse_format_version_from_metadata(&metadata).unwrap(),
+            InvertedListFormatVersion::V3
+        );
     }
 
     #[test]
@@ -5433,7 +7943,7 @@ mod tests {
         let mut builder = index.into_builder().await.unwrap();
 
         let mapping = HashMap::from([(0, None), (2, Some(3))]);
-        builder.remap(&mapping).await.unwrap();
+        builder.remap(&RowAddrRemap::direct(mapping)).await.unwrap();
 
         // after remap, the doc 0 is removed, and the doc 2 is updated to 3
         assert_eq!(builder.tokens.len(), 1);
@@ -5448,7 +7958,7 @@ mod tests {
 
         // remap to delete all docs
         let mapping = HashMap::from([(1, None), (3, None)]);
-        builder.remap(&mapping).await.unwrap();
+        builder.remap(&RowAddrRemap::direct(mapping)).await.unwrap();
 
         assert_eq!(builder.tokens.len(), 0);
         assert_eq!(builder.posting_lists.len(), 0);
@@ -5583,7 +8093,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_modern_prewarm_shrinks_cached_posting_buffers() {
+    async fn test_modern_prewarm_packs_group_with_shared_posting_buffer() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -5635,82 +8145,552 @@ mod tests {
             !inverted_list.is_legacy_layout(),
             "test should use modern posting layout"
         );
+        assert!(
+            inverted_list.has_impacts,
+            "modern posting fixture should include impact skip data"
+        );
 
-        inverted_list.prewarm_posting_lists(false).await.unwrap();
+        inverted_list.prewarm_posting_lists(false, 2).await.unwrap();
 
         // The two tiny tokens land in a single cache group [0, 2) (issue
         // #7040); both postings are read out of that group entry.
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
 
-        let PostingList::Compressed(alpha) = group.get(0).unwrap() else {
+        assert!(
+            group.is_packed(),
+            "no-position prewarm should pack v2 groups"
+        );
+        assert!(
+            group.needs_external_metadata(),
+            "prewarmed packed groups must not duplicate reader score/length metadata"
+        );
+        let (alpha_score, alpha_len) = inverted_list.bulk_metadata_for_token(0);
+        let PostingList::Compressed(alpha) = group
+            .posting_list(0, alpha_score, alpha_len)
+            .unwrap()
+            .unwrap()
+        else {
             panic!("expected compressed posting list for token 0");
         };
-        let PostingList::Compressed(beta) = group.get(1).unwrap() else {
+        let PostingList::Compressed(alpha_again) = group
+            .posting_list(0, alpha_score, alpha_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list for repeated token 0 access");
+        };
+        let (beta_score, beta_len) = inverted_list.bulk_metadata_for_token(1);
+        let PostingList::Compressed(beta) = group
+            .posting_list(1, beta_score, beta_len)
+            .unwrap()
+            .unwrap()
+        else {
             panic!("expected compressed posting list for token 1");
         };
 
-        assert_ne!(
+        assert!(
+            alpha.impacts.is_some() && beta.impacts.is_some(),
+            "packed prewarm must preserve impact skip data"
+        );
+        assert!(
+            alpha
+                .impacts
+                .as_ref()
+                .unwrap()
+                .shares_derived_state_with(alpha_again.impacts.as_ref().unwrap()),
+            "repeated packed slot access must share decoded impact state"
+        );
+        assert!(
+            alpha.shares_first_docs_with(&alpha_again),
+            "repeated packed slot access must share decoded block heads"
+        );
+        assert_eq!(
+            alpha.block_first_docs().as_ptr(),
+            alpha_again.block_first_docs().as_ptr(),
+            "packed block heads should be decoded only once per slot"
+        );
+        assert_eq!(
             alpha.blocks.values().as_ptr(),
             beta.blocks.values().as_ptr(),
-            "prewarm should not leave cached posting lists sharing the same values buffer"
+            "packed posting views should share the group's values buffer"
         );
     }
 
-    /// IO accounting for the IO-counting stats test below: tracks bytes
-    /// pulled from the posting file so we can assert that the stats path is
-    /// O(1) in num_unique_tokens.
-    #[derive(Debug, Default)]
-    struct PostingMetadataCounter {
-        rows_read: std::sync::atomic::AtomicUsize,
-        metadata_rows_read: std::sync::atomic::AtomicUsize,
-        read_range_calls: std::sync::atomic::AtomicUsize,
-    }
+    #[tokio::test]
+    async fn test_packed_prewarm_groups_do_not_retain_the_full_chunk() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
 
-    impl PostingMetadataCounter {
-        fn rows_read(&self) -> usize {
-            self.rows_read.load(std::sync::atomic::Ordering::Relaxed)
-        }
-        fn metadata_rows_read(&self) -> usize {
-            self.metadata_rows_read
-                .load(std::sync::atomic::Ordering::Relaxed)
-        }
-        fn read_range_calls(&self) -> usize {
-            self.read_range_calls
-                .load(std::sync::atomic::Ordering::Relaxed)
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token_id in 0..4u32 {
+            builder.tokens.add(format!("t{token_id}"));
+            let mut posting = PostingListBuilder::new(false);
+            posting.add(token_id, PositionRecorder::Count(1));
+            builder.posting_lists.push(posting);
+            builder.docs.append(1000 + token_id as u64, 1);
         }
+        builder.write(store.as_ref()).await.unwrap();
+
+        let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
+        let cache = LanceCache::with_capacity(1 << 20);
+        let mut posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
+        posting_reader.grouping = PostingGrouping::SyntheticFixed { group_size: 2 };
+
+        assert_eq!(
+            posting_reader
+                .prewarm_posting_lists_chunked(false, Some(4), 1)
+                .await
+                .unwrap(),
+            1,
+            "the test must read both groups in one prewarm chunk"
+        );
+
+        let first_group = posting_reader
+            .index_cache
+            .get_with_key(&posting_list_group_cache_key(
+                0,
+                2,
+                posting_reader.has_impacts,
+            ))
+            .await
+            .unwrap();
+        let second_group = posting_reader
+            .index_cache
+            .get_with_key(&posting_list_group_cache_key(
+                2,
+                4,
+                posting_reader.has_impacts,
+            ))
+            .await
+            .unwrap();
+        let (first_score, first_len) = posting_reader.bulk_metadata_for_token(0);
+        let PostingList::Compressed(first) = first_group
+            .posting_list(0, first_score, first_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in first group");
+        };
+        let (neighbor_score, neighbor_len) = posting_reader.bulk_metadata_for_token(1);
+        let PostingList::Compressed(first_neighbor) = first_group
+            .posting_list(1, neighbor_score, neighbor_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in first group");
+        };
+        let (second_score, second_len) = posting_reader.bulk_metadata_for_token(2);
+        let PostingList::Compressed(second) = second_group
+            .posting_list(0, second_score, second_len)
+            .unwrap()
+            .unwrap()
+        else {
+            panic!("expected compressed posting list in second group");
+        };
+
+        assert_eq!(
+            first.blocks.values().as_ptr(),
+            first_neighbor.blocks.values().as_ptr(),
+            "postings in one group should share the group's values buffer"
+        );
+        assert_ne!(
+            first.blocks.values().as_ptr(),
+            second.blocks.values().as_ptr(),
+            "each group must own a compact buffer instead of retaining the full chunk"
+        );
     }
 
-    struct CountingPostingReader {
-        inner: Arc<dyn IndexReader>,
-        counter: Arc<PostingMetadataCounter>,
+    #[test]
+    fn test_prewarm_chunk_ranges_preserve_group_boundaries() {
+        let grouping = PostingGrouping::SyntheticFixed { group_size: 4 };
+        assert_eq!(
+            prewarm_chunk_ranges(&grouping, 13, 5),
+            vec![(0, 4), (4, 8), (8, 13)],
+            "grouped chunks may contain multiple groups but must never split one"
+        );
+        assert_eq!(
+            prewarm_chunk_ranges(&PostingGrouping::None, 13, 5),
+            vec![(0, 5), (5, 10), (10, 13)],
+            "ungrouped chunk ranges should use plain token ranges"
+        );
     }
 
-    #[async_trait]
-    impl IndexReader for CountingPostingReader {
-        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
-            self.inner.read_record_batch(n, batch_size).await
-        }
-        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
-            self.inner.read_global_buffer(index).await
-        }
-        async fn read_range(
-            &self,
-            range: std::ops::Range<usize>,
-            projection: Option<&[&str]>,
-        ) -> Result<RecordBatch> {
-            let n = range.end - range.start;
-            self.counter
-                .read_range_calls
-                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-            self.counter
-                .rows_read
-                .fetch_add(n, std::sync::atomic::Ordering::Relaxed);
-            let touches_metadata = projection
+    #[test]
+    fn test_synthetic_grouping_preserves_fixed_boundaries() {
+        let grouping = PostingGrouping::SyntheticFixed { group_size: 4 };
+        assert_eq!(
+            grouping.range_for_token(5, 10),
+            Some((4, 8)),
+            "synthetic token groups should be fixed-size ranges"
+        );
+        assert_eq!(
+            grouping.range_for_token(9, 10),
+            Some((8, 10)),
+            "the final synthetic group should end at token_count"
+        );
+        assert_eq!(
+            prewarm_chunk_ranges(&grouping, 10, 6),
+            vec![(0, 4), (4, 10)],
+            "prewarm chunks may contain multiple synthetic groups but must not split one"
+        );
+        assert_eq!(
+            grouping.ranges_for_chunk(4, 10, 10),
+            vec![(4, 8), (8, 10)],
+            "publish selection should enumerate synthetic groups in a chunk"
+        );
+    }
+
+    /// Prewarming a large partition in multiple chunks must end up holding exactly the
+    /// same per-token posting lists (doc ids and frequencies) as the whole-file path.
+    /// Parametrized over layout: the legacy-v1 chunk path rebases global offsets to
+    /// chunk-local rows, while the modern one-row-per-token path covers both
+    /// legacy-sized v2 and 256-doc v3 posting blocks.
+    #[rstest::rstest]
+    #[case::v1(InvertedListFormatVersion::V1, LEGACY_BLOCK_SIZE)]
+    #[case::v2(InvertedListFormatVersion::V2, LEGACY_BLOCK_SIZE)]
+    #[case::v3(InvertedListFormatVersion::V3, 256)]
+    #[tokio::test]
+    async fn test_prewarm_streams_in_chunks_preserves_content(
+        #[case] format_version: InvertedListFormatVersion,
+        #[case] block_size: usize,
+    ) {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // One partition with enough tokens to span multiple runtime synthetic
+        // groups and several docs per token.
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
+        const DOCS_PER_TOKEN: u32 = 3;
+        let posting_tail_codec = format_version.posting_tail_codec();
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+            block_size,
+        );
+        // expected[token] = [(doc_id, frequency)] in stored (doc-id) order.
+        let mut expected: Vec<Vec<(u32, u32)>> = Vec::new();
+        let mut doc_id = 0u64;
+        for t in 0..num_tokens {
+            builder.tokens.add(format!("tok_{t:03}"));
+            let mut posting = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                false,
+                posting_tail_codec,
+                block_size,
+            );
+            let mut docs = Vec::new();
+            for _ in 0..DOCS_PER_TOKEN {
+                posting.add(doc_id as u32, PositionRecorder::Count(1));
+                builder.docs.append(doc_id, 1);
+                docs.push((doc_id as u32, 1));
+                doc_id += 1;
+            }
+            expected.push(docs);
+            builder.posting_lists.push(posting);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)
+            .unwrap();
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
+            ),
+            ("params".to_owned(), serde_json::to_string(&params).unwrap()),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                posting_tail_codec.as_str().to_owned(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(1 << 20));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        let inverted_list = &index.partitions[0].inverted_list;
+        assert_eq!(inverted_list.len(), num_tokens as usize);
+        assert_eq!(inverted_list.block_size(), block_size);
+
+        // Force a small target chunk. Since CHUNK_TOKENS is below the runtime
+        // group size, synthetic group alignment should still split only at
+        // group boundaries.
+        const CHUNK_TOKENS: usize = 6;
+        let chunk_count = inverted_list
+            .prewarm_posting_lists_chunked(false, Some(CHUNK_TOKENS), 2)
+            .await
+            .unwrap();
+
+        // (1) The partition was streamed in multiple chunks. The exact count is
+        // group-alignment-dependent (chunks snap to whole groups), so just
+        // require more than one.
+        assert!(
+            chunk_count > 1,
+            "single partition must be streamed in more than one chunk, got {chunk_count}"
+        );
+
+        if format_version == InvertedListFormatVersion::V3 {
+            let (start, end) = inverted_list.group_range_for_token(0).unwrap();
+            let group = inverted_list
+                .index_cache
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    inverted_list.has_impacts,
+                ))
+                .await
+                .expect("v3 prewarm should populate the packed group cache");
+            assert!(group.is_packed());
+            let (max_score, length) = inverted_list.bulk_metadata_for_token(0);
+            let PostingList::Compressed(posting) =
+                group.posting_list(0, max_score, length).unwrap().unwrap()
+            else {
+                panic!("expected compressed v3 posting list");
+            };
+            assert_eq!(posting.block_size, 256);
+            assert!(
+                posting.impacts.is_some(),
+                "v3 packed prewarm must preserve impact skip data"
+            );
+        }
+
+        // (2) Correctness: every token's posting list round-trips with exactly
+        // the doc ids and frequencies of the whole-file path.
+        for token_id in 0..num_tokens {
+            let actual = inverted_list
+                .posting_list(token_id, false, &NoOpMetricsCollector)
+                .await
+                .unwrap()
+                .iter()
+                .map(|(doc_id, freq, _positions)| (doc_id as u32, freq))
+                .collect::<Vec<_>>();
+            assert_eq!(
+                actual, expected[token_id as usize],
+                "token {token_id} posting list mismatch after chunked prewarm"
+            );
+        }
+    }
+
+    /// With positions, the chunked prewarm must strip positions into their own
+    /// per-token cache entries (leaving the posting cache positions-free) and still
+    /// round-trip exact doc ids, frequencies, and positions across chunk boundaries.
+    #[tokio::test]
+    async fn test_prewarm_streams_in_chunks_with_positions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let format_version = InvertedListFormatVersion::V2;
+        let posting_tail_codec = format_version.posting_tail_codec();
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
+        const DOCS_PER_TOKEN: u32 = 3;
+        let mut builder = InnerBuilder::new_with_format_version(
+            0,
+            true,
+            TokenSetFormat::default(),
+            format_version,
+        );
+        // expected[token] = [(doc_id, frequency, positions)].
+        let mut expected: Vec<Vec<(u32, u32, Vec<u32>)>> = Vec::new();
+        let mut doc_id = 0u64;
+        for t in 0..num_tokens {
+            builder.tokens.add(format!("tok_{t:03}"));
+            let mut posting =
+                PostingListBuilder::new_with_posting_tail_codec(true, posting_tail_codec);
+            let mut docs = Vec::new();
+            for _ in 0..DOCS_PER_TOKEN {
+                let positions = vec![t % 3, t % 3 + 2, t % 3 + 5];
+                posting.add(
+                    doc_id as u32,
+                    PositionRecorder::Position(positions.clone().into()),
+                );
+                builder.docs.append(doc_id, positions.len() as u32);
+                docs.push((doc_id as u32, positions.len() as u32, positions));
+                doc_id += 1;
+            }
+            expected.push(docs);
+            builder.posting_lists.push(posting);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                posting_tail_codec.as_str().to_owned(),
+            ),
+            (
+                POSITIONS_LAYOUT_KEY.to_owned(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_owned(),
+            ),
+            (
+                POSITIONS_CODEC_KEY.to_owned(),
+                PositionStreamCodec::PackedDelta.as_str().to_owned(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(1 << 20));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        let inverted_list = &index.partitions[0].inverted_list;
+
+        const CHUNK_TOKENS: usize = 5;
+        let chunk_count = inverted_list
+            .prewarm_posting_lists_chunked(true, Some(CHUNK_TOKENS), 2)
+            .await
+            .unwrap();
+        assert!(
+            chunk_count > 1,
+            "partition must be streamed in more than one chunk, got {chunk_count}"
+        );
+
+        for token_id in 0..num_tokens {
+            // The prewarmed posting cache entry is positions-free.
+            let (start, end) = inverted_list.group_range_for_token(token_id).unwrap();
+            let group = inverted_list
+                .index_cache
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    inverted_list.has_impacts,
+                ))
+                .await
+                .unwrap();
+            let slot = (token_id - start) as usize;
+            assert!(
+                !group.is_packed(),
+                "with-position prewarm should retain the materialized fallback"
+            );
+            assert!(
+                !group
+                    .posting_list(slot, None, None)
+                    .unwrap()
+                    .unwrap()
+                    .has_position(),
+                "token {token_id} posting cache entry must be positions-free after prewarm"
+            );
+
+            // Full content (doc ids, frequencies, positions) round-trips; the
+            // positions come from the dedicated per-token cache prewarm populated.
+            let actual = inverted_list
+                .posting_list(token_id, true, &NoOpMetricsCollector)
+                .await
+                .unwrap()
+                .iter()
+                .map(|(doc_id, freq, positions)| {
+                    (doc_id as u32, freq, positions.unwrap().collect::<Vec<_>>())
+                })
+                .collect::<Vec<_>>();
+            assert_eq!(
+                actual, expected[token_id as usize],
+                "token {token_id} posting list / positions mismatch after chunked prewarm"
+            );
+        }
+    }
+
+    /// IO accounting for the IO-counting stats test below: tracks bytes
+    /// pulled from the posting file so we can assert that the stats path is
+    /// O(1) in num_unique_tokens.
+    #[derive(Debug, Default)]
+    struct PostingMetadataCounter {
+        rows_read: std::sync::atomic::AtomicUsize,
+        metadata_rows_read: std::sync::atomic::AtomicUsize,
+        read_range_calls: std::sync::atomic::AtomicUsize,
+    }
+
+    impl PostingMetadataCounter {
+        fn rows_read(&self) -> usize {
+            self.rows_read.load(std::sync::atomic::Ordering::Relaxed)
+        }
+        fn metadata_rows_read(&self) -> usize {
+            self.metadata_rows_read
+                .load(std::sync::atomic::Ordering::Relaxed)
+        }
+        fn read_range_calls(&self) -> usize {
+            self.read_range_calls
+                .load(std::sync::atomic::Ordering::Relaxed)
+        }
+    }
+
+    struct CountingPostingReader {
+        inner: Arc<dyn IndexReader>,
+        counter: Arc<PostingMetadataCounter>,
+    }
+
+    #[async_trait]
+    impl IndexReader for CountingPostingReader {
+        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
+            self.inner.read_record_batch(n, batch_size).await
+        }
+        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
+            self.inner.read_global_buffer(index).await
+        }
+        async fn read_range(
+            &self,
+            range: std::ops::Range<usize>,
+            projection: Option<&[&str]>,
+        ) -> Result<RecordBatch> {
+            let n = range.end - range.start;
+            self.counter
+                .read_range_calls
+                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            self.counter
+                .rows_read
+                .fetch_add(n, std::sync::atomic::Ordering::Relaxed);
+            let touches_metadata = projection
                 .map(|cols| cols.contains(&MAX_SCORE_COL) || cols.contains(&LENGTH_COL))
                 .unwrap_or(false);
             if touches_metadata {
@@ -5733,13 +8713,13 @@ mod tests {
 
     #[derive(Debug)]
     struct CountingStore {
-        inner: Arc<LanceIndexStore>,
+        inner: Arc<dyn IndexStore>,
         posting_file: String,
         counter: Arc<PostingMetadataCounter>,
     }
 
     impl DeepSizeOf for CountingStore {
-        fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+        fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
             self.inner.deep_size_of_children(context)
         }
     }
@@ -5759,6 +8739,13 @@ mod tests {
         fn io_parallelism(&self) -> usize {
             self.inner.io_parallelism()
         }
+        fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+            Arc::new(Self {
+                inner: self.inner.with_io_priority(io_priority),
+                posting_file: self.posting_file.clone(),
+                counter: self.counter.clone(),
+            })
+        }
         async fn new_index_file(
             &self,
             name: &str,
@@ -5777,10 +8764,28 @@ mod tests {
                 Ok(reader)
             }
         }
-        async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<()> {
+        async fn copy_index_file(
+            &self,
+            name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<crate::scalar::IndexFile> {
             self.inner.copy_index_file(name, dest_store).await
         }
-        async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<()> {
+        async fn copy_index_file_to(
+            &self,
+            name: &str,
+            new_name: &str,
+            dest_store: &dyn IndexStore,
+        ) -> Result<crate::scalar::IndexFile> {
+            self.inner
+                .copy_index_file_to(name, new_name, dest_store)
+                .await
+        }
+        async fn rename_index_file(
+            &self,
+            name: &str,
+            new_name: &str,
+        ) -> Result<crate::scalar::IndexFile> {
             self.inner.rename_index_file(name, new_name).await
         }
         async fn delete_index_file(&self, name: &str) -> Result<()> {
@@ -5791,9 +8796,14 @@ mod tests {
         }
     }
 
+    // Returns the `TempObjDir` guard so callers keep the backing store alive
+    // for the index's lifetime: the deferred DocSet re-opens the docs file on
+    // demand (it does not pin an open handle), so the files must still exist
+    // when the test exercises a scoring path.
     async fn load_counted_v2_index(
         num_tokens: usize,
-    ) -> (Arc<InvertedIndex>, Arc<PostingMetadataCounter>) {
+        cache: LanceCache,
+    ) -> (Arc<InvertedIndex>, Arc<PostingMetadataCounter>, TempObjDir) {
         let tmpdir = TempObjDir::default();
         let inner_store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -5837,10 +8847,10 @@ mod tests {
             posting_file: posting_file_path(0),
             counter: counter.clone(),
         });
-        let index = InvertedIndex::load(counting_store, None, &LanceCache::no_cache())
+        let index = InvertedIndex::load(counting_store, None, &cache)
             .await
             .unwrap();
-        (index, counter)
+        (index, counter, tmpdir)
     }
 
     /// IO regression test for the lazy posting-metadata refactor. Builds a
@@ -5850,9 +8860,9 @@ mod tests {
     ///
     /// * `InvertedIndex::load` does not touch the posting file at all
     ///   (`InvertedPartition::load` only needs the token file and docs file).
-    /// * `bm25_stats_for_terms(["t0"])` reads exactly one row from the
-    ///   posting file (the single LENGTH_COL entry for token 0) regardless
-    ///   of how many unique tokens the partition has.
+    /// * `bm25_stats_for_terms(["t0"])` reads exactly one metadata row from
+    ///   the posting file for token 0 regardless of how many unique tokens the
+    ///   partition has.
     ///
     /// Before this refactor, `PostingListReader::try_new` did
     /// `read_range(0..num_rows, [MAX_SCORE_COL, LENGTH_COL])`, so the
@@ -5865,7 +8875,8 @@ mod tests {
     #[case::tokens_1000(1000)]
     #[tokio::test]
     async fn test_bm25_stats_for_terms_is_lazy(#[case] num_tokens: usize) {
-        let (index, counter) = load_counted_v2_index(num_tokens).await;
+        let (index, counter, _tmpdir) =
+            load_counted_v2_index(num_tokens, LanceCache::no_cache()).await;
         assert!(
             !index.partitions[0].inverted_list.is_legacy_layout(),
             "this test only proves the lazy path for v2 indexes",
@@ -5906,30 +8917,124 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_grouped_posting_lists_read_one_group_per_neighborhood() {
-        // Cold-start scoring must not bulk-read the full `0..num_tokens`
-        // metadata table. With small-posting grouping (issue #7040), scoring
-        // K adjacent cold tokens shares a single group cache entry: one
-        // read_range bounded by the group size, independent of the partition's
-        // total token count.
-        let num_tokens = 500;
-        let queried_tokens: [u32; 4] = [0, 1, 2, 3];
-        let (index, counter) = load_counted_v2_index(num_tokens).await;
-        let inverted_list = index.partitions[0].inverted_list.clone();
-        assert!(
-            !inverted_list.is_legacy_layout(),
-            "this test only proves the lazy path for v2 indexes",
-        );
-        assert!(
-            inverted_list.group_starts.is_some(),
-            "freshly written v2 index should carry posting group offsets",
+    async fn test_bm25_stats_for_terms_reuses_posting_metadata_cache() {
+        let cache = LanceCache::with_capacity(1024 * 1024);
+        let (index, counter, _tmpdir) = load_counted_v2_index(100, cache.clone()).await;
+
+        let terms = ["t0".to_string()];
+        let first = index.bm25_stats_for_terms(&terms).await.unwrap();
+        assert_eq!(first, (100, 100, vec![1]));
+        assert_eq!(counter.metadata_rows_read(), 1);
+
+        let second = index.bm25_stats_for_terms(&terms).await.unwrap();
+        assert_eq!(second, first);
+        assert_eq!(
+            counter.metadata_rows_read(),
+            1,
+            "repeated stats for the same token should reuse cached posting metadata",
         );
+    }
 
-        // This fixture uses a no-op cache, so each call re-reads; that isolates
-        // the per-query read shape. Each posting_list call reads exactly its
+    #[tokio::test]
+    async fn test_aggregate_corpus_stats_reuses_cached_value() {
+        let (index, _counter, _tmpdir) = load_counted_v2_index(100, LanceCache::no_cache()).await;
+        assert!(index.corpus_stats.get().is_none());
+
+        let first = index.aggregate_corpus_stats().await.unwrap();
+        assert_eq!(first, (100, 100));
+        assert_eq!(index.corpus_stats.get().copied(), Some(first));
+
+        let second = index.aggregate_corpus_stats().await.unwrap();
+        assert_eq!(second, first);
+    }
+
+    #[tokio::test]
+    async fn test_stats_then_num_tokens_view_reuses_shared_storage() {
+        let (index, _counter, _tmpdir) = load_counted_v2_index(100, LanceCache::no_cache()).await;
+        let partition = index.partitions[0].clone();
+
+        assert_eq!(index.aggregate_corpus_stats().await.unwrap(), (100, 100));
+        assert_eq!(partition.docs.total_tokens_cached(), Some(100));
+
+        let views =
+            futures::future::join_all((0..8).map(|_| partition.docs.ensure_num_tokens_loaded()))
+                .await
+                .into_iter()
+                .collect::<Result<Vec<_>>>()
+                .unwrap();
+        let first = &views[0];
+        assert!(views.iter().all(|view| Arc::ptr_eq(first, view)));
+        assert!(!first.has_row_ids());
+        assert!(matches!(&first.num_tokens, NumTokens::Shared(_)));
+        assert_eq!(first.total_tokens_num(), 100);
+
+        let all_rows = RowAddrMask::all_rows();
+        let wand_view = partition.docs.docs_for_wand(&all_rows).await.unwrap();
+        assert!(Arc::ptr_eq(first, &wand_view));
+
+        let filtered = RowAddrMask::allow_nothing();
+        let full = partition.docs.docs_for_wand(&filtered).await.unwrap();
+        assert!(full.has_row_ids());
+        assert!(matches!(&full.num_tokens, NumTokens::Owned(_)));
+        assert_eq!(full.total_tokens_num(), 100);
+        assert_eq!(
+            partition.docs.resolve_row_ids(&[0, 99]).await.unwrap(),
+            [0, 99]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_concurrent_total_and_num_tokens_view_initialization() {
+        let (index, _counter, _tmpdir) = load_counted_v2_index(100, LanceCache::no_cache()).await;
+        let docs = index.partitions[0].docs.clone();
+
+        let totals = futures::future::join_all((0..8).map(|_| docs.total_tokens_num()));
+        let views = futures::future::join_all((0..8).map(|_| docs.ensure_num_tokens_loaded()));
+        let (totals, views) = tokio::join!(totals, views);
+
+        let totals = totals.into_iter().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(totals, vec![100; 8]);
+        let views = views.into_iter().collect::<Result<Vec<_>>>().unwrap();
+        let first = &views[0];
+        assert!(views.iter().all(|view| Arc::ptr_eq(first, view)));
+        assert!(matches!(&first.num_tokens, NumTokens::Shared(_)));
+        assert_eq!(docs.total_tokens_cached(), Some(100));
+    }
+
+    #[tokio::test]
+    async fn test_grouped_posting_lists_read_one_group_per_neighborhood() {
+        // Cold-start scoring must not bulk-read the full `0..num_tokens`
+        // metadata table. With small-posting grouping (issue #7040), scoring
+        // K adjacent cold tokens shares a single group cache entry: one
+        // read_range bounded by the group size, independent of the partition's
+        // total token count.
+        let runtime_group_size = runtime_posting_group_tokens().max(1);
+        let queried_token_count = runtime_group_size.min(4);
+        let queried_tokens = (0..queried_token_count as u32).collect::<Vec<_>>();
+        let num_tokens = runtime_group_size
+            .saturating_mul(2)
+            .max(queried_token_count + 1)
+            .min(1024);
+        let (index, counter, _tmpdir) =
+            load_counted_v2_index(num_tokens, LanceCache::no_cache()).await;
+        let inverted_list = index.partitions[0].inverted_list.clone();
+        assert!(
+            !inverted_list.is_legacy_layout(),
+            "this test only proves the lazy path for v2 indexes",
+        );
+        assert!(
+            matches!(
+                &inverted_list.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "freshly written v2 index should use runtime synthetic groups",
+        );
+
+        // This fixture uses a no-op cache, so each call re-reads; that isolates
+        // the per-query read shape. Each posting_list call reads exactly its
         // own group — bounded by the group size, never the full token table.
         let metrics = Arc::new(NoOpMetricsCollector);
-        for token_id in queried_tokens {
+        for &token_id in &queried_tokens {
             inverted_list
                 .posting_list(token_id, false, metrics.as_ref())
                 .await
@@ -5939,9 +9044,9 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group_len = (end - start) as usize;
         assert!(
-            (queried_tokens.len()..num_tokens).contains(&group_len),
-            "group [{start}, {end}) should cover the queried neighborhood but be \
-             far smaller than the {num_tokens}-token table",
+            (queried_tokens.len()..=num_tokens).contains(&group_len),
+            "group [{start}, {end}) should cover the queried neighborhood and \
+             stay bounded by the {num_tokens}-token table",
         );
         assert_eq!(
             counter.read_range_calls(),
@@ -5957,8 +9062,8 @@ mod tests {
     }
 
     /// Build a single-partition v2 index where every token's posting list spans
-    /// `docs_per_token` docs. Small `docs_per_token` yields tiny posting lists
-    /// that the writer packs densely into shared cache groups.
+    /// `docs_per_token` docs. Runtime grouping packs consecutive token rows
+    /// into shared cache groups.
     async fn load_v2_index_with_grouped_postings(
         num_tokens: usize,
         docs_per_token: usize,
@@ -6015,25 +9120,18 @@ mod tests {
         (index, cache)
     }
 
-    /// The read path decodes a posting-list group by slicing one buffer read for
-    /// the whole `[start, end)` row range, so every posting list in a cached
-    /// group shares a single `blocks` buffer. `DeepSizeOf` must count each
-    /// posting's slice of that buffer, not the whole buffer once per posting —
-    /// otherwise a group of N postings reports ~N times its real footprint.
-    #[rstest::rstest]
-    #[case::single_doc_terms(512, 1)]
-    #[case::small_terms(512, 4)]
-    #[case::medium_terms(256, 32)]
+    /// Packed groups charge their Arrow buffers and contiguous metadata once,
+    /// avoiding the per-member enum/array object graph of a materialized group.
     #[tokio::test]
-    async fn test_read_path_group_size_counts_slices_not_shared_buffer(
-        #[case] num_tokens: usize,
-        #[case] docs_per_token: usize,
-    ) {
-        let (index, _cache) = load_v2_index_with_grouped_postings(num_tokens, docs_per_token).await;
+    async fn test_packed_group_deep_size_is_smaller_than_materialized_graph() {
+        let (index, _cache) = load_v2_index_with_grouped_postings(512, 1).await;
         let inverted_list = index.partitions[0].inverted_list.clone();
         assert!(!inverted_list.is_legacy_layout(), "expected v2 layout");
         assert!(
-            inverted_list.group_starts.is_some(),
+            matches!(
+                &inverted_list.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
             "expected grouped posting lists"
         );
 
@@ -6045,22 +9143,31 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
+        assert!(group.is_packed(), "cold v2 group should use packed storage");
+        inverted_list.ensure_metadata_loaded().await.unwrap();
 
-        // Sum what counting the full backing buffer once per posting list would
-        // charge, and confirm the postings really do share a single buffer.
         let mut distinct_buffers = std::collections::HashSet::new();
-        let mut charged_if_counted_per_posting = 0usize;
-        for posting in &group.posting_lists {
+        let mut materialized = Vec::with_capacity(group.len());
+        for slot in 0..group.len() {
+            let (max_score, length) = inverted_list.bulk_metadata_for_token(start + slot as u32);
+            let posting = group
+                .posting_list(slot, max_score, length)
+                .unwrap()
+                .unwrap();
             let PostingList::Compressed(compressed) = posting else {
                 panic!("expected compressed posting lists");
             };
-            charged_if_counted_per_posting += compressed.blocks.get_buffer_memory_size();
             distinct_buffers.insert(compressed.blocks.values().as_ptr());
+            materialized.push(PostingList::Compressed(compressed));
         }
-        let posting_count = group.posting_lists.len();
+        let posting_count = materialized.len();
 
         assert!(
             posting_count > 1,
@@ -6071,13 +9178,12 @@ mod tests {
             1,
             "read-path postings in a group should share one backing buffer"
         );
-        // With slice-aware accounting the shared buffer is counted ~once, so the
-        // whole group costs far less than counting it once per posting list.
-        let reported = group.deep_size_of();
+        let packed_size = group.deep_size_of();
+        let materialized_size = PostingListGroup::new(materialized).deep_size_of();
         assert!(
-            reported < charged_if_counted_per_posting / 2,
-            "group deep_size_of {reported}B should not scale with the {posting_count}x-counted \
-             shared buffer ({charged_if_counted_per_posting}B)"
+            packed_size * 4 < materialized_size * 3,
+            "packed group deep_size_of {packed_size}B should be at least 25% smaller than the \
+             {materialized_size}B materialized graph for {posting_count} postings"
         );
     }
 
@@ -6132,6 +9238,8 @@ mod tests {
             1.0,
             SLICE_LEN as u32,
             PostingTailCodec::Fixed32,
+            LEGACY_BLOCK_SIZE,
+            None,
             None,
         );
 
@@ -6279,11 +9387,23 @@ mod tests {
         let (start, end) = inverted_list.group_range_for_token(0).unwrap();
         let group = inverted_list
             .index_cache
-            .get_with_key(&PostingListGroupKey { start, end })
+            .get_with_key(&posting_list_group_cache_key(
+                start,
+                end,
+                inverted_list.has_impacts,
+            ))
             .await
             .unwrap();
         assert!(
-            !group.get(0).unwrap().has_position(),
+            !group.is_packed(),
+            "with-position prewarm should retain the materialized fallback"
+        );
+        assert!(
+            !group
+                .posting_list(0, None, None)
+                .unwrap()
+                .unwrap()
+                .has_position(),
             "posting cache should remain positions-free after prewarm"
         );
 
@@ -6487,86 +9607,1156 @@ mod tests {
         }
     }
 
-    #[tokio::test]
-    async fn test_phrase_query_reads_legacy_per_doc_positions() {
-        let tmpdir = TempObjDir::default();
-        let store = Arc::new(LanceIndexStore::new(
-            ObjectStore::local().into(),
-            tmpdir.clone(),
-            Arc::new(LanceCache::no_cache()),
-        ));
-
-        let mut builder = InnerBuilder::new_with_format_version(
-            0,
-            true,
-            TokenSetFormat::default(),
-            InvertedListFormatVersion::V1,
-        );
-        builder.tokens.add("hello".to_owned());
-        builder.tokens.add("world".to_owned());
-        builder
-            .posting_lists
-            .push(PostingListBuilder::new_with_posting_tail_codec(
-                true,
-                PostingTailCodec::Fixed32,
-            ));
-        builder
-            .posting_lists
-            .push(PostingListBuilder::new_with_posting_tail_codec(
-                true,
-                PostingTailCodec::Fixed32,
-            ));
-        builder.posting_lists[0].add(0, PositionRecorder::Position(vec![0].into()));
-        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![1].into()));
-        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
-        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![2].into()));
-        builder.docs.append(100, 2);
-        builder.docs.append(101, 2);
-        builder.write(store.as_ref()).await.unwrap();
-
-        let metadata = std::collections::HashMap::from_iter(vec![
+    async fn write_test_metadata(
+        store: &Arc<LanceIndexStore>,
+        partition_ids: Vec<u64>,
+        params: InvertedIndexParams,
+    ) {
+        let format_version = params.resolved_format_version();
+        let metadata = HashMap::from([
             (
                 "partitions".to_owned(),
-                serde_json::to_string(&vec![0_u64]).unwrap(),
-            ),
-            (
-                "params".to_owned(),
-                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
+                serde_json::to_string(&partition_ids).unwrap(),
             ),
+            ("params".to_owned(), serde_json::to_string(&params).unwrap()),
             (
                 TOKEN_SET_FORMAT_KEY.to_owned(),
                 TokenSetFormat::default().to_string(),
             ),
+            (
+                POSTING_TAIL_CODEC_KEY.to_owned(),
+                format_version.posting_tail_codec().as_str().to_owned(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_owned(),
+                format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_owned(),
+                params.posting_block_size().to_string(),
+            ),
         ]);
         let mut writer = store
             .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
             .await
             .unwrap();
         writer.finish_with_metadata(metadata).await.unwrap();
+    }
+
+    async fn write_test_partition_with_optional_impacts(
+        store: &Arc<LanceIndexStore>,
+        partition_id: u64,
+        mut builder: InnerBuilder,
+        token_set_format: TokenSetFormat,
+        with_impacts: bool,
+    ) {
+        let format_version = InvertedListFormatVersion::V1;
+        let block_size = LEGACY_BLOCK_SIZE;
+        let docs = std::mem::take(&mut builder.docs);
+        let schema = inverted_list_schema_for_version_with_block_size_and_impacts(
+            false,
+            format_version,
+            block_size,
+            with_impacts,
+        );
+
+        let mut posting_writer = store
+            .new_index_file(&posting_file_path(partition_id), schema.clone())
+            .await
+            .unwrap();
+        for posting_list in std::mem::take(&mut builder.posting_lists) {
+            let batch = posting_list
+                .to_batch_with_docs(&docs, schema.clone())
+                .unwrap();
+            posting_writer.write_record_batch(batch).await.unwrap();
+        }
+        posting_writer.finish().await.unwrap();
+
+        let token_batch = std::mem::take(&mut builder.tokens)
+            .to_batch(token_set_format)
+            .unwrap();
+        let mut token_writer = store
+            .new_index_file(&token_file_path(partition_id), token_batch.schema())
+            .await
+            .unwrap();
+        token_writer.write_record_batch(token_batch).await.unwrap();
+        token_writer.finish().await.unwrap();
+
+        let doc_batch = docs.to_batch().unwrap();
+        let mut doc_writer = store
+            .new_index_file(&doc_file_path(partition_id), doc_batch.schema())
+            .await
+            .unwrap();
+        doc_writer.write_record_batch(doc_batch).await.unwrap();
+        doc_writer.finish().await.unwrap();
+    }
+
+    async fn load_global_scoring_test_index(
+        second_partition_has_impacts: bool,
+    ) -> (TempObjDir, Arc<InvertedIndex>) {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let partition_specs = [
+            (0, 100, 5_000, 101..111, 5_000, true),
+            (1, 200, 1_000, 201..301, 1, second_partition_has_impacts),
+        ];
+        for (
+            partition_id,
+            matching_row_id,
+            matching_doc_length,
+            other_row_ids,
+            other_doc_length,
+            with_impacts,
+        ) in partition_specs
+        {
+            let mut builder = InnerBuilder::new_with_format_version(
+                partition_id,
+                false,
+                TokenSetFormat::default(),
+                InvertedListFormatVersion::V1,
+            );
+            builder.tokens.add("alpha".to_owned());
+            builder
+                .posting_lists
+                .push(PostingListBuilder::new_with_posting_tail_codec(
+                    false,
+                    InvertedListFormatVersion::V1.posting_tail_codec(),
+                ));
+            builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+            builder.docs.append(matching_row_id, matching_doc_length);
+            for row_id in other_row_ids {
+                builder.docs.append(row_id, other_doc_length);
+            }
+            write_test_partition_with_optional_impacts(
+                &store,
+                partition_id,
+                builder,
+                TokenSetFormat::default(),
+                with_impacts,
+            )
+            .await;
+        }
+
+        write_test_metadata(&store, vec![0, 1], InvertedIndexParams::default()).await;
+        let cache = LanceCache::with_capacity(4096);
+        let index = InvertedIndex::load(store, None, &cache).await.unwrap();
+        (tmpdir, index)
+    }
+
+    async fn search_test_impact_partition(
+        partition: &InvertedPartition,
+        tokens: &Tokens,
+        params: &FtsSearchParams,
+        scorer: Arc<MemBM25Scorer>,
+        shared_threshold: Arc<AtomicU32>,
+    ) -> Vec<DocCandidate> {
+        let LoadedPostings {
+            postings,
+            grouped_expansions,
+            impact_safe,
+        } = partition
+            .load_posting_lists(
+                tokens,
+                params,
+                Operator::Or,
+                scorer.as_ref(),
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        assert!(impact_safe);
+        assert!(grouped_expansions.is_empty());
+
+        let mask = NoFilter.mask();
+        let docs_for_wand = partition.docs.docs_for_wand(mask.as_ref()).await.unwrap();
+        let mut candidates = partition
+            .bm25_search(
+                docs_for_wand.as_ref(),
+                params,
+                Operator::Or,
+                mask,
+                postings,
+                Some(scorer),
+                &NoOpMetricsCollector,
+                shared_threshold,
+            )
+            .unwrap();
+        resolve_deferred_candidates(&partition.docs, &mut candidates)
+            .await
+            .unwrap();
+        candidates
+    }
+
+    #[tokio::test]
+    async fn test_impact_partitions_share_global_threshold_without_pruning_winner() {
+        // Partition 0 wins under its local corpus statistics but loses under
+        // the global statistics. If its local score escapes into the shared
+        // floor, partition 1 will incorrectly prune the real global winner.
+        let (_tmpdir, index) = load_global_scoring_test_index(true).await;
+        let first_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 0)
+            .unwrap();
+        let second_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 1)
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(vec!["alpha".to_owned()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let scorer = Arc::new(
+            index
+                .bm25_base_scorer(tokens.as_ref(), params.as_ref())
+                .await
+                .unwrap(),
+        );
+        first_partition
+            .inverted_list
+            .ensure_metadata_loaded()
+            .await
+            .unwrap();
+        second_partition
+            .inverted_list
+            .ensure_metadata_loaded()
+            .await
+            .unwrap();
+        let first_local_scorer = IndexBM25Scorer::new(std::iter::once(first_partition.as_ref()));
+        let second_local_scorer = IndexBM25Scorer::new(std::iter::once(second_partition.as_ref()));
+        let first_local_score =
+            first_local_scorer.query_weight("alpha") * first_local_scorer.doc_weight(1, 5_000);
+        let second_local_score =
+            second_local_scorer.query_weight("alpha") * second_local_scorer.doc_weight(1, 1_000);
+        assert!(first_local_score > second_local_score);
+        let shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+
+        // Search sequentially so partition 0 deterministically publishes its
+        // score before partition 1 evaluates its impact upper bound.
+        let first_candidates = search_test_impact_partition(
+            first_partition,
+            tokens.as_ref(),
+            params.as_ref(),
+            scorer.clone(),
+            shared_threshold.clone(),
+        )
+        .await;
+        assert_eq!(first_candidates.len(), 1);
+        assert!(matches!(
+            first_candidates[0].addr,
+            CandidateAddr::RowId(100)
+        ));
+        let first_score =
+            scorer.query_weight("alpha") * scorer.doc_weight(1, first_candidates[0].doc_length);
+        let published_threshold = f32::from_bits(shared_threshold.load(Ordering::Relaxed));
+        assert!(
+            (published_threshold - first_score).abs() < 1e-6,
+            "published threshold: {published_threshold}, expected global score: {first_score}"
+        );
+
+        let second_candidates = search_test_impact_partition(
+            second_partition,
+            tokens.as_ref(),
+            params.as_ref(),
+            scorer.clone(),
+            shared_threshold.clone(),
+        )
+        .await;
+        assert_eq!(second_candidates.len(), 1);
+        assert!(matches!(
+            second_candidates[0].addr,
+            CandidateAddr::RowId(200)
+        ));
+        let second_score =
+            scorer.query_weight("alpha") * scorer.doc_weight(1, second_candidates[0].doc_length);
+        assert!(
+            second_score > first_score,
+            "second score: {second_score}, first score: {first_score}"
+        );
+        assert!(
+            (f32::from_bits(shared_threshold.load(Ordering::Relaxed)) - second_score).abs() < 1e-6
+        );
+
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(row_ids, vec![200]);
+        assert_eq!(scores.len(), 1);
+        assert!((scores[0] - second_score).abs() < 1e-6);
+    }
+
+    #[tokio::test]
+    async fn test_mixed_impact_and_legacy_partitions_use_global_final_scores() {
+        let (_tmpdir, index) = load_global_scoring_test_index(false).await;
+
+        let impact_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 0)
+            .unwrap();
+        let legacy_partition = index
+            .partitions
+            .iter()
+            .find(|partition| partition.id() == 1)
+            .unwrap();
+
+        let impact_posting = impact_partition
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(impact_posting.has_impacts());
+
+        let legacy_posting = legacy_partition
+            .inverted_list
+            .posting_list(0, false, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(!legacy_posting.has_impacts());
+
+        let tokens = Arc::new(Tokens::new(vec!["alpha".to_string()], DocType::Text));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(1)));
+        let (row_ids, scores) = index
+            .bm25_search(
+                tokens.clone(),
+                params.clone(),
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids, vec![200]);
+        assert_eq!(row_ids.len(), scores.len());
+
+        let scorer = index
+            .bm25_base_scorer(tokens.as_ref(), params.as_ref())
+            .await
+            .unwrap();
+        let expected_score = scorer.query_weight("alpha") * scorer.doc_weight(1, 1_000);
+        assert!(
+            (scores[0] - expected_score).abs() < 1e-6,
+            "score: {}, expected: {}",
+            scores[0],
+            expected_score
+        );
+    }
+
+    #[tokio::test]
+    async fn test_and_query_returns_empty_when_exact_term_missing() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.docs.append(100, 1);
+        builder.write(store.as_ref()).await.unwrap();
 
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
         let cache = Arc::new(LanceCache::with_capacity(4096));
         let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
             .await
             .unwrap();
 
         let tokens = Arc::new(Tokens::new(
-            vec!["hello".to_owned(), "world".to_owned()],
+            vec!["alpha".to_owned(), "missing".to_owned()],
             DocType::Text,
         ));
-        let params = Arc::new(
-            FtsSearchParams::new()
-                .with_limit(Some(10))
-                .with_phrase_slop(Some(0)),
-        );
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
         let prefilter = Arc::new(NoFilter);
         let metrics = Arc::new(NoOpMetricsCollector);
 
-        let (row_ids, _scores) = index
-            .bm25_search(tokens, params, Operator::And, prefilter, metrics, None)
+        let (and_row_ids, _) = index
+            .bm25_search(
+                tokens.clone(),
+                params.clone(),
+                Operator::And,
+                prefilter.clone(),
+                metrics.clone(),
+                None,
+            )
+            .await
+            .unwrap();
+        assert!(
+            and_row_ids.is_empty(),
+            "AND must not match when any required term is missing"
+        );
+
+        let (or_row_ids, _) = index
+            .bm25_search(tokens, params, Operator::Or, prefilter, metrics, None)
+            .await
+            .unwrap();
+        assert_eq!(
+            or_row_ids,
+            vec![100],
+            "OR should still match the present term"
+        );
+    }
+
+    // Enough distinct tokens that `write_posting_lists` emits several posting-list
+    // batches (the default batch size is 256 rows), exercising the restructured
+    // producer and async send path.
+    const MANY_BATCH_TOKENS: u64 = 1000;
+    const MANY_BATCH_ROW_ID_BASE: u64 = 1000;
+
+    // Writes a single partition whose posting lists span many output batches. Each
+    // token `tok{i:05}` maps to row id `MANY_BATCH_ROW_ID_BASE + i`.
+    async fn write_partition_spanning_many_batches(store: &dyn IndexStore) {
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for i in 0..MANY_BATCH_TOKENS {
+            // Zero-padded so tokens are inserted in sorted order, as the set expects.
+            builder.tokens.add(format!("tok{i:05}"));
+            let doc_id = builder.docs.append(MANY_BATCH_ROW_ID_BASE + i, 1);
+            let mut posting_list = PostingListBuilder::new(false);
+            posting_list.add(doc_id, PositionRecorder::Count(1));
+            builder.posting_lists.push(posting_list);
+        }
+        builder
+            .write(store)
+            .await
+            .expect("writing posting lists should succeed");
+    }
+
+    // Correctness guard for the restructured posting-list writer. The producer now
+    // builds each batch in its own `spawn_cpu` call, handing the builder and the
+    // remaining posting lists back out so state (the cross-batch cache-group
+    // accumulator) is preserved, and dispatches with an async `send().await`. This
+    // verifies that path over many batches by checking representative tokens after
+    // they cross the producer/consumer boundary.
+    //
+    // Note: this does not reproduce the single-thread-pool deadlock the async send
+    // fixes -- that requires a 1-thread CPU pool (a process-global singleton) plus
+    // ~8MB of buffered posting data to trigger a consumer-side encoder flush, which
+    // is impractical as a lightweight unit test.
+    #[tokio::test]
+    async fn test_write_many_posting_list_batches_preserves_all_batches() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_partition_spanning_many_batches(store.as_ref()).await;
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        // Probe tokens from the first, a middle, and the last batch to confirm they
+        // remain queryable after crossing the producer/consumer boundary.
+        for token_idx in [0u64, MANY_BATCH_TOKENS / 2, MANY_BATCH_TOKENS - 1] {
+            let tokens = Arc::new(Tokens::new(
+                vec![format!("tok{token_idx:05}")],
+                DocType::Text,
+            ));
+            let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+            let (row_ids, _) = index
+                .bm25_search(
+                    tokens,
+                    params,
+                    Operator::Or,
+                    Arc::new(NoFilter),
+                    Arc::new(NoOpMetricsCollector),
+                    None,
+                )
+                .await
+                .unwrap();
+            assert_eq!(
+                row_ids,
+                vec![MANY_BATCH_ROW_ID_BASE + token_idx],
+                "token tok{token_idx:05} should map to its single document"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_and_query_skips_partition_missing_required_term() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder0 = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder0.tokens.add("alpha".to_owned());
+        builder0.posting_lists.push(PostingListBuilder::new(false));
+        builder0.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder0.docs.append(100, 1);
+        builder0.write(store.as_ref()).await.unwrap();
+
+        let mut builder1 = InnerBuilder::new(1, false, TokenSetFormat::default());
+        builder1.tokens.add("alpha".to_owned());
+        builder1.tokens.add("beta".to_owned());
+        builder1.posting_lists.push(PostingListBuilder::new(false));
+        builder1.posting_lists.push(PostingListBuilder::new(false));
+        builder1.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder1.posting_lists[1].add(0, PositionRecorder::Count(1));
+        builder1.docs.append(200, 2);
+        builder1.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0, 1], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(
+            vec!["alpha".to_owned(), "beta".to_owned()],
+            DocType::Text,
+        ));
+        let params = Arc::new(FtsSearchParams::new().with_limit(Some(10)));
+        let (mut row_ids, _) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        row_ids.sort_unstable();
+        assert_eq!(
+            row_ids,
+            vec![200],
+            "partition missing beta must not contribute alpha-only hits"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_and_groups_expansions_by_original_position() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.tokens.add("alphi".to_owned());
+        builder.tokens.add("beta".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[1].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(1, PositionRecorder::Count(1));
+        builder.docs.append(100, 2);
+        builder.docs.append(101, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_fuzziness(Some(1)),
+        );
+
+        let missing_position_tokens = Arc::new(Tokens::new(
+            vec!["betx".to_owned(), "zzzzz".to_owned()],
+            DocType::Text,
+        ));
+        let (missing_and_row_ids, _) = index
+            .bm25_search(
+                missing_position_tokens.clone(),
+                params.clone(),
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        assert!(
+            missing_and_row_ids.is_empty(),
+            "fuzzy AND must require at least one expansion for every original position"
+        );
+
+        let (mut or_row_ids, _) = index
+            .bm25_search(
+                missing_position_tokens,
+                params.clone(),
+                Operator::Or,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        or_row_ids.sort_unstable();
+        assert_eq!(
+            or_row_ids,
+            vec![100, 101],
+            "OR should still match present fuzzy expansions"
+        );
+
+        let grouped_tokens = Arc::new(Tokens::new(
+            vec!["alphx".to_owned(), "betx".to_owned()],
+            DocType::Text,
+        ));
+        let (mut grouped_row_ids, _) = index
+            .bm25_search(
+                grouped_tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+        grouped_row_ids.sort_unstable();
+        assert_eq!(
+            grouped_row_ids,
+            vec![100, 101],
+            "each original fuzzy position should match any one of its expansions"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_expansion_cap_applies_to_whole_query() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        for token in ["alpha", "alphi", "beta", "beti"] {
+            builder.tokens.add(token.to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+        }
+        for token_id in 0..4 {
+            builder.posting_lists[token_id].add(token_id as u32, PositionRecorder::Count(1));
+            builder.docs.append(100 + token_id as u64, 1);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+        let partition = index.partitions[0].clone();
+        let params = FtsSearchParams::new()
+            .with_fuzziness(Some(1))
+            .with_max_expansions(3);
+        let tokens = Tokens::new(vec!["alphx".to_owned(), "betx".to_owned()], DocType::Text);
+
+        let expanded = partition.expand_fuzzy(&tokens, &params).unwrap();
+        let expanded_terms = (0..expanded.len())
+            .map(|idx| (expanded.get_token(idx).to_owned(), expanded.position(idx)))
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            expanded_terms,
+            vec![
+                ("alpha".to_owned(), 0),
+                ("alphi".to_owned(), 0),
+                ("beta".to_owned(), 1),
+            ],
+            "max_expansions should cap the whole fuzzy query, not each token"
+        );
+    }
+
+    /// Write one partition holding `variants` in order, with one
+    /// single-token doc per variant taken from `row_ids`.
+    async fn write_variant_partition(
+        store: &Arc<LanceIndexStore>,
+        partition_id: u64,
+        variants: &[&str],
+        row_ids: &[u64],
+    ) {
+        let mut builder = InnerBuilder::new(partition_id, false, TokenSetFormat::default());
+        for token in variants {
+            builder.tokens.add((*token).to_owned());
+            builder.posting_lists.push(PostingListBuilder::new(false));
+        }
+        for (local_idx, row_id) in row_ids.iter().enumerate() {
+            builder.posting_lists[local_idx].add(local_idx as u32, PositionRecorder::Count(1));
+            builder.docs.append(*row_id, 1);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_expansion_cap_is_global_across_partitions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        write_variant_partition(&store, 0, &["alpha", "alphb"], &[100, 101]).await;
+        write_variant_partition(&store, 1, &["alphc", "alphd"], &[102, 103]).await;
+        write_test_metadata(&store, vec![0, 1], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let params = FtsSearchParams::new()
+            .with_fuzziness(Some(1))
+            .with_max_expansions(3);
+        let tokens = Tokens::new(vec!["alphx".to_owned()], DocType::Text);
+
+        let expanded = index.expand_fuzzy_tokens(&tokens, &params).unwrap();
+        let expanded_terms = (0..expanded.len())
+            .map(|idx| expanded.get_token(idx).to_owned())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            expanded_terms,
+            vec!["alpha".to_owned(), "alphb".to_owned(), "alphc".to_owned()],
+            "max_expansions must cap the whole query across partitions, \
+             in lexicographic order"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_results_independent_of_partition_shape() {
+        // The same four single-variant docs, laid out as one partition and
+        // as two. With a binding max_expansions the two shapes must still
+        // match the same documents with the same scores.
+        let single_dir = TempObjDir::default();
+        let single_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            single_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        write_variant_partition(
+            &single_store,
+            0,
+            &["alpha", "alphb", "alphc", "alphd"],
+            &[100, 101, 102, 103],
+        )
+        .await;
+        write_test_metadata(&single_store, vec![0], InvertedIndexParams::default()).await;
+
+        let split_dir = TempObjDir::default();
+        let split_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            split_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        write_variant_partition(&split_store, 0, &["alpha", "alphb"], &[100, 101]).await;
+        write_variant_partition(&split_store, 1, &["alphc", "alphd"], &[102, 103]).await;
+        write_test_metadata(&split_store, vec![0, 1], InvertedIndexParams::default()).await;
+
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_fuzziness(Some(1))
+                .with_max_expansions(3),
+        );
+
+        let mut results = Vec::new();
+        for store in [single_store, split_store] {
+            let cache = LanceCache::with_capacity(4096);
+            let index = InvertedIndex::load(store, None, &cache).await.unwrap();
+            let tokens = Arc::new(Tokens::new(vec!["alphx".to_owned()], DocType::Text));
+            let (row_ids, scores) = index
+                .bm25_search(
+                    tokens,
+                    params.clone(),
+                    Operator::Or,
+                    Arc::new(NoFilter),
+                    Arc::new(NoOpMetricsCollector),
+                    None,
+                )
+                .await
+                .unwrap();
+            let mut scored = row_ids.into_iter().zip(scores).collect::<Vec<_>>();
+            scored.sort_unstable_by_key(|(row_id, _)| *row_id);
+            results.push(scored);
+        }
+
+        assert_eq!(
+            results[0]
+                .iter()
+                .map(|(row_id, _)| *row_id)
+                .collect::<Vec<_>>(),
+            vec![100, 101, 102],
+            "a binding cap keeps the three lexicographically smallest variants"
+        );
+        assert_eq!(
+            results[0], results[1],
+            "fuzzy results must not depend on the partition shape"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_and_scores_grouped_expansions_by_matched_token() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.tokens.add("alphi".to_owned());
+        builder.tokens.add("beta".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(2, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(3, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(4, PositionRecorder::Count(1));
+        builder.posting_lists[0].add(5, PositionRecorder::Count(1));
+        builder.posting_lists[1].add(1, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[2].add(1, PositionRecorder::Count(1));
+        builder.docs.append(100, 2);
+        builder.docs.append(101, 2);
+        builder.docs.append(102, 1);
+        builder.docs.append(103, 1);
+        builder.docs.append(104, 1);
+        builder.docs.append(105, 1);
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(
+            vec!["alphx".to_owned(), "betx".to_owned()],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(1))
+                .with_fuzziness(Some(1)),
+        );
+        let (row_ids, _scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                Arc::new(NoOpMetricsCollector),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(
+            row_ids,
+            vec![101],
+            "the rare matched expansion should outrank the common expansion"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fuzzy_and_grouped_rescore_keeps_wand_limit_bounded() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let num_docs = BLOCK_SIZE * 2 + 4;
+        let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
+        builder.tokens.add("alpha".to_owned());
+        builder.tokens.add("alphi".to_owned());
+        builder.tokens.add("beta".to_owned());
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+        builder.posting_lists.push(PostingListBuilder::new(false));
+
+        builder.posting_lists[0].add(0, PositionRecorder::Count(1));
+        builder.posting_lists[1].add(1, PositionRecorder::Count(1));
+        for doc_id in 0..num_docs {
+            builder.posting_lists[2].add(doc_id as u32, PositionRecorder::Count(1));
+            if doc_id >= 2 {
+                builder.posting_lists[0].add(doc_id as u32, PositionRecorder::Count(1));
+            }
+            let num_tokens = if doc_id < 2 { 2 } else { 100 };
+            builder.docs.append(100 + doc_id as u64, num_tokens);
+        }
+        builder.write(store.as_ref()).await.unwrap();
+
+        write_test_metadata(&store, vec![0], InvertedIndexParams::default()).await;
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(
+            vec!["alphx".to_owned(), "betx".to_owned()],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(1))
+                .with_fuzziness(Some(1)),
+        );
+        let metrics = Arc::new(LocalMetricsCollector::default());
+        let (row_ids, _scores) = index
+            .bm25_search(
+                tokens,
+                params,
+                Operator::And,
+                Arc::new(NoFilter),
+                metrics.clone(),
+                None,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(
+            row_ids,
+            vec![101],
+            "final rescoring should still rank by the matched expansion"
+        );
+        let comparisons = metrics.comparisons.load(Ordering::Relaxed);
+        assert!(
+            comparisons < num_docs,
+            "grouped fuzzy AND should not clear the WAND top-k bound and scan every candidate; comparisons={comparisons}, num_docs={num_docs}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_phrase_query_reads_legacy_per_doc_positions() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let mut builder = InnerBuilder::new_with_format_version(
+            0,
+            true,
+            TokenSetFormat::default(),
+            InvertedListFormatVersion::V1,
+        );
+        builder.tokens.add("hello".to_owned());
+        builder.tokens.add("world".to_owned());
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder
+            .posting_lists
+            .push(PostingListBuilder::new_with_posting_tail_codec(
+                true,
+                PostingTailCodec::Fixed32,
+            ));
+        builder.posting_lists[0].add(0, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(0, PositionRecorder::Position(vec![1].into()));
+        builder.posting_lists[0].add(1, PositionRecorder::Position(vec![0].into()));
+        builder.posting_lists[1].add(1, PositionRecorder::Position(vec![2].into()));
+        builder.docs.append(100, 2);
+        builder.docs.append(101, 2);
+        builder.write(store.as_ref()).await.unwrap();
+
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&vec![0_u64]).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default().with_position(true)).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        let cache = Arc::new(LanceCache::with_capacity(4096));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
+            .await
+            .unwrap();
+
+        let tokens = Arc::new(Tokens::new(
+            vec!["hello".to_owned(), "world".to_owned()],
+            DocType::Text,
+        ));
+        let params = Arc::new(
+            FtsSearchParams::new()
+                .with_limit(Some(10))
+                .with_phrase_slop(Some(0)),
+        );
+        let prefilter = Arc::new(NoFilter);
+        let metrics = Arc::new(NoOpMetricsCollector);
+
+        let (row_ids, _scores) = index
+            .bm25_search(tokens, params, Operator::And, prefilter, metrics, None)
+            .await
+            .unwrap();
+
+        assert_eq!(row_ids, vec![100]);
+    }
+
+    /// Build a multi-partition inverted index in `store` with `num_partitions`
+    /// partitions, each carrying a handful of tokens/docs.
+    async fn build_multi_partition_index(
+        store: &Arc<LanceIndexStore>,
+        num_partitions: u64,
+    ) -> (Arc<InvertedIndex>, Arc<LanceCache>) {
+        for id in 0..num_partitions {
+            let mut builder = InnerBuilder::new_with_format_version(
+                id,
+                false,
+                TokenSetFormat::default(),
+                InvertedListFormatVersion::V1,
+            );
+            // A few distinct tokens per partition so each posting file has real
+            // content to read and materialize during prewarm.
+            for t in 0..4u32 {
+                builder.tokens.add(format!("tok_{id}_{t}"));
+                let mut posting = PostingListBuilder::new_with_posting_tail_codec(
+                    false,
+                    PostingTailCodec::Fixed32,
+                );
+                let base = id * 1000 + t as u64 * 10;
+                for d in 0..5u32 {
+                    posting.add(d, PositionRecorder::Count(1));
+                    builder.docs.append(base + d as u64, 4);
+                }
+                builder.posting_lists.push(posting);
+            }
+            builder.write(store.as_ref()).await.unwrap();
+        }
+
+        let partition_ids: Vec<u64> = (0..num_partitions).collect();
+        let metadata = std::collections::HashMap::from_iter(vec![
+            (
+                "partitions".to_owned(),
+                serde_json::to_string(&partition_ids).unwrap(),
+            ),
+            (
+                "params".to_owned(),
+                serde_json::to_string(&InvertedIndexParams::default()).unwrap(),
+            ),
+            (
+                TOKEN_SET_FORMAT_KEY.to_owned(),
+                TokenSetFormat::default().to_string(),
+            ),
+        ]);
+        let mut writer = store
+            .new_index_file(METADATA_FILE, Arc::new(arrow_schema::Schema::empty()))
+            .await
+            .unwrap();
+        writer.finish_with_metadata(metadata).await.unwrap();
+
+        // Keep the cache alive and return it: the partition readers hold only a
+        // WeakLanceCache, so the prewarmed entries vanish if this Arc is dropped.
+        let cache = Arc::new(LanceCache::with_capacity(1 << 20));
+        let index = InvertedIndex::load(store.clone(), None, cache.as_ref())
             .await
             .unwrap();
+        (index, cache)
+    }
 
-        assert_eq!(row_ids, vec![100]);
+    /// The prewarm cost estimate must come from cheap object metadata (the
+    /// posting file length) without reading the posting data, and must be
+    /// monotonic in the partition's content.
+    #[tokio::test]
+    async fn test_posting_data_size_bytes_uses_file_length() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let (index, _cache) = build_multi_partition_index(&store, 3).await;
+        for part in &index.partitions {
+            // File length is reported by object metadata at open time; it must be
+            // non-trivial for a partition that actually holds postings.
+            let est = part.inverted_list.posting_data_size_bytes();
+            assert!(
+                est > 0,
+                "expected a non-zero posting-data size estimate, got {est}"
+            );
+        }
+    }
+
+    /// Each partition must read through the shared scheduler at a distinct base
+    /// priority. Tied priorities (every partition at 0) break the scheduler's
+    /// backpressure deadlock-break — which admits the lowest-priority in-flight
+    /// request — because there is no unique lowest request to advance, so a
+    /// concurrent multi-partition read (e.g. prewarm) can wedge. Distinct
+    /// per-partition priorities keep the in-flight set totally ordered.
+    #[tokio::test]
+    async fn test_partitions_load_with_distinct_priorities() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let (index, _cache) = build_multi_partition_index(&store, 5).await;
+
+        let mut priorities: Vec<u64> = index
+            .partitions
+            .iter()
+            .map(|part| {
+                part.store
+                    .as_any()
+                    .downcast_ref::<LanceIndexStore>()
+                    .expect("partition store should be a LanceIndexStore")
+                    .io_priority()
+            })
+            .collect();
+
+        // Distinct and dense (0..N): every partition reads at its own priority,
+        // so the shared scheduler sees a total order across all partitions. The
+        // partitions may finish loading in any order, so sort before comparing —
+        // what matters is that the priorities form a contiguous, collision-free
+        // set, not which partition ended up at which slot.
+        priorities.sort_unstable();
+        assert_eq!(
+            priorities,
+            (0..index.partitions.len() as u64).collect::<Vec<_>>()
+        );
     }
 
     #[tokio::test]
@@ -6654,6 +10844,70 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn test_block_size_256_writes_v3_metadata_and_index_version() -> Result<()> {
+        let src_dir = TempObjDir::default();
+        let dest_dir = TempObjDir::default();
+        let src_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            src_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let dest_store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            dest_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let params = InvertedIndexParams::default().block_size(256)?;
+        let format_version = params.resolved_format_version();
+        assert_eq!(format_version, InvertedListFormatVersion::V3);
+
+        let mut partition = InnerBuilder::new_with_format_version_and_block_size(
+            0,
+            false,
+            TokenSetFormat::default(),
+            format_version,
+            params.posting_block_size(),
+        );
+        partition.tokens.add("hello".to_owned());
+        let mut posting_list = PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+            false,
+            format_version.posting_tail_codec(),
+            params.posting_block_size(),
+        );
+        posting_list.add(0, PositionRecorder::Count(1));
+        partition.posting_lists.push(posting_list);
+        partition.docs.append(100, 1);
+        partition.write(src_store.as_ref()).await?;
+
+        write_test_metadata(&src_store, vec![0], params).await;
+
+        let index = InvertedIndex::load(src_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(index.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(
+            index.index_version(),
+            InvertedListFormatVersion::V3.index_version()
+        );
+
+        let created = index
+            .update(empty_doc_stream(), dest_store.as_ref(), None)
+            .await?;
+        assert_eq!(
+            created.index_version,
+            InvertedListFormatVersion::V3.index_version()
+        );
+
+        let updated = InvertedIndex::load(dest_store, None, &LanceCache::no_cache()).await?;
+        assert_eq!(updated.format_version(), InvertedListFormatVersion::V3);
+        assert_eq!(
+            updated.index_version(),
+            InvertedListFormatVersion::V3.index_version()
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_merge_segments_preserves_arrow_token_set_format() -> Result<()> {
         let src_dir = TempObjDir::default();
@@ -6875,58 +11129,119 @@ mod tests {
         );
     }
 
-    /// An [`IndexReader`] wrapper that hides the posting-group-offsets schema
-    /// metadata key, so a [`PostingListReader`] opened on it takes the
-    /// pre-grouping per-token fallback path (issue #7040).
-    struct GroupKeyStrippingReader {
-        inner: Arc<dyn IndexReader>,
-        schema: lance_core::datatypes::Schema,
-    }
+    #[tokio::test]
+    async fn flat_bm25_search_uses_full_document_length_for_normalization() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("text", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1])),
+                Arc::new(StringArray::from(vec![
+                    "alpha",
+                    "alpha filler filler filler filler filler filler filler filler filler",
+                ])),
+            ],
+        )
+        .unwrap();
 
-    impl GroupKeyStrippingReader {
-        fn new(inner: Arc<dyn IndexReader>) -> Self {
-            let mut schema = inner.schema().clone();
-            schema.metadata.remove(POSTING_GROUP_OFFSETS_BUF_KEY);
-            Self { inner, schema }
-        }
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer: Box<dyn LanceTokenizer> = Box::new(TextTokenizer::new(
+            TextAnalyzer::builder(SimpleTokenizer::default()).build(),
+        ));
+
+        let result_stream = flat_bm25_search_stream_with_metrics(
+            input,
+            "text".to_string(),
+            "alpha".to_string(),
+            tokenizer,
+            None,
+            100,
+            None,
+        )
+        .await
+        .unwrap();
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let row_ids = scored[ROW_ID].as_primitive::<UInt64Type>();
+        let scores = scored[SCORE_COL].as_primitive::<Float32Type>();
+
+        assert_eq!(row_ids.values(), &[0, 1]);
+        assert!(
+            scores.value(0) > scores.value(1),
+            "same term frequency should score shorter document higher; short={}, long={}",
+            scores.value(0),
+            scores.value(1)
+        );
     }
 
-    #[async_trait]
-    impl IndexReader for GroupKeyStrippingReader {
-        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
-            self.inner.read_record_batch(n, batch_size).await
-        }
-        async fn read_global_buffer(&self, index: u32) -> Result<bytes::Bytes> {
-            self.inner.read_global_buffer(index).await
-        }
-        async fn read_range(
-            &self,
-            range: std::ops::Range<usize>,
-            projection: Option<&[&str]>,
-        ) -> Result<RecordBatch> {
-            self.inner.read_range(range, projection).await
-        }
-        async fn num_batches(&self, batch_size: u64) -> u32 {
-            self.inner.num_batches(batch_size).await
-        }
-        fn num_rows(&self) -> usize {
-            self.inner.num_rows()
-        }
-        fn schema(&self) -> &lance_core::datatypes::Schema {
-            &self.schema
-        }
+    #[tokio::test]
+    async fn flat_bm25_search_treats_string_lists_as_row_documents() {
+        let mut docs_builder =
+            GenericListBuilder::<i32, _>::new(GenericStringBuilder::<i32>::new());
+        docs_builder.values().append_value("alpha");
+        docs_builder.values().append_value("alpha beta");
+        docs_builder.append(true);
+        docs_builder.values().append_value("beta");
+        docs_builder.append(true);
+        docs_builder.append(true);
+        docs_builder.values().append_null();
+        docs_builder.append(true);
+        docs_builder.append(false);
+
+        let docs = Arc::new(docs_builder.finish()) as ArrayRef;
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("text", docs.data_type().clone(), true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3, 4])) as ArrayRef,
+                docs,
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+        let tokenizer: Box<dyn LanceTokenizer> = Box::new(TextTokenizer::new(
+            TextAnalyzer::builder(SimpleTokenizer::default()).build(),
+        ));
+
+        let result_stream = flat_bm25_search_stream_with_metrics(
+            input,
+            "text".to_string(),
+            "alpha".to_string(),
+            tokenizer,
+            None,
+            100,
+            None,
+        )
+        .await
+        .unwrap();
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let scored = arrow::compute::concat_batches(&FTS_SCHEMA, &batches).unwrap();
+        let row_ids = scored[ROW_ID].as_primitive::<UInt64Type>();
+
+        assert_eq!(row_ids.values(), &[0]);
     }
 
     fn posting_entries(posting: &PostingList) -> Vec<(u64, u32)> {
         posting.iter().map(|(doc, freq, _)| (doc, freq)).collect()
     }
 
-    /// The grouped read path and the legacy per-token fallback must return
-    /// identical posting lists for every token, including at group
-    /// boundaries. Builds a single v2 partition that spans several groups,
-    /// then reads it both with and without the group offsets present.
+    /// Runtime synthetic grouping must return correct posting lists for every
+    /// token, including across synthetic group boundaries.
     #[tokio::test]
-    async fn test_posting_list_fallback_matches_grouped() {
+    async fn test_posting_list_synthetic_grouping_reads_group_boundaries() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -6934,15 +11249,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // A small token cap forces several groups regardless of the default,
-        // so the comparison exercises the partition_point math at group
-        // boundaries.
-        let num_tokens = 150u32;
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_tokens {
             builder.tokens.add(format!("t{t}"));
             let mut pl = PostingListBuilder::new(false);
@@ -6954,36 +11262,27 @@ mod tests {
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
         let cache = LanceCache::no_cache();
-        let grouped = PostingListReader::try_new(reader.clone(), &cache)
-            .await
-            .unwrap();
-        assert!(
-            grouped.group_starts.as_ref().is_some_and(|s| s.len() > 1),
-            "fixture should span multiple groups",
-        );
-
-        let stripped: Arc<dyn IndexReader> = Arc::new(GroupKeyStrippingReader::new(reader));
-        let fallback = PostingListReader::try_new(stripped, &cache).await.unwrap();
+        let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
         assert!(
-            fallback.group_starts.is_none(),
-            "stripped reader must take the per-token fallback path",
+            matches!(
+                &posting_reader.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "v2 reader must synthesize runtime posting groups",
         );
 
         let metrics = NoOpMetricsCollector;
         for token in 0..num_tokens {
-            let g = grouped.posting_list(token, false, &metrics).await.unwrap();
-            let f = fallback.posting_list(token, false, &metrics).await.unwrap();
-            assert_eq!(
-                posting_entries(&g),
-                posting_entries(&f),
-                "grouped vs fallback mismatch for token {token}",
-            );
-            assert_eq!(g.len(), f.len(), "length mismatch for token {token}");
+            let posting = posting_reader
+                .posting_list(token, false, &metrics)
+                .await
+                .unwrap();
             assert_eq!(
-                g.max_score(),
-                f.max_score(),
-                "max_score mismatch for token {token}",
+                posting_entries(&posting),
+                vec![(token as u64, 1)],
+                "synthetic grouping mismatch for token {token}",
             );
+            assert_eq!(posting.len(), 1, "length mismatch for token {token}");
         }
     }
 
@@ -7001,14 +11300,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // Small token cap so the partition spans several groups regardless of
-        // the default, exercising every group boundary including the last.
-        let num_tokens = 150u32;
+        let num_tokens = runtime_posting_group_tokens() as u32 + 4;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_tokens {
             builder.tokens.add(format!("t{t}"));
             let mut pl = PostingListBuilder::new(false);
@@ -7024,21 +11317,28 @@ mod tests {
         let cache = LanceCache::with_capacity(1 << 20);
         let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
         assert!(
-            posting_reader
-                .group_starts
-                .as_ref()
-                .is_some_and(|s| s.len() > 1),
-            "fixture should span multiple groups",
+            matches!(
+                &posting_reader.grouping,
+                PostingGrouping::SyntheticFixed { .. }
+            ),
+            "v2 reader should use runtime synthetic groups",
         );
 
-        posting_reader.prewarm_posting_lists(false).await.unwrap();
+        posting_reader
+            .prewarm_posting_lists(false, 2)
+            .await
+            .unwrap();
 
         for token in 0..num_tokens {
             let (start, end) = posting_reader.group_range_for_token(token).unwrap();
             assert!(
                 posting_reader
                     .index_cache
-                    .get_with_key(&PostingListGroupKey { start, end })
+                    .get_with_key(&posting_list_group_cache_key(
+                        start,
+                        end,
+                        posting_reader.has_impacts,
+                    ))
                     .await
                     .is_some(),
                 "prewarm did not populate group [{start}, {end}) that the read \
@@ -7055,10 +11355,10 @@ mod tests {
         );
     }
 
-    /// An empty partition writes no group-offsets buffer, so its reader takes
-    /// the per-token fallback path (issue #7040).
+    /// An empty partition has no synthetic groups because there are no token
+    /// rows to cache.
     #[tokio::test]
-    async fn test_empty_partition_has_no_group_offsets() {
+    async fn test_empty_partition_has_no_synthetic_groups() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -7070,28 +11370,20 @@ mod tests {
         builder.write(store.as_ref()).await.unwrap();
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
-        assert!(
-            !reader
-                .schema()
-                .metadata
-                .contains_key(POSTING_GROUP_OFFSETS_BUF_KEY),
-            "empty partition must not write the group-offsets metadata key",
-        );
-
         let posting_reader = PostingListReader::try_new(reader, &LanceCache::no_cache())
             .await
             .unwrap();
         assert!(
-            posting_reader.group_starts.is_none(),
-            "reader for an empty partition must use the per-token fallback path",
+            matches!(&posting_reader.grouping, PostingGrouping::None),
+            "reader for an empty partition must not create cache groups",
         );
         assert!(posting_reader.is_empty());
     }
 
-    /// A posting list that alone exceeds the group target lands in its own
-    /// `[t, t+1)` group (the clamp case) and reads back intact (issue #7040).
+    /// A large posting list can share a runtime synthetic group with neighbors;
+    /// grouping is token-count based and should still read every member intact.
     #[tokio::test]
-    async fn test_oversized_term_is_own_group_on_read() {
+    async fn test_large_posting_reads_inside_synthetic_group() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -7099,14 +11391,8 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // A tiny byte target so a modest posting trips the clamp without
-        // needing a huge fixture; the surrounding tiny terms regroup after it.
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 50,
-            max_tokens: 1000,
-        };
-        let big_docs = 30u32;
+        let big_docs = (BLOCK_SIZE * 3 + 5) as u32;
         builder.tokens.add("big".to_owned());
         let mut big = PostingListBuilder::new(false);
         for d in 0..big_docs {
@@ -7128,11 +11414,12 @@ mod tests {
         let posting_reader = PostingListReader::try_new(reader, &LanceCache::no_cache())
             .await
             .unwrap();
+        let expected_end = runtime_posting_group_tokens().min(5) as u32;
 
         assert_eq!(
             posting_reader.group_range_for_token(0),
-            Some((0, 1)),
-            "an oversized term must occupy its own single-row group",
+            Some((0, expected_end)),
+            "runtime synthetic grouping should group by token count, not posting bytes",
         );
         let big = posting_reader
             .posting_list(0, false, &NoOpMetricsCollector)
@@ -7147,11 +11434,11 @@ mod tests {
         assert_eq!(tiny.len(), 1);
     }
 
-    /// When the group offsets are absent, prewarm populates per-token
-    /// `PostingListKey` entries (the fallback path), matching what the read
-    /// path then looks up (issue #7040).
+    /// Non-empty v2 indexes should prewarm synthetic `PostingListGroupKey`
+    /// entries, matching what the read path then looks up without persisted
+    /// grouping metadata.
     #[tokio::test]
-    async fn test_prewarm_fallback_populates_per_token_entries() {
+    async fn test_prewarm_synthetic_grouping_populates_group_entries() {
         let tmpdir = TempObjDir::default();
         let store = Arc::new(LanceIndexStore::new(
             ObjectStore::local().into(),
@@ -7171,21 +11458,47 @@ mod tests {
         builder.write(store.as_ref()).await.unwrap();
 
         let reader = store.open_index_file(&posting_file_path(0)).await.unwrap();
-        let stripped: Arc<dyn IndexReader> = Arc::new(GroupKeyStrippingReader::new(reader));
         let cache = LanceCache::with_capacity(1 << 20);
-        let posting_reader = PostingListReader::try_new(stripped, &cache).await.unwrap();
-        assert!(posting_reader.group_starts.is_none());
+        let posting_reader = PostingListReader::try_new(reader, &cache).await.unwrap();
+        assert!(matches!(
+            &posting_reader.grouping,
+            PostingGrouping::SyntheticFixed { .. }
+        ));
 
-        posting_reader.prewarm_posting_lists(false).await.unwrap();
+        posting_reader
+            .prewarm_posting_lists(false, 2)
+            .await
+            .unwrap();
 
         for token_id in 0..num_tokens {
+            let (start, end) = posting_reader.group_range_for_token(token_id).unwrap();
+            let group = posting_reader
+                .index_cache
+                .get_with_key(&posting_list_group_cache_key(
+                    start,
+                    end,
+                    posting_reader.has_impacts,
+                ))
+                .await
+                .unwrap_or_else(|| {
+                    panic!(
+                        "synthetic prewarm should populate group [{start}, {end}) for token {token_id}"
+                    )
+                });
+            assert!(
+                group.is_packed(),
+                "no-position synthetic prewarm should insert a packed group"
+            );
             assert!(
                 posting_reader
                     .index_cache
-                    .get_with_key(&PostingListKey { token_id })
+                    .get_with_key(&posting_list_cache_key(
+                        token_id,
+                        posting_reader.has_impacts,
+                    ))
                     .await
-                    .is_some(),
-                "fallback prewarm should populate per-token entry {token_id}",
+                    .is_none(),
+                "synthetic prewarm should not populate per-token entry {token_id}",
             );
         }
     }
@@ -7202,15 +11515,11 @@ mod tests {
             Arc::new(LanceCache::no_cache()),
         ));
 
-        // 130 rare tokens (one doc each) plus one common token in every doc; a
-        // small token cap spreads them across several groups so scoring must
-        // index into the right group slot.
-        let num_rare = 130u32;
+        // Rare tokens (one doc each) plus one common token in every doc. The
+        // token count exceeds the runtime group size so scoring must index
+        // into the right synthetic group slot.
+        let num_rare = runtime_posting_group_tokens() as u32 + 2;
         let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default());
-        builder.group_config = PostingGroupConfig {
-            target_bytes: 4096,
-            max_tokens: 32,
-        };
         for t in 0..num_rare {
             builder.tokens.add(format!("t{t}"));
             builder.posting_lists.push(PostingListBuilder::new(false));
@@ -7257,7 +11566,7 @@ mod tests {
                 index
                     .bm25_search(
                         Arc::new(Tokens::new(vec![term], DocType::Text)),
-                        Arc::new(FtsSearchParams::new().with_limit(Some(200))),
+                        Arc::new(FtsSearchParams::new().with_limit(Some(num_rare as usize))),
                         Operator::Or,
                         Arc::new(NoFilter),
                         Arc::new(NoOpMetricsCollector),
@@ -7268,8 +11577,13 @@ mod tests {
             }
         };
 
-        let (rows_70, _) = query("t70").await;
-        assert_eq!(rows_70, vec![1070], "rare token must map to its single doc");
+        let rare_query_id = num_rare / 2;
+        let (rare_rows, _) = query(&format!("t{rare_query_id}")).await;
+        assert_eq!(
+            rare_rows,
+            vec![1000 + rare_query_id as u64],
+            "rare token must map to its single doc",
+        );
 
         // Cold vs warm cache must agree for the common (large) token.
         let (cold_rows, cold_scores) = query("common").await;
@@ -7281,4 +11595,56 @@ mod tests {
             "warm-cache scores must match cold"
         );
     }
+
+    #[tokio::test]
+    async fn flat_bm25_search_stop_word_query_over_unindexed_rows_returns_empty() {
+        let schema = Arc::new(Schema::new(vec![
+            ROW_ID_FIELD.clone(),
+            Field::new("text", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![0u64, 1, 2])),
+                Arc::new(StringArray::from(vec![
+                    "the quick brown fox",
+                    "a lazy dog",
+                    "for the win",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let input: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(vec![Ok(batch)]),
+        ));
+
+        // Analyzer with an English stop-word filter, so the query "the"
+        // tokenizes to zero terms -- exactly the production trigger.
+        let tokenizer: Box<dyn LanceTokenizer> = Box::new(TextTokenizer::new(
+            TextAnalyzer::builder(SimpleTokenizer::default())
+                .filter(StopWordFilter::new(Language::English).unwrap())
+                .build(),
+        ));
+
+        let result_stream = flat_bm25_search_stream_with_metrics(
+            input,
+            "text".to_string(),
+            "the".to_string(),
+            tokenizer,
+            None,
+            100,
+            None,
+        )
+        .await
+        .unwrap();
+
+        let batches: Vec<_> = result_stream.try_collect().await.unwrap();
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            total_rows, 0,
+            "a stop-word-only query has no searchable terms and must match nothing"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/iter.rs b/rust/lance-index/src/scalar/inverted/iter.rs
index dc07b15769c..3755fa41d80 100644
--- a/rust/lance-index/src/scalar/inverted/iter.rs
+++ b/rust/lance-index/src/scalar/inverted/iter.rs
@@ -6,10 +6,9 @@ use arrow_array::{Array, LargeBinaryArray};
 
 use super::{
     CompressedPositionStorage, PostingList, PostingTailCodec,
-    builder::BLOCK_SIZE,
     encoding::{
-        decode_position_stream_block, decompress_positions, decompress_posting_block,
-        decompress_posting_remainder,
+        MAX_POSTING_BLOCK_SIZE, decode_position_stream_block, decompress_positions,
+        decompress_posting_block, decompress_posting_remainder,
     },
 };
 
@@ -28,6 +27,7 @@ impl<'a> PostingListIterator<'a> {
                     posting.blocks.clone(),
                     posting.posting_tail_codec,
                     posting.positions.clone(),
+                    posting.block_size,
                 )))
             }
         }
@@ -82,13 +82,14 @@ pub struct CompressedPostingListIterator {
     next_block_idx: usize,
     posting_tail_codec: PostingTailCodec,
     positions: Option<CompressedPositionStorage>,
+    block_size: usize,
     idx: usize,
     doc_ids: Vec<u32>,
     frequencies: Vec<u32>,
     doc_idx_in_block: usize,
     decoded_positions: Vec<u32>,
     position_offsets: Vec<usize>,
-    buffer: [u32; BLOCK_SIZE],
+    buffer: [u32; MAX_POSTING_BLOCK_SIZE],
 }
 
 impl CompressedPostingListIterator {
@@ -97,10 +98,11 @@ impl CompressedPostingListIterator {
         blocks: LargeBinaryArray,
         posting_tail_codec: PostingTailCodec,
         positions: Option<CompressedPositionStorage>,
+        block_size: usize,
     ) -> Self {
         debug_assert!(length > 0, "length: {}", length);
         debug_assert_eq!(
-            length.div_ceil(BLOCK_SIZE),
+            length.div_ceil(block_size),
             blocks.len(),
             "length: {}, num_blocks: {}",
             length,
@@ -108,18 +110,19 @@ impl CompressedPostingListIterator {
         );
 
         Self {
-            remainder: length % BLOCK_SIZE,
+            remainder: length % block_size,
             blocks,
             next_block_idx: 0,
             posting_tail_codec,
             positions,
+            block_size,
             idx: 0,
             doc_ids: Vec::new(),
             frequencies: Vec::new(),
             doc_idx_in_block: 0,
             decoded_positions: Vec::new(),
             position_offsets: Vec::new(),
-            buffer: [0; BLOCK_SIZE],
+            buffer: [0; MAX_POSTING_BLOCK_SIZE],
         }
     }
 }
@@ -163,6 +166,7 @@ impl Iterator for CompressedPostingListIterator {
                 compressed,
                 self.remainder,
                 self.posting_tail_codec,
+                self.block_size,
                 &mut self.doc_ids,
                 &mut self.frequencies,
             );
@@ -172,6 +176,7 @@ impl Iterator for CompressedPostingListIterator {
                 &mut self.buffer,
                 &mut self.doc_ids,
                 &mut self.frequencies,
+                self.block_size,
             );
         }
         self.doc_idx_in_block = 0;
diff --git a/rust/lance-index/src/scalar/inverted/lazy_docset.rs b/rust/lance-index/src/scalar/inverted/lazy_docset.rs
new file mode 100644
index 00000000000..d062cca4f63
--- /dev/null
+++ b/rust/lance-index/src/scalar/inverted/lazy_docset.rs
@@ -0,0 +1,405 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Deferred-load wrapper around [`DocSet`].
+//!
+//! The inverted-index `DocSet` holds the per-doc `row_id` and `num_tokens`
+//! arrays for a partition. Eager loading on partition open pulls roughly
+//! 12 bytes × num_docs per partition; across thousands of partitions on
+//! cold object storage that's tens of GiB of IO before a query has even
+//! checked whether a partition contains the term it's looking for.
+//!
+//! [`LazyDocSet`] defers the load. Cheap sync getters (`len`,
+//! `total_tokens_cached`) work without IO; async getters fetch on
+//! demand and cache. Wand scoring still needs per-doc num_tokens, but
+//! only partitions that actually contribute hits pay
+//! `ensure_num_tokens_loaded`/`ensure_loaded`.
+
+use std::sync::Arc;
+
+use arrow::array::AsArray;
+use arrow::datatypes::{UInt32Type, UInt64Type};
+use arrow_array::{Array, UInt32Array, UInt64Array};
+use lance_core::ROW_ID;
+use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
+use tokio::sync::OnceCell;
+
+use crate::scalar::RowIdRemapper;
+use crate::scalar::inverted::index::{DocSet, NUM_TOKEN_COL};
+use crate::scalar::{IndexReader, IndexStore};
+use lance_select::mask::RowAddrMask;
+
+/// Lazy view over an inverted-index partition's `DocSet`.
+///
+/// Two variants:
+/// - `Loaded`: a pre-materialized DocSet (legacy paths, tests).
+///   Sync accessors return cached values; async accessors return
+///   the same DocSet.
+/// - `Deferred`: backed by an [`IndexReader`]; columns are read and
+///   cached on first request.
+pub enum LazyDocSet {
+    Loaded(LoadedDocSet),
+    Deferred(Box<DeferredDocSet>),
+}
+
+/// Pre-materialized DocSet view -- no reader, no IO.
+pub struct LoadedDocSet {
+    docs: Arc<DocSet>,
+    num_rows: usize,
+    total_tokens: u64,
+}
+
+/// Atomically published num-tokens state for deferred scoring.
+///
+/// Keeping the Arrow column and the zero-copy `DocSet` view that carries its
+/// total in one `OnceCell` prevents cancellation from exposing a partially
+/// initialized scoring cache.
+struct NumTokensSnapshot {
+    column: Arc<UInt32Array>,
+    docs: Arc<DocSet>,
+}
+
+/// Store-backed DocSet view that loads on demand and caches.
+///
+/// Holds the [`IndexStore`] and docs-file path rather than an open
+/// [`IndexReader`], so a cached partition does not pin a docs-file
+/// handle for its whole lifetime. The reader is re-opened on demand
+/// inside each column accessor and dropped when that read completes;
+/// because the resulting buffers are cached in the `OnceCell`s below,
+/// a contributing partition re-opens only on a cold miss, and a
+/// partition that never scores never opens the docs file at all after
+/// construction.
+pub struct DeferredDocSet {
+    store: Arc<dyn IndexStore>,
+    docs_path: String,
+    is_legacy: bool,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+    /// V3 (256-doc block) partitions score with quantized doc lengths; the
+    /// flag is applied to every `DocSet` this deferred set materializes.
+    quantized_scoring: bool,
+    /// Doc count cached at construction so `len()` stays sync + IO-free.
+    num_rows: usize,
+    /// `NUM_TOKEN_COL` and its zero-copy scoring view carrying the cached sum,
+    /// published together on first read.
+    num_tokens: OnceCell<NumTokensSnapshot>,
+    /// `ROW_ID` arrow buffer cached on first read.
+    row_ids_col: OnceCell<Arc<UInt64Array>>,
+    /// Full DocSet, materialized on first `ensure_loaded`.
+    full: OnceCell<Arc<DocSet>>,
+}
+
+impl std::fmt::Debug for LazyDocSet {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Loaded(l) => f
+                .debug_struct("LazyDocSet::Loaded")
+                .field("num_rows", &l.num_rows)
+                .field("total_tokens", &l.total_tokens)
+                .finish(),
+            Self::Deferred(d) => f
+                .debug_struct("LazyDocSet::Deferred")
+                .field("num_rows", &d.num_rows)
+                .field(
+                    "total_tokens_loaded",
+                    &(d.num_tokens.initialized() || d.full.initialized()),
+                )
+                .field("num_tokens_loaded", &d.num_tokens.initialized())
+                .field("full_loaded", &d.full.initialized())
+                .finish(),
+        }
+    }
+}
+
+impl DeepSizeOf for LazyDocSet {
+    fn deep_size_of_children(&self, ctx: &mut lance_core::deepsize::Context) -> usize {
+        match self {
+            Self::Loaded(l) => l.docs.deep_size_of_children(ctx),
+            Self::Deferred(d) => {
+                d.full
+                    .get()
+                    .map(|d| d.deep_size_of_children(ctx))
+                    .unwrap_or(0)
+                    + d.num_tokens
+                        .get()
+                        .map(|snapshot| {
+                            let arr: &dyn Array = snapshot.column.as_ref();
+                            snapshot.docs.deep_size_of_children(ctx)
+                                + arr.deep_size_of_children(ctx)
+                        })
+                        .unwrap_or(0)
+                    + d.row_ids_col
+                        .get()
+                        .map(|arr| {
+                            let arr: &dyn Array = arr.as_ref();
+                            arr.deep_size_of_children(ctx)
+                        })
+                        .unwrap_or(0)
+            }
+        }
+    }
+}
+
+impl LazyDocSet {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        store: Arc<dyn IndexStore>,
+        docs_path: String,
+        num_rows: usize,
+        is_legacy: bool,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        quantized_scoring: bool,
+    ) -> Self {
+        Self::Deferred(Box::new(DeferredDocSet {
+            store,
+            docs_path,
+            is_legacy,
+            frag_reuse_index,
+            quantized_scoring,
+            num_rows,
+            num_tokens: OnceCell::new(),
+            row_ids_col: OnceCell::new(),
+            full: OnceCell::new(),
+        }))
+    }
+
+    /// Wrap an already-materialized [`DocSet`]. Used by legacy paths
+    /// and tests that need to seed a partition without a reader.
+    pub fn from_loaded(docs: DocSet) -> Self {
+        let num_rows = docs.len();
+        let total_tokens = docs.total_tokens_num();
+        Self::Loaded(LoadedDocSet {
+            docs: Arc::new(docs),
+            num_rows,
+            total_tokens,
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        match self {
+            Self::Loaded(l) => l.num_rows,
+            Self::Deferred(d) => d.num_rows,
+        }
+    }
+
+    /// Sync read of cached `total_tokens`. Returns `None` for a
+    /// `Deferred` LazyDocSet that hasn't yet had any of
+    /// `total_tokens_num` / `ensure_num_tokens_loaded` / `ensure_loaded`
+    /// run. Used by sync scoring code that has already paid for one
+    /// of those async calls.
+    pub fn total_tokens_cached(&self) -> Option<u64> {
+        match self {
+            Self::Loaded(l) => Some(l.total_tokens),
+            Self::Deferred(d) => d
+                .full
+                .get()
+                .map(|docs| docs.total_tokens_num())
+                .or_else(|| {
+                    d.num_tokens
+                        .get()
+                        .map(|snapshot| snapshot.docs.total_tokens_num())
+                }),
+        }
+    }
+
+    /// True if this DocSet carries a FragReuseIndex. Callers MUST
+    /// avoid the deferred-row_id path when this is set: targeted
+    /// row_id reads return raw stored ids, bypassing the per-id
+    /// `remap_row_id` filter that `DocSet::from_columns` applies.
+    pub fn has_frag_reuse_remap(&self) -> bool {
+        match self {
+            Self::Loaded(_) => false,
+            Self::Deferred(d) => d.frag_reuse_index.is_some(),
+        }
+    }
+
+    /// Sum of `num_tokens` across all docs.
+    pub async fn total_tokens_num(&self) -> Result<u64> {
+        match self {
+            Self::Loaded(l) => Ok(l.total_tokens),
+            Self::Deferred(d) => d.total_tokens_num().await,
+        }
+    }
+
+    /// Materialize the full DocSet, including row_ids.
+    pub async fn ensure_loaded(&self) -> Result<Arc<DocSet>> {
+        match self {
+            Self::Loaded(l) => Ok(l.docs.clone()),
+            Self::Deferred(d) => d.ensure_loaded().await,
+        }
+    }
+
+    /// Materialize a DocSet that carries num_tokens but no row_ids.
+    /// Used by the deferred-row_id scoring path; the per-partition
+    /// caller resolves surviving doc_ids -> row_ids post-wand via
+    /// [`Self::resolve_row_ids`]. The tokens-only result is cached separately;
+    /// a later `ensure_loaded` must still produce a full DocSet.
+    pub async fn ensure_num_tokens_loaded(&self) -> Result<Arc<DocSet>> {
+        match self {
+            Self::Loaded(l) => Ok(l.docs.clone()),
+            Self::Deferred(d) => d.ensure_num_tokens_loaded().await,
+        }
+    }
+
+    /// Pick the right DocSet shape for a wand walk under `mask`:
+    /// the num_tokens-only deferred form when the mask is trivial
+    /// AND no FragReuseIndex needs to filter row_ids; otherwise the
+    /// full DocSet. Encapsulates the policy so callers don't have to
+    /// rederive the conditions for the targeted-read fast path.
+    pub async fn docs_for_wand(&self, mask: &RowAddrMask) -> Result<Arc<DocSet>> {
+        if mask.is_select_all() && !self.has_frag_reuse_remap() {
+            self.ensure_num_tokens_loaded().await
+        } else {
+            self.ensure_loaded().await
+        }
+    }
+
+    /// Resolve a batch of `doc_id`s to their `row_id`s. Used by the
+    /// deferred-row_id scoring path to map post-wand top-K candidates
+    /// without going through a full DocSet build.
+    ///
+    /// Not safe with a FragReuseIndex (see
+    /// [`Self::has_frag_reuse_remap`]): the targeted reads return
+    /// raw stored ids without applying the remap/skip.
+    pub async fn resolve_row_ids(&self, doc_ids: &[u32]) -> Result<Vec<u64>> {
+        match self {
+            Self::Loaded(l) => Ok(doc_ids.iter().map(|&d| l.docs.row_id(d)).collect()),
+            Self::Deferred(d) => d.resolve_row_ids(doc_ids).await,
+        }
+    }
+}
+
+impl DeferredDocSet {
+    /// Open a fresh docs-file reader. Dropped by the caller once its read
+    /// completes, so no handle is pinned across the partition's lifetime.
+    async fn reader(&self) -> Result<Arc<dyn IndexReader>> {
+        self.store.open_index_file(&self.docs_path).await
+    }
+
+    async fn total_tokens_num(&self) -> Result<u64> {
+        if let Some(full) = self.full.get() {
+            return Ok(full.total_tokens_num());
+        }
+        Ok(self.num_tokens_snapshot().await?.docs.total_tokens_num())
+    }
+
+    async fn num_tokens_snapshot(&self) -> Result<&NumTokensSnapshot> {
+        self.num_tokens
+            .get_or_try_init(|| async {
+                let reader = self.reader().await?;
+                let batch = reader
+                    .read_range(0..self.num_rows, Some(&[NUM_TOKEN_COL]))
+                    .await?;
+                let column = Arc::new(batch[NUM_TOKEN_COL].as_primitive::<UInt32Type>().clone());
+                let total_tokens = column.values().iter().map(|&n| n as u64).sum();
+                let mut docs = DocSet::from_cached_num_tokens(column.as_ref(), total_tokens);
+                docs.set_quantized_scoring(self.quantized_scoring);
+                Result::Ok(NumTokensSnapshot {
+                    column,
+                    docs: Arc::new(docs),
+                })
+            })
+            .await
+    }
+
+    async fn row_ids_column(&self) -> Result<Arc<UInt64Array>> {
+        self.row_ids_col
+            .get_or_try_init(|| async {
+                let reader = self.reader().await?;
+                let batch = reader.read_range(0..self.num_rows, Some(&[ROW_ID])).await?;
+                Result::Ok(Arc::new(batch[ROW_ID].as_primitive::<UInt64Type>().clone()))
+            })
+            .await
+            .cloned()
+    }
+
+    async fn ensure_loaded(&self) -> Result<Arc<DocSet>> {
+        let docs = self
+            .full
+            .get_or_try_init(|| async {
+                // If the stats path already pulled NUM_TOKEN_COL,
+                // read only ROW_ID and rebuild from the two columns.
+                let mut docs = if let Some(num_tokens) = self.num_tokens.get() {
+                    let row_ids = self.row_ids_column().await?;
+                    DocSet::from_columns(
+                        row_ids.as_ref(),
+                        num_tokens.column.as_ref(),
+                        self.is_legacy,
+                        self.frag_reuse_index.clone(),
+                    )?
+                } else {
+                    DocSet::load(
+                        self.reader().await?,
+                        self.is_legacy,
+                        self.frag_reuse_index.clone(),
+                    )
+                    .await?
+                };
+                docs.set_quantized_scoring(self.quantized_scoring);
+                Result::Ok(Arc::new(docs))
+            })
+            .await?
+            .clone();
+        Ok(docs)
+    }
+
+    async fn ensure_num_tokens_loaded(&self) -> Result<Arc<DocSet>> {
+        if let Some(full) = self.full.get() {
+            return Ok(full.clone());
+        }
+        Ok(self.num_tokens_snapshot().await?.docs.clone())
+    }
+
+    async fn resolve_row_ids(&self, doc_ids: &[u32]) -> Result<Vec<u64>> {
+        if let Some(full) = self.full.get()
+            && full.has_row_ids()
+        {
+            return Ok(doc_ids.iter().map(|&d| full.row_id(d)).collect());
+        }
+        if let Some(arr) = self.row_ids_col.get() {
+            return Ok(doc_ids.iter().map(|&d| arr.value(d as usize)).collect());
+        }
+        let ranges: Vec<std::ops::Range<usize>> = doc_ids
+            .iter()
+            .map(|&d| d as usize..d as usize + 1)
+            .collect();
+        let reader = self.reader().await?;
+        let batch = reader.read_ranges(&ranges, Some(&[ROW_ID])).await?;
+        let arr = batch[ROW_ID].as_primitive::<UInt64Type>();
+        Ok((0..arr.len()).map(|i| arr.value(i)).collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::scalar::lance_format::LanceIndexStore;
+    use lance_core::cache::LanceCache;
+    use lance_core::utils::tempfile::TempObjDir;
+    use lance_io::object_store::ObjectStore;
+
+    #[tokio::test]
+    async fn test_full_docset_is_a_complete_cached_snapshot() {
+        let temp_dir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            ObjectStore::local().into(),
+            temp_dir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let docs = LazyDocSet::new(store, "unused".to_owned(), 3, false, None, false);
+        assert_eq!(docs.total_tokens_cached(), None);
+
+        let row_ids = UInt64Array::from(vec![10, 20, 30]);
+        let num_tokens = UInt32Array::from(vec![3, 5, 8]);
+        let full = Arc::new(DocSet::from_columns(&row_ids, &num_tokens, false, None).unwrap());
+        let LazyDocSet::Deferred(deferred) = &docs else {
+            panic!("expected a deferred DocSet");
+        };
+        deferred.full.set(full.clone()).unwrap();
+
+        let wand_docs = docs.ensure_num_tokens_loaded().await.unwrap();
+        assert!(Arc::ptr_eq(&wand_docs, &full));
+        assert_eq!(wand_docs.total_tokens_num(), 16);
+        assert_eq!(docs.total_tokens_cached(), Some(16));
+    }
+}
diff --git a/rust/lance-index/src/scalar/inverted/scorer.rs b/rust/lance-index/src/scalar/inverted/scorer.rs
index e3fb81871ef..3a33a67ff7a 100644
--- a/rust/lance-index/src/scalar/inverted/scorer.rs
+++ b/rust/lance-index/src/scalar/inverted/scorer.rs
@@ -3,6 +3,7 @@
 
 use super::InvertedPartition;
 use std::collections::HashMap;
+use std::sync::Arc;
 
 // the Scorer trait is used to calculate the score of a token in a document
 // in general, the score is calculated as:
@@ -10,12 +11,74 @@ use std::collections::HashMap;
 pub trait Scorer: Send + Sync {
     fn query_weight(&self, token: &str) -> f32;
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32;
+
+    /// Finite upper bound for every non-negative value returned by
+    /// [`Self::doc_weight`]. Returning `None` disables score-independent
+    /// pruning where the posting format has no stored impact bounds.
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        None
+    }
+
+    /// Stable identity for the corpus-level inputs used by [`Self::doc_weight`].
+    ///
+    /// Implementations should return `Some` only when equal keys guarantee the
+    /// same document weight for every `(freq, doc_tokens)` pair. Scorers without
+    /// such an identity keep impact bounds in the query-local cache only.
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        None
+    }
+
+    /// The doc-length-dependent BM25 denominator addend, when `doc_weight`
+    /// factors as `(K1 + 1) * freq / (freq + addend)`; `None` for scorers
+    /// without that shape. Scoring hot loops use this to bake a per-norm-code
+    /// addend cache (Lucene's norm cache), which is bit-identical to calling
+    /// `doc_weight` because both paths evaluate the same expressions.
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        let _ = doc_tokens;
+        None
+    }
+}
+
+impl<T: Scorer + ?Sized> Scorer for Arc<T> {
+    fn query_weight(&self, token: &str) -> f32 {
+        self.as_ref().query_weight(token)
+    }
+
+    fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+        self.as_ref().doc_weight(freq, doc_tokens)
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        self.as_ref().doc_weight_upper_bound()
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        self.as_ref().doc_weight_cache_key()
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        self.as_ref().doc_norm(doc_tokens)
+    }
+}
+
+/// The frequency-dependent half of the BM25 doc weight; `doc_norm` is the
+/// doc-length addend (from [`Scorer::doc_norm`] or a per-norm-code cache).
+#[inline]
+pub(super) fn bm25_doc_weight_with_norm(freq: u32, doc_norm: f32) -> f32 {
+    let freq = freq as f32;
+    (K1 + 1.0) * freq / (freq + doc_norm)
 }
 
 // BM25 parameters
 pub const K1: f32 = 1.2;
 pub const B: f32 = 0.75;
 
+#[inline]
+fn bm25_doc_norm(doc_tokens: u32, avg_doc_length: f32) -> f32 {
+    let doc_tokens = doc_tokens as f32;
+    K1 * (1.0 - B + B * doc_tokens / avg_doc_length)
+}
+
 #[derive(Debug, Clone)]
 pub struct MemBM25Scorer {
     pub total_tokens: u64,
@@ -77,45 +140,56 @@ impl Scorer for MemBM25Scorer {
     }
 
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
-        let freq = freq as f32;
-        let doc_tokens = doc_tokens as f32;
-        let doc_norm = K1 * (1.0 - B + B * doc_tokens / self.avg_doc_length());
-        (K1 + 1.0) * freq / (freq + doc_norm)
+        let doc_norm = bm25_doc_norm(doc_tokens, self.avg_doc_length());
+        bm25_doc_weight_with_norm(freq, doc_norm)
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        Some(bm25_doc_norm(doc_tokens, self.avg_doc_length()))
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        Some(K1 + 1.0)
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        Some(u64::from(self.avg_doc_length().to_bits()))
     }
 }
 
 pub struct IndexBM25Scorer<'a> {
     partitions: Vec<&'a InvertedPartition>,
     num_docs: usize,
-    total_tokens: u64,
     avg_doc_length: f32,
 }
 
 impl<'a> IndexBM25Scorer<'a> {
+    /// Sync constructor. Reads each partition's cached `total_tokens` via
+    /// `LazyDocSet::total_tokens_cached()`; callers must have already
+    /// populated it (via `ensure_loaded`, `ensure_num_tokens_loaded`, or
+    /// `total_tokens_num`). Panics with a clear message otherwise — this
+    /// is the wand-scoring path where the contract is statically known.
     pub fn new(partitions: impl Iterator<Item = &'a InvertedPartition>) -> Self {
         let partitions = partitions.collect::<Vec<_>>();
         let num_docs = partitions.iter().map(|p| p.docs.len()).sum();
-        let total_tokens = partitions
+        let total_tokens: u64 = partitions
             .iter()
-            .map(|part| part.docs.total_tokens_num())
-            .sum::<u64>();
+            .map(|p| {
+                p.docs.total_tokens_cached().expect(
+                    "IndexBM25Scorer::new requires each partition's total_tokens to be \
+                     cached; call `ensure_loaded` / `ensure_num_tokens_loaded` / \
+                     `total_tokens_num` first",
+                )
+            })
+            .sum();
         let avgdl = total_tokens as f32 / num_docs as f32;
         Self {
             partitions,
             num_docs,
-            total_tokens,
             avg_doc_length: avgdl,
         }
     }
 
-    pub fn num_docs(&self) -> usize {
-        self.num_docs
-    }
-
-    pub fn total_tokens(&self) -> u64 {
-        self.total_tokens
-    }
-
     pub fn num_docs_containing_token(&self, token: &str) -> usize {
         self.partitions
             .iter()
@@ -140,10 +214,20 @@ impl Scorer for IndexBM25Scorer<'_> {
     }
 
     fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
-        let freq = freq as f32;
-        let doc_tokens = doc_tokens as f32;
-        let doc_norm = K1 * (1.0 - B + B * doc_tokens / self.avg_doc_length);
-        (K1 + 1.0) * freq / (freq + doc_norm)
+        let doc_norm = bm25_doc_norm(doc_tokens, self.avg_doc_length);
+        bm25_doc_weight_with_norm(freq, doc_norm)
+    }
+
+    fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+        Some(bm25_doc_norm(doc_tokens, self.avg_doc_length))
+    }
+
+    fn doc_weight_upper_bound(&self) -> Option<f32> {
+        Some(K1 + 1.0)
+    }
+
+    fn doc_weight_cache_key(&self) -> Option<u64> {
+        Some(u64::from(self.avg_doc_length.to_bits()))
     }
 }
 
diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs
index 6024747025b..429ad965419 100644
--- a/rust/lance-index/src/scalar/inverted/tokenizer.rs
+++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs
@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use lance_core::{Error, Result};
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Deserializer, Serialize};
 use std::{env, path::PathBuf};
 
 #[cfg(feature = "tokenizer-jieba")]
@@ -22,6 +22,10 @@ use crate::pbold;
 use crate::scalar::inverted::tokenizer::document_tokenizer::{
     JsonTokenizer, LanceTokenizer, TextTokenizer,
 };
+use crate::scalar::inverted::{
+    InvertedListFormatVersion, default_fts_format_version_for_block_size,
+    resolve_fts_format_version, validate_format_version_block_size,
+};
 pub use lance_tokenizer::Language;
 use lance_tokenizer::{
     AsciiFoldingFilter, IcuTokenizer, LowerCaser, NgramTokenizer, RawTokenizer, RemoveLongFilter,
@@ -29,6 +33,16 @@ use lance_tokenizer::{
     WhitespaceTokenizer,
 };
 
+/// Posting block size for indexes whose metadata predates configurable block sizes.
+///
+/// This must remain 128 so that legacy on-disk data is decoded correctly.
+pub const LEGACY_BLOCK_SIZE: usize = 128;
+/// Default posting block size for newly created indexes when none is configured.
+///
+/// This intentionally matches [`LEGACY_BLOCK_SIZE`] today but may evolve independently.
+pub const DEFAULT_BLOCK_SIZE: usize = 128;
+pub const VALID_BLOCK_SIZES: [usize; 2] = [128, 256];
+
 /// Tokenizer configs
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct InvertedIndexParams {
@@ -42,6 +56,7 @@ pub struct InvertedIndexParams {
     /// - `whitespace`: splits tokens on whitespace
     /// - `raw`: no tokenization
     /// - `icu`: ICU dictionary-based word segmentation
+    /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
     /// - `lindera/*`: Lindera tokenizer
     /// - `jieba/*`: Jieba tokenizer
     ///
@@ -97,6 +112,17 @@ pub struct InvertedIndexParams {
     #[serde(default)]
     pub(crate) prefix_only: bool,
 
+    /// Number of documents in each compressed posting block.
+    ///
+    /// Missing serialized values come from indexes written before this
+    /// parameter existed and must read as 128 for backwards compatibility. New
+    /// indexes currently default to 128.
+    #[serde(
+        default = "legacy_block_size",
+        deserialize_with = "deserialize_block_size"
+    )]
+    pub(crate) block_size: usize,
+
     /// Total memory limit in MiB for the build stage.
     ///
     /// This is split evenly across FTS workers at build time. By default Lance
@@ -119,6 +145,21 @@ pub struct InvertedIndexParams {
     /// The effective worker count is clamped to `[1, num_cpus - 2]`.
     #[serde(rename = "num_workers", skip_serializing, default)]
     pub(crate) num_workers: Option<usize>,
+
+    /// On-disk FTS format version to write when creating a new index.
+    ///
+    /// This is a build-time only parameter and is not persisted with the index.
+    /// If unset, Lance writes v2 for `block_size = 128` and v3 for
+    /// `block_size = 256`.
+    /// `format_version = 3` is experimental and is only valid with
+    /// `block_size = 256`.
+    #[serde(
+        rename = "format_version",
+        skip_serializing,
+        default,
+        deserialize_with = "deserialize_format_version"
+    )]
+    pub(crate) format_version: Option<InvertedListFormatVersion>,
 }
 
 impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
@@ -137,6 +178,7 @@ impl TryFrom<&InvertedIndexParams> for pbold::InvertedIndexDetails {
             min_ngram_length: params.min_ngram_length,
             max_ngram_length: params.max_ngram_length,
             prefix_only: params.prefix_only,
+            block_size: Some(params.block_size as u32),
         })
     }
 }
@@ -164,8 +206,13 @@ impl TryFrom<&pbold::InvertedIndexDetails> for InvertedIndexParams {
             min_ngram_length: details.min_ngram_length,
             max_ngram_length: details.max_ngram_length,
             prefix_only: details.prefix_only,
+            block_size: match details.block_size {
+                Some(block_size) => validate_block_size(block_size as usize)?,
+                None => LEGACY_BLOCK_SIZE,
+            },
             memory_limit_mb: defaults.memory_limit_mb,
             num_workers: defaults.num_workers,
+            format_version: defaults.format_version,
         })
     }
 }
@@ -182,6 +229,61 @@ fn default_max_ngram_length() -> u32 {
     3
 }
 
+fn legacy_block_size() -> usize {
+    LEGACY_BLOCK_SIZE
+}
+
+fn invalid_block_size_message(block_size: usize) -> String {
+    format!("FTS inverted index block_size must be one of 128 or 256, got {block_size}")
+}
+
+pub fn validate_block_size(block_size: usize) -> Result<usize> {
+    if VALID_BLOCK_SIZES.contains(&block_size) {
+        Ok(block_size)
+    } else {
+        Err(Error::invalid_input(invalid_block_size_message(block_size)))
+    }
+}
+
+fn deserialize_block_size<'de, D>(deserializer: D) -> std::result::Result<usize, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let block_size = usize::deserialize(deserializer)?;
+    validate_block_size(block_size).map_err(serde::de::Error::custom)
+}
+
+fn deserialize_format_version<'de, D>(
+    deserializer: D,
+) -> std::result::Result<Option<InvertedListFormatVersion>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let value = Option::<serde_json::Value>::deserialize(deserializer)?;
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    match value {
+        serde_json::Value::Null => Ok(None),
+        serde_json::Value::String(value) => resolve_fts_format_version(Some(&value))
+            .map(Some)
+            .map_err(serde::de::Error::custom),
+        serde_json::Value::Number(value) => {
+            let Some(format_version) = value.as_u64() else {
+                return Err(serde::de::Error::custom(format!(
+                    "FTS format_version must be 1, 2, or 3, got {value}"
+                )));
+            };
+            resolve_fts_format_version(Some(&format_version.to_string()))
+                .map(Some)
+                .map_err(serde::de::Error::custom)
+        }
+        other => Err(serde::de::Error::custom(format!(
+            "FTS format_version must be 1, 2, or 3, got {other}"
+        ))),
+    }
+}
+
 impl Default for InvertedIndexParams {
     fn default() -> Self {
         Self::new("simple".to_owned(), Language::English)
@@ -197,6 +299,7 @@ impl InvertedIndexParams {
     /// - `raw`: no tokenization
     /// - `ngram`: N-Gram tokenizer
     /// - `icu`: ICU dictionary-based word segmentation
+    /// - `icu/split`: ICU segmentation with simple-style delimiter splitting
     /// - `lindera/*`: Lindera tokenizer
     /// - `jieba/*`: Jieba tokenizer
     ///
@@ -218,8 +321,10 @@ impl InvertedIndexParams {
             min_ngram_length: default_min_ngram_length(),
             max_ngram_length: default_max_ngram_length(),
             prefix_only: false,
+            block_size: DEFAULT_BLOCK_SIZE,
             memory_limit_mb: None,
             num_workers: None,
+            format_version: None,
         }
     }
 
@@ -308,6 +413,25 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set the compressed posting block size.
+    ///
+    /// Supported values are 128 and 256. Larger values reduce block-max metadata
+    /// and WAND skip granularity; smaller values preserve the legacy layout.
+    ///
+    /// `block_size = 256` is experimental and may introduce breaking changes.
+    /// Use `128` when stable compatibility with the legacy posting layout is required.
+    pub fn block_size(mut self, block_size: usize) -> Result<Self> {
+        self.block_size = validate_block_size(block_size)?;
+        Ok(self)
+    }
+
+    /// Get the compressed posting block size.
+    ///
+    /// `256` is experimental and may introduce breaking changes.
+    pub fn posting_block_size(&self) -> usize {
+        self.block_size
+    }
+
     pub fn memory_limit_mb(mut self, memory_limit_mb: u64) -> Self {
         self.memory_limit_mb = Some(memory_limit_mb);
         self
@@ -322,6 +446,33 @@ impl InvertedIndexParams {
         self
     }
 
+    /// Set the on-disk FTS format version to use when creating a new index.
+    ///
+    /// If unset, Lance writes v2 for `block_size = 128` and v3 for
+    /// `block_size = 256`. Existing indexes keep their own on-disk format
+    /// during update and optimize operations.
+    /// `format_version = 3` is experimental and is only valid with
+    /// `block_size = 256`.
+    pub fn format_version(mut self, format_version: InvertedListFormatVersion) -> Self {
+        self.format_version = Some(format_version);
+        self
+    }
+
+    /// Resolve the requested FTS format version, falling back to the default for
+    /// the configured block size.
+    pub fn resolved_format_version(&self) -> InvertedListFormatVersion {
+        self.format_version.unwrap_or_else(|| {
+            default_fts_format_version_for_block_size(self.block_size)
+                .expect("InvertedIndexParams block_size must be validated before use")
+        })
+    }
+
+    /// Validate that the requested FTS format version can safely encode the
+    /// configured posting block size.
+    pub fn validate_format_version(&self) -> Result<()> {
+        validate_format_version_block_size(self.resolved_format_version(), self.block_size)
+    }
+
     /// Serialize params for the build/training path, including build-only fields.
     pub fn to_training_json(&self) -> serde_json::Result<serde_json::Value> {
         let mut value = serde_json::to_value(self)?;
@@ -340,9 +491,34 @@ impl InvertedIndexParams {
                 serde_json::Value::from(num_workers),
             );
         }
+        if let Some(format_version) = self.format_version {
+            object.insert(
+                "format_version".to_string(),
+                serde_json::Value::from(format_version.index_version()),
+            );
+        }
         Ok(value)
     }
 
+    /// Deserialize params for new index training, using current creation defaults
+    /// for omitted fields.
+    pub(crate) fn from_training_json(params: &str) -> Result<Self> {
+        let supplied = serde_json::from_str::<serde_json::Value>(params)?;
+        let mut value = serde_json::to_value(Self::default())?;
+
+        let supplied = supplied.as_object().ok_or_else(|| {
+            Error::invalid_input("FTS inverted index params must be a JSON object".to_string())
+        })?;
+        let object = value
+            .as_object_mut()
+            .expect("inverted index params should serialize to a JSON object");
+        object.extend(supplied.clone());
+
+        let params: Self = serde_json::from_value(value)?;
+        params.validate_format_version()?;
+        Ok(params)
+    }
+
     pub fn build(&self) -> Result<Box<dyn LanceTokenizer>> {
         let mut builder = self.build_base_tokenizer()?;
         if let Some(max_token_length) = self.max_token_length {
@@ -355,16 +531,7 @@ impl InvertedIndexParams {
             builder = builder.filter_dynamic(Stemmer::new(self.language));
         }
         if self.remove_stop_words {
-            let stop_word_filter = match &self.custom_stop_words {
-                Some(words) => StopWordFilter::remove(words.iter().cloned()),
-                None => StopWordFilter::new(self.language).ok_or_else(|| {
-                    Error::invalid_input(format!(
-                        "removing stop words for language {:?} is not supported yet",
-                        self.language
-                    ))
-                })?,
-            };
-            builder = builder.filter_dynamic(stop_word_filter);
+            builder = builder.filter_dynamic(self.stop_word_filter()?);
         }
         if self.ascii_folding {
             builder = builder.filter_dynamic(AsciiFoldingFilter);
@@ -382,12 +549,30 @@ impl InvertedIndexParams {
         }
     }
 
+    fn stop_word_filter(&self) -> Result<StopWordFilter> {
+        match &self.custom_stop_words {
+            Some(words) => Ok(StopWordFilter::remove(words.iter().cloned())),
+            None if self.base_tokenizer == "icu" || self.base_tokenizer == "icu/split" => {
+                Ok(StopWordFilter::all())
+            }
+            None => StopWordFilter::new(self.language).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "removing stop words for language {:?} is not supported yet",
+                    self.language
+                ))
+            }),
+        }
+    }
+
     fn build_base_tokenizer(&self) -> Result<TextAnalyzerBuilder> {
         match self.base_tokenizer.as_str() {
             "simple" => Ok(TextAnalyzer::builder(SimpleTokenizer::default()).dynamic()),
             "whitespace" => Ok(TextAnalyzer::builder(WhitespaceTokenizer::default()).dynamic()),
             "raw" => Ok(TextAnalyzer::builder(RawTokenizer::default()).dynamic()),
             "icu" => Ok(TextAnalyzer::builder(IcuTokenizer::default()).dynamic()),
+            "icu/split" => {
+                Ok(TextAnalyzer::builder(IcuTokenizer::default().with_simple_split()).dynamic())
+            }
             "ngram" => {
                 let tokenizer = NgramTokenizer::new(
                     self.min_ngram_length as usize,
@@ -439,17 +624,22 @@ pub fn language_model_home() -> Option<PathBuf> {
 
 #[cfg(test)]
 mod tests {
-    use super::InvertedIndexParams;
-    use lance_tokenizer::TokenStream;
+    use crate::pbold;
+
+    use super::{InvertedIndexParams, InvertedListFormatVersion};
+    use lance_tokenizer::{Language, TokenStream};
+    use rstest::rstest;
 
     #[test]
     fn test_build_only_fields_are_not_serialized() {
         let params = InvertedIndexParams::default()
             .memory_limit_mb(4096)
-            .num_workers(7);
+            .num_workers(7)
+            .format_version(InvertedListFormatVersion::V1);
         let json = serde_json::to_value(&params).unwrap();
         assert!(json.get("memory_limit").is_none());
         assert!(json.get("num_workers").is_none());
+        assert!(json.get("format_version").is_none());
     }
 
     #[test]
@@ -471,23 +661,162 @@ mod tests {
         let obj = json.as_object_mut().unwrap();
         obj.insert("memory_limit".to_string(), serde_json::Value::from(4096));
         obj.insert("num_workers".to_string(), serde_json::Value::from(3));
+        obj.insert("format_version".to_string(), serde_json::Value::from("v1"));
 
         let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
         assert_eq!(params.memory_limit_mb, Some(4096));
         assert_eq!(params.num_workers, Some(3));
+        assert_eq!(params.format_version, Some(InvertedListFormatVersion::V1));
     }
 
     #[test]
     fn test_training_json_serializes_build_only_fields() {
         let params = InvertedIndexParams::default()
             .memory_limit_mb(4096)
-            .num_workers(3);
+            .num_workers(3)
+            .format_version(InvertedListFormatVersion::V1);
         let json = params.to_training_json().unwrap();
         assert_eq!(
             json.get("memory_limit"),
             Some(&serde_json::Value::from(4096))
         );
         assert_eq!(json.get("num_workers"), Some(&serde_json::Value::from(3)));
+        assert_eq!(
+            json.get("format_version"),
+            Some(&serde_json::Value::from(1))
+        );
+    }
+
+    #[test]
+    fn test_default_format_version_resolves_to_v2() {
+        assert_eq!(
+            InvertedIndexParams::default().resolved_format_version(),
+            InvertedListFormatVersion::V2
+        );
+    }
+
+    #[test]
+    fn test_block_size_256_defaults_to_v3() {
+        assert_eq!(
+            InvertedIndexParams::default()
+                .block_size(256)
+                .unwrap()
+                .resolved_format_version(),
+            InvertedListFormatVersion::V3
+        );
+    }
+
+    #[test]
+    fn test_format_version_must_match_block_size() {
+        InvertedIndexParams::default()
+            .format_version(InvertedListFormatVersion::V2)
+            .validate_format_version()
+            .unwrap();
+        InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .validate_format_version()
+            .unwrap();
+
+        let err = InvertedIndexParams::default()
+            .block_size(256)
+            .unwrap()
+            .format_version(InvertedListFormatVersion::V2)
+            .validate_format_version()
+            .unwrap_err();
+        assert!(err.to_string().contains("block_size=256"));
+
+        let err = InvertedIndexParams::default()
+            .format_version(InvertedListFormatVersion::V3)
+            .validate_format_version()
+            .unwrap_err();
+        assert!(err.to_string().contains("format_version=3"));
+    }
+
+    #[test]
+    fn test_training_json_rejects_incompatible_format_version_and_block_size() {
+        let err =
+            InvertedIndexParams::from_training_json(r#"{"block_size": 256, "format_version": 2}"#)
+                .unwrap_err();
+        assert!(err.to_string().contains("block_size=256"));
+    }
+
+    #[test]
+    fn test_training_json_invalid_numeric_format_version_includes_value() {
+        let err = InvertedIndexParams::from_training_json(r#"{"format_version": -1}"#).unwrap_err();
+        assert!(matches!(&err, lance_core::Error::Arrow { .. }));
+        assert!(err.to_string().contains("got -1"));
+    }
+
+    #[test]
+    fn test_block_size_default_serializes() {
+        let params = InvertedIndexParams::default();
+        assert_eq!(params.block_size, 128);
+        let json = serde_json::to_value(&params).unwrap();
+        assert_eq!(json.get("block_size"), Some(&serde_json::Value::from(128)));
+    }
+
+    #[test]
+    fn test_block_size_missing_metadata_falls_back_to_128() {
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        json.as_object_mut().unwrap().remove("block_size");
+
+        let params: InvertedIndexParams = serde_json::from_value(json).unwrap();
+        assert_eq!(params.block_size, 128);
+    }
+
+    #[test]
+    fn test_block_size_details_conversion() {
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let details = pbold::InvertedIndexDetails::try_from(&params).unwrap();
+        assert_eq!(details.block_size, Some(256));
+
+        let old_details = pbold::InvertedIndexDetails {
+            base_tokenizer: Some("simple".to_string()),
+            language: serde_json::to_string(&Language::English).unwrap(),
+            with_position: false,
+            max_token_length: Some(40),
+            lower_case: true,
+            stem: true,
+            remove_stop_words: true,
+            ascii_folding: true,
+            min_ngram_length: 3,
+            max_ngram_length: 3,
+            prefix_only: false,
+            block_size: None,
+        };
+        let params = InvertedIndexParams::try_from(&old_details).unwrap();
+        assert_eq!(params.block_size, 128);
+    }
+
+    #[rstest]
+    #[case::block_size_128(128)]
+    #[case::block_size_256(256)]
+    fn test_block_size_accepts_supported_values(#[case] block_size: usize) {
+        let params = InvertedIndexParams::default()
+            .block_size(block_size)
+            .unwrap();
+        assert_eq!(params.block_size, block_size);
+
+        let roundtrip: InvertedIndexParams =
+            serde_json::from_value(serde_json::to_value(&params).unwrap()).unwrap();
+        assert_eq!(roundtrip.block_size, block_size);
+    }
+
+    #[test]
+    fn test_block_size_rejects_invalid_values() {
+        let err = InvertedIndexParams::default().block_size(129).unwrap_err();
+        assert!(err.to_string().contains("block_size"));
+
+        let err = InvertedIndexParams::default().block_size(512).unwrap_err();
+        assert!(err.to_string().contains("128 or 256"));
+
+        let mut json = serde_json::to_value(InvertedIndexParams::default()).unwrap();
+        json.as_object_mut()
+            .unwrap()
+            .insert("block_size".to_string(), serde_json::Value::from(1024));
+        let err = serde_json::from_value::<InvertedIndexParams>(json).unwrap_err();
+        assert!(err.to_string().contains("128 or 256"));
     }
 
     #[test]
@@ -503,4 +832,142 @@ mod tests {
         stream.process(&mut |token| tokens.push(token.text.clone()));
         assert_eq!(tokens, vec!["hello", "こんにちは", "世界"]);
     }
+
+    #[test]
+    fn test_build_icu_tokenizer_with_split_on_non_alphanumeric() {
+        let mut tokenizer = InvertedIndexParams::default()
+            .base_tokenizer("icu/split".to_string())
+            .stem(false)
+            .remove_stop_words(false)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_doc("hello_world こんにちは世界 alpha.beta");
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.text.clone()));
+        assert_eq!(
+            tokens,
+            vec!["hello", "world", "こんにちは", "世界", "alpha", "beta"]
+        );
+    }
+
+    #[test]
+    fn test_remove_stop_words_respects_language_for_non_icu_tokenizer() {
+        let mut tokenizer = InvertedIndexParams::default()
+            .stem(false)
+            .base_tokenizer("simple".to_string())
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("the 的 lance data");
+        let mut tokens = Vec::new();
+        while let Some(token) = stream.next() {
+            tokens.push(token.text.clone());
+        }
+        assert_eq!(
+            tokens,
+            vec!["的".to_string(), "lance".to_string(), "data".to_string()]
+        );
+    }
+
+    #[test]
+    fn test_custom_stop_words_replace_language_builtins() {
+        let mut tokenizer = InvertedIndexParams::default()
+            .stem(false)
+            .custom_stop_words(Some(vec!["lance".to_string()]))
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("the lance data");
+        let mut tokens = Vec::new();
+        while let Some(token) = stream.next() {
+            tokens.push(token.text.clone());
+        }
+        assert_eq!(tokens, vec!["the".to_string(), "data".to_string()]);
+    }
+
+    #[rstest]
+    #[case::icu("icu")]
+    #[case::icu_split("icu/split")]
+    fn test_icu_stop_words_use_all_builtin_lists(#[case] base_tokenizer: &str) {
+        let mut tokenizer = InvertedIndexParams::default()
+            .stem(false)
+            .base_tokenizer(base_tokenizer.to_string())
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("the 的 lance data");
+        let mut tokens = Vec::new();
+        while let Some(token) = stream.next() {
+            tokens.push(token.text.clone());
+        }
+        assert_eq!(tokens, vec!["lance".to_string(), "data".to_string()]);
+    }
+
+    // Common English pronouns/function words such as `you`/`my`/`your`/`we`
+    // must be removed by the ICU `all()` stop-word path. These are among the
+    // highest-frequency tokens, so leaking them builds pathologically large
+    // single-term posting lists (and previously overflowed the u32 posting-list
+    // size counter, panicking the whole index build). The leak is independent
+    // of stemming, so we assert it for both stem=false and stem=true.
+    #[rstest]
+    #[case::icu_no_stem("icu", false)]
+    #[case::icu_stem("icu", true)]
+    #[case::icu_split_no_stem("icu/split", false)]
+    #[case::icu_split_stem("icu/split", true)]
+    // `simple` is the recommended tokenizer for monolingual English corpora and
+    // uses StopWordFilter::new(English) rather than the ICU all() path, so it
+    // must be covered too.
+    #[case::simple_no_stem("simple", false)]
+    #[case::simple_stem("simple", true)]
+    fn test_icu_common_english_stop_words_do_not_leak(
+        #[case] base_tokenizer: &str,
+        #[case] stem: bool,
+    ) {
+        let mut tokenizer = InvertedIndexParams::default()
+            .base_tokenizer(base_tokenizer.to_string())
+            .stem(stem)
+            .remove_stop_words(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("you my your we lance data");
+        let tokens: Vec<String> = std::iter::from_fn(|| stream.next().map(|t| t.text.clone()))
+            .filter(|t| matches!(t.as_str(), "you" | "my" | "your" | "we"))
+            .collect();
+        assert!(
+            tokens.is_empty(),
+            "common English stop words leaked through the icu pipeline (stem={stem}): {tokens:?}"
+        );
+    }
+
+    // Common Chinese function words/particles (了 是 在 的 和 有 我) are the
+    // highest-frequency Chinese tokens; like the English pronouns they must be
+    // removed by the ICU `all()` stop-word path so they don't build huge
+    // posting lists. Real content words (英语 = "English", 数据 = "data") must
+    // survive. ICU dictionary segmentation splits the input into words, so this
+    // exercises the CJK stop-word path end to end.
+    #[rstest]
+    #[case::icu("icu")]
+    #[case::icu_split("icu/split")]
+    fn test_icu_common_chinese_stop_words_do_not_leak(#[case] base_tokenizer: &str) {
+        let mut tokenizer = InvertedIndexParams::default()
+            .base_tokenizer(base_tokenizer.to_string())
+            .stem(true)
+            .remove_stop_words(true)
+            .build()
+            .unwrap();
+        let mut stream = tokenizer.token_stream_for_search("我 在 有 了 是 的 和 英语 数据");
+        let tokens: Vec<String> =
+            std::iter::from_fn(|| stream.next().map(|t| t.text.clone())).collect();
+        let stop = ["我", "在", "有", "了", "是", "的", "和"];
+        let leaked: Vec<&String> = tokens
+            .iter()
+            .filter(|t| stop.contains(&t.as_str()))
+            .collect();
+        assert!(
+            leaked.is_empty(),
+            "common Chinese stop words leaked through the icu pipeline: {leaked:?} (all tokens: {tokens:?})"
+        );
+        // The real content words must still be indexed.
+        assert!(
+            tokens.iter().any(|t| t == "英语"),
+            "content word 英语 was dropped: {tokens:?}"
+        );
+    }
 }
diff --git a/rust/lance-index/src/scalar/inverted/wand.rs b/rust/lance-index/src/scalar/inverted/wand.rs
index 2f14ecc7da0..465d0f8e2ab 100644
--- a/rust/lance-index/src/scalar/inverted/wand.rs
+++ b/rust/lance-index/src/scalar/inverted/wand.rs
@@ -2,31 +2,37 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use std::ops::Deref;
+use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, LazyLock};
-use std::{cell::UnsafeCell, collections::BinaryHeap};
+use std::{
+    cell::{RefCell, UnsafeCell},
+    collections::{BinaryHeap, VecDeque},
+};
 use std::{cmp::Reverse, fmt::Debug};
 
 use arrow::array::AsArray;
 use arrow::datatypes::Int32Type;
 use arrow_array::Array;
 use itertools::Itertools;
-use lance_core::Result;
 use lance_core::utils::address::RowAddress;
+use lance_core::{Error, Result};
 use lance_select::RowAddrMask;
 
 use crate::metrics::MetricsCollector;
 
 use super::{
     CompressedPositionStorage,
+    impact::{IMPACT_LEVEL1_BLOCKS, ImpactScoreCache, ImpactSkipData},
+    index::{PositionStreamCodec, dequantize_doc_length},
     query::Operator,
-    scorer::{K1, idf},
+    scorer::{K1, bm25_doc_weight_with_norm, idf},
 };
 use super::{
     CompressedPostingList, DocSet, PostingList, RawDocInfo,
     builder::ScoredDoc,
     encoding::{
-        decode_position_stream_block, decompress_positions, decompress_posting_block,
-        decompress_posting_remainder,
+        MAX_POSTING_BLOCK_SIZE, decode_position_stream_block, decompress_positions,
+        decompress_posting_block, decompress_posting_remainder, seek_packed_doc_positions,
     },
     query::FtsSearchParams,
     scorer::Scorer,
@@ -35,12 +41,303 @@ use super::{DocInfo, builder::BLOCK_SIZE};
 
 const TERMINATED_DOC_ID: u64 = u64::MAX;
 
+/// Top-k heap entry: (scored doc, doc length, posting doc id, frequency slot).
+/// The (term, freq) pairs live outside the heap so heap churn moves a compact
+/// tuple instead of a `Vec`.
+type TopKHeap = BinaryHeap<Reverse<(ScoredDoc, u32, u64, u32)>>;
+
+/// Reusable (term, freq) storage for live heap candidates. Replacing the k-th
+/// result reuses its slot and retained `Vec` capacity, so memory stays bounded
+/// by the live top-k rather than every historical heap admission.
+struct FrequencySlots {
+    slots: Vec<Vec<(u32, u32)>>,
+}
+
+impl FrequencySlots {
+    fn with_capacity(capacity: usize) -> Self {
+        Self {
+            slots: Vec::with_capacity(capacity),
+        }
+    }
+
+    fn push(&mut self, pairs: impl Iterator<Item = (u32, u32)>) -> Result<u32> {
+        let slot = u32::try_from(self.slots.len()).map_err(|_| {
+            Error::internal(format!(
+                "FTS top-k frequency slot count {} exceeds u32::MAX",
+                self.slots.len()
+            ))
+        })?;
+        self.slots.push(pairs.collect());
+        Ok(slot)
+    }
+
+    fn replace(&mut self, slot: u32, pairs: impl Iterator<Item = (u32, u32)>) -> Result<()> {
+        let num_slots = self.slots.len();
+        let slot = self.slots.get_mut(slot as usize).ok_or_else(|| {
+            Error::internal(format!(
+                "FTS top-k frequency slot {slot} is out of bounds for {num_slots} slots"
+            ))
+        })?;
+        slot.clear();
+        slot.extend(pairs);
+        Ok(())
+    }
+
+    fn take(&mut self, slot: u32) -> Result<Vec<(u32, u32)>> {
+        let num_slots = self.slots.len();
+        self.slots
+            .get_mut(slot as usize)
+            .map(std::mem::take)
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "FTS top-k frequency slot {slot} is out of bounds for {num_slots} slots"
+                ))
+            })
+    }
+}
+
+/// Owns the top-k heap and the frequency slots referenced by its entries.
+struct TopKCollector {
+    limit: usize,
+    heap: TopKHeap,
+    frequency_slots: FrequencySlots,
+}
+
+impl TopKCollector {
+    fn new(limit: usize, initial_capacity: usize) -> Self {
+        let initial_capacity = initial_capacity.min(limit);
+        Self {
+            limit,
+            heap: BinaryHeap::with_capacity(initial_capacity),
+            frequency_slots: FrequencySlots::with_capacity(initial_capacity),
+        }
+    }
+
+    /// Insert a competitive result. When the heap is full, its evicted entry's
+    /// frequency slot is cleared and reused before the replacement is pushed.
+    fn insert(
+        &mut self,
+        doc: ScoredDoc,
+        doc_length: u32,
+        posting_doc_id: u64,
+        pairs: impl Iterator<Item = (u32, u32)>,
+    ) -> Result<bool> {
+        if self.limit == 0 {
+            return Ok(false);
+        }
+        if self.heap.len() > self.limit {
+            return Err(Error::internal(format!(
+                "FTS top-k heap length {} exceeds limit {}",
+                self.heap.len(),
+                self.limit
+            )));
+        }
+
+        let frequency_slot = if self.heap.len() == self.limit {
+            let Some(kth_score) = self.heap.peek().map(|entry| entry.0.0.score.0) else {
+                return Err(Error::internal(
+                    "FTS top-k heap is empty while its nonzero limit is reached",
+                ));
+            };
+            // Preserve the existing collector semantics for non-finite custom
+            // scorer output: only a strictly greater raw f32 replaces k-th.
+            if doc.score.0.partial_cmp(&kth_score) != Some(std::cmp::Ordering::Greater) {
+                return Ok(false);
+            }
+            let Some(Reverse((_, _, _, frequency_slot))) = self.heap.pop() else {
+                return Err(Error::internal(
+                    "FTS top-k heap entry disappeared during replacement",
+                ));
+            };
+            self.frequency_slots.replace(frequency_slot, pairs)?;
+            frequency_slot
+        } else {
+            self.frequency_slots.push(pairs)?
+        };
+
+        self.heap
+            .push(Reverse((doc, doc_length, posting_doc_id, frequency_slot)));
+        Ok(true)
+    }
+
+    fn kth_score_if_full(&self) -> Option<f32> {
+        if self.heap.len() == self.limit {
+            self.heap.peek().map(|entry| entry.0.0.score.0)
+        } else {
+            None
+        }
+    }
+
+    fn into_candidates(
+        self,
+        mut to_addr: impl FnMut(u64) -> CandidateAddr,
+    ) -> Result<Vec<DocCandidate>> {
+        let Self {
+            heap,
+            mut frequency_slots,
+            ..
+        } = self;
+        heap.into_iter()
+            .map(
+                |Reverse((doc, doc_length, posting_doc_id, frequency_slot))| {
+                    Ok(DocCandidate {
+                        addr: to_addr(doc.row_id),
+                        posting_doc_id,
+                        freqs: frequency_slots.take(frequency_slot)?,
+                        doc_length,
+                    })
+                },
+            )
+            .collect()
+    }
+
+    #[cfg(test)]
+    fn num_frequency_slots(&self) -> usize {
+        self.frequency_slots.slots.len()
+    }
+}
+const LINEAR_BLOCK_SKIP_LIMIT: usize = 8;
 pub static FLAT_SEARCH_PERCENT_THRESHOLD: LazyLock<u64> = LazyLock::new(|| {
     std::env::var("LANCE_FLAT_SEARCH_PERCENT_THRESHOLD")
         .unwrap_or_else(|_| "10".to_string())
         .parse::<u64>()
         .unwrap_or(10)
 });
+// Bulk MAXSCORE path for top-k disjunctions (Lucene MaxScoreBulkScorer
+// style). Default on: with right-sized partitions it wins by a wide margin
+// (Lucene-parity latency) and its results are score-identical to the classic
+// WAND loop. LANCE_FTS_MAXSCORE=0 opts back into the classic loop.
+static USE_MAXSCORE_SEARCH: LazyLock<bool> =
+    LazyLock::new(|| std::env::var("LANCE_FTS_MAXSCORE").as_deref() != Ok("0"));
+// Bulk conjunction path for top-k AND / phrase queries: block-max window
+// skipping plus a slice-level merge over decompressed blocks, replacing the
+// per-doc `next()` leapfrog. Results are identical to the classic AND loop.
+// LANCE_FTS_BULK_AND accepts auto (default), on/1, or off/0. Auto enables the
+// bulk path only for its consistently faster two- and three-clause kernels.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+enum BulkAndMode {
+    #[default]
+    Auto,
+    On,
+    Off,
+}
+
+impl BulkAndMode {
+    fn parse(value: &str) -> Option<Self> {
+        let value = value.trim();
+        if value.eq_ignore_ascii_case("auto") {
+            Some(Self::Auto)
+        } else if value.eq_ignore_ascii_case("on") || value == "1" {
+            Some(Self::On)
+        } else if value.eq_ignore_ascii_case("off") || value == "0" {
+            Some(Self::Off)
+        } else {
+            None
+        }
+    }
+
+    const fn enabled_for(self, num_clauses: usize) -> bool {
+        match self {
+            Self::Auto => matches!(num_clauses, 2 | 3),
+            Self::On => true,
+            Self::Off => false,
+        }
+    }
+}
+
+fn bulk_and_mode_from_env() -> BulkAndMode {
+    match std::env::var("LANCE_FTS_BULK_AND") {
+        Ok(value) => BulkAndMode::parse(&value).unwrap_or_else(|| {
+            log::warn!(
+                "Invalid LANCE_FTS_BULK_AND value {value:?}; expected auto, on/1, or off/0; \
+                 falling back to auto"
+            );
+            BulkAndMode::Auto
+        }),
+        Err(std::env::VarError::NotPresent) => BulkAndMode::Auto,
+        Err(std::env::VarError::NotUnicode(value)) => {
+            log::warn!(
+                "Invalid non-Unicode LANCE_FTS_BULK_AND value {value:?}; expected auto, on/1, \
+                 or off/0; falling back to auto"
+            );
+            BulkAndMode::Auto
+        }
+    }
+}
+
+static BULK_AND_MODE: LazyLock<BulkAndMode> = LazyLock::new(bulk_and_mode_from_env);
+
+#[cfg(target_arch = "x86_64")]
+static HAS_AVX2: LazyLock<bool> = LazyLock::new(|| std::arch::is_x86_feature_detected!("avx2"));
+
+/// First index in `[pos, end)` where `docs[index] >= target` (scalar).
+/// Posting-block doc ids stay below 2^31, which the AVX2 variant relies on.
+#[inline]
+unsafe fn find_next_geq_scalar(docs: *const u32, mut pos: usize, end: usize, target: u32) -> usize {
+    unsafe {
+        while pos < end && *docs.add(pos) < target {
+            pos += 1;
+        }
+    }
+    pos
+}
+
+/// AVX2 `find_next_geq` (the analogue of Lucene's VectorUtil.findNextGEQ):
+/// branchless 8-wide compare+movemask kills the mispredicted exits that
+/// dominate the scalar catch-up scan on irregular doc gaps.
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn find_next_geq_avx2(docs: *const u32, mut pos: usize, end: usize, target: u32) -> usize {
+    use core::arch::x86_64::*;
+    debug_assert!(target <= i32::MAX as u32);
+    unsafe {
+        let target_lanes = _mm256_set1_epi32(target as i32);
+        while pos + 8 <= end {
+            let docs_lanes = _mm256_loadu_si256(docs.add(pos) as *const __m256i);
+            // lane mask of docs[i] < target; doc ids < 2^31 keep the signed
+            // compare equivalent to unsigned.
+            let below = _mm256_cmpgt_epi32(target_lanes, docs_lanes);
+            let mask = _mm256_movemask_ps(_mm256_castsi256_ps(below)) as u32;
+            if mask != 0xFF {
+                return pos + mask.trailing_ones() as usize;
+            }
+            pos += 8;
+        }
+        find_next_geq_scalar(docs, pos, end, target)
+    }
+}
+
+#[inline]
+unsafe fn find_next_geq(docs: *const u32, pos: usize, end: usize, target: u32) -> usize {
+    #[cfg(target_arch = "x86_64")]
+    if *HAS_AVX2 {
+        return unsafe { find_next_geq_avx2(docs, pos, end, target) };
+    }
+    unsafe { find_next_geq_scalar(docs, pos, end, target) }
+}
+
+#[inline]
+fn conservative_bm25_upper_bound(query_weight: f32) -> f32 {
+    if query_weight <= 0.0 {
+        0.0
+    } else {
+        query_weight * (K1 + 1.0)
+    }
+}
+
+#[inline]
+fn scorer_upper_bound<S: Scorer + ?Sized>(query_weight: f32, scorer: &S) -> f32 {
+    if query_weight.is_nan() {
+        return f32::INFINITY;
+    }
+    if query_weight <= 0.0 {
+        return 0.0;
+    }
+    match scorer.doc_weight_upper_bound() {
+        Some(bound) if bound.is_finite() && bound >= 0.0 => query_weight * bound,
+        _ => f32::INFINITY,
+    }
+}
 
 pub struct PostingIterator {
     token: String,
@@ -52,7 +349,12 @@ pub struct PostingIterator {
     index: usize,
     // the index of current block, this can be changed by `next() and shallow_next()`
     block_idx: usize,
+    current_doc: Option<DocInfo>,
     approximate_upper_bound: f32,
+    // Position cursors temporarily own this buffer and return it on drop. This
+    // keeps repeated cursor creation allocation-free without lending a slice
+    // out of the interior-mutable compressed state.
+    position_scratch: RefCell<Option<Vec<u32>>>,
 
     // for compressed posting list
     compressed: Option<UnsafeCell<CompressedState>>,
@@ -63,22 +365,46 @@ struct CompressedState {
     block_idx: usize,
     doc_ids: Vec<u32>,
     freqs: Vec<u32>,
-    buffer: Box<[u32; BLOCK_SIZE]>,
+    buffer: Box<[u32; MAX_POSTING_BLOCK_SIZE]>,
     position_block_idx: Option<usize>,
     position_values: Vec<u32>,
     position_offsets: Vec<usize>,
+    // Seek state for PackedDelta position blocks: the lazily-built group
+    // header index, the last unpacked group (memoized), the decoded varint
+    // tail, and the block's total delta count. Together these let a phrase
+    // check decode just the candidate doc's positions instead of the whole
+    // 256-doc position block.
+    position_group_offsets: Vec<usize>,
+    position_unpacked_group: Box<[u32; BLOCK_SIZE]>,
+    position_unpacked_group_idx: Option<usize>,
+    position_tail: Vec<u32>,
+    position_total_deltas: usize,
+    block_max_window: BlockMaxWindow,
+    // Lucene-style anchored impact score caches: one slot per level, keyed by
+    // the entry the block cursor currently sits in. Each holds
+    // (entry_idx, doc_up_to, max_score). See `impact_level0`/`impact_level1`.
+    level0_cache: Option<(usize, u32, f32)>,
+    level1_cache: Option<(usize, u32, f32)>,
 }
 
 impl CompressedState {
-    fn new() -> Self {
+    fn new(block_size: usize) -> Self {
         Self {
             block_idx: 0,
-            doc_ids: Vec::with_capacity(BLOCK_SIZE),
-            freqs: Vec::with_capacity(BLOCK_SIZE),
-            buffer: Box::new([0; BLOCK_SIZE]),
+            doc_ids: Vec::with_capacity(block_size),
+            freqs: Vec::with_capacity(block_size),
+            buffer: Box::new([0; MAX_POSTING_BLOCK_SIZE]),
             position_block_idx: None,
             position_values: Vec::new(),
             position_offsets: Vec::new(),
+            position_group_offsets: Vec::new(),
+            position_unpacked_group: Box::new([0; BLOCK_SIZE]),
+            position_unpacked_group_idx: None,
+            position_tail: Vec::new(),
+            position_total_deltas: 0,
+            block_max_window: BlockMaxWindow::new(),
+            level0_cache: None,
+            level1_cache: None,
         }
     }
 
@@ -90,21 +416,29 @@ impl CompressedState {
         num_blocks: usize,
         length: u32,
         tail_codec: super::PostingTailCodec,
+        block_size: usize,
     ) {
         self.doc_ids.clear();
         self.freqs.clear();
 
-        let remainder = length as usize % BLOCK_SIZE;
+        let remainder = length as usize % block_size;
         if block_idx + 1 == num_blocks && remainder != 0 {
             decompress_posting_remainder(
                 block,
                 remainder,
                 tail_codec,
+                block_size,
                 &mut self.doc_ids,
                 &mut self.freqs,
             );
         } else {
-            decompress_posting_block(block, &mut self.buffer, &mut self.doc_ids, &mut self.freqs);
+            decompress_posting_block(
+                block,
+                &mut self.buffer[..],
+                &mut self.doc_ids,
+                &mut self.freqs,
+                block_size,
+            );
         }
         self.block_idx = block_idx;
         self.position_block_idx = None;
@@ -113,6 +447,116 @@ impl CompressedState {
     }
 }
 
+#[derive(Clone)]
+struct BlockMaxWindow {
+    // Sliding block range used for Lucene-style getMaxScore(upTo). The deque is
+    // monotonic by score and covers blocks in [start_block_idx, next_block_idx).
+    // Only used for compressed lists without impact skip data; impact lists
+    // answer window max scores from the anchored level caches instead.
+    start_block_idx: usize,
+    next_block_idx: usize,
+    max_scores: VecDeque<(usize, f32)>,
+    impact_score_cache: ImpactScoreCache,
+}
+
+struct BlockMaxScore {
+    score: f32,
+    blocks_scanned: usize,
+}
+
+impl BlockMaxWindow {
+    fn new() -> Self {
+        Self {
+            start_block_idx: 0,
+            next_block_idx: 0,
+            max_scores: VecDeque::new(),
+            impact_score_cache: ImpactScoreCache::default(),
+        }
+    }
+
+    fn reset(&mut self, start_block_idx: usize) {
+        self.start_block_idx = start_block_idx;
+        self.next_block_idx = start_block_idx;
+        self.max_scores.clear();
+    }
+
+    fn max_score_up_to<S: Scorer + ?Sized>(
+        &mut self,
+        list: &CompressedPostingList,
+        start_block_idx: usize,
+        up_to: u64,
+        query_weight: f32,
+        scorer: &S,
+    ) -> BlockMaxScore {
+        if start_block_idx >= list.blocks.len() {
+            self.reset(start_block_idx);
+            return BlockMaxScore {
+                score: 0.0,
+                blocks_scanned: 0,
+            };
+        }
+        if start_block_idx < self.start_block_idx || start_block_idx > self.next_block_idx {
+            self.reset(start_block_idx);
+        }
+        self.start_block_idx = start_block_idx;
+        while matches!(self.max_scores.front(), Some((block_idx, _)) if *block_idx < start_block_idx)
+        {
+            self.max_scores.pop_front();
+        }
+
+        if list.block_least_doc_id(start_block_idx) as u64 > up_to {
+            self.reset(start_block_idx);
+            return BlockMaxScore {
+                score: 0.0,
+                blocks_scanned: 0,
+            };
+        }
+
+        // V3 postings score quantized doc lengths, which can be shorter than
+        // the exact lengths used to bake a legacy max score. Without impacts,
+        // use the score-independent BM25 ceiling instead of that stale bound.
+        if list.block_size == MAX_POSTING_BLOCK_SIZE {
+            self.reset(start_block_idx);
+            return BlockMaxScore {
+                score: scorer_upper_bound(query_weight, scorer),
+                blocks_scanned: 0,
+            };
+        }
+
+        self.next_block_idx = self.next_block_idx.max(start_block_idx);
+        let mut blocks_scanned = 0;
+        while self.next_block_idx < list.blocks.len()
+            && list.block_least_doc_id(self.next_block_idx) as u64 <= up_to
+        {
+            let score = match list.impacts.as_ref() {
+                Some(impacts) => impacts.level0_score_cached(
+                    self.next_block_idx,
+                    query_weight,
+                    scorer,
+                    &mut self.impact_score_cache,
+                ),
+                None => list.block_max_score(self.next_block_idx),
+            };
+            while matches!(self.max_scores.back(), Some((_, old_score)) if *old_score <= score) {
+                self.max_scores.pop_back();
+            }
+            self.max_scores.push_back((self.next_block_idx, score));
+            self.next_block_idx += 1;
+            blocks_scanned += 1;
+        }
+
+        let score = self
+            .max_scores
+            .front()
+            .map(|(_, score)| *score)
+            .unwrap_or(0.0);
+        BlockMaxScore {
+            score,
+            blocks_scanned,
+        }
+    }
+}
+
 impl Debug for PostingIterator {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("PostingIterator")
@@ -166,6 +610,104 @@ impl Ord for PostingIterator {
 }
 
 impl PostingIterator {
+    fn block_idx_for_doc(
+        &self,
+        list: &CompressedPostingList,
+        mut block_idx: usize,
+        least_id: u32,
+    ) -> usize {
+        let mut linear_skips = 0;
+        while block_idx + 1 < list.blocks.len() && linear_skips < LINEAR_BLOCK_SKIP_LIMIT {
+            if list.block_least_doc_id(block_idx + 1) > least_id {
+                return block_idx;
+            }
+            block_idx += 1;
+            linear_skips += 1;
+        }
+
+        if block_idx + 1 >= list.blocks.len() {
+            return block_idx;
+        }
+
+        if let Some(impacts) = list.impacts.as_ref()
+            && let Some(block_idx) =
+                self.block_idx_for_doc_with_impacts(list, impacts, block_idx, least_id)
+        {
+            return block_idx;
+        }
+
+        self.block_idx_for_doc_by_least_doc_id(list, block_idx, least_id, list.blocks.len())
+    }
+
+    fn block_idx_for_doc_with_impacts(
+        &self,
+        list: &CompressedPostingList,
+        impacts: &ImpactSkipData,
+        mut block_idx: usize,
+        least_id: u32,
+    ) -> Option<usize> {
+        while block_idx + 1 < list.blocks.len() {
+            let group_idx = (block_idx + 1) / IMPACT_LEVEL1_BLOCKS;
+            let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(list.blocks.len());
+            let group_doc_up_to = impacts.level1_doc_up_to(group_idx)?;
+            if group_doc_up_to < least_id {
+                block_idx = group_end - 1;
+                continue;
+            }
+            if group_doc_up_to == least_id {
+                return Some(group_end - 1);
+            }
+            return Some(
+                self.block_idx_for_doc_by_least_doc_id(list, block_idx, least_id, group_end),
+            );
+        }
+        Some(block_idx)
+    }
+
+    fn block_idx_for_doc_by_least_doc_id(
+        &self,
+        list: &CompressedPostingList,
+        block_idx: usize,
+        least_id: u32,
+        right: usize,
+    ) -> usize {
+        let mut left = block_idx + 1;
+        let mut right = right;
+        while left < right {
+            let mid = left + (right - left) / 2;
+            if list.block_least_doc_id(mid) <= least_id {
+                left = mid + 1;
+            } else {
+                right = mid;
+            }
+        }
+        left - 1
+    }
+
+    #[inline]
+    fn block_end_doc(&self) -> u64 {
+        self.next_block_first_doc()
+            .map(|doc| doc.saturating_sub(1))
+            .unwrap_or(TERMINATED_DOC_ID)
+    }
+
+    /// Level1 bound of the group holding the current block, for group-wide
+    /// skipping: (group doc_up_to, group max score). `None` when the list has
+    /// no impact skip data or the group entry is missing/malformed.
+    fn impact_group_bound<S: Scorer + ?Sized>(&self, scorer: &S) -> Option<(u64, f32)> {
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                let impacts = list.impacts.as_ref()?;
+                let (doc_up_to, score) = self.impact_level1(impacts, scorer);
+                if doc_up_to == u32::MAX {
+                    return None;
+                }
+                Some((u64::from(doc_up_to), score))
+            }
+            PostingList::Plain(_) => None,
+        }
+    }
+
     #[inline]
     fn compressed_state_ptr(&self) -> *mut CompressedState {
         debug_assert!(self.compressed.is_some());
@@ -189,6 +731,7 @@ impl PostingIterator {
                 list.blocks.len(),
                 list.length,
                 list.posting_tail_codec,
+                list.block_size,
             );
         }
         compressed as *mut CompressedState
@@ -213,14 +756,29 @@ impl PostingIterator {
         list: PostingList,
         num_doc: usize,
     ) -> Self {
-        let approximate_upper_bound = match list.max_score() {
-            Some(max_score) => max_score,
-            None => idf(list.len(), num_doc) * (K1 + 1.0),
+        // BM25's doc weight is bounded by K1 + 1 for any freq and doc length,
+        // so query_weight * (K1 + 1) is a valid global bound even when index
+        // stats drift after appends. Keeping it finite matters: an INFINITY
+        // bound can never park the iterator in the WAND tail, forcing a deep
+        // advance on every candidate.
+        let approximate_upper_bound = match &list {
+            PostingList::Compressed(posting) if posting.impacts.is_some() => f32::INFINITY,
+            PostingList::Compressed(posting) if posting.block_size == MAX_POSTING_BLOCK_SIZE => {
+                conservative_bm25_upper_bound(query_weight)
+            }
+            _ => match list.max_score() {
+                Some(max_score) => max_score,
+                None => idf(list.len(), num_doc) * (K1 + 1.0),
+            },
+        };
+        let compressed = match &list {
+            PostingList::Compressed(list) => {
+                Some(UnsafeCell::new(CompressedState::new(list.block_size)))
+            }
+            PostingList::Plain(_) => None,
         };
 
-        let is_compressed = matches!(list, PostingList::Compressed(_));
-
-        Self {
+        let mut posting = Self {
             token,
             token_id,
             position,
@@ -228,9 +786,13 @@ impl PostingIterator {
             list,
             index: 0,
             block_idx: 0,
+            current_doc: None,
             approximate_upper_bound,
-            compressed: is_compressed.then(|| UnsafeCell::new(CompressedState::new())),
-        }
+            position_scratch: RefCell::new(Some(Vec::new())),
+            compressed,
+        };
+        posting.refresh_current_doc();
+        posting
     }
 
     #[inline]
@@ -248,8 +810,36 @@ impl PostingIterator {
         self.approximate_upper_bound
     }
 
+    /// Tightest known list-wide score bound. Impact lists answer from the
+    /// baked doc-weight slab (the data-driven equivalent of the max_score the
+    /// non-impact format bakes at build time); everything else falls back to
+    /// `approximate_upper_bound`. A finite, tight global bound lets lagging
+    /// iterators park in the WAND tail instead of being force-advanced.
+    #[inline]
+    fn global_upper_bound<S: Scorer + ?Sized>(&self, scorer: &S) -> f32 {
+        if self.query_weight <= 0.0 {
+            return 0.0;
+        }
+        if let PostingList::Compressed(ref list) = self.list
+            && let Some(impacts) = list.impacts.as_ref()
+        {
+            let compressed = unsafe { &mut *self.compressed_state_ptr() };
+            return self.query_weight
+                * impacts.global_max_doc_weight_cached(
+                    scorer,
+                    &mut compressed.block_max_window.impact_score_cache,
+                );
+        }
+        if let PostingList::Compressed(ref list) = self.list
+            && list.block_size == MAX_POSTING_BLOCK_SIZE
+        {
+            return scorer_upper_bound(self.query_weight, scorer);
+        }
+        self.approximate_upper_bound
+    }
+
     #[inline]
-    fn score<S: Scorer>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
+    fn score<S: Scorer + ?Sized>(&self, scorer: &S, freq: u32, doc_length: u32) -> f32 {
         self.query_weight * scorer.doc_weight(freq, doc_length)
     }
 
@@ -265,14 +855,19 @@ impl PostingIterator {
 
     #[inline]
     fn doc(&self) -> Option<DocInfo> {
+        self.current_doc
+    }
+
+    fn refresh_current_doc(&mut self) {
         if self.empty() {
-            return None;
+            self.current_doc = None;
+            return;
         }
 
-        match self.list {
+        let current_doc = match self.list {
             PostingList::Compressed(ref list) => {
-                let block_idx = self.index / BLOCK_SIZE;
-                let block_offset = self.index % BLOCK_SIZE;
+                let block_idx = self.index >> list.block_shift();
+                let block_offset = self.index & list.block_mask();
                 let compressed = unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
 
                 // Read from the decompressed block
@@ -282,15 +877,22 @@ impl PostingIterator {
                 Some(doc)
             }
             PostingList::Plain(ref list) => Some(DocInfo::Located(list.doc(self.index))),
-        }
+        };
+        self.current_doc = current_doc;
     }
 
-    fn position_cursor(&self) -> Option<PositionCursor<'_>> {
+    fn position_cursor(&self) -> Result<PositionCursor<'_>> {
         match self.list {
-            PostingList::Plain(ref list) => list.positions.as_ref().map(|positions| {
+            PostingList::Plain(ref list) => {
+                let positions = list.positions.as_ref().ok_or_else(|| {
+                    Error::index(format!(
+                        "positions are missing for token {:?} (token id {}, query position {})",
+                        self.token, self.token_id, self.position
+                    ))
+                })?;
                 let start = positions.value_offsets()[self.index] as usize;
                 let end = positions.value_offsets()[self.index + 1] as usize;
-                PositionCursor::new(
+                Ok(PositionCursor::new(
                     PositionValues::Owned(
                         positions.values().as_primitive::<Int32Type>().values()[start..end]
                             .iter()
@@ -298,48 +900,129 @@ impl PostingIterator {
                             .collect(),
                     ),
                     self.position as i32,
-                )
-            }),
-            PostingList::Compressed(ref list) => match list.positions.as_ref()? {
+                ))
+            }
+            PostingList::Compressed(ref list) => match list.positions.as_ref().ok_or_else(|| {
+                Error::index(format!(
+                    "positions are missing for token {:?} (token id {}, query position {})",
+                    self.token, self.token_id, self.position
+                ))
+            })? {
                 CompressedPositionStorage::LegacyPerDoc(positions) => {
                     let positions = positions.value(self.index);
                     let positions = decompress_positions(positions.as_binary());
-                    Some(PositionCursor::new(
+                    Ok(PositionCursor::new(
                         PositionValues::Owned(positions),
                         self.position as i32,
                     ))
                 }
                 CompressedPositionStorage::SharedStream(stream) => {
-                    let block_idx = self.index / BLOCK_SIZE;
-                    let block_offset = self.index % BLOCK_SIZE;
+                    let block_idx = self.index >> list.block_shift();
+                    let block_offset = self.index & list.block_mask();
                     let compressed =
                         unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
-                    if compressed.position_block_idx != Some(block_idx) {
-                        decode_position_stream_block(
-                            stream.block(block_idx),
-                            compressed.freqs.as_slice(),
-                            stream.codec(),
-                            &mut compressed.position_values,
-                        )
-                        .expect("shared position stream decoding should succeed");
-                        compressed.position_offsets.clear();
-                        compressed
-                            .position_offsets
-                            .reserve(compressed.freqs.len() + 1);
-                        compressed.position_offsets.push(0);
-                        let mut offset = 0usize;
-                        for &freq in &compressed.freqs {
-                            offset += freq as usize;
-                            compressed.position_offsets.push(offset);
+                    match stream.codec() {
+                        PositionStreamCodec::PackedDelta => {
+                            // Seekable layout: decode only the candidate doc's
+                            // positions. Per-block seek state resets when the
+                            // block cursor moves; the group header index and
+                            // varint tail fill in lazily as candidates touch
+                            // them.
+                            if compressed.position_block_idx != Some(block_idx) {
+                                compressed.position_group_offsets.clear();
+                                compressed.position_group_offsets.push(0);
+                                compressed.position_tail.clear();
+                                compressed.position_unpacked_group_idx = None;
+                                compressed.position_offsets.clear();
+                                compressed
+                                    .position_offsets
+                                    .reserve(compressed.freqs.len() + 1);
+                                compressed.position_offsets.push(0);
+                                let mut offset = 0usize;
+                                for &freq in &compressed.freqs {
+                                    offset += freq as usize;
+                                    compressed.position_offsets.push(offset);
+                                }
+                                compressed.position_total_deltas = offset;
+                                compressed.position_block_idx = Some(block_idx);
+                            }
+                            let delta_start = compressed.position_offsets[block_offset];
+                            let delta_end = compressed.position_offsets[block_offset + 1];
+                            let mut position_values = self
+                                .position_scratch
+                                .borrow_mut()
+                                .take()
+                                .unwrap_or_default();
+                            if let Err(error) = seek_packed_doc_positions(
+                                stream.block(block_idx),
+                                compressed.position_total_deltas,
+                                delta_start..delta_end,
+                                &mut compressed.position_group_offsets,
+                                &mut compressed.position_unpacked_group,
+                                &mut compressed.position_unpacked_group_idx,
+                                &mut compressed.position_tail,
+                                &mut position_values,
+                            ) {
+                                *self.position_scratch.borrow_mut() = Some(position_values);
+                                return Err(Error::index(format!(
+                                    "failed to decode positions for token {:?} (token id {}, query position {}) at posting index {}: {error}",
+                                    self.token, self.token_id, self.position, self.index
+                                )));
+                            }
+                            Ok(PositionCursor::new(
+                                PositionValues::Recycled(RecycledPositionValues::new(
+                                    position_values,
+                                    &self.position_scratch,
+                                )),
+                                self.position as i32,
+                            ))
+                        }
+                        PositionStreamCodec::VarintDocDelta => {
+                            if compressed.position_block_idx != Some(block_idx) {
+                                compressed.position_values.clear();
+                                decode_position_stream_block(
+                                    stream.block(block_idx),
+                                    compressed.freqs.as_slice(),
+                                    stream.codec(),
+                                    &mut compressed.position_values,
+                                )
+                                .map_err(|error| {
+                                    Error::index(format!(
+                                        "failed to decode positions for token {:?} (token id {}, query position {}) in block {block_idx}: {error}",
+                                        self.token, self.token_id, self.position
+                                    ))
+                                })?;
+                                compressed.position_offsets.clear();
+                                compressed
+                                    .position_offsets
+                                    .reserve(compressed.freqs.len() + 1);
+                                compressed.position_offsets.push(0);
+                                let mut offset = 0usize;
+                                for &freq in &compressed.freqs {
+                                    offset += freq as usize;
+                                    compressed.position_offsets.push(offset);
+                                }
+                                compressed.position_block_idx = Some(block_idx);
+                            }
+                            let start = compressed.position_offsets[block_offset];
+                            let end = compressed.position_offsets[block_offset + 1];
+                            let mut position_values = self
+                                .position_scratch
+                                .borrow_mut()
+                                .take()
+                                .unwrap_or_default();
+                            position_values.clear();
+                            position_values
+                                .extend_from_slice(&compressed.position_values[start..end]);
+                            Ok(PositionCursor::new(
+                                PositionValues::Recycled(RecycledPositionValues::new(
+                                    position_values,
+                                    &self.position_scratch,
+                                )),
+                                self.position as i32,
+                            ))
                         }
-                        compressed.position_block_idx = Some(block_idx);
                     }
-                    let start = compressed.position_offsets[block_offset];
-                    let end = compressed.position_offsets[block_offset + 1];
-                    Some(PositionCursor::new(
-                        PositionValues::Borrowed(&compressed.position_values[start..end]),
-                        self.position as i32,
-                    ))
                 }
             },
         }
@@ -351,36 +1034,41 @@ impl PostingIterator {
             PostingList::Compressed(ref list) => {
                 debug_assert!(least_id <= u32::MAX as u64);
                 let least_id = least_id as u32;
-                let mut block_idx = self.index / BLOCK_SIZE;
-                while block_idx + 1 < list.blocks.len()
-                    && list.block_least_doc_id(block_idx + 1) <= least_id
-                {
-                    block_idx += 1;
-                }
-                self.index = self.index.max(block_idx * BLOCK_SIZE);
+                let shift = list.block_shift();
+                let block_idx = self.block_idx_for_doc(list, self.index >> shift, least_id);
+                self.index = self.index.max(block_idx << shift);
                 let length = list.length as usize;
                 while self.index < length {
-                    let block_idx = self.index / BLOCK_SIZE;
-                    let block_offset = self.index % BLOCK_SIZE;
+                    let block_idx = self.index >> shift;
+                    let block_offset = self.index & list.block_mask();
                     let compressed =
                         unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
                     let in_block = &compressed.doc_ids[block_offset..];
                     let offset_in_block = in_block.partition_point(|&doc_id| doc_id < least_id);
                     let new_offset = block_offset + offset_in_block;
                     if new_offset < compressed.doc_ids.len() {
-                        self.index = block_idx * BLOCK_SIZE + new_offset;
-                        break;
+                        self.index = (block_idx << shift) + new_offset;
+                        self.block_idx = block_idx;
+                        self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                            doc_id: compressed.doc_ids[new_offset],
+                            frequency: compressed.freqs[new_offset],
+                        }));
+                        return;
                     }
                     if block_idx + 1 >= list.blocks.len() {
                         self.index = length;
+                        self.block_idx = self.index >> shift;
+                        self.current_doc = None;
                         break;
                     }
-                    self.index = (block_idx + 1) * BLOCK_SIZE;
+                    self.index = (block_idx + 1) << shift;
                 }
-                self.block_idx = self.index / BLOCK_SIZE;
+                self.block_idx = self.index >> shift;
+                self.current_doc = None;
             }
             PostingList::Plain(ref list) => {
                 self.index += list.row_ids[self.index..].partition_point(|&id| id < least_id);
+                self.current_doc = (!self.empty()).then(|| DocInfo::Located(list.doc(self.index)));
             }
         }
     }
@@ -390,11 +1078,7 @@ impl PostingIterator {
             PostingList::Compressed(ref list) => {
                 debug_assert!(least_id <= u32::MAX as u64);
                 let least_id = least_id as u32;
-                while self.block_idx + 1 < list.blocks.len()
-                    && list.block_least_doc_id(self.block_idx + 1) <= least_id
-                {
-                    self.block_idx += 1;
-                }
+                self.block_idx = self.block_idx_for_doc(list, self.block_idx, least_id);
             }
             PostingList::Plain(_) => {
                 // we don't have block max score for legacy index,
@@ -403,52 +1087,348 @@ impl PostingIterator {
         }
     }
 
+    /// Anchored level0 impact bound of the current block: (doc_up_to, max_score),
+    /// memoized until the block cursor moves. Malformed entries degrade to
+    /// (u32::MAX, INFINITY), which keeps pruning safe by making the block look
+    /// unskippable.
     #[inline]
-    fn block_max_score(&self) -> f32 {
-        match self.list {
-            PostingList::Compressed(ref list) => list.block_max_score(self.block_idx),
-            PostingList::Plain(_) => self.approximate_upper_bound,
+    fn impact_level0<S: Scorer + ?Sized>(
+        &self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> (u32, f32) {
+        let compressed = unsafe { &mut *self.compressed_state_ptr() };
+        if let Some((block_idx, doc_up_to, score)) = compressed.level0_cache
+            && block_idx == self.block_idx
+        {
+            return (doc_up_to, score);
         }
+        let doc_up_to = impacts.level0_doc_up_to(self.block_idx).unwrap_or(u32::MAX);
+        let score = impacts.level0_score_cached(
+            self.block_idx,
+            self.query_weight,
+            scorer,
+            &mut compressed.block_max_window.impact_score_cache,
+        );
+        compressed.level0_cache = Some((self.block_idx, doc_up_to, score));
+        (doc_up_to, score)
     }
 
-    fn block_first_doc(&self) -> Option<u64> {
+    /// Anchored level1 impact bound of the group holding the current block,
+    /// memoized until the cursor crosses a group boundary.
+    #[inline]
+    fn impact_level1<S: Scorer + ?Sized>(
+        &self,
+        impacts: &ImpactSkipData,
+        scorer: &S,
+    ) -> (u32, f32) {
+        let group_idx = self.block_idx / IMPACT_LEVEL1_BLOCKS;
+        let compressed = unsafe { &mut *self.compressed_state_ptr() };
+        if let Some((cached_group_idx, doc_up_to, score)) = compressed.level1_cache
+            && cached_group_idx == group_idx
+        {
+            return (doc_up_to, score);
+        }
+        let doc_up_to = impacts.level1_doc_up_to(group_idx).unwrap_or(u32::MAX);
+        let score = impacts.level1_score_cached(
+            group_idx,
+            self.query_weight,
+            scorer,
+            &mut compressed.block_max_window.impact_score_cache,
+        );
+        compressed.level1_cache = Some((group_idx, doc_up_to, score));
+        (doc_up_to, score)
+    }
+
+    #[inline]
+    fn block_max_score<S: Scorer + ?Sized>(&self, scorer: &S) -> f32 {
         match self.list {
             PostingList::Compressed(ref list) => {
-                Some(list.block_least_doc_id(self.block_idx) as u64)
+                if let Some(impacts) = list.impacts.as_ref() {
+                    return self.impact_level0(impacts, scorer).1;
+                }
+                if list.block_size == MAX_POSTING_BLOCK_SIZE {
+                    return scorer_upper_bound(self.query_weight, scorer);
+                }
+                list.block_max_score(self.block_idx)
             }
-            PostingList::Plain(ref plain) => plain.row_ids.get(self.index).cloned(),
+            PostingList::Plain(_) => self.approximate_upper_bound,
         }
     }
 
+    /// Tight max-score bound over `[current block, up_to]`. The common case —
+    /// a window ending inside the current block — answers from the anchored
+    /// level0 memo; wider windows fall back to the sliding block-max deque,
+    /// which scores each block once as it slides forward.
     #[inline]
-    fn next_block_first_doc(&self) -> Option<u64> {
+    fn block_max_score_up_to_with_stats<S: Scorer + ?Sized>(
+        &self,
+        up_to: u64,
+        scorer: &S,
+    ) -> BlockMaxScore {
         match self.list {
             PostingList::Compressed(ref list) => {
-                if self.block_idx + 1 >= list.blocks.len() {
-                    return None;
+                if let Some(impacts) = list.impacts.as_ref() {
+                    let (level0_up_to, level0_score) = self.impact_level0(impacts, scorer);
+                    if up_to <= u64::from(level0_up_to) {
+                        return BlockMaxScore {
+                            score: level0_score,
+                            blocks_scanned: 0,
+                        };
+                    }
                 }
-                Some(list.block_least_doc_id(self.block_idx + 1) as u64)
+                let compressed = unsafe { &mut *self.compressed_state_ptr() };
+                compressed.block_max_window.max_score_up_to(
+                    list,
+                    self.block_idx,
+                    up_to,
+                    self.query_weight,
+                    scorer,
+                )
             }
-            PostingList::Plain(ref plain) => plain.row_ids.get(self.index + 1).cloned(),
+            PostingList::Plain(_) => BlockMaxScore {
+                score: self.approximate_upper_bound,
+                blocks_scanned: 0,
+            },
         }
     }
-}
-
-#[derive(Debug)]
-pub struct DocCandidate {
-    pub row_id: u64,
-    /// (term_index, freq)
-    pub freqs: Vec<(u32, u32)>,
-    pub doc_length: u32,
-}
 
-struct HeadPosting {
-    // Iterators that are already positioned on or after the next candidate doc.
-    // The heap is ordered by smallest doc id so the top element determines
-    // the next target doc to consider.
-    doc_id: u64,
-    posting: Box<PostingIterator>,
-}
+    fn window_max_score<S: Scorer + ?Sized>(&self, up_to: Option<u64>, scorer: &S) -> f32 {
+        if let Some(up_to) = up_to
+            && let PostingList::Compressed(ref list) = self.list
+            && list.impacts.is_some()
+        {
+            return self.block_max_score_up_to_with_stats(up_to, scorer).score;
+        }
+        self.block_max_score(scorer)
+    }
+
+    #[inline]
+    fn is_compressed(&self) -> bool {
+        matches!(self.list, PostingList::Compressed(_))
+    }
+
+    #[inline]
+    fn has_next_compressed_block(&self) -> bool {
+        match self.list {
+            PostingList::Compressed(ref list) => self.block_idx + 1 < list.blocks.len(),
+            PostingList::Plain(_) => false,
+        }
+    }
+
+    fn block_first_doc(&self) -> Option<u64> {
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                Some(list.block_least_doc_id(self.block_idx) as u64)
+            }
+            PostingList::Plain(ref plain) => plain.row_ids.get(self.index).cloned(),
+        }
+    }
+
+    /// Bulk-score every posting in `[current doc, up_to]` into the window
+    /// accumulator (slot = doc - window_min) and leave the iterator on the
+    /// first doc beyond `up_to`. This is the Lucene `nextDocsAndScores`
+    /// equivalent: it walks the decompressed block arrays directly, with no
+    /// per-doc heap traffic.
+    // The norm cache arg tips this hot-path fn over the limit; bundling the
+    // scoring inputs isn't worth the churn here.
+    #[allow(clippy::too_many_arguments)]
+    fn collect_window_scores<S: Scorer + ?Sized>(
+        &mut self,
+        window_min: u64,
+        up_to: u64,
+        clause_idx: usize,
+        docs: &DocSet,
+        scorer: &S,
+        norm_k: Option<(&[u8], &[f32; 256])>,
+        acc: &mut WindowAccumulator,
+    ) {
+        if self.doc().is_some_and(|doc| doc.doc_id() < window_min) {
+            self.next(window_min);
+        }
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                let shift = list.block_shift();
+                let mask = list.block_mask();
+                'blocks: while let Some(doc) = self.current_doc {
+                    if doc.doc_id() > up_to {
+                        break;
+                    }
+                    let block_idx = self.index >> shift;
+                    let block_offset = self.index & mask;
+                    let compressed =
+                        unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx) };
+                    for offset in block_offset..compressed.doc_ids.len() {
+                        let doc_id = compressed.doc_ids[offset];
+                        if u64::from(doc_id) > up_to {
+                            self.index = (block_idx << shift) + offset;
+                            self.block_idx = block_idx;
+                            self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id,
+                                frequency: compressed.freqs[offset],
+                            }));
+                            break 'blocks;
+                        }
+                        let freq = compressed.freqs[offset];
+                        // One byte-norm load plus a cached addend replaces
+                        // recomputing the BM25 denominator per doc.
+                        let doc_weight = match norm_k {
+                            Some((norms, cache)) => bm25_doc_weight_with_norm(
+                                freq,
+                                cache[norms[doc_id as usize] as usize],
+                            ),
+                            None => scorer.doc_weight(freq, docs.scoring_num_tokens(doc_id)),
+                        };
+                        let score = self.query_weight * doc_weight;
+                        let slot = (u64::from(doc_id) - window_min) as usize;
+                        acc.add(clause_idx, slot, score, freq);
+                    }
+                    // Block exhausted: step into the next block (or finish).
+                    let next_start = (block_idx + 1) << shift;
+                    if next_start >= list.length as usize {
+                        self.index = list.length as usize;
+                        self.block_idx = self.index >> shift;
+                        self.current_doc = None;
+                        break;
+                    }
+                    self.index = next_start;
+                    self.block_idx = block_idx + 1;
+                    let compressed =
+                        unsafe { &mut *self.ensure_compressed_block_ptr(list, block_idx + 1) };
+                    self.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                        doc_id: compressed.doc_ids[0],
+                        frequency: compressed.freqs[0],
+                    }));
+                }
+            }
+            PostingList::Plain(_) => {
+                while let Some(doc) = self.doc() {
+                    let doc_id = doc.doc_id();
+                    if doc_id > up_to {
+                        break;
+                    }
+                    let doc_length = match &doc {
+                        DocInfo::Raw(raw) => docs.scoring_num_tokens(raw.doc_id),
+                        DocInfo::Located(located) => docs.num_tokens_by_row_id(located.row_id),
+                    };
+                    let score = self.score(scorer, doc.frequency(), doc_length);
+                    let slot = (doc_id - window_min) as usize;
+                    acc.add(clause_idx, slot, score, doc.frequency());
+                    self.next(doc_id + 1);
+                }
+            }
+        }
+    }
+
+    #[inline]
+    fn next_block_first_doc(&self) -> Option<u64> {
+        match self.list {
+            PostingList::Compressed(ref list) => {
+                if self.block_idx + 1 >= list.blocks.len() {
+                    return None;
+                }
+                Some(list.block_least_doc_id(self.block_idx + 1) as u64)
+            }
+            PostingList::Plain(ref plain) => plain.row_ids.get(self.index + 1).cloned(),
+        }
+    }
+}
+
+/// Inner window span (in doc ids) of the bulk MAXSCORE path. Same as Lucene's
+/// `MaxScoreBulkScorer.INNER_WINDOW_SIZE`.
+const MAXSCORE_INNER_WINDOW: usize = 1 << 12;
+
+/// Lucene's `MathUtil.sumUpperBound` factor, adapted to Lance's `f32` score
+/// accumulation. Prefix bounds are summed in `f64`, then widened enough to
+/// cover any recursive `f32` summation order of the same non-negative values.
+#[inline]
+fn score_sum_upper_bound_factor(num_values: usize) -> f64 {
+    if num_values <= 2 {
+        1.0
+    } else {
+        let relative_error_bound = (num_values - 1) as f64 * f64::from(f32::EPSILON);
+        1.0 + 2.0 * relative_error_bound
+    }
+}
+
+#[inline]
+fn score_sum_cannot_exceed(
+    partial_score: f32,
+    remaining_upper_bound: f64,
+    threshold: f32,
+    upper_bound_factor: f64,
+) -> bool {
+    ((f64::from(partial_score) + remaining_upper_bound) * upper_bound_factor) as f32 <= threshold
+}
+
+/// Per-window score/frequency accumulator for the bulk MAXSCORE path. Slot i
+/// covers doc id `window_min + i`; `freqs` is laid out clause-major so accept
+/// paths can recover (term, freq) pairs of the essential clauses.
+struct WindowAccumulator {
+    scores: Vec<f32>,
+    freqs: Vec<u32>,
+    words: Vec<u64>,
+    num_clauses: usize,
+}
+
+impl WindowAccumulator {
+    fn new(num_clauses: usize) -> Self {
+        Self {
+            scores: vec![0.0; MAXSCORE_INNER_WINDOW],
+            freqs: vec![0; num_clauses * MAXSCORE_INNER_WINDOW],
+            words: vec![0; MAXSCORE_INNER_WINDOW / 64],
+            num_clauses,
+        }
+    }
+
+    #[inline]
+    fn add(&mut self, clause_idx: usize, slot: usize, score: f32, freq: u32) {
+        self.scores[slot] += score;
+        // Doc-major layout: one slot's clause frequencies share a cache line.
+        self.freqs[slot * self.num_clauses + clause_idx] = freq;
+        self.words[slot >> 6] |= 1u64 << (slot & 63);
+    }
+
+    #[inline]
+    fn clause_freq(&self, clause_idx: usize, slot: usize) -> u32 {
+        self.freqs[slot * self.num_clauses + clause_idx]
+    }
+
+    #[inline]
+    fn clear_slot(&mut self, slot: usize) {
+        self.scores[slot] = 0.0;
+        self.freqs[slot * self.num_clauses..(slot + 1) * self.num_clauses].fill(0);
+    }
+}
+
+/// How wand identified a candidate: either it already had the real
+/// row_id (DocSet carried row_ids), or only the partition-local
+/// doc_id (deferred-row_id path; the caller must resolve via
+/// [`super::lazy_docset::LazyDocSet::resolve_row_ids`]).
+#[derive(Debug, Clone, Copy)]
+pub enum CandidateAddr {
+    RowId(u64),
+    Pending(u32),
+}
+
+#[derive(Debug)]
+pub struct DocCandidate {
+    pub addr: CandidateAddr,
+    /// The document key used by the posting lists: doc_id for compressed
+    /// postings, row_id for legacy plain postings.
+    pub posting_doc_id: u64,
+    /// (term_index, freq)
+    pub freqs: Vec<(u32, u32)>,
+    pub doc_length: u32,
+}
+
+struct HeadPosting {
+    // Iterators that are already positioned on or after the next candidate doc.
+    // The heap is ordered by smallest doc id so the top element determines
+    // the next target doc to consider.
+    doc_id: u64,
+    posting: Box<PostingIterator>,
+}
 
 impl HeadPosting {
     fn new(posting: Box<PostingIterator>) -> Self {
@@ -526,6 +1506,23 @@ impl PartialEq for TailPosting {
     }
 }
 
+#[derive(Default)]
+struct AndWindowStats {
+    windows_wide: usize,
+    windows_narrow: usize,
+    windows_skipped: usize,
+    range_blocks_scanned: usize,
+    candidates_returned: usize,
+}
+
+#[derive(Default)]
+struct AndSearchStats {
+    pruned_before_return_start: usize,
+    candidates_seen: usize,
+    full_scores: usize,
+    freqs_collected: usize,
+}
+
 impl Eq for TailPosting {}
 
 impl PartialOrd for TailPosting {
@@ -574,8 +1571,31 @@ pub struct Wand<'a, S: Scorer> {
     // Last conjunction doc returned to the caller. The next conjunction search
     // resumes strictly after this doc, like Lucene's `nextDoc()/advance()`.
     and_last_doc: Option<u64>,
+    and_window_stats: AndWindowStats,
+    and_candidates_pruned_before_return: usize,
+    // Test-only override for comparing bulk and classic conjunctions without
+    // mutating the process-wide environment.
+    bulk_and_mode_override: Option<BulkAndMode>,
+    #[cfg(test)]
+    bulk_and_searches: usize,
     docs: &'a DocSet,
     scorer: S,
+    // Shared cross-partition top-k floor. Each partition publishes its local
+    // k-th score (`atomic_store_max_f32`) and prunes against the running value
+    // -- a lower bound on the global k-th, so it never drops a real top-k doc.
+    shared_threshold: Option<Arc<AtomicU32>>,
+}
+
+/// Monotonically raise an f32 stored in an `AtomicU32` to `val`. CAS loop (not a
+/// bit-max) so it stays correct for negative scores -- BM25 idf can go negative.
+fn atomic_store_max_f32(slot: &AtomicU32, val: f32) {
+    let mut cur = slot.load(Ordering::Relaxed);
+    while val > f32::from_bits(cur) {
+        match slot.compare_exchange_weak(cur, val.to_bits(), Ordering::Relaxed, Ordering::Relaxed) {
+            Ok(_) => break,
+            Err(actual) => cur = actual,
+        }
+    }
 }
 
 // we were using row id as doc id in the past, which is u64,
@@ -620,8 +1640,69 @@ impl<'a, S: Scorer> Wand<'a, S> {
             up_to: None,
             and_max_score: f32::INFINITY,
             and_last_doc: None,
+            and_window_stats: AndWindowStats::default(),
+            and_candidates_pruned_before_return: 0,
+            bulk_and_mode_override: None,
+            #[cfg(test)]
+            bulk_and_searches: 0,
             docs,
             scorer,
+            shared_threshold: None,
+        }
+    }
+
+    /// Test hook: force one conjunction mode so parity tests can compare bulk
+    /// and classic search within one process.
+    #[cfg(test)]
+    fn with_bulk_and_mode(mut self, mode: BulkAndMode) -> Self {
+        self.bulk_and_mode_override = Some(mode);
+        self
+    }
+
+    /// Share one cross-partition top-k floor across a query's partitions.
+    pub(crate) fn with_shared_threshold(mut self, shared: Arc<AtomicU32>) -> Self {
+        self.shared_threshold = Some(shared);
+        self
+    }
+
+    /// Per-search norm→BM25-denominator cache (Lucene's norm cache): the doc
+    /// byte-norm slab plus the 256 possible denominator addends. Available
+    /// when the DocSet scores quantized (V3 partitions) and the scorer
+    /// factors `doc_weight` as `(K1+1)*freq/(freq + addend)`. Scoring through
+    /// the cache is bit-identical to `scorer.doc_weight`, because both
+    /// evaluate the same expressions on the same quantized lengths.
+    fn norm_k_cache(&self) -> Option<(&'a [u8], Box<[f32; 256]>)> {
+        let docs: &'a DocSet = self.docs;
+        let norms = docs.scoring_norms()?;
+        let mut cache = Box::new([0f32; 256]);
+        for (code, slot) in cache.iter_mut().enumerate() {
+            *slot = self.scorer.doc_norm(dequantize_doc_length(code as u8))?;
+        }
+        Some((norms, cache))
+    }
+
+    /// Set the pruning threshold from this partition's k-th best, raised to the
+    /// shared cross-partition floor when one is attached.
+    fn update_threshold(&mut self, local_kth: f32, wand_factor: f32) {
+        let mut t = local_kth * wand_factor;
+        if let Some(shared) = self.shared_threshold.as_ref() {
+            atomic_store_max_f32(shared, local_kth);
+            let g = f32::from_bits(shared.load(Ordering::Relaxed)) * wand_factor;
+            if g > t {
+                t = g;
+            }
+        }
+        self.threshold = t;
+    }
+
+    /// Raise the local threshold to the shared cross-partition floor, picking up
+    /// updates published by sibling partitions.
+    fn raise_to_shared_floor(&mut self, wand_factor: f32) {
+        if let Some(shared) = self.shared_threshold.as_ref() {
+            let g = f32::from_bits(shared.load(Ordering::Relaxed)) * wand_factor;
+            if g > self.threshold {
+                self.threshold = g;
+            }
         }
     }
 
@@ -649,19 +1730,87 @@ impl<'a, S: Scorer> Wand<'a, S> {
             _ => {}
         }
 
-        let mut candidates = BinaryHeap::with_capacity(std::cmp::min(limit, BLOCK_SIZE * 10));
+        // Top-k disjunctions over compressed lists can opt into the bulk
+        // MAXSCORE path (Lucene MaxScoreBulkScorer style): it streams whole
+        // blocks of the essential clauses into a window accumulator instead of
+        // advancing doc-at-a-time through a heap.
+        if *USE_MAXSCORE_SEARCH
+            && self.operator == Operator::Or
+            && params.phrase_slop.is_none()
+            && !self.head.is_empty()
+            && self
+                .head
+                .iter()
+                .all(|posting| posting.posting.is_compressed())
+        {
+            return self.maxscore_search(params, mask, metrics);
+        }
+
+        // Top-k conjunctions (AND and phrase) over compressed lists use the
+        // bulk path: the same block-max window pruning, but candidates come
+        // from a slice-level merge over decompressed blocks instead of per-doc
+        // `next()` leapfrogging through boxed iterators.
+        if self.operator == Operator::And
+            && !self.lead.is_empty()
+            && self.lead.iter().all(|posting| posting.is_compressed())
+            && self
+                .bulk_and_mode_override
+                .unwrap_or_else(|| *BULK_AND_MODE)
+                .enabled_for(self.lead.len())
+        {
+            #[cfg(test)]
+            {
+                self.bulk_and_searches += 1;
+            }
+            return self.and_bulk_search(params, mask, metrics);
+        }
+
+        // Deferred-row_id path: when the DocSet was built without
+        // row_ids, wand emits candidates carrying just the
+        // partition-local doc_id; the outer caller resolves them to
+        // row_ids post-wand.
+        let docs_has_row_ids = self.docs.has_row_ids();
+
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
         let mut num_comparisons = 0;
-        while let Some((doc, mut score)) = self.next()? {
+        let mut and_search_stats = (self.operator == Operator::And).then_some(AndSearchStats {
+            pruned_before_return_start: self.and_candidates_pruned_before_return,
+            ..Default::default()
+        });
+        loop {
+            self.raise_to_shared_floor(params.wand_factor);
+            let Some((doc, mut score)) = self.next()? else {
+                break;
+            };
             num_comparisons += 1;
+            if let Some(and_stats) = and_search_stats.as_mut() {
+                and_stats.candidates_seen += 1;
+            }
 
+            // Either a real row_id (so we can run the mask check
+            // inline) or the doc_id widened to u64 (deferred path;
+            // the outer caller will resolve it post-wand).
+            let posting_doc_id = doc.doc_id();
             let row_id = match &doc {
                 DocInfo::Raw(doc) => {
-                    // if the doc is not located, we need to find the row id
-                    self.docs.row_id(doc.doc_id)
+                    if docs_has_row_ids {
+                        self.docs.row_id(doc.doc_id)
+                    } else {
+                        doc.doc_id as u64
+                    }
                 }
                 DocInfo::Located(doc) => doc.row_id,
             };
-            if !mask.selected(row_id) {
+            // Skip docs the fragment-reuse remap deleted. They are tombstoned
+            // in the DocSet (slot kept so posting-list doc_ids stay aligned)
+            // and must not surface in results.
+            if docs_has_row_ids && row_id == RowAddress::TOMBSTONE_ROW {
+                if self.operator == Operator::Or {
+                    self.push_back_leads(doc.doc_id() + 1);
+                }
+                continue;
+            }
+            if docs_has_row_ids && !mask.selected(row_id) {
                 if self.operator == Operator::Or {
                     self.push_back_leads(doc.doc_id() + 1);
                 }
@@ -669,14 +1818,14 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             let doc_length = match &doc {
-                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
 
             let score = if self.operator == Operator::Or {
                 self.advance_all_tail(doc.doc_id(), Some(doc_length), Some(&mut score));
                 if params.phrase_slop.is_some()
-                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)?
                 {
                     self.push_back_leads(doc.doc_id() + 1);
                     continue;
@@ -685,38 +1834,66 @@ impl<'a, S: Scorer> Wand<'a, S> {
             } else {
                 self.advance_all_tail(doc.doc_id(), None, None);
                 if params.phrase_slop.is_some()
-                    && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                    && !self.check_positions(params.phrase_slop.unwrap() as i32)?
                 {
                     continue;
                 }
+                if let Some(and_stats) = and_search_stats.as_mut() {
+                    and_stats.full_scores += 1;
+                }
                 self.score(doc_length)
             };
 
-            let freqs = self.iter_term_freqs().collect();
-            if candidates.len() < limit {
-                candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
-                if candidates.len() == limit {
-                    self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
+            if candidates.insert(
+                ScoredDoc::new(row_id, score),
+                doc_length,
+                posting_doc_id,
+                self.iter_term_freqs(),
+            )? {
+                if let Some(and_stats) = and_search_stats.as_mut() {
+                    and_stats.freqs_collected += 1;
+                }
+                if let Some(kth) = candidates.kth_score_if_full() {
+                    self.update_threshold(kth, params.wand_factor);
                 }
-            } else if score > candidates.peek().unwrap().0.0.score.0 {
-                candidates.pop();
-                candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
-                self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
             }
             if self.operator == Operator::Or {
                 self.push_back_leads(doc.doc_id() + 1);
             }
         }
+        if self.operator == Operator::And {
+            tracing::debug!(
+                and_windows_wide = self.and_window_stats.windows_wide,
+                and_windows_narrow = self.and_window_stats.windows_narrow,
+                and_windows_skipped = self.and_window_stats.windows_skipped,
+                and_range_blocks_scanned = self.and_window_stats.range_blocks_scanned,
+                and_candidates_returned = self.and_window_stats.candidates_returned,
+                "fts conjunction block-max window stats"
+            );
+        }
         metrics.record_comparisons(num_comparisons);
+        if let Some(and_stats) = and_search_stats {
+            let and_candidates_pruned_before_return = self
+                .and_candidates_pruned_before_return
+                .saturating_sub(and_stats.pruned_before_return_start);
+            metrics.record_and_candidates_seen(and_stats.candidates_seen);
+            metrics.record_and_candidates_pruned_before_return(and_candidates_pruned_before_return);
+            metrics.record_and_full_scores(and_stats.full_scores);
+            metrics.record_freqs_collected(and_stats.freqs_collected);
+        }
 
-        Ok(candidates
-            .into_iter()
-            .map(|Reverse((doc, freqs, doc_length))| DocCandidate {
-                row_id: doc.row_id,
-                freqs,
-                doc_length,
-            })
-            .collect())
+        // The heap entry's `row_id` slot is either a real row_id
+        // (DocSet had row_ids) or the doc_id widened to u64
+        // (deferred). Tag it accordingly so the caller can match
+        // rather than guess.
+        let to_addr = |row_id_slot: u64| {
+            if docs_has_row_ids {
+                CandidateAddr::RowId(row_id_slot)
+            } else {
+                CandidateAddr::Pending(row_id_slot as u32)
+            }
+        };
+        candidates.into_candidates(to_addr)
     }
 
     fn flat_search(
@@ -731,11 +1908,16 @@ impl<'a, S: Scorer> Wand<'a, S> {
         }
 
         // we need to map the row ids to doc ids, and sort them,
-        // because WAND PostingIterator can't go back to the previous doc id
+        // because WAND PostingIterator can't go back to the previous doc id.
+        // A list column maps one row id to several doc ids, so expand every
+        // document the row owns — keying on a single doc id would drop matches
+        // at non-last list positions (lancedb#3352).
         let doc_ids = row_ids
-            .filter_map(|row_addr| {
+            .flat_map(|row_addr| {
                 let row_id: u64 = row_addr.into();
-                self.docs.doc_id(row_id).map(|doc_id| (doc_id, row_id))
+                self.docs
+                    .doc_ids(row_id)
+                    .map(move |doc_id| (doc_id, row_id))
             })
             .sorted_unstable()
             .collect::<Vec<_>>();
@@ -751,7 +1933,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             .unwrap_or(false);
 
         let mut num_comparisons = 0;
-        let mut candidates = BinaryHeap::new();
+        let mut candidates = TopKCollector::new(limit, 0);
         for (doc_id, row_id) in doc_ids {
             num_comparisons += 1;
             self.move_head_before_target_to_tail(doc_id);
@@ -775,7 +1957,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
             // check positions
             if params.phrase_slop.is_some()
-                && !self.check_positions(params.phrase_slop.unwrap() as i32)
+                && !self.check_positions(params.phrase_slop.unwrap() as i32)?
             {
                 self.advance_lead_to_head(doc_id + 1);
                 continue;
@@ -783,7 +1965,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
             // score the doc
             let doc_length = match is_compressed {
-                true => self.docs.num_tokens(doc_id as u32),
+                true => self.docs.scoring_num_tokens(doc_id as u32),
                 false => self.docs.num_tokens_by_row_id(row_id),
             };
             if self.operator == Operator::Or && !self.refine_or_candidate(doc_id, doc_length) {
@@ -797,31 +1979,534 @@ impl<'a, S: Scorer> Wand<'a, S> {
 
             self.collect_tail_matches(doc_id);
             let score = self.score(doc_length);
-            let freqs = self.iter_term_freqs().collect();
 
-            if candidates.len() < limit {
-                candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
-                if candidates.len() == limit {
-                    self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
-                }
-            } else if score > candidates.peek().unwrap().0.0.score.0 {
-                candidates.pop();
-                candidates.push(Reverse((ScoredDoc::new(row_id, score), freqs, doc_length)));
-                self.threshold = candidates.peek().unwrap().0.0.score.0 * params.wand_factor;
+            if candidates.insert(
+                ScoredDoc::new(row_id, score),
+                doc_length,
+                doc_id,
+                self.iter_term_freqs(),
+            )? && let Some(kth) = candidates.kth_score_if_full()
+            {
+                self.update_threshold(kth, params.wand_factor);
             }
 
             self.advance_lead_to_head(doc_id + 1);
         }
         metrics.record_comparisons(num_comparisons);
 
-        Ok(candidates
+        // flat_search is driven by an explicit row_ids iterator, so
+        // every candidate already has a real row_id.
+        candidates.into_candidates(CandidateAddr::RowId)
+    }
+
+    /// Bulk MAXSCORE top-k disjunction, mirroring Lucene's MaxScoreBulkScorer.
+    ///
+    /// Per outer window (bounded by the essential clauses' block boundaries):
+    /// clauses are partitioned into a non-essential prefix — sorted by window
+    /// max score, as many as fit under the threshold — and the essential rest.
+    /// Essential clauses stream their postings into a window accumulator in
+    /// bulk; only accumulated candidates that could still beat the threshold
+    /// probe the non-essential clauses. No per-doc heap maintenance happens
+    /// anywhere on this path.
+    fn maxscore_search(
+        &mut self,
+        params: &FtsSearchParams,
+        mask: Arc<RowAddrMask>,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<Vec<DocCandidate>> {
+        struct MaxScoreClause {
+            posting: Box<PostingIterator>,
+            bound: f32,
+            prefix_bound: f64,
+        }
+
+        let limit = params.limit.unwrap_or(usize::MAX);
+        let docs_has_row_ids = self.docs.has_row_ids();
+        let mut clauses = std::mem::take(&mut self.head)
+            .into_vec()
             .into_iter()
-            .map(|Reverse((doc, freqs, doc_length))| DocCandidate {
-                row_id: doc.row_id,
-                freqs,
-                doc_length,
+            .map(|head| MaxScoreClause {
+                posting: head.posting,
+                bound: 0.0,
+                prefix_bound: 0.0,
             })
-            .collect())
+            .collect::<Vec<_>>();
+        let total_sum_upper_bound_factor = score_sum_upper_bound_factor(clauses.len());
+
+        let mut acc = WindowAccumulator::new(clauses.len());
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
+        let norm_k = self.norm_k_cache();
+        let norm_k_ref = norm_k
+            .as_ref()
+            .map(|(norms, cache)| (*norms, cache.as_ref()));
+        let mut num_comparisons = 0usize;
+        // Adaptive minimum window size (Lucene): grow windows when they yield
+        // too few candidates to amortize the per-window bound computations.
+        let mut min_window_size = 1u64;
+        let mut num_windows = 0u64;
+        let mut prev_first_essential = 0usize;
+
+        let mut window_min = clauses
+            .iter()
+            .filter_map(|clause| clause.posting.doc().map(|doc| doc.doc_id()))
+            .min()
+            .unwrap_or(TERMINATED_DOC_ID);
+
+        while window_min < TERMINATED_DOC_ID {
+            clauses.retain(|clause| clause.posting.doc().is_some());
+            if clauses.is_empty() {
+                break;
+            }
+            self.raise_to_shared_floor(params.wand_factor);
+
+            // Window boundary from the previous window's essential clauses
+            // only: dense non-essential clauses must not fragment the window.
+            let first_window_lead = prev_first_essential.min(clauses.len() - 1);
+            let mut window_max = TERMINATED_DOC_ID;
+            for clause in &mut clauses {
+                let doc = clause
+                    .posting
+                    .doc()
+                    .map(|doc| doc.doc_id())
+                    .expect("exhausted clauses were retained out");
+                clause.posting.shallow_next(doc.max(window_min));
+            }
+            for clause in &clauses[first_window_lead..] {
+                window_max = window_max.min(clause.posting.block_end_doc());
+            }
+            if clauses.len() > 1 {
+                // Target at least 32 candidates per clause per window on
+                // average before shrinking windows back to block granularity.
+                if (num_comparisons as u64) < num_windows * 32 * clauses.len() as u64 {
+                    min_window_size = (min_window_size * 2).min(MAXSCORE_INNER_WINDOW as u64);
+                } else {
+                    min_window_size = 1;
+                }
+                window_max = window_max.max(window_min.saturating_add(min_window_size - 1));
+            }
+
+            for clause in &mut clauses {
+                let doc = clause
+                    .posting
+                    .doc()
+                    .map(|doc| doc.doc_id())
+                    .expect("exhausted clauses were retained out");
+                clause.bound = if doc > window_max {
+                    0.0
+                } else {
+                    clause
+                        .posting
+                        .block_max_score_up_to_with_stats(window_max, &self.scorer)
+                        .score
+                };
+            }
+            clauses.sort_unstable_by(|a, b| a.bound.total_cmp(&b.bound));
+            let mut first_essential = 0;
+            let mut prefix = 0.0_f64;
+            if self.threshold > 0.0 {
+                for (i, clause) in clauses.iter_mut().enumerate() {
+                    let next_prefix = prefix + f64::from(clause.bound);
+                    let widened = (next_prefix * score_sum_upper_bound_factor(i + 1)) as f32;
+                    if widened > self.threshold {
+                        break;
+                    }
+                    prefix = next_prefix;
+                    clause.prefix_bound = prefix;
+                    first_essential = i + 1;
+                }
+            }
+            prev_first_essential = first_essential;
+            num_windows += 1;
+
+            if first_essential == clauses.len() {
+                // No clause combination inside this window can beat the
+                // threshold: skip it wholesale.
+                window_min = match window_max {
+                    TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                    max => max + 1,
+                };
+                // Single live clause: instead of re-running the window
+                // machinery once per block, scan the baked per-block bounds
+                // for the next block that can beat the threshold. This is the
+                // slab form of Lucene's getSkipUpTo and turns dead stretches
+                // of a dominant term into a tight load-mul-compare loop.
+                if clauses.len() == 1 && window_min != TERMINATED_DOC_ID && self.threshold > 0.0 {
+                    let posting = &clauses[0].posting;
+                    if let PostingList::Compressed(ref list) = posting.list
+                        && let Some(impacts) = list.impacts.as_ref()
+                    {
+                        let compressed = unsafe { &mut *posting.compressed_state_ptr() };
+                        let bounds = impacts.level0_doc_weight_bounds_cached(
+                            &self.scorer,
+                            &mut compressed.block_max_window.impact_score_cache,
+                        );
+                        let query_weight = posting.query_weight;
+                        // Position by binary search on the first-doc slab: the
+                        // deep cursor lags arbitrarily far behind during long
+                        // skip runs, and walking from it re-scans the same
+                        // blocks on every dead window.
+                        let first_docs = list.block_first_docs();
+                        let mut block_idx = first_docs
+                            .partition_point(|&first| u64::from(first) <= window_min)
+                            .saturating_sub(1);
+                        while block_idx < bounds.len()
+                            && query_weight * bounds[block_idx] <= self.threshold
+                        {
+                            block_idx += 1;
+                        }
+                        window_min = if block_idx < bounds.len() {
+                            window_min.max(u64::from(list.block_least_doc_id(block_idx)))
+                        } else {
+                            TERMINATED_DOC_ID
+                        };
+                    }
+                }
+                continue;
+            }
+
+            let total_non_essential_bound = if first_essential > 0 {
+                clauses[first_essential - 1].prefix_bound
+            } else {
+                0.0
+            };
+
+            // Single essential clause (the common case once the threshold is
+            // competitive): stream it directly against the non-essential
+            // prefix, skipping the accumulator entirely.
+            if first_essential + 1 == clauses.len() {
+                let (non_essential, essential) = clauses.split_at_mut(first_essential);
+                let posting = &mut essential[0].posting;
+                if posting.doc().is_some_and(|doc| doc.doc_id() < window_min) {
+                    posting.next(window_min);
+                }
+                let essential_term = posting.term_index();
+                let essential_weight = posting.query_weight;
+
+                macro_rules! consider_candidate {
+                    ($doc:expr, $freq:expr) => {{
+                        let doc = $doc;
+                        let freq = $freq;
+                        num_comparisons += 1;
+                        // One byte-norm load + cached addend when available;
+                        // the exact doc length is only needed at insert time.
+                        let norm_addend =
+                            norm_k_ref.map(|(norms, cache)| cache[norms[doc as usize] as usize]);
+                        let score = match norm_addend {
+                            Some(addend) => {
+                                essential_weight * bm25_doc_weight_with_norm(freq, addend)
+                            }
+                            None => {
+                                essential_weight
+                                    * self
+                                        .scorer
+                                        .doc_weight(freq, self.docs.scoring_num_tokens(doc as u32))
+                            }
+                        };
+                        if !(self.threshold > 0.0
+                            && score_sum_cannot_exceed(
+                                score,
+                                total_non_essential_bound,
+                                self.threshold,
+                                total_sum_upper_bound_factor,
+                            ))
+                        {
+                            let row_id = if docs_has_row_ids {
+                                self.docs.row_id(doc as u32)
+                            } else {
+                                doc
+                            };
+                            let masked_out = docs_has_row_ids
+                                && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id));
+                            if !masked_out {
+                                let mut total = score;
+                                let mut rejected = false;
+                                for i in (0..non_essential.len()).rev() {
+                                    if self.threshold > 0.0
+                                        && score_sum_cannot_exceed(
+                                            total,
+                                            non_essential[i].prefix_bound,
+                                            self.threshold,
+                                            total_sum_upper_bound_factor,
+                                        )
+                                    {
+                                        rejected = true;
+                                        break;
+                                    }
+                                    let probe = &mut non_essential[i].posting;
+                                    if probe.doc().is_some_and(|d| d.doc_id() < doc) {
+                                        probe.next(doc);
+                                    }
+                                    if let Some(d) = probe.doc()
+                                        && d.doc_id() == doc
+                                    {
+                                        total += match norm_addend {
+                                            Some(addend) => {
+                                                probe.query_weight
+                                                    * bm25_doc_weight_with_norm(
+                                                        d.frequency(),
+                                                        addend,
+                                                    )
+                                            }
+                                            None => probe.score(
+                                                &self.scorer,
+                                                d.frequency(),
+                                                self.docs.scoring_num_tokens(doc as u32),
+                                            ),
+                                        };
+                                    }
+                                }
+
+                                // Match the classic path's emission rule: a
+                                // candidate must beat the running threshold,
+                                // which drops zero-score matches (e.g. terms
+                                // with idf 0) exactly like Wand::next does.
+                                if !rejected && total > self.threshold {
+                                    let doc_length = self.docs.scoring_num_tokens(doc as u32);
+                                    if candidates.insert(
+                                        ScoredDoc::new(row_id, total),
+                                        doc_length,
+                                        doc,
+                                        std::iter::once((essential_term, freq)).chain(
+                                            non_essential.iter().filter_map(|clause| {
+                                                clause.posting.doc().and_then(|d| {
+                                                    (d.doc_id() == doc).then(|| {
+                                                        (clause.posting.term_index(), d.frequency())
+                                                    })
+                                                })
+                                            }),
+                                        ),
+                                    )? && let Some(kth) = candidates.kth_score_if_full()
+                                    {
+                                        self.update_threshold(kth, params.wand_factor);
+                                    }
+                                }
+                            }
+                        }
+                    }};
+                }
+
+                match posting.list {
+                    PostingList::Compressed(ref list) => {
+                        let shift = list.block_shift();
+                        let mask = list.block_mask();
+                        'stream: while let Some(cur) = posting.current_doc {
+                            if cur.doc_id() > window_max {
+                                break;
+                            }
+                            let block_idx = posting.index >> shift;
+                            let block_offset = posting.index & mask;
+                            let compressed = unsafe {
+                                &mut *posting.ensure_compressed_block_ptr(list, block_idx)
+                            };
+                            for offset in block_offset..compressed.doc_ids.len() {
+                                let doc_id = compressed.doc_ids[offset];
+                                if u64::from(doc_id) > window_max {
+                                    posting.index = (block_idx << shift) + offset;
+                                    posting.block_idx = block_idx;
+                                    posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                        doc_id,
+                                        frequency: compressed.freqs[offset],
+                                    }));
+                                    break 'stream;
+                                }
+                                consider_candidate!(u64::from(doc_id), compressed.freqs[offset]);
+                            }
+                            let next_start = (block_idx + 1) << shift;
+                            if next_start >= list.length as usize {
+                                posting.index = list.length as usize;
+                                posting.block_idx = posting.index >> shift;
+                                posting.current_doc = None;
+                                break;
+                            }
+                            posting.index = next_start;
+                            posting.block_idx = block_idx + 1;
+                            let compressed = unsafe {
+                                &mut *posting.ensure_compressed_block_ptr(list, block_idx + 1)
+                            };
+                            posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id: compressed.doc_ids[0],
+                                frequency: compressed.freqs[0],
+                            }));
+                        }
+                    }
+                    PostingList::Plain(_) => {
+                        while let Some(cur) = posting.doc() {
+                            let doc = cur.doc_id();
+                            if doc > window_max {
+                                break;
+                            }
+                            consider_candidate!(doc, cur.frequency());
+                            posting.next(doc + 1);
+                        }
+                    }
+                }
+
+                window_min = match window_max {
+                    TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                    max => max + 1,
+                };
+                continue;
+            }
+
+            // Stream the essential clauses through inner windows.
+            let mut inner_min = window_min;
+            loop {
+                let mut next_essential_doc = TERMINATED_DOC_ID;
+                for clause in &clauses[first_essential..] {
+                    if let Some(doc) = clause.posting.doc() {
+                        next_essential_doc = next_essential_doc.min(doc.doc_id());
+                    }
+                }
+                inner_min = inner_min.max(next_essential_doc);
+                if inner_min == TERMINATED_DOC_ID || inner_min > window_max {
+                    break;
+                }
+                let inner_max =
+                    window_max.min(inner_min.saturating_add(MAXSCORE_INNER_WINDOW as u64 - 1));
+
+                for (clause_idx, clause) in clauses.iter_mut().enumerate().skip(first_essential) {
+                    clause.posting.collect_window_scores(
+                        inner_min,
+                        inner_max,
+                        clause_idx,
+                        self.docs,
+                        &self.scorer,
+                        norm_k_ref,
+                        &mut acc,
+                    );
+                }
+
+                // Drain candidates in doc order, completing them with the
+                // non-essential clauses ordered by descending bound.
+                for word_idx in 0..acc.words.len() {
+                    let mut word = acc.words[word_idx];
+                    if word == 0 {
+                        continue;
+                    }
+                    acc.words[word_idx] = 0;
+                    while word != 0 {
+                        let bit = word.trailing_zeros() as usize;
+                        word &= word - 1;
+                        let slot = (word_idx << 6) | bit;
+                        let doc = inner_min + slot as u64;
+                        let mut score = acc.scores[slot];
+                        num_comparisons += 1;
+
+                        if self.threshold > 0.0
+                            && score_sum_cannot_exceed(
+                                score,
+                                total_non_essential_bound,
+                                self.threshold,
+                                total_sum_upper_bound_factor,
+                            )
+                        {
+                            acc.clear_slot(slot);
+                            continue;
+                        }
+
+                        let row_id = if docs_has_row_ids {
+                            self.docs.row_id(doc as u32)
+                        } else {
+                            doc
+                        };
+                        if docs_has_row_ids
+                            && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id))
+                        {
+                            acc.clear_slot(slot);
+                            continue;
+                        }
+
+                        // Doc length is only needed at heap-insert time; the
+                        // non-essential completion scores go through the norm
+                        // cache when available.
+                        let norm_addend =
+                            norm_k_ref.map(|(norms, cache)| cache[norms[doc as usize] as usize]);
+                        let mut doc_length_cell: Option<u32> = None;
+                        let mut rejected = false;
+                        for i in (0..first_essential).rev() {
+                            if self.threshold > 0.0
+                                && score_sum_cannot_exceed(
+                                    score,
+                                    clauses[i].prefix_bound,
+                                    self.threshold,
+                                    total_sum_upper_bound_factor,
+                                )
+                            {
+                                rejected = true;
+                                break;
+                            }
+                            let posting = &mut clauses[i].posting;
+                            if posting.doc().is_some_and(|d| d.doc_id() < doc) {
+                                posting.next(doc);
+                            }
+                            if let Some(d) = posting.doc()
+                                && d.doc_id() == doc
+                            {
+                                score += match norm_addend {
+                                    Some(addend) => {
+                                        posting.query_weight
+                                            * bm25_doc_weight_with_norm(d.frequency(), addend)
+                                    }
+                                    None => {
+                                        let doc_length =
+                                            *doc_length_cell.get_or_insert_with(|| {
+                                                self.docs.scoring_num_tokens(doc as u32)
+                                            });
+                                        posting.score(&self.scorer, d.frequency(), doc_length)
+                                    }
+                                };
+                            }
+                        }
+
+                        if !rejected && score > self.threshold {
+                            let doc_length = doc_length_cell
+                                .unwrap_or_else(|| self.docs.scoring_num_tokens(doc as u32));
+                            if candidates.insert(
+                                ScoredDoc::new(row_id, score),
+                                doc_length,
+                                doc,
+                                clauses.iter().enumerate().filter_map(|(i, clause)| {
+                                    if i >= first_essential {
+                                        let freq = acc.clause_freq(i, slot);
+                                        (freq > 0).then(|| (clause.posting.term_index(), freq))
+                                    } else {
+                                        clause.posting.doc().and_then(|d| {
+                                            (d.doc_id() == doc).then(|| {
+                                                (clause.posting.term_index(), d.frequency())
+                                            })
+                                        })
+                                    }
+                                }),
+                            )? && let Some(kth) = candidates.kth_score_if_full()
+                            {
+                                self.update_threshold(kth, params.wand_factor);
+                            }
+                        }
+                        acc.clear_slot(slot);
+                    }
+                }
+                if inner_max >= window_max {
+                    break;
+                }
+                inner_min = inner_max + 1;
+            }
+
+            window_min = match window_max {
+                TERMINATED_DOC_ID => TERMINATED_DOC_ID,
+                max => max + 1,
+            };
+        }
+
+        metrics.record_comparisons(num_comparisons);
+
+        let to_addr = |row_id_slot: u64| {
+            if docs_has_row_ids {
+                CandidateAddr::RowId(row_id_slot)
+            } else {
+                CandidateAddr::Pending(row_id_slot as u32)
+            }
+        };
+        candidates.into_candidates(to_addr)
     }
 
     // calculate the score of the current document
@@ -844,63 +2529,111 @@ impl<'a, S: Scorer> Wand<'a, S> {
         })
     }
 
-    // find the next doc candidate
-    // Find the next term-level candidate doc. The returned score is the exact
-    // contribution from the current `lead` set; additional score can still come
-    // from `tail` iterators that are advanced to the same doc later.
-    fn next(&mut self) -> Result<Option<(DocInfo, f32)>> {
-        if self.operator == Operator::And {
-            return Ok(self.next_and_candidate().map(|doc| (doc, 0.0)));
+    fn and_candidate_cannot_beat_threshold(&self, doc_length: u32) -> bool {
+        if self.operator != Operator::And
+            || self.threshold <= 0.0
+            || self.num_terms < 2
+            || self.lead.len() != self.num_terms
+        {
+            return false;
         }
 
-        while let Some(target) = self.head_doc() {
-            if self.up_to.is_none_or(|up_to| target > up_to) {
-                self.update_max_scores(target);
+        let Some((first, remaining)) = self.lead.split_first() else {
+            return false;
+        };
+        let Some(doc) = first.doc() else {
+            return false;
+        };
+
+        let remaining_upper_bound = remaining
+            .iter()
+            .map(|posting| posting.block_max_score(&self.scorer))
+            .sum::<f32>();
+        first.score(&self.scorer, doc.frequency(), doc_length) + remaining_upper_bound
+            <= self.threshold
+    }
+
+    // find the next doc candidate
+    // Find the next term-level candidate doc. The returned score is the exact
+    // contribution from the current `lead` set; additional score can still come
+    // from `tail` iterators that are advanced to the same doc later.
+    fn next(&mut self) -> Result<Option<(DocInfo, f32)>> {
+        if self.operator == Operator::And {
+            let candidate = self.next_and_candidate();
+            if candidate.is_some() {
+                self.and_window_stats.candidates_returned += 1;
+            }
+            return Ok(candidate.map(|doc| (doc, 0.0)));
+        }
+
+        loop {
+            let Some(target) = self.head_doc() else {
+                if self.advance_tail_to_next_or_window() {
+                    continue;
+                }
+                return Ok(None);
+            };
+
+            if self.up_to.is_none_or(|up_to| target > up_to) {
+                self.update_max_scores(target);
             }
             self.move_head_doc_to_lead(target);
             if self.lead.is_empty() {
                 continue;
             }
 
-            let Some(doc) = self.lead.first().and_then(|posting| posting.doc()) else {
+            // Block-Max WAND pruning: skip the whole window when its score upper
+            // bound cannot reach the top-k threshold.
+            if self.threshold > 0.0 && self.or_block_window_max() <= self.threshold {
+                // On the final block `up_to` is the `u64::MAX` sentinel; step once
+                // there to avoid seeking past the valid doc id range.
+                let mut skip_to = match self.up_to {
+                    Some(up_to) if up_to < u32::MAX as u64 => up_to + 1,
+                    _ => target + 1,
+                };
+                // The narrow window is dead; if the whole level1 group is dead
+                // too, hop over it in one advance.
+                let group_skip = self.or_group_skip_to();
+                if let Some(group_skip_to) = group_skip {
+                    skip_to = skip_to.max(group_skip_to);
+                }
+                self.push_back_leads(skip_to);
+                continue;
+            }
+
+            let Some(first_doc) = self.lead.first().and_then(|posting| posting.doc()) else {
                 self.push_back_leads(target + 1);
                 continue;
             };
-            let doc_length = match &doc {
-                DocInfo::Raw(doc) => self.docs.num_tokens(doc.doc_id),
+            let doc_length = match &first_doc {
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
                 DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
             };
-            let mut lead_score = self
-                .lead
-                .iter()
-                .filter_map(|posting| {
-                    posting.doc().map(|lead_doc| {
-                        posting.score(&self.scorer, lead_doc.frequency(), doc_length)
-                    })
-                })
-                .sum::<f32>();
+            let mut lead_score = 0.0;
+            if let Some(first_posting) = self.lead.first() {
+                lead_score += first_posting.score(&self.scorer, first_doc.frequency(), doc_length);
+            }
+            for posting in self.lead.iter().skip(1) {
+                if let Some(lead_doc) = posting.doc() {
+                    lead_score += posting.score(&self.scorer, lead_doc.frequency(), doc_length);
+                }
+            }
 
             while lead_score <= self.threshold {
                 if lead_score + self.tail_max_score <= self.threshold {
-                    self.push_back_leads(doc.doc_id() + 1);
+                    self.push_back_leads(first_doc.doc_id() + 1);
                     break;
                 }
                 if !self.advance_tail_top(target, doc_length, &mut lead_score) {
-                    self.push_back_leads(doc.doc_id() + 1);
+                    self.push_back_leads(first_doc.doc_id() + 1);
                     break;
                 }
             }
 
             if !self.lead.is_empty() {
-                return Ok(self
-                    .lead
-                    .first()
-                    .and_then(|posting| posting.doc())
-                    .map(|doc| (doc, lead_score)));
+                return Ok(Some((first_doc, lead_score)));
             }
         }
-
-        Ok(None)
     }
     fn next_and_candidate(&mut self) -> Option<DocInfo> {
         if self.lead.len() < self.num_terms {
@@ -953,9 +2686,603 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 }
             }
 
+            let lead_doc = self.lead.first().and_then(|posting| posting.doc())?;
+            let doc_length = match &lead_doc {
+                DocInfo::Raw(doc) => self.docs.scoring_num_tokens(doc.doc_id),
+                DocInfo::Located(doc) => self.docs.num_tokens_by_row_id(doc.row_id),
+            };
+            if self.and_candidate_cannot_beat_threshold(doc_length) {
+                self.and_candidates_pruned_before_return += 1;
+                let next_target = self.and_advance_target(doc.saturating_add(1));
+                if next_target == TERMINATED_DOC_ID {
+                    return None;
+                }
+                self.lead[0].next(next_target);
+                continue;
+            }
+
             self.and_last_doc = Some(doc);
-            return self.lead.first().and_then(|posting| posting.doc());
+            return Some(lead_doc);
+        }
+    }
+
+    fn posting_block_up_to(posting: &PostingIterator, target: u64) -> u64 {
+        posting
+            .next_block_first_doc()
+            .map(|doc| doc.saturating_sub(1))
+            .unwrap_or(TERMINATED_DOC_ID)
+            .max(target)
+    }
+
+    /// Bulk conjunction search. The window ends at the nearest next-block
+    /// boundary across the clauses, so within a window every clause
+    /// contributes exactly one decompressed block and the intersection is a
+    /// plain merge over `u32` slices — the per-candidate cost drops from a
+    /// full `PostingIterator::next` call per clause to a couple of loads.
+    /// Window skipping, per-candidate pruning, scoring, and heap semantics
+    /// mirror the classic loop exactly, so results are identical.
+    fn and_bulk_search(
+        &mut self,
+        params: &FtsSearchParams,
+        mask: Arc<RowAddrMask>,
+        metrics: &dyn MetricsCollector,
+    ) -> Result<Vec<DocCandidate>> {
+        let limit = params.limit.unwrap_or(usize::MAX);
+        if limit == 0 {
+            return Ok(vec![]);
+        }
+        let docs_has_row_ids = self.docs.has_row_ids();
+        let num_lists = self.lead.len();
+        let phrase_slop = params.phrase_slop;
+
+        // Per-window view of one clause's current block. Raw pointers into the
+        // clause's `CompressedState`; valid for the whole window because the
+        // block cursor does not move within a window (position decoding writes
+        // to separate fields of the same state).
+        struct WindowList {
+            docs: *const u32,
+            freqs: *const u32,
+            // Cursor and exclusive end, as offsets within the block.
+            pos: usize,
+            end: usize,
+            // Absolute posting index of the block's first entry.
+            block_start: usize,
+        }
+
+        // Merge kernels: intersect the window's per-clause slices and record
+        // each match as (doc, per-clause block offsets). Hand-specialized for
+        // two and three clauses so cursors and bounds stay in registers; the
+        // generic kernel covers other widths. Offsets fit u8 because a block
+        // holds at most `MAX_POSTING_BLOCK_SIZE` (256) entries. The macro
+        // stamps a scalar and an AVX2 variant — `#[target_feature]` must
+        // cover the whole kernel for the vector catch-up scans to inline.
+        // Kernels prune a lead doc before any follower advance when its
+        // frequency-bucketed score bound (plus the other clauses' block
+        // maxes) cannot beat the threshold — the score-first ordering
+        // Lucene's conjunction scorer uses, with doc length dropped from the
+        // bound so no doc-length load is involved. The bound is monotone
+        // (doc length only shrinks a BM25 weight, and the last bucket holds
+        // the clause sup), so every skipped doc would also fail the exact
+        // per-candidate prune: results are unchanged, the work never happens.
+        macro_rules! merge_kernels {
+            ($name2:ident, $name3:ident, $geq:ident $(, #[$feat:meta])?) => {
+                $(#[$feat])?
+                unsafe fn $name2(
+                    wins: &[WindowList],
+                    lut: &[f32; FREQ_LUT_BUCKETS],
+                    others_block_max: f32,
+                    threshold: f32,
+                    docs_out: &mut Vec<u32>,
+                    offs_out: &mut Vec<u8>,
+                ) {
+                    let (d0, mut p0, e0) = (wins[0].docs, wins[0].pos, wins[0].end);
+                    let (d1, mut p1, e1) = (wins[1].docs, wins[1].pos, wins[1].end);
+                    let f0 = wins[0].freqs;
+                    let prune = threshold > f32::NEG_INFINITY;
+                    unsafe {
+                        while p0 < e0 {
+                            let doc = *d0.add(p0);
+                            if prune {
+                                let freq = (*f0.add(p0) as usize).min(FREQ_LUT_BUCKETS - 1);
+                                if lut[freq] + others_block_max <= threshold {
+                                    p0 += 1;
+                                    continue;
+                                }
+                            }
+                            p1 = $geq(d1, p1, e1, doc);
+                            if p1 >= e1 {
+                                return;
+                            }
+                            let second = *d1.add(p1);
+                            if second > doc {
+                                p0 = $geq(d0, p0 + 1, e0, second);
+                                continue;
+                            }
+                            docs_out.push(doc);
+                            offs_out.push(p0 as u8);
+                            offs_out.push(p1 as u8);
+                            p0 += 1;
+                        }
+                    }
+                }
+
+                $(#[$feat])?
+                unsafe fn $name3(
+                    wins: &[WindowList],
+                    lut: &[f32; FREQ_LUT_BUCKETS],
+                    others_block_max: f32,
+                    threshold: f32,
+                    docs_out: &mut Vec<u32>,
+                    offs_out: &mut Vec<u8>,
+                ) {
+                    let (d0, mut p0, e0) = (wins[0].docs, wins[0].pos, wins[0].end);
+                    let (d1, mut p1, e1) = (wins[1].docs, wins[1].pos, wins[1].end);
+                    let (d2, mut p2, e2) = (wins[2].docs, wins[2].pos, wins[2].end);
+                    let f0 = wins[0].freqs;
+                    let prune = threshold > f32::NEG_INFINITY;
+                    unsafe {
+                        'outer: while p0 < e0 {
+                            let doc = *d0.add(p0);
+                            if prune {
+                                let freq = (*f0.add(p0) as usize).min(FREQ_LUT_BUCKETS - 1);
+                                if lut[freq] + others_block_max <= threshold {
+                                    p0 += 1;
+                                    continue 'outer;
+                                }
+                            }
+                            p1 = $geq(d1, p1, e1, doc);
+                            if p1 >= e1 {
+                                return;
+                            }
+                            let second = *d1.add(p1);
+                            if second > doc {
+                                p0 = $geq(d0, p0 + 1, e0, second);
+                                continue 'outer;
+                            }
+                            p2 = $geq(d2, p2, e2, doc);
+                            if p2 >= e2 {
+                                return;
+                            }
+                            let third = *d2.add(p2);
+                            if third > doc {
+                                p0 = $geq(d0, p0 + 1, e0, third);
+                                continue 'outer;
+                            }
+                            docs_out.push(doc);
+                            offs_out.push(p0 as u8);
+                            offs_out.push(p1 as u8);
+                            offs_out.push(p2 as u8);
+                            p0 += 1;
+                        }
+                    }
+                }
+            };
+        }
+        merge_kernels!(merge_window_2, merge_window_3, find_next_geq_scalar);
+        #[cfg(target_arch = "x86_64")]
+        merge_kernels!(
+            merge_window_2_avx2,
+            merge_window_3_avx2,
+            find_next_geq_avx2,
+            #[target_feature(enable = "avx2")]
+        );
+
+        #[inline]
+        fn merge_window_1(wins: &[WindowList], docs_out: &mut Vec<u32>, offs_out: &mut Vec<u8>) {
+            let win = &wins[0];
+            for pos in win.pos..win.end {
+                docs_out.push(unsafe { *win.docs.add(pos) });
+                offs_out.push(pos as u8);
+            }
+        }
+
+        #[inline]
+        #[allow(clippy::too_many_arguments)]
+        fn merge_window_n(
+            wins: &[WindowList],
+            lut: &[f32; FREQ_LUT_BUCKETS],
+            others_block_max: f32,
+            threshold: f32,
+            cursors: &mut Vec<usize>,
+            docs_out: &mut Vec<u32>,
+            offs_out: &mut Vec<u8>,
+        ) {
+            let prune = threshold > f32::NEG_INFINITY;
+            cursors.clear();
+            cursors.extend(wins.iter().map(|win| win.pos));
+            'outer: while cursors[0] < wins[0].end {
+                let doc = unsafe { *wins[0].docs.add(cursors[0]) };
+                if prune {
+                    let freq = unsafe { *wins[0].freqs.add(cursors[0]) as usize }
+                        .min(FREQ_LUT_BUCKETS - 1);
+                    if lut[freq] + others_block_max <= threshold {
+                        cursors[0] += 1;
+                        continue 'outer;
+                    }
+                }
+                for j in 1..wins.len() {
+                    let win = &wins[j];
+                    let pos = unsafe { find_next_geq(win.docs, cursors[j], win.end, doc) };
+                    cursors[j] = pos;
+                    if pos >= win.end {
+                        return;
+                    }
+                    let clause_doc = unsafe { *win.docs.add(pos) };
+                    if clause_doc > doc {
+                        cursors[0] = unsafe {
+                            find_next_geq(wins[0].docs, cursors[0] + 1, wins[0].end, clause_doc)
+                        };
+                        continue 'outer;
+                    }
+                }
+                docs_out.push(doc);
+                for &pos in cursors.iter() {
+                    offs_out.push(pos as u8);
+                }
+                cursors[0] += 1;
+            }
+        }
+
+        let mut candidates = TopKCollector::new(limit, std::cmp::min(limit, BLOCK_SIZE * 10));
+        let mut num_comparisons: usize = 0;
+        let mut stats = AndSearchStats {
+            pruned_before_return_start: self.and_candidates_pruned_before_return,
+            ..Default::default()
+        };
+        let mut wins: Vec<WindowList> = Vec::with_capacity(num_lists);
+        // Per-window candidate batch. The merge kernel only records matches;
+        // scoring then runs in two passes so the doc-length gather issues
+        // independent loads (their cache misses overlap) instead of
+        // serializing behind per-candidate branching. A window spans at most
+        // one block per clause, so a batch holds at most one block's worth.
+        let mut batch_docs: Vec<u32> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut batch_offs: Vec<u8> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE * num_lists);
+        let mut batch_lens: Vec<u32> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut batch_norms: Vec<u8> = Vec::with_capacity(MAX_POSTING_BLOCK_SIZE);
+        let mut cursor_scratch: Vec<usize> = Vec::with_capacity(num_lists);
+        // Norm cache: one byte-norm load plus a cached addend replaces the
+        // per-clause BM25 denominator recompute in pass B.
+        let norm_k = self.norm_k_cache();
+        let norm_k_ref = norm_k
+            .as_ref()
+            .map(|(norms, cache)| (*norms, cache.as_ref()));
+
+        // Per-window prune LUT for the merge kernels: an upper bound of the
+        // first (rarest) clause's score by clamped frequency. Lead docs whose
+        // bound plus the remaining clauses' block maxes cannot beat the
+        // threshold are skipped before any follower advances — the same
+        // score-first ordering Lucene's conjunction scorer uses, with the
+        // doc-length dropped from the bound so no doc-length load is needed.
+        // The last bucket holds the frequency-independent sup, so clamping
+        // stays a valid upper bound. Skips only avoid work; every emitted
+        // candidate still goes through the exact per-candidate prune below.
+        const FREQ_LUT_BUCKETS: usize = 64;
+        let mut freq_bound_lut = [f32::INFINITY; FREQ_LUT_BUCKETS];
+        for (freq, slot) in freq_bound_lut
+            .iter_mut()
+            .enumerate()
+            .take(FREQ_LUT_BUCKETS - 1)
+        {
+            *slot = self.lead[0].score(&self.scorer, freq as u32, 0);
+        }
+        // The clamp bucket must bound every frequency it absorbs; the
+        // clause-wide sup does.
+        freq_bound_lut[FREQ_LUT_BUCKETS - 1] = self.lead[0].approximate_upper_bound();
+
+        // The conjunction can only start at the max of the clauses' first docs.
+        let mut target: u64 = 0;
+        for posting in &self.lead {
+            match posting.doc() {
+                Some(doc) => target = target.max(doc.doc_id()),
+                None => return Ok(vec![]),
+            }
+        }
+
+        'window: loop {
+            self.raise_to_shared_floor(params.wand_factor);
+            if self.threshold > 0.0 {
+                let advanced = self.and_advance_target(target);
+                if advanced == TERMINATED_DOC_ID {
+                    break;
+                }
+                target = advanced;
+            }
+            debug_assert!(target <= u32::MAX as u64);
+            let target32 = target as u32;
+
+            // Position every clause's block cursor at the block that can hold
+            // `target`, and end the window at the nearest next-block boundary.
+            let mut win_end = TERMINATED_DOC_ID;
+            for j in 0..num_lists {
+                let (block_idx, block_up_to) = {
+                    let posting = &self.lead[j];
+                    let PostingList::Compressed(ref list) = posting.list else {
+                        unreachable!("bulk AND requires compressed postings");
+                    };
+                    let block_idx = posting.block_idx_for_doc(list, posting.block_idx, target32);
+                    let block_up_to = if block_idx + 1 < list.blocks.len() {
+                        u64::from(list.block_least_doc_id(block_idx + 1)).saturating_sub(1)
+                    } else {
+                        TERMINATED_DOC_ID
+                    };
+                    (block_idx, block_up_to.max(target))
+                };
+                self.lead[j].block_idx = block_idx;
+                win_end = win_end.min(block_up_to);
+            }
+            let win_end32 = u32::try_from(win_end).unwrap_or(u32::MAX);
+
+            // Decompress each clause's block and slice it to [target, win_end].
+            wins.clear();
+            let mut skip_window = false;
+            let mut exhausted = false;
+            for posting in &self.lead {
+                let PostingList::Compressed(ref list) = posting.list else {
+                    unreachable!("bulk AND requires compressed postings");
+                };
+                let block_idx = posting.block_idx;
+                let state = unsafe { &mut *posting.ensure_compressed_block_ptr(list, block_idx) };
+                let lo = state.doc_ids.partition_point(|&doc| doc < target32);
+                let hi = if win_end32 == u32::MAX {
+                    state.doc_ids.len()
+                } else {
+                    lo + state.doc_ids[lo..].partition_point(|&doc| doc <= win_end32)
+                };
+                if lo == hi {
+                    // No docs of this clause in the window: the whole window
+                    // has no conjunction match. If this was the clause's last
+                    // block and it is fully behind the target, the clause is
+                    // exhausted and the conjunction is done.
+                    if block_idx + 1 >= list.blocks.len()
+                        && state.doc_ids.last().is_none_or(|&doc| doc < target32)
+                    {
+                        exhausted = true;
+                    }
+                    skip_window = true;
+                    break;
+                }
+                wins.push(WindowList {
+                    docs: state.doc_ids.as_ptr(),
+                    freqs: state.freqs.as_ptr(),
+                    pos: lo,
+                    end: hi,
+                    block_start: block_idx << list.block_shift(),
+                });
+            }
+            if exhausted {
+                break 'window;
+            }
+            if !skip_window {
+                // Constant within the window (block-anchored); mirrors
+                // `and_candidate_cannot_beat_threshold`'s remaining-clause
+                // bound of first-clause-exact + rest-block-max.
+                let others_block_max: f32 = self.lead[1..]
+                    .iter()
+                    .map(|posting| posting.block_max_score(&self.scorer))
+                    .sum();
+
+                batch_docs.clear();
+                batch_offs.clear();
+                // NEG_INFINITY disables the kernel-level freq-bound prune
+                // (single clause, or no threshold yet).
+                let kernel_threshold = if self.threshold > 0.0 && num_lists >= 2 {
+                    self.threshold
+                } else {
+                    f32::NEG_INFINITY
+                };
+                #[cfg(target_arch = "x86_64")]
+                let use_avx2 = *HAS_AVX2;
+                #[cfg(not(target_arch = "x86_64"))]
+                let use_avx2 = false;
+                match (num_lists, use_avx2) {
+                    (1, _) => merge_window_1(&wins, &mut batch_docs, &mut batch_offs),
+                    #[cfg(target_arch = "x86_64")]
+                    (2, true) => unsafe {
+                        merge_window_2_avx2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    #[cfg(target_arch = "x86_64")]
+                    (3, true) => unsafe {
+                        merge_window_3_avx2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    (2, _) => unsafe {
+                        merge_window_2(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    (3, _) => unsafe {
+                        merge_window_3(
+                            &wins,
+                            &freq_bound_lut,
+                            others_block_max,
+                            kernel_threshold,
+                            &mut batch_docs,
+                            &mut batch_offs,
+                        )
+                    },
+                    _ => merge_window_n(
+                        &wins,
+                        &freq_bound_lut,
+                        others_block_max,
+                        kernel_threshold,
+                        &mut cursor_scratch,
+                        &mut batch_docs,
+                        &mut batch_offs,
+                    ),
+                }
+
+                // Pass A: gather doc norms/lengths for the whole batch up
+                // front so the loads issue back-to-back and their cache
+                // misses overlap. With the norm cache, only the byte code is
+                // gathered; the exact length decodes from a tiny LUT.
+                batch_lens.clear();
+                batch_norms.clear();
+                match norm_k_ref {
+                    Some((norms, _)) => {
+                        for &doc in batch_docs.iter() {
+                            batch_norms.push(norms[doc as usize]);
+                        }
+                    }
+                    None => match self.docs.scoring_norms() {
+                        Some(norms) => {
+                            for &doc in batch_docs.iter() {
+                                batch_lens.push(dequantize_doc_length(norms[doc as usize]));
+                            }
+                        }
+                        None => {
+                            for &doc in batch_docs.iter() {
+                                batch_lens.push(self.docs.scoring_num_tokens(doc));
+                            }
+                        }
+                    },
+                }
+
+                // Pass B: prune / verify / score / insert, in doc order, with
+                // exactly the classic loop's semantics.
+                for (index, &doc) in batch_docs.iter().enumerate() {
+                    let (norm_addend, doc_length) = match norm_k_ref {
+                        Some((_, cache)) => {
+                            let code = batch_norms[index];
+                            (Some(cache[code as usize]), dequantize_doc_length(code))
+                        }
+                        None => (None, batch_lens[index]),
+                    };
+                    let offs = &batch_offs[index * num_lists..(index + 1) * num_lists];
+                    if self.threshold > 0.0 && num_lists >= 2 {
+                        let first_freq = unsafe { *wins[0].freqs.add(offs[0] as usize) };
+                        let first_score = match norm_addend {
+                            Some(addend) => {
+                                self.lead[0].query_weight
+                                    * bm25_doc_weight_with_norm(first_freq, addend)
+                            }
+                            None => self.lead[0].score(&self.scorer, first_freq, doc_length),
+                        };
+                        if first_score + others_block_max <= self.threshold {
+                            self.and_candidates_pruned_before_return += 1;
+                            continue;
+                        }
+                    }
+                    stats.candidates_seen += 1;
+                    self.and_window_stats.candidates_returned += 1;
+                    num_comparisons += 1;
+
+                    let row_id = if docs_has_row_ids {
+                        self.docs.row_id(doc)
+                    } else {
+                        u64::from(doc)
+                    };
+                    if docs_has_row_ids
+                        && (row_id == RowAddress::TOMBSTONE_ROW || !mask.selected(row_id))
+                    {
+                        continue;
+                    }
+
+                    if let Some(slop) = phrase_slop {
+                        // Park every clause's iterator on this doc so
+                        // `position_cursor` reads the right posting entry. The
+                        // window block is already decompressed; position blocks
+                        // decode lazily and are cached per block.
+                        for ((win, posting), &off) in
+                            wins.iter().zip(self.lead.iter_mut()).zip(offs.iter())
+                        {
+                            posting.index = win.block_start + off as usize;
+                            posting.current_doc = Some(DocInfo::Raw(RawDocInfo {
+                                doc_id: doc,
+                                frequency: unsafe { *win.freqs.add(off as usize) },
+                            }));
+                        }
+                        let matched = if slop == 0 {
+                            self.check_exact_positions_bulk()?
+                        } else {
+                            self.check_positions(slop as i32)?
+                        };
+                        if !matched {
+                            continue;
+                        }
+                    }
+                    stats.full_scores += 1;
+
+                    let mut score = 0.0f32;
+                    for ((win, posting), &off) in wins.iter().zip(self.lead.iter()).zip(offs.iter())
+                    {
+                        let freq = unsafe { *win.freqs.add(off as usize) };
+                        score += match norm_addend {
+                            Some(addend) => {
+                                posting.query_weight * bm25_doc_weight_with_norm(freq, addend)
+                            }
+                            None => posting.score(&self.scorer, freq, doc_length),
+                        };
+                    }
+
+                    if candidates.insert(
+                        ScoredDoc::new(row_id, score),
+                        doc_length,
+                        u64::from(doc),
+                        wins.iter().zip(self.lead.iter()).zip(offs.iter()).map(
+                            |((win, posting), &off)| {
+                                (posting.term_index(), unsafe {
+                                    *win.freqs.add(off as usize)
+                                })
+                            },
+                        ),
+                    )? {
+                        stats.freqs_collected += 1;
+                        if let Some(kth) = candidates.kth_score_if_full() {
+                            self.update_threshold(kth, params.wand_factor);
+                        }
+                    }
+                }
+            }
+
+            if win_end == TERMINATED_DOC_ID {
+                break;
+            }
+            target = win_end + 1;
         }
+
+        tracing::debug!(
+            and_windows_wide = self.and_window_stats.windows_wide,
+            and_windows_narrow = self.and_window_stats.windows_narrow,
+            and_windows_skipped = self.and_window_stats.windows_skipped,
+            and_range_blocks_scanned = self.and_window_stats.range_blocks_scanned,
+            and_candidates_returned = self.and_window_stats.candidates_returned,
+            "fts conjunction block-max window stats (bulk)"
+        );
+        metrics.record_comparisons(num_comparisons);
+        let pruned_before_return = self
+            .and_candidates_pruned_before_return
+            .saturating_sub(stats.pruned_before_return_start);
+        metrics.record_and_candidates_seen(stats.candidates_seen);
+        metrics.record_and_candidates_pruned_before_return(pruned_before_return);
+        metrics.record_and_full_scores(stats.full_scores);
+        metrics.record_freqs_collected(stats.freqs_collected);
+
+        let to_addr = |row_id_slot: u64| {
+            if docs_has_row_ids {
+                CandidateAddr::RowId(row_id_slot)
+            } else {
+                CandidateAddr::Pending(row_id_slot as u32)
+            }
+        };
+        candidates.into_candidates(to_addr)
     }
 
     fn and_move_to_next_block(&mut self, target: u64) {
@@ -965,19 +3292,65 @@ impl<'a, S: Scorer> Wand<'a, S> {
             return;
         }
 
-        let mut up_to = TERMINATED_DOC_ID;
-        let mut max_score = 0.0;
+        if self.lead.is_empty() {
+            self.up_to = Some(TERMINATED_DOC_ID);
+            self.and_max_score = 0.0;
+            return;
+        }
+
         for posting in &mut self.lead {
             posting.shallow_next(target);
-            let block_end = posting
-                .next_block_first_doc()
-                .map(|doc| doc.saturating_sub(1))
-                .unwrap_or(TERMINATED_DOC_ID);
-            up_to = up_to.min(block_end.max(target));
-            max_score += posting.block_max_score();
         }
-        self.up_to = Some(up_to);
-        self.and_max_score = max_score;
+
+        let narrow_up_to = self
+            .lead
+            .iter()
+            .map(|posting| Self::posting_block_up_to(posting, target))
+            .min()
+            .unwrap_or(TERMINATED_DOC_ID);
+        let narrow_max_score = self
+            .lead
+            .iter()
+            .map(|posting| posting.block_max_score(&self.scorer))
+            .sum::<f32>();
+
+        if narrow_max_score >= self.threshold {
+            self.up_to = Some(narrow_up_to);
+            self.and_max_score = narrow_max_score;
+            self.and_window_stats.windows_narrow += 1;
+            return;
+        }
+
+        let lead_up_to = self
+            .lead
+            .first()
+            .map(|posting| Self::posting_block_up_to(posting, target))
+            .unwrap_or(TERMINATED_DOC_ID);
+        let can_try_wide = lead_up_to > narrow_up_to
+            && lead_up_to != TERMINATED_DOC_ID
+            && self.lead.iter().all(|posting| posting.is_compressed());
+
+        if can_try_wide {
+            let mut wide_max_score = 0.0;
+            let mut range_blocks_scanned = 0;
+            for posting in &mut self.lead {
+                let block_max = posting.block_max_score_up_to_with_stats(lead_up_to, &self.scorer);
+                wide_max_score += block_max.score;
+                range_blocks_scanned += block_max.blocks_scanned;
+            }
+            self.and_window_stats.range_blocks_scanned += range_blocks_scanned;
+
+            if wide_max_score < self.threshold {
+                self.up_to = Some(lead_up_to);
+                self.and_max_score = wide_max_score;
+                self.and_window_stats.windows_wide += 1;
+                return;
+            }
+        }
+
+        self.up_to = Some(narrow_up_to);
+        self.and_max_score = narrow_max_score;
+        self.and_window_stats.windows_narrow += 1;
     }
 
     fn and_advance_target(&mut self, mut target: u64) -> u64 {
@@ -992,6 +3365,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             if self.and_max_score >= self.threshold {
                 return target;
             }
+            self.and_window_stats.windows_skipped += 1;
             if up_to == TERMINATED_DOC_ID {
                 return TERMINATED_DOC_ID;
             }
@@ -1022,9 +3396,19 @@ impl<'a, S: Scorer> Wand<'a, S> {
     // Move all head iterators that are already known to be behind `target`
     // into `tail`, possibly overflowing low-value entries back into `head`.
     fn move_head_before_target_to_tail(&mut self, target: u64) {
+        if self.threshold <= 0.0 {
+            while matches!(self.head_doc(), Some(doc_id) if doc_id < target) {
+                if let Some(mut posting) = self.head.pop().map(|posting| posting.posting) {
+                    posting.next(target);
+                    self.push_head(posting);
+                }
+            }
+            return;
+        }
+
         while matches!(self.head_doc(), Some(doc_id) if doc_id < target) {
             if let Some(posting) = self.head.pop() {
-                let upper_bound = posting.posting.approximate_upper_bound();
+                let upper_bound = posting.posting.global_upper_bound(&self.scorer);
                 if let Some(mut evicted) =
                     self.insert_tail_with_overflow(posting.posting, upper_bound)
                 {
@@ -1035,6 +3419,24 @@ impl<'a, S: Scorer> Wand<'a, S> {
         }
     }
 
+    /// Upper bound on the score of any document in the window `[target, up_to]`
+    /// for a disjunction. Sums the block-max of every overlapping iterator:
+    /// `lead`, `head` (later docs still in the window, which
+    /// `can_target_beat_threshold` omits), and the tail via `tail_max_score`.
+    fn or_block_window_max(&self) -> f32 {
+        let lead: f32 = self
+            .lead
+            .iter()
+            .map(|posting| posting.window_max_score(self.up_to, &self.scorer))
+            .sum();
+        let head: f32 = self
+            .head
+            .iter()
+            .map(|posting| posting.posting.window_max_score(self.up_to, &self.scorer))
+            .sum();
+        lead + head + self.tail_max_score
+    }
+
     fn can_target_beat_threshold(&mut self, target: u64) -> bool {
         if self.up_to.is_none_or(|up_to| target > up_to) {
             self.update_max_scores(target);
@@ -1043,12 +3445,12 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let mut sum = self
             .lead
             .iter()
-            .map(|posting| posting.block_max_score())
+            .map(|posting| posting.window_max_score(self.up_to, &self.scorer))
             .sum::<f32>();
         let mut possible_matches = self.lead.len();
         for posting in &self.tail {
             if matches!(posting.posting.block_first_doc(), Some(block_doc) if block_doc <= target) {
-                sum += posting.posting.block_max_score();
+                sum += posting.posting.window_max_score(self.up_to, &self.scorer);
                 possible_matches += 1;
             }
         }
@@ -1062,67 +3464,121 @@ impl<'a, S: Scorer> Wand<'a, S> {
     fn update_max_scores(&mut self, target: u64) {
         // Refresh the block-max window for the current target. The resulting
         // `up_to` is the furthest doc id for which this block-max view remains
-        // valid.
+        // valid. Like Lucene's WANDScorer, the boundary comes from the cheap
+        // clauses only, and the refresh avoids allocating: heaps are recycled
+        // through their backing vectors (shallow_next never changes the doc a
+        // head entry is ordered by, so heapify restores the same shape).
         let lead_cost = self
             .lead
             .iter()
             .map(|posting| posting.cost())
             .min()
             .unwrap_or(usize::MAX);
-        let mut up_to = TERMINATED_DOC_ID;
+        let mut narrow_up_to = TERMINATED_DOC_ID;
         for posting in &mut self.lead {
             posting.shallow_next(target);
-            let block_end = posting
-                .next_block_first_doc()
-                .map(|doc| doc.saturating_sub(1))
-                .unwrap_or(TERMINATED_DOC_ID);
-            up_to = up_to.min(block_end);
-        }
-        let head = std::mem::take(&mut self.head);
-        let mut rebuilt_head = BinaryHeap::with_capacity(head.len());
-        for mut posting in head.into_vec() {
-            if posting.posting.cost() <= lead_cost {
-                posting.posting.shallow_next(posting.doc_id());
-                let block_end = posting
-                    .posting
-                    .next_block_first_doc()
-                    .map(|doc| doc.saturating_sub(1))
-                    .unwrap_or(TERMINATED_DOC_ID);
-                up_to = up_to.min(block_end);
-            }
-            rebuilt_head.push(posting);
+            narrow_up_to = narrow_up_to.min(posting.block_end_doc());
+        }
+
+        let mut head_postings = std::mem::take(&mut self.head).into_vec();
+        for posting in &mut head_postings {
+            // Unlike Lucene, every head clause participates in the boundary:
+            // the refresh is allocation-free and answers from the anchored
+            // level caches, so frequent refreshes are cheap, while keeping the
+            // window inside every clause's current block keeps all the bounds
+            // at tight level0 values.
+            let doc_id = posting.doc_id();
+            posting.posting.shallow_next(doc_id);
+            narrow_up_to = narrow_up_to.min(posting.posting.block_end_doc());
+        }
+
+        let mut tail_postings = std::mem::take(&mut self.tail).into_vec();
+        for tail_posting in &mut tail_postings {
+            tail_posting.posting.shallow_next(target);
         }
-        self.head = rebuilt_head;
-        if up_to == TERMINATED_DOC_ID
-            && let Some(top) = self.tail.peek()
-            && top.cost <= lead_cost
+
+        if narrow_up_to == TERMINATED_DOC_ID
+            && let Some(top) = tail_postings
+                .iter()
+                .min_by_key(|posting| posting.posting.cost())
+            && top.posting.cost() <= lead_cost
         {
-            let block_end = top
-                .posting
-                .next_block_first_doc()
-                .map(|doc| doc.saturating_sub(1))
-                .unwrap_or(TERMINATED_DOC_ID);
-            up_to = up_to.min(block_end.max(target));
+            narrow_up_to = narrow_up_to.min(top.posting.block_end_doc().max(target));
         }
-        self.up_to = Some(up_to);
 
-        let tail = std::mem::take(&mut self.tail);
+        self.up_to = Some(narrow_up_to);
+        self.head = BinaryHeap::from(head_postings);
+
         self.tail_max_score = 0.0;
-        for mut tail_posting in tail.into_vec() {
-            tail_posting.posting.shallow_next(target);
-            let upper_bound = match tail_posting.posting.block_first_doc() {
-                Some(block_doc) if block_doc <= target => tail_posting.posting.block_max_score(),
+        for tail_posting in tail_postings {
+            let posting = tail_posting.posting;
+            let upper_bound = match posting.block_first_doc() {
+                Some(block_doc) if block_doc <= target => {
+                    posting.window_max_score(self.up_to, &self.scorer)
+                }
                 _ => 0.0,
             };
-            if let Some(mut evicted) =
-                self.insert_tail_with_overflow(tail_posting.posting, upper_bound)
-            {
+            if let Some(mut evicted) = self.insert_tail_with_overflow(posting, upper_bound) {
                 evicted.next(target);
                 self.push_head(evicted);
             }
         }
     }
 
+    /// After the narrow window proved skippable, try widening the skip to the
+    /// level1 group boundary, in the spirit of Lucene's `getSkipUpTo`. All
+    /// bounds come from the anchored level caches, so a failed attempt costs a
+    /// few loads and float adds — unlike an eager wide-window probe.
+    ///
+    /// The group boundary is the minimum current-group end over every live
+    /// iterator, so each iterator's level1 score is a valid bound over
+    /// `[target, group_up_to]`.
+    fn or_group_skip_to(&self) -> Option<u64> {
+        let mut group_up_to = TERMINATED_DOC_ID;
+        for posting in &self.lead {
+            let (doc_up_to, _) = posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        for posting in self.head.iter() {
+            let (doc_up_to, _) = posting.posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        for tail_posting in self.tail.iter() {
+            let (doc_up_to, _) = tail_posting.posting.impact_group_bound(&self.scorer)?;
+            group_up_to = group_up_to.min(doc_up_to);
+        }
+        if self.up_to.is_some_and(|up_to| group_up_to <= up_to) {
+            // No gain over the narrow window skip.
+            return None;
+        }
+
+        // Second pass over the memoized bounds: sum only the iterators that
+        // can produce a doc inside the skipped range.
+        let mut bounds_sum = 0.0_f32;
+        for posting in &self.lead {
+            let (_, score) = posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        for posting in self.head.iter() {
+            if posting.doc_id() > group_up_to {
+                continue;
+            }
+            let (_, score) = posting.posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        for tail_posting in self.tail.iter() {
+            if !matches!(
+                tail_posting.posting.block_first_doc(),
+                Some(block_doc) if block_doc <= group_up_to
+            ) {
+                continue;
+            }
+            let (_, score) = tail_posting.posting.impact_group_bound(&self.scorer)?;
+            bounds_sum += score;
+        }
+        (bounds_sum <= self.threshold).then_some(group_up_to.saturating_add(1))
+    }
+
     fn refine_or_candidate(&mut self, target: u64, doc_length: u32) -> bool {
         if self.threshold <= 0.0 {
             return true;
@@ -1241,13 +3697,58 @@ impl<'a, S: Scorer> Wand<'a, S> {
         Some(candidate.posting)
     }
 
+    fn lead_to_tail_upper_bound(&self, posting: &PostingIterator, target: u64) -> f32 {
+        if self.operator == Operator::Or
+            && posting.is_compressed()
+            && self.up_to.is_some_and(|up_to| target <= up_to)
+        {
+            posting.window_max_score(self.up_to, &self.scorer)
+        } else {
+            posting.global_upper_bound(&self.scorer)
+        }
+    }
+
+    fn advance_tail_to_next_or_window(&mut self) -> bool {
+        if self.operator != Operator::Or || self.tail.is_empty() {
+            return false;
+        }
+
+        let Some(up_to) = self.up_to else {
+            return false;
+        };
+        if up_to >= u32::MAX as u64 {
+            return false;
+        }
+        if !self
+            .tail
+            .iter()
+            .any(|tail| tail.posting.has_next_compressed_block())
+        {
+            return false;
+        }
+
+        // A low-scoring tail can be the only iterator left in the current
+        // window. Move to the next window so a later high-scoring block is still
+        // reachable instead of ending the disjunction early.
+        self.update_max_scores(up_to + 1);
+        true
+    }
+
     fn push_back_leads(&mut self, target: u64) {
         // After finishing a candidate doc, convert the aligned iterators back
         // into lagging iterators. Entries that do not stay in `tail` are
         // advanced to `target` and returned to `head`.
-        let leads = std::mem::take(&mut self.lead);
-        for posting in leads {
-            let upper_bound = posting.approximate_upper_bound();
+        // pop() drains in place, keeping self.lead's capacity for reuse.
+        if self.threshold <= 0.0 {
+            while let Some(mut posting) = self.lead.pop() {
+                posting.next(target);
+                self.push_head(posting);
+            }
+            return;
+        }
+
+        while let Some(posting) = self.lead.pop() {
+            let upper_bound = self.lead_to_tail_upper_bound(&posting, target);
             if let Some(mut evicted) = self.insert_tail_with_overflow(posting, upper_bound) {
                 evicted.next(target);
                 self.push_head(evicted);
@@ -1269,10 +3770,9 @@ impl<'a, S: Scorer> Wand<'a, S> {
         };
         self.tail_max_score -= upper_bound;
         posting.next(target);
-        match posting.doc().map(|doc| doc.doc_id()) {
-            Some(doc_id) if doc_id == target => {
-                let frequency = posting.doc().expect("posting must exist").frequency();
-                *lead_score += posting.score(&self.scorer, frequency, doc_length);
+        match posting.doc() {
+            Some(doc) if doc.doc_id() == target => {
+                *lead_score += posting.score(&self.scorer, doc.frequency(), doc_length);
                 self.lead.push(posting);
             }
             Some(_) => self.push_head(posting),
@@ -1295,14 +3795,10 @@ impl<'a, S: Scorer> Wand<'a, S> {
         for tail_posting in tail.into_vec() {
             let mut posting = tail_posting.posting;
             posting.next(target);
-            match posting.doc().map(|doc| doc.doc_id()) {
-                Some(doc_id) if doc_id == target => {
+            match posting.doc() {
+                Some(doc) if doc.doc_id() == target => {
                     if let (Some(doc_length), Some(score)) = (doc_length, score.as_deref_mut()) {
-                        let frequency = posting
-                            .doc()
-                            .expect("posting moved to target should have doc")
-                            .frequency();
-                        *score += posting.score(&self.scorer, frequency, doc_length);
+                        *score += posting.score(&self.scorer, doc.frequency(), doc_length);
                     }
                     self.lead.push(posting)
                 }
@@ -1327,7 +3823,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             .collect()
     }
 
-    fn check_positions(&self, slop: i32) -> bool {
+    fn check_positions(&self, slop: i32) -> Result<bool> {
         if slop == 0 {
             return self.check_exact_positions();
         }
@@ -1335,8 +3831,8 @@ impl<'a, S: Scorer> Wand<'a, S> {
         let mut position_iters = self
             .current_doc_postings()
             .into_iter()
-            .map(|posting| posting.position_cursor().expect("positions must exist"))
-            .collect::<Vec<_>>();
+            .map(PostingIterator::position_cursor)
+            .collect::<Result<Vec<_>>>()?;
         position_iters.sort_unstable_by_key(|iter| iter.position_in_query);
 
         loop {
@@ -1346,7 +3842,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
                 let last = window[0].relative_position();
                 let next = window[1].relative_position();
                 let (Some(last), Some(next)) = (last, next) else {
-                    return false;
+                    return Ok(false);
                 };
 
                 let move_to = if last > next {
@@ -1362,7 +3858,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             if all_same {
-                return true;
+                return Ok(true);
             }
 
             position_iters.iter_mut().for_each(|iter| {
@@ -1371,21 +3867,72 @@ impl<'a, S: Scorer> Wand<'a, S> {
         }
     }
 
-    fn check_exact_positions(&self) -> bool {
+    /// Allocation-free exact-phrase check for the bulk conjunction path,
+    /// where every clause is a parked `lead` iterator. Semantically identical
+    /// to [`Self::check_exact_positions`] — some base position must align all
+    /// clauses at their query offsets — without the per-candidate cursor vec
+    /// and sort.
+    fn check_exact_positions_bulk(&self) -> Result<bool> {
+        const MAX_INLINE_CLAUSES: usize = 16;
+        let num_clauses = self.lead.len();
+        if num_clauses > MAX_INLINE_CLAUSES {
+            return self.check_exact_positions();
+        }
+        // Cursors stay alive in the stack array so owned position buffers
+        // (legacy per-doc storage) remain valid while we scan.
+        let mut cursors: [Option<PositionCursor<'_>>; MAX_INLINE_CLAUSES] =
+            std::array::from_fn(|_| None);
+        let mut anchor_idx = 0usize;
+        let mut anchor_len = usize::MAX;
+        for (index, (slot, posting)) in cursors.iter_mut().zip(self.lead.iter()).enumerate() {
+            let cursor = posting.position_cursor()?;
+            if cursor.len() < anchor_len {
+                anchor_len = cursor.len();
+                anchor_idx = index;
+            }
+            *slot = Some(cursor);
+        }
+
+        let anchor = cursors[anchor_idx]
+            .as_ref()
+            .expect("anchor cursor was just populated");
+        let anchor_offset = anchor.position_in_query as u32;
+        'anchor: for &anchor_position in anchor.positions.as_slice() {
+            let Some(base) = anchor_position.checked_sub(anchor_offset) else {
+                continue;
+            };
+            for (index, slot) in cursors[..num_clauses].iter().enumerate() {
+                if index == anchor_idx {
+                    continue;
+                }
+                let cursor = slot.as_ref().expect("clause cursor was just populated");
+                let Some(target) = base.checked_add(cursor.position_in_query as u32) else {
+                    return Ok(false);
+                };
+                if cursor.positions.as_slice().binary_search(&target).is_err() {
+                    continue 'anchor;
+                }
+            }
+            return Ok(true);
+        }
+        Ok(false)
+    }
+
+    fn check_exact_positions(&self) -> Result<bool> {
         let mut position_iters = self
             .current_doc_postings()
             .into_iter()
-            .map(|posting| posting.position_cursor().expect("positions must exist"))
-            .collect::<Vec<_>>();
+            .map(PostingIterator::position_cursor)
+            .collect::<Result<Vec<_>>>()?;
         position_iters.sort_unstable_by_key(|iter| iter.len());
         let Some(lead) = position_iters.first() else {
-            return false;
+            return Ok(false);
         };
         let lead_position = lead.position_in_query;
 
         loop {
             let Some(anchor) = position_iters[0].absolute_position() else {
-                return false;
+                return Ok(false);
             };
             let Some(base) = anchor.checked_sub(lead_position as u32) else {
                 position_iters[0].advance_next();
@@ -1396,10 +3943,10 @@ impl<'a, S: Scorer> Wand<'a, S> {
             let mut matched = true;
             for follower in position_iters.iter_mut().skip(1) {
                 let Some(target) = base.checked_add(follower.position_in_query as u32) else {
-                    return false;
+                    return Ok(false);
                 };
                 let Some(position) = follower.advance_to_absolute(target) else {
-                    return false;
+                    return Ok(false);
                 };
                 if position != target {
                     next_lead_relative = Some(position as i32 - follower.position_in_query);
@@ -1409,7 +3956,7 @@ impl<'a, S: Scorer> Wand<'a, S> {
             }
 
             if matched {
-                return true;
+                return Ok(true);
             }
 
             position_iters[0].advance_to_relative(next_lead_relative.unwrap());
@@ -1417,16 +3964,50 @@ impl<'a, S: Scorer> Wand<'a, S> {
     }
 }
 
+#[derive(Debug)]
+struct RecycledPositionValues<'a> {
+    values: Option<Vec<u32>>,
+    pool: &'a RefCell<Option<Vec<u32>>>,
+}
+
+impl<'a> RecycledPositionValues<'a> {
+    fn new(values: Vec<u32>, pool: &'a RefCell<Option<Vec<u32>>>) -> Self {
+        Self {
+            values: Some(values),
+            pool,
+        }
+    }
+
+    fn as_slice(&self) -> &[u32] {
+        self.values
+            .as_deref()
+            .expect("position values are present until drop")
+    }
+}
+
+impl Drop for RecycledPositionValues<'_> {
+    fn drop(&mut self) {
+        let values = self
+            .values
+            .take()
+            .expect("position values are present until drop");
+        let mut pool = self.pool.borrow_mut();
+        if pool.is_none() {
+            *pool = Some(values);
+        }
+    }
+}
+
 #[derive(Debug)]
 enum PositionValues<'a> {
-    Borrowed(&'a [u32]),
+    Recycled(RecycledPositionValues<'a>),
     Owned(Vec<u32>),
 }
 
 impl<'a> PositionValues<'a> {
     fn as_slice(&self) -> &[u32] {
         match self {
-            Self::Borrowed(values) => values,
+            Self::Recycled(values) => values.as_slice(),
             Self::Owned(values) => values.as_slice(),
         }
     }
@@ -1496,16 +4077,48 @@ mod tests {
     use arrow::buffer::ScalarBuffer;
     use rstest::rstest;
 
+    use std::sync::atomic::{AtomicUsize, Ordering};
+
+    use super::super::impact::build_impact_skip_data;
     use super::*;
-    use crate::scalar::inverted::scorer::IndexBM25Scorer;
+    use crate::scalar::inverted::scorer::{IndexBM25Scorer, MemBM25Scorer};
     use crate::{
-        metrics::NoOpMetricsCollector,
+        metrics::{MetricsCollector, NoOpMetricsCollector},
         scalar::inverted::{
-            CompressedPostingList, PlainPostingList, PostingListBuilder, builder::PositionRecorder,
-            encoding::compress_posting_list,
+            CompressedPostingList, PlainPostingList, PostingListBuilder, SharedPositionStream,
+            builder::PositionRecorder,
+            encoding::{
+                compress_posting_list, compress_posting_list_with_tail_codec_and_block_size,
+                encode_position_stream_block_into,
+            },
         },
     };
 
+    #[test]
+    fn test_maxscore_prefix_bound_covers_f32_summation_rounding() {
+        let remaining_bounds = [6.286_838_4e-7_f32, 0.015_441_144_f32];
+        let essential_score = 2.762_496_2_f32;
+        let threshold = f32::from_bits(0x4031_c9bc);
+
+        // Raw f32 prefix accumulation rounds down to an apparent tie, while
+        // scoring the clauses individually produces a competitive document.
+        let raw_prefix = remaining_bounds.into_iter().sum::<f32>();
+        assert_eq!(essential_score + raw_prefix, threshold);
+        let actual_score = remaining_bounds
+            .into_iter()
+            .rev()
+            .fold(essential_score, |score, bound| score + bound);
+        assert!(actual_score > threshold);
+
+        let prefix_bound = remaining_bounds.into_iter().map(f64::from).sum::<f64>();
+        assert!(!score_sum_cannot_exceed(
+            essential_score,
+            prefix_bound,
+            threshold,
+            score_sum_upper_bound_factor(3),
+        ));
+    }
+
     struct UnitScorer;
 
     impl Scorer for UnitScorer {
@@ -1518,6 +4131,98 @@ mod tests {
         }
     }
 
+    struct PartialNormScorer;
+
+    impl Scorer for PartialNormScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, _doc_tokens: u32) -> f32 {
+            freq as f32
+        }
+
+        fn doc_norm(&self, doc_tokens: u32) -> Option<f32> {
+            (doc_tokens == 0).then_some(0.0)
+        }
+    }
+
+    #[test]
+    fn test_norm_cache_requires_all_norm_codes() {
+        let mut docs = DocSet::default();
+        docs.append(0, 1);
+        docs.set_quantized_scoring(true);
+        let wand = Wand::new(Operator::Or, std::iter::empty(), &docs, PartialNormScorer);
+
+        assert!(wand.norm_k_cache().is_none());
+    }
+
+    #[test]
+    fn test_top_k_collector_reuses_frequency_slots() -> Result<()> {
+        const LIMIT: usize = 8;
+        const NUM_DOCS: usize = 10_000;
+        let mut collector = TopKCollector::new(LIMIT, LIMIT);
+
+        for doc in 0..NUM_DOCS {
+            let num_terms = doc % 4 + 1;
+            let inserted = collector.insert(
+                ScoredDoc::new(doc as u64, doc as f32),
+                num_terms as u32,
+                doc as u64,
+                (0..num_terms).map(|term| (term as u32, doc as u32)),
+            )?;
+            assert!(inserted);
+            assert!(collector.num_frequency_slots() <= LIMIT);
+        }
+        assert_eq!(collector.num_frequency_slots(), LIMIT);
+
+        let mut candidates = collector.into_candidates(CandidateAddr::RowId)?;
+        candidates.sort_unstable_by_key(|candidate| candidate.posting_doc_id);
+        assert_eq!(candidates.len(), LIMIT);
+        for (candidate, expected_doc) in candidates.iter().zip(NUM_DOCS - LIMIT..NUM_DOCS) {
+            assert_eq!(candidate.posting_doc_id, expected_doc as u64);
+            assert!(matches!(
+                candidate.addr,
+                CandidateAddr::RowId(row_id) if row_id == expected_doc as u64
+            ));
+            let expected_freqs = (0..expected_doc % 4 + 1)
+                .map(|term| (term as u32, expected_doc as u32))
+                .collect::<Vec<_>>();
+            assert_eq!(candidate.freqs, expected_freqs);
+        }
+        Ok(())
+    }
+
+    #[rstest]
+    #[case::auto("auto", Some(BulkAndMode::Auto))]
+    #[case::auto_case_and_whitespace(" AUTO ", Some(BulkAndMode::Auto))]
+    #[case::on("on", Some(BulkAndMode::On))]
+    #[case::on_legacy("1", Some(BulkAndMode::On))]
+    #[case::off("off", Some(BulkAndMode::Off))]
+    #[case::off_legacy("0", Some(BulkAndMode::Off))]
+    #[case::invalid("true", None)]
+    #[case::empty("", None)]
+    fn test_bulk_and_mode_parse(#[case] value: &str, #[case] expected: Option<BulkAndMode>) {
+        assert_eq!(BulkAndMode::parse(value), expected);
+    }
+
+    #[rstest]
+    #[case::auto_one(BulkAndMode::Auto, 1, false)]
+    #[case::auto_two(BulkAndMode::Auto, 2, true)]
+    #[case::auto_three(BulkAndMode::Auto, 3, true)]
+    #[case::auto_four(BulkAndMode::Auto, 4, false)]
+    #[case::on_one(BulkAndMode::On, 1, true)]
+    #[case::on_five(BulkAndMode::On, 5, true)]
+    #[case::off_two(BulkAndMode::Off, 2, false)]
+    #[case::off_five(BulkAndMode::Off, 5, false)]
+    fn test_bulk_and_mode_enabled_for(
+        #[case] mode: BulkAndMode,
+        #[case] num_clauses: usize,
+        #[case] expected: bool,
+    ) {
+        assert_eq!(mode.enabled_for(num_clauses), expected);
+    }
+
     struct PanicQueryWeightScorer;
 
     impl Scorer for PanicQueryWeightScorer {
@@ -1542,6 +4247,97 @@ mod tests {
         }
     }
 
+    // Inverse-doc-length scorer that counts scored documents, so a test can
+    // assert that block-max pruning skipped blocks.
+    struct CountingScorer {
+        scored: Arc<AtomicUsize>,
+    }
+
+    impl Scorer for CountingScorer {
+        fn query_weight(&self, _token: &str) -> f32 {
+            1.0
+        }
+
+        fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 {
+            self.scored.fetch_add(1, Ordering::Relaxed);
+            freq as f32 / doc_tokens as f32
+        }
+    }
+
+    #[derive(Default)]
+    struct CountAndSearchStats {
+        comparisons: AtomicUsize,
+        candidates_seen: AtomicUsize,
+        candidates_pruned_before_return: AtomicUsize,
+        full_scores: AtomicUsize,
+        freqs_collected: AtomicUsize,
+    }
+
+    impl MetricsCollector for CountAndSearchStats {
+        fn record_parts_loaded(&self, _: usize) {}
+
+        fn record_index_loads(&self, _: usize) {}
+
+        fn record_comparisons(&self, n: usize) {
+            self.comparisons.fetch_add(n, Ordering::Relaxed);
+        }
+
+        fn record_and_candidates_seen(&self, n: usize) {
+            self.candidates_seen.fetch_add(n, Ordering::Relaxed);
+        }
+
+        fn record_and_candidates_pruned_before_return(&self, n: usize) {
+            self.candidates_pruned_before_return
+                .fetch_add(n, Ordering::Relaxed);
+        }
+
+        fn record_and_full_scores(&self, n: usize) {
+            self.full_scores.fetch_add(n, Ordering::Relaxed);
+        }
+
+        fn record_freqs_collected(&self, n: usize) {
+            self.freqs_collected.fetch_add(n, Ordering::Relaxed);
+        }
+    }
+
+    struct PanicOnAndMetrics {
+        comparisons: AtomicUsize,
+    }
+
+    impl PanicOnAndMetrics {
+        fn new() -> Self {
+            Self {
+                comparisons: AtomicUsize::new(0),
+            }
+        }
+    }
+
+    impl MetricsCollector for PanicOnAndMetrics {
+        fn record_parts_loaded(&self, _: usize) {}
+
+        fn record_index_loads(&self, _: usize) {}
+
+        fn record_comparisons(&self, n: usize) {
+            self.comparisons.fetch_add(n, Ordering::Relaxed);
+        }
+
+        fn record_and_candidates_seen(&self, _: usize) {
+            panic!("OR search should not record AND candidate metrics");
+        }
+
+        fn record_and_candidates_pruned_before_return(&self, _: usize) {
+            panic!("OR search should not record AND prune metrics");
+        }
+
+        fn record_and_full_scores(&self, _: usize) {
+            panic!("OR search should not record AND scoring metrics");
+        }
+
+        fn record_freqs_collected(&self, _: usize) {
+            panic!("OR search should not record AND frequency metrics");
+        }
+    }
+
     fn generate_posting_list(
         doc_ids: Vec<u32>,
         max_score: f32,
@@ -1549,6 +4345,17 @@ mod tests {
         is_compressed: bool,
     ) -> PostingList {
         let freqs = vec![1; doc_ids.len()];
+        generate_posting_list_with_freqs(doc_ids, freqs, max_score, block_max_scores, is_compressed)
+    }
+
+    fn generate_posting_list_with_freqs(
+        doc_ids: Vec<u32>,
+        freqs: Vec<u32>,
+        max_score: f32,
+        block_max_scores: Option<Vec<f32>>,
+        is_compressed: bool,
+    ) -> PostingList {
+        assert_eq!(doc_ids.len(), freqs.len());
         let block_max_scores = block_max_scores.unwrap_or_else(|| vec![max_score; doc_ids.len()]);
         if is_compressed {
             let blocks = compress_posting_list(
@@ -1563,6 +4370,8 @@ mod tests {
                 max_score,
                 doc_ids.len() as u32,
                 crate::scalar::inverted::PostingTailCodec::VarintDelta,
+                crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+                None,
                 None,
             ))
         } else {
@@ -1575,6 +4384,75 @@ mod tests {
         }
     }
 
+    fn generate_impact_posting_list_with_freqs(
+        doc_ids: Vec<u32>,
+        freqs: Vec<u32>,
+        doc_lengths: Vec<u32>,
+    ) -> PostingList {
+        generate_impact_posting_list_with_freqs_and_block_size(
+            doc_ids,
+            freqs,
+            doc_lengths,
+            crate::scalar::inverted::LEGACY_BLOCK_SIZE,
+        )
+    }
+
+    fn generate_impact_posting_list_with_freqs_and_block_size(
+        doc_ids: Vec<u32>,
+        freqs: Vec<u32>,
+        doc_lengths: Vec<u32>,
+        block_size: usize,
+    ) -> PostingList {
+        assert_eq!(doc_ids.len(), freqs.len());
+        assert_eq!(doc_ids.len(), doc_lengths.len());
+        let block_max_scores = vec![0.0; doc_ids.len().div_ceil(block_size)];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            freqs.iter(),
+            block_max_scores.into_iter(),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            block_size,
+        )
+        .unwrap();
+        let impact_blocks = doc_ids
+            .chunks(block_size)
+            .zip(freqs.chunks(block_size))
+            .zip(doc_lengths.chunks(block_size))
+            .map(|((doc_ids, freqs), doc_lengths)| {
+                doc_ids
+                    .iter()
+                    .copied()
+                    .zip(freqs.iter().copied())
+                    .zip(doc_lengths.iter().copied())
+                    .map(|((doc_id, freq), doc_length)| (doc_id, freq, doc_length))
+                    .collect::<Vec<_>>()
+            })
+            .collect::<Vec<_>>();
+        let impacts = build_impact_skip_data(impact_blocks.as_slice()).unwrap();
+        PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            0.0,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            block_size,
+            None,
+            Some(impacts),
+        ))
+    }
+
+    fn generate_contiguous_impact_posting_list_with_block_size(
+        total: usize,
+        block_size: usize,
+    ) -> PostingList {
+        generate_impact_posting_list_with_freqs_and_block_size(
+            (0..total as u32).collect(),
+            vec![1; total],
+            vec![1; total],
+            block_size,
+        )
+    }
+
     fn generate_posting_list_with_positions(
         doc_ids: Vec<u32>,
         positions_by_doc: Vec<Vec<u32>>,
@@ -1612,6 +4490,107 @@ mod tests {
         }
     }
 
+    #[rstest]
+    #[case::packed_delta(PositionStreamCodec::PackedDelta)]
+    #[case::varint_doc_delta(PositionStreamCodec::VarintDocDelta)]
+    fn test_shared_position_cursors_use_independent_scratch(
+        #[case] codec: PositionStreamCodec,
+    ) -> Result<()> {
+        let mut posting_list =
+            generate_posting_list_with_positions(vec![0], vec![vec![1_u32, 3, 10]], 1.0, true);
+        let PostingList::Compressed(ref mut list) = posting_list else {
+            unreachable!("the helper was asked for a compressed posting list");
+        };
+        let mut encoded = Vec::new();
+        encode_position_stream_block_into(&[1, 3, 10], &[3], codec, &mut encoded)?;
+        list.positions = Some(CompressedPositionStorage::SharedStream(
+            SharedPositionStream::new(codec, vec![0], bytes::Bytes::from(encoded)),
+        ));
+
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, 1);
+        let first = posting.position_cursor()?;
+        let second = posting.position_cursor()?;
+
+        assert_eq!(second.positions.as_slice(), &[1, 3, 10]);
+        assert_eq!(first.positions.as_slice(), &[1, 3, 10]);
+        assert!(posting.position_scratch.borrow().is_none());
+        drop(second);
+        assert!(posting.position_scratch.borrow().is_some());
+        drop(first);
+        assert!(posting.position_scratch.borrow().is_some());
+        Ok(())
+    }
+
+    #[test]
+    fn test_phrase_search_propagates_corrupt_packed_positions() {
+        let mut docs = DocSet::default();
+        docs.append(0, BLOCK_SIZE as u32 + 1);
+
+        let mut corrupt_list = generate_posting_list_with_positions(
+            vec![0],
+            vec![(0..BLOCK_SIZE as u32).collect()],
+            1.0,
+            true,
+        );
+        let PostingList::Compressed(ref mut list) = corrupt_list else {
+            unreachable!("the helper was asked for a compressed posting list");
+        };
+        // A bit width of one requires a 16-byte payload. Keep only the header
+        // to verify malformed on-disk data becomes a search error, not a panic.
+        list.positions = Some(CompressedPositionStorage::SharedStream(
+            SharedPositionStream::new(
+                PositionStreamCodec::PackedDelta,
+                vec![0],
+                bytes::Bytes::from_static(&[1]),
+            ),
+        ));
+
+        let postings = vec![
+            PostingIterator::new(String::from("corrupt"), 0, 0, corrupt_list, docs.len()),
+            PostingIterator::new(
+                String::from("valid"),
+                1,
+                1,
+                generate_posting_list_with_positions(
+                    vec![0],
+                    vec![vec![BLOCK_SIZE as u32]],
+                    1.0,
+                    true,
+                ),
+                docs.len(),
+            ),
+        ];
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        let mut params = FtsSearchParams::default().with_limit(Some(10));
+        params.phrase_slop = Some(0);
+
+        let error = wand
+            .search(
+                &params,
+                Arc::new(RowAddrMask::default()),
+                &NoOpMetricsCollector,
+            )
+            .expect_err("corrupt packed positions should fail the phrase search");
+        let message = error.to_string();
+        assert!(
+            message.contains("packed position group payload"),
+            "{message}"
+        );
+        assert!(message.contains("corrupt"), "{message}");
+    }
+
+    fn sorted_candidate_row_ids(candidates: Vec<DocCandidate>) -> Vec<u64> {
+        let mut row_ids = candidates
+            .into_iter()
+            .map(|candidate| match candidate.addr {
+                CandidateAddr::RowId(row_id) => row_id,
+                CandidateAddr::Pending(doc_id) => doc_id as u64,
+            })
+            .collect::<Vec<_>>();
+        row_ids.sort_unstable();
+        row_ids
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_wand(#[values(false, true)] is_compressed: bool) {
@@ -1656,6 +4635,70 @@ mod tests {
         assert_eq!(result.len(), 0); // Should not panic
     }
 
+    /// The shared floor prunes partitions that can't reach the global top-k: a
+    /// high-scoring partition sets the floor and the rest skip their blocks.
+    /// (Result correctness is covered by the FTS search tests, since sharing is
+    /// always on.)
+    #[test]
+    fn cross_partition_threshold_sharing_prunes() {
+        use crate::metrics::MetricsCollector;
+        use std::sync::atomic::AtomicUsize;
+
+        #[derive(Default)]
+        struct CountComparisons(AtomicUsize);
+        impl MetricsCollector for CountComparisons {
+            fn record_parts_loaded(&self, _: usize) {}
+            fn record_index_loads(&self, _: usize) {}
+            fn record_comparisons(&self, n: usize) {
+                self.0.fetch_add(n, Ordering::Relaxed);
+            }
+        }
+
+        let params = FtsSearchParams::default().with_limit(Some(10));
+        let part_docs = 4 * BLOCK_SIZE as u32;
+        // One high-scoring partition (weight 10) then 7 low-scoring ones.
+        let parts: Vec<(f32, std::ops::Range<u32>)> = std::iter::once((10.0, 0..part_docs))
+            .chain((1..8).map(|i| (1.0, i * part_docs..(i + 1) * part_docs)))
+            .collect();
+
+        let new_floor = || Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits()));
+
+        // Total comparisons across all partitions. `Some(floor)` makes every
+        // partition share that one floor; `None` gives each its own.
+        let total_comparisons = |shared_floor: Option<&Arc<AtomicU32>>| -> usize {
+            let metrics = CountComparisons::default();
+            for (qw, rows) in &parts {
+                let mut docs = DocSet::default();
+                for d in rows.clone() {
+                    docs.append(d as u64, 1);
+                }
+                let postings = vec![PostingIterator::with_query_weight(
+                    String::from("t"),
+                    0,
+                    0,
+                    *qw,
+                    generate_posting_list(rows.clone().collect(), *qw, None, false),
+                    docs.len(),
+                )];
+                let floor = shared_floor.cloned().unwrap_or_else(new_floor);
+                Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer)
+                    .with_shared_threshold(floor)
+                    .search(&params, Arc::new(RowAddrMask::default()), &metrics)
+                    .unwrap();
+            }
+            metrics.0.load(Ordering::Relaxed)
+        };
+
+        let one_floor = new_floor();
+        let with_shared_floor = total_comparisons(Some(&one_floor));
+        let with_private_floors = total_comparisons(None);
+        assert!(
+            with_shared_floor < with_private_floors,
+            "shared floor should prune comparisons: \
+             shared={with_shared_floor} private={with_private_floors}"
+        );
+    }
+
     #[test]
     fn test_posting_iterator_next_compressed_partition_point() {
         let mut docs = DocSet::default();
@@ -1721,6 +4764,122 @@ mod tests {
         assert!(result.is_ok());
     }
 
+    #[test]
+    fn test_or_single_term_block_skip_matches_and() {
+        // Hot docs occupy the middle block; the flanking blocks score far below
+        // the threshold. A single-term disjunction must skip them yet return what
+        // the conjunctive path returns.
+        let total = 3 * BLOCK_SIZE as u32;
+        let hot = BLOCK_SIZE as u32..BLOCK_SIZE as u32 + 12;
+
+        let mut docs = DocSet::default();
+        for row_id in 0..total {
+            // hot docs get distinct scores 1/1..1/12; the rest score 0.001
+            let doc_tokens = if hot.contains(&row_id) {
+                row_id - hot.start + 1
+            } else {
+                1000
+            };
+            docs.append(row_id as u64, doc_tokens);
+        }
+
+        let params = FtsSearchParams::new().with_limit(Some(10));
+        let run = |operator| {
+            let scored = Arc::new(AtomicUsize::new(0));
+            let posting = PostingIterator::with_query_weight(
+                String::from("term"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(
+                    (0..total).collect(),
+                    1.0,
+                    Some(vec![0.001, 1.0, 0.001]),
+                    true,
+                ),
+                docs.len(),
+            );
+            let mut wand = Wand::new(
+                operator,
+                std::iter::once(posting),
+                &docs,
+                CountingScorer {
+                    scored: scored.clone(),
+                },
+            );
+            let hits = wand
+                .search(
+                    &params,
+                    Arc::new(RowAddrMask::default()),
+                    &NoOpMetricsCollector,
+                )
+                .unwrap();
+            let mut row_ids = hits
+                .iter()
+                .map(|hit| match hit.addr {
+                    CandidateAddr::RowId(r) => r,
+                    CandidateAddr::Pending(_) => panic!("row_id should be set in this path"),
+                })
+                .collect::<Vec<_>>();
+            row_ids.sort_unstable();
+            (row_ids, scored.load(Ordering::Relaxed))
+        };
+
+        let (or_hits, or_scored) = run(Operator::Or);
+        let (and_hits, _) = run(Operator::And);
+
+        let expected = (hot.start..hot.start + 10)
+            .map(u64::from)
+            .collect::<Vec<_>>();
+        assert_eq!(or_hits, expected, "OR must return the top-k");
+        assert_eq!(or_hits, and_hits, "OR and AND must agree for a single term");
+        // Without pruning OR scores all `total` docs; with it the cold blocks are skipped.
+        assert!(
+            or_scored <= 2 * BLOCK_SIZE,
+            "expected pruning to skip a block, but scored {or_scored} of {total}",
+        );
+    }
+
+    #[rstest]
+    fn test_or_search_does_not_record_and_metrics(#[values(false, true)] is_compressed: bool) {
+        let mut docs = DocSet::default();
+        for row_id in 0..6 {
+            docs.append(row_id, 1);
+        }
+
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("alpha"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(vec![0, 1, 4], 1.0, None, is_compressed),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("beta"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(vec![1, 2, 5], 1.0, None, is_compressed),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        let metrics = PanicOnAndMetrics::new();
+        let candidates = wand
+            .search(
+                &FtsSearchParams::default(),
+                Arc::new(RowAddrMask::default()),
+                &metrics,
+            )
+            .unwrap();
+
+        assert_eq!(sorted_candidate_row_ids(candidates), vec![0, 1, 2, 4, 5]);
+        assert!(metrics.comparisons.load(Ordering::Relaxed) > 0);
+    }
+
     #[test]
     fn test_wand_new_uses_precomputed_query_weight() {
         let mut docs = DocSet::default();
@@ -1802,32 +4961,247 @@ mod tests {
     }
 
     #[test]
-    fn test_and_search_prunes_with_threshold_and_keeps_candidate() {
+    fn test_or_push_back_lead_uses_current_block_max_for_tail_bound() {
+        let total = 2 * BLOCK_SIZE as u32;
         let mut docs = DocSet::default();
-        for i in 0..(2 * BLOCK_SIZE as u64) {
-            let doc_tokens = if i < BLOCK_SIZE as u64 { 100 } else { 1 };
-            docs.append(i, doc_tokens);
+        for doc_id in 0..total {
+            docs.append(doc_id as u64, 1);
         }
-        let all_docs = (0..2 * BLOCK_SIZE as u32).collect::<Vec<_>>();
 
-        let postings = vec![
-            PostingIterator::with_query_weight(
-                String::from("a"),
-                0,
-                0,
-                1.0,
-                generate_posting_list(all_docs.clone(), 1.0, Some(vec![0.02, 1.0]), true),
-                docs.len(),
-            ),
-            PostingIterator::with_query_weight(
-                String::from("b"),
-                1,
-                1,
-                1.0,
-                generate_posting_list(all_docs, 1.0, Some(vec![0.02, 1.0]), true),
-                docs.len(),
-            ),
-        ];
+        let postings = vec![PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            generate_posting_list((0..total).collect(), 10.0, Some(vec![1.0, 10.0]), true),
+            docs.len(),
+        )];
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 1.5;
+
+        wand.update_max_scores(0);
+        wand.move_head_doc_to_lead(0);
+        assert_eq!(wand.up_to, Some((BLOCK_SIZE - 1) as u64));
+
+        wand.push_back_leads(1);
+
+        assert_eq!(wand.tail.len(), 1);
+        assert!(
+            (wand.tail_max_score - 1.0).abs() < 1e-6,
+            "tail should use the current block max, got {}",
+            wand.tail_max_score
+        );
+        assert!(wand.head_doc().is_none());
+    }
+
+    #[test]
+    fn test_or_push_back_lead_falls_back_after_block_window_expires() {
+        let total = 2 * BLOCK_SIZE as u32;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total {
+            docs.append(doc_id as u64, 1);
+        }
+
+        let freqs = (0..total)
+            .map(|doc_id| if doc_id >= BLOCK_SIZE as u32 { 10 } else { 1 })
+            .collect::<Vec<_>>();
+        let mut posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            generate_posting_list_with_freqs(
+                (0..total).collect(),
+                freqs,
+                10.0,
+                Some(vec![1.0, 10.0]),
+                true,
+            ),
+            docs.len(),
+        );
+        posting.next((BLOCK_SIZE - 1) as u64);
+        let mut wand = Wand::new(Operator::Or, std::iter::once(posting), &docs, UnitScorer);
+        wand.threshold = 1.5;
+
+        let block_end = (BLOCK_SIZE - 1) as u64;
+        wand.update_max_scores(block_end);
+        wand.move_head_doc_to_lead(block_end);
+        assert_eq!(wand.up_to, Some(block_end));
+
+        wand.push_back_leads(BLOCK_SIZE as u64);
+
+        assert!(wand.tail.is_empty());
+        assert_eq!(wand.head_doc(), Some(BLOCK_SIZE as u64));
+        let candidate = wand.next().unwrap().unwrap();
+        assert_eq!(candidate.0.doc_id(), BLOCK_SIZE as u64);
+    }
+
+    #[test]
+    fn test_non_positive_threshold_advances_without_impact_bound_scoring() {
+        let mut docs = DocSet::default();
+        for doc_id in 0..3 {
+            docs.append(doc_id, 1);
+        }
+        let make_posting = || {
+            PostingIterator::with_query_weight(
+                String::from("term"),
+                0,
+                0,
+                1.0,
+                generate_impact_posting_list_with_freqs(
+                    vec![0, 1, 2],
+                    vec![1, 1, 1],
+                    vec![1, 1, 1],
+                ),
+                docs.len(),
+            )
+        };
+        let scored = Arc::new(AtomicUsize::new(0));
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(make_posting()),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.move_head_before_target_to_tail(1);
+        assert_eq!(wand.head_doc(), Some(1));
+        assert!(wand.tail.is_empty());
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(make_posting()),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.move_head_doc_to_lead(0);
+        wand.push_back_leads(1);
+        assert_eq!(wand.head_doc(), Some(1));
+        assert!(wand.tail.is_empty());
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn test_or_plain_tail_does_not_advance_headless_window() {
+        let mut docs = DocSet::default();
+        for doc_id in 0..4 {
+            docs.append(doc_id, 1);
+        }
+
+        let postings = vec![PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            generate_posting_list(vec![0, 1, 2, 3], 1.0, None, false),
+            docs.len(),
+        )];
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 2.0;
+
+        wand.update_max_scores(0);
+        wand.move_head_doc_to_lead(0);
+        wand.push_back_leads(1);
+
+        assert_eq!(wand.tail.len(), 1);
+        assert!(wand.head_doc().is_none());
+        assert!(!wand.advance_tail_to_next_or_window());
+    }
+
+    #[test]
+    fn test_or_headless_tail_window_scans_past_final_top_tail() {
+        let total = 3 * BLOCK_SIZE as u32;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total {
+            docs.append(doc_id as u64, 1);
+        }
+
+        let future_docs = (0..BLOCK_SIZE as u32)
+            .chain(2 * BLOCK_SIZE as u32..3 * BLOCK_SIZE as u32)
+            .collect::<Vec<_>>();
+        let mut future_freqs = vec![1; future_docs.len()];
+        future_freqs[0] = 4;
+        future_freqs[BLOCK_SIZE] = 20;
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("future"),
+                0,
+                0,
+                1.0,
+                generate_posting_list_with_freqs(
+                    future_docs,
+                    future_freqs,
+                    20.0,
+                    Some(vec![4.0, 20.0]),
+                    true,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("final_tail"),
+                1,
+                1,
+                1.0,
+                generate_posting_list_with_freqs(vec![0], vec![6], 6.0, Some(vec![6.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("booster"),
+                2,
+                2,
+                1.0,
+                generate_posting_list_with_freqs(vec![0], vec![7], 7.0, Some(vec![7.0]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::Or, postings.into_iter(), &docs, UnitScorer);
+        let result = wand
+            .search(
+                &FtsSearchParams::new().with_limit(Some(1)),
+                Arc::new(RowAddrMask::default()),
+                &NoOpMetricsCollector,
+            )
+            .unwrap();
+
+        assert_eq!(
+            sorted_candidate_row_ids(result),
+            vec![(2 * BLOCK_SIZE) as u64]
+        );
+    }
+
+    #[test]
+    fn test_and_search_prunes_with_threshold_and_keeps_candidate() {
+        let mut docs = DocSet::default();
+        for i in 0..(2 * BLOCK_SIZE as u64) {
+            let doc_tokens = if i < BLOCK_SIZE as u64 { 100 } else { 1 };
+            docs.append(i, doc_tokens);
+        }
+        let all_docs = (0..2 * BLOCK_SIZE as u32).collect::<Vec<_>>();
+
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(all_docs.clone(), 1.0, Some(vec![0.02, 1.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(all_docs, 1.0, Some(vec![0.02, 1.0]), true),
+                docs.len(),
+            ),
+        ];
 
         let mut wand = Wand::new(
             Operator::And,
@@ -1841,6 +5215,607 @@ mod tests {
         assert_eq!(candidate.0.doc_id(), BLOCK_SIZE as u64);
     }
 
+    #[test]
+    fn test_and_advance_falls_back_to_narrow_when_range_max_loosens_bound() {
+        let total = 4 * BLOCK_SIZE as u32;
+        let mut docs = DocSet::default();
+        for i in 0..total {
+            docs.append(i as u64, 1);
+        }
+
+        let lead_docs = (0..total).step_by(2).collect::<Vec<_>>();
+        let follower_docs = (0..total).collect::<Vec<_>>();
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("lead"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(lead_docs, 1.0, Some(vec![1.0, 1.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("follower"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(follower_docs, 10.0, Some(vec![0.1, 10.0, 0.1, 0.1]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 5.0;
+
+        let target = wand.and_advance_target(0);
+
+        assert_eq!(target, BLOCK_SIZE as u64);
+        assert_eq!(wand.up_to, Some((2 * BLOCK_SIZE - 1) as u64));
+        assert!(
+            (wand.and_max_score - 11.0).abs() < 1e-6,
+            "expected the second narrow window to include the high follower block, got {}",
+            wand.and_max_score
+        );
+        assert_eq!(wand.and_window_stats.windows_wide, 0);
+        assert_eq!(wand.and_window_stats.windows_narrow, 2);
+        assert_eq!(wand.and_window_stats.windows_skipped, 1);
+    }
+
+    #[test]
+    fn test_and_advance_uses_narrow_window_for_candidate_ranges() {
+        let total = 4 * BLOCK_SIZE as u32;
+        let mut docs = DocSet::default();
+        for i in 0..total {
+            docs.append(i as u64, 1);
+        }
+
+        let lead_docs = (0..total).step_by(2).collect::<Vec<_>>();
+        let follower_docs = (0..total).collect::<Vec<_>>();
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("lead"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(lead_docs, 1.0, Some(vec![1.0, 1.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("follower"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(follower_docs, 1.0, Some(vec![1.0, 1.0, 1.0, 1.0]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 1.5;
+
+        let target = wand.and_advance_target(0);
+
+        assert_eq!(target, 0);
+        assert_eq!(wand.up_to, Some((BLOCK_SIZE - 1) as u64));
+        assert!((wand.and_max_score - 2.0).abs() < 1e-6);
+        assert_eq!(wand.and_window_stats.windows_wide, 0);
+        assert_eq!(wand.and_window_stats.windows_narrow, 1);
+        assert_eq!(wand.and_window_stats.range_blocks_scanned, 0);
+    }
+
+    #[test]
+    fn test_and_wide_window_only_skips_and_does_not_return_candidates() {
+        let total = 4 * BLOCK_SIZE as u32;
+        let mut docs = DocSet::default();
+        for i in 0..total {
+            docs.append(i as u64, 1);
+        }
+
+        let lead_docs = (0..total).step_by(2).collect::<Vec<_>>();
+        let follower_docs = (0..total).collect::<Vec<_>>();
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("lead"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(lead_docs, 3.0, Some(vec![1.0, 3.0]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("follower"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(follower_docs, 3.0, Some(vec![0.1, 0.1, 3.0, 3.0]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        wand.threshold = 2.0;
+
+        let candidate = wand.next().unwrap().unwrap();
+
+        assert_eq!(candidate.0.doc_id(), (2 * BLOCK_SIZE) as u64);
+        assert_eq!(wand.up_to, Some((3 * BLOCK_SIZE - 1) as u64));
+        assert_eq!(wand.and_window_stats.windows_wide, 1);
+        assert_eq!(wand.and_window_stats.windows_skipped, 1);
+        assert_eq!(wand.and_window_stats.windows_narrow, 1);
+        assert_eq!(wand.and_window_stats.candidates_returned, 1);
+    }
+
+    #[test]
+    fn test_and_range_max_preserves_exact_top_k() {
+        let total = 4 * BLOCK_SIZE as u32;
+        let hot = BLOCK_SIZE as u32..BLOCK_SIZE as u32 + 16;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total {
+            let doc_tokens = if hot.contains(&doc_id) { 1 } else { 1000 };
+            docs.append(doc_id as u64, doc_tokens);
+        }
+
+        let params = FtsSearchParams::new().with_limit(Some(8));
+        let run = |is_compressed: bool| {
+            let lead_docs = (0..total).step_by(2).collect::<Vec<_>>();
+            let follower_docs = (0..total).collect::<Vec<_>>();
+            let lead_scores = is_compressed.then_some(vec![1.0, 0.001]);
+            let follower_scores = is_compressed.then_some(vec![0.001, 1.0, 0.001, 0.001]);
+            let postings = vec![
+                PostingIterator::with_query_weight(
+                    String::from("lead"),
+                    0,
+                    0,
+                    1.0,
+                    generate_posting_list(lead_docs, 1.0, lead_scores, is_compressed),
+                    docs.len(),
+                ),
+                PostingIterator::with_query_weight(
+                    String::from("follower"),
+                    1,
+                    1,
+                    1.0,
+                    generate_posting_list(follower_docs, 1.0, follower_scores, is_compressed),
+                    docs.len(),
+                ),
+            ];
+            let mut wand = Wand::new(
+                Operator::And,
+                postings.into_iter(),
+                &docs,
+                InverseDocLengthScorer,
+            );
+            sorted_candidate_row_ids(
+                wand.search(
+                    &params,
+                    Arc::new(RowAddrMask::default()),
+                    &NoOpMetricsCollector,
+                )
+                .unwrap(),
+            )
+        };
+
+        let compressed = run(true);
+        let plain = run(false);
+        let expected = hot.step_by(2).map(u64::from).collect::<Vec<_>>();
+        assert_eq!(compressed, expected);
+        assert_eq!(compressed, plain);
+    }
+
+    #[test]
+    fn test_block_max_score_up_to_slides_and_expires_old_max() {
+        let total = 5 * BLOCK_SIZE as u32;
+        let posting = generate_posting_list(
+            (0..total).collect(),
+            5.0,
+            Some(vec![1.0, 4.0, 2.0, 5.0, 3.0]),
+            true,
+        );
+        let mut posting = PostingIterator::new(String::from("term"), 0, 0, posting, total as usize);
+
+        posting.shallow_next(0);
+        assert_eq!(
+            posting
+                .block_max_score_up_to_with_stats((3 * BLOCK_SIZE - 1) as u64, &UnitScorer)
+                .score,
+            4.0
+        );
+
+        posting.shallow_next((2 * BLOCK_SIZE) as u64);
+        assert_eq!(
+            posting
+                .block_max_score_up_to_with_stats((4 * BLOCK_SIZE - 1) as u64, &UnitScorer)
+                .score,
+            5.0
+        );
+
+        posting.shallow_next((4 * BLOCK_SIZE) as u64);
+        assert_eq!(
+            posting
+                .block_max_score_up_to_with_stats((5 * BLOCK_SIZE - 1) as u64, &UnitScorer)
+                .score,
+            3.0
+        );
+    }
+
+    #[test]
+    fn test_impact_level1_skip_keeps_boundary_equality_in_group() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS + 1) * block_size;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+            let target = (IMPACT_LEVEL1_BLOCKS * block_size - 1) as u64;
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, IMPACT_LEVEL1_BLOCKS - 1);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, IMPACT_LEVEL1_BLOCKS - 1);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_impact_level1_skip_handles_partial_final_group() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS + 3) * block_size + 17;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+            let target = (total - 1) as u64;
+            let expected_block = total.div_ceil(block_size) - 1;
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, expected_block);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, expected_block);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_impact_level1_skip_reaches_far_target_doc() {
+        for block_size in [crate::scalar::inverted::LEGACY_BLOCK_SIZE, 256] {
+            let total = (IMPACT_LEVEL1_BLOCKS * 3 + 5) * block_size;
+            let target_block = IMPACT_LEVEL1_BLOCKS * 2 + 2;
+            let target = (target_block * block_size + 17) as u64;
+            let mut posting = PostingIterator::new(
+                String::from("term"),
+                0,
+                0,
+                generate_contiguous_impact_posting_list_with_block_size(total, block_size),
+                total,
+            );
+
+            posting.shallow_next(target);
+            assert_eq!(posting.block_idx, target_block);
+
+            posting.next(target);
+            assert_eq!(posting.block_idx, target_block);
+            assert_eq!(posting.doc().map(|doc| doc.doc_id()), Some(target));
+        }
+    }
+
+    #[test]
+    fn test_or_impact_level1_window_skips_low_group_with_single_score() {
+        let total = (IMPACT_LEVEL1_BLOCKS + 1) * BLOCK_SIZE;
+        let target = (IMPACT_LEVEL1_BLOCKS * BLOCK_SIZE) as u64;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total as u64 {
+            docs.append(doc_id, 1);
+        }
+
+        let doc_ids = (0..total as u32).collect::<Vec<_>>();
+        let freqs = doc_ids
+            .iter()
+            .map(|doc_id| if u64::from(*doc_id) < target { 1 } else { 10 })
+            .collect::<Vec<_>>();
+        let posting_list = generate_impact_posting_list_with_freqs(doc_ids, freqs, vec![1; total]);
+        let mut probe = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            posting_list.clone(),
+            docs.len(),
+        );
+        probe.shallow_next(0);
+        let counting_scorer = CountingScorer {
+            scored: Arc::new(AtomicUsize::new(0)),
+        };
+        let (group_up_to, group_score) = probe.impact_group_bound(&counting_scorer).unwrap();
+        assert_eq!(group_up_to, target - 1);
+        assert_eq!(group_score, 1.0);
+        // A window ending past the current block must answer from level1.
+        assert_eq!(
+            probe.window_max_score(Some(target - 1), &counting_scorer),
+            1.0
+        );
+
+        let posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            1.0,
+            posting_list,
+            docs.len(),
+        );
+        let scored = Arc::new(AtomicUsize::new(0));
+        let mut wand = Wand::new(
+            Operator::Or,
+            std::iter::once(posting),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+        wand.threshold = 2.0;
+
+        let (candidate, score) = wand.next().unwrap().unwrap();
+        assert_eq!(candidate.doc_id(), target);
+        assert_eq!(score, 10.0);
+        // The doc-weight bounds bake exactly once (one doc_weight call per
+        // frontier pair across all entries); beyond that only the returned
+        // candidate is scored.
+        let total_entries = (IMPACT_LEVEL1_BLOCKS + 1) + 2;
+        assert!(
+            scored.load(Ordering::Relaxed) <= total_entries + 8,
+            "bounds should be baked once instead of recomputed per window; scored={}",
+            scored.load(Ordering::Relaxed)
+        );
+    }
+
+    #[test]
+    fn test_compressed_impact_block_max_score_memoizes_current_block() {
+        let total = 2 * BLOCK_SIZE as u32;
+        let doc_ids = (0..total).collect::<Vec<_>>();
+        let freqs = doc_ids
+            .iter()
+            .map(|doc_id| if *doc_id < BLOCK_SIZE as u32 { 1 } else { 2 })
+            .collect::<Vec<_>>();
+        let doc_lengths = vec![1; total as usize];
+        let posting_list = generate_impact_posting_list_with_freqs(doc_ids, freqs, doc_lengths);
+        let mut posting =
+            PostingIterator::new(String::from("term"), 0, 0, posting_list, total as usize);
+        let scored = Arc::new(AtomicUsize::new(0));
+        let scorer = CountingScorer {
+            scored: scored.clone(),
+        };
+
+        let first_score = posting.block_max_score(&scorer);
+        assert_eq!(first_score, 1.0);
+        // Baking the query-local doc-weight bounds visits every frontier pair
+        // once (two level0 entries plus one level1 entry for this list).
+        let baked = scored.load(Ordering::Relaxed);
+        assert!(baked >= 2);
+        {
+            let compressed = unsafe { &mut *posting.compressed_state_ptr() };
+            assert_eq!(
+                compressed.level0_cache,
+                Some((0, BLOCK_SIZE as u32 - 1, first_score))
+            );
+        }
+
+        let second_score = posting.block_max_score(&scorer);
+        assert_eq!(second_score, first_score);
+        assert_eq!(
+            scored.load(Ordering::Relaxed),
+            baked,
+            "repeated block max scores must not recompute doc weights"
+        );
+
+        posting.shallow_next(BLOCK_SIZE as u64);
+        let next_block_score = posting.block_max_score(&scorer);
+        assert_eq!(next_block_score, 2.0);
+        assert_eq!(
+            scored.load(Ordering::Relaxed),
+            baked,
+            "other blocks answer from the baked bounds without rescoring"
+        );
+    }
+
+    #[rstest]
+    #[case(0.0)]
+    #[case(-1.0)]
+    fn test_non_positive_query_weight_skips_global_impact_bound(#[case] query_weight: f32) {
+        let posting = PostingIterator::with_query_weight(
+            String::from("term"),
+            0,
+            0,
+            query_weight,
+            generate_impact_posting_list_with_freqs(vec![0], vec![1], vec![1]),
+            1,
+        );
+        let scored = Arc::new(AtomicUsize::new(0));
+        let scorer = CountingScorer {
+            scored: scored.clone(),
+        };
+
+        assert_eq!(posting.global_upper_bound(&scorer), 0.0);
+        assert_eq!(scored.load(Ordering::Relaxed), 0);
+    }
+
+    #[test]
+    fn test_and_candidate_prune_scores_first_term_before_full_score() {
+        let total_docs = 2 * BLOCK_SIZE as u32 + 1;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total_docs {
+            let doc_tokens = if doc_id == 0 { 1 } else { 1000 };
+            docs.append(doc_id as u64, doc_tokens);
+        }
+
+        let first_docs = (0..2 * BLOCK_SIZE as u32).collect::<Vec<_>>();
+        let second_docs = (0..total_docs).collect::<Vec<_>>();
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(first_docs, 1.0, Some(vec![1.0, 0.001]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(second_docs, 1.0, Some(vec![1.0, 0.001, 0.001]), true),
+                docs.len(),
+            ),
+        ];
+
+        let scored = Arc::new(AtomicUsize::new(0));
+        let mut wand = Wand::new(
+            Operator::And,
+            postings.into_iter(),
+            &docs,
+            CountingScorer {
+                scored: scored.clone(),
+            },
+        );
+
+        let result = wand
+            .search(
+                &FtsSearchParams::new().with_limit(Some(1)),
+                Arc::new(RowAddrMask::default()),
+                &NoOpMetricsCollector,
+            )
+            .unwrap();
+
+        let addrs = result.into_iter().map(|doc| doc.addr).collect::<Vec<_>>();
+        assert!(matches!(addrs.as_slice(), [CandidateAddr::RowId(0)]));
+        let scored = scored.load(Ordering::Relaxed);
+        // The bulk path evaluates 63 doc weights up front to fill its
+        // frequency-bound prune LUT; those are bound computations, not
+        // per-candidate scoring.
+        assert!(
+            scored <= BLOCK_SIZE + 1 + 63,
+            "expected candidate pruning to avoid full scoring in the first block, scored {scored}"
+        );
+    }
+
+    #[test]
+    fn test_and_candidate_prune_records_scoring_counters() {
+        let total_docs = 2 * BLOCK_SIZE as u32 + 1;
+        let mut docs = DocSet::default();
+        for doc_id in 0..total_docs {
+            let doc_tokens = if doc_id == 0 { 1 } else { 1000 };
+            docs.append(doc_id as u64, doc_tokens);
+        }
+
+        let first_docs = (0..2 * BLOCK_SIZE as u32).collect::<Vec<_>>();
+        let second_docs = (0..total_docs).collect::<Vec<_>>();
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list(first_docs, 1.0, Some(vec![1.0, 0.001]), true),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list(second_docs, 1.0, Some(vec![1.0, 0.001, 0.001]), true),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(
+            Operator::And,
+            postings.into_iter(),
+            &docs,
+            InverseDocLengthScorer,
+        );
+        let metrics = CountAndSearchStats::default();
+        let result = wand
+            .search(
+                &FtsSearchParams::new().with_limit(Some(1)),
+                Arc::new(RowAddrMask::default()),
+                &metrics,
+            )
+            .unwrap();
+
+        let addrs = result.into_iter().map(|doc| doc.addr).collect::<Vec<_>>();
+        assert!(matches!(addrs.as_slice(), [CandidateAddr::RowId(0)]));
+
+        let candidates_seen = metrics.candidates_seen.load(Ordering::Relaxed);
+        let candidates_pruned_before_return = metrics
+            .candidates_pruned_before_return
+            .load(Ordering::Relaxed);
+        let full_scores = metrics.full_scores.load(Ordering::Relaxed);
+        assert_eq!(metrics.comparisons.load(Ordering::Relaxed), 1);
+        assert_eq!(candidates_seen, 1);
+        assert!(candidates_pruned_before_return > 0);
+        assert_eq!(full_scores, 1);
+        assert_eq!(metrics.freqs_collected.load(Ordering::Relaxed), 1);
+    }
+
+    #[test]
+    fn test_and_candidate_prune_keeps_later_high_score_candidate() {
+        let mut docs = DocSet::default();
+        for doc_id in 0..3 {
+            docs.append(doc_id, 1);
+        }
+
+        let postings = vec![
+            PostingIterator::with_query_weight(
+                String::from("a"),
+                0,
+                0,
+                1.0,
+                generate_posting_list_with_freqs(
+                    vec![0, 1],
+                    vec![10, 1],
+                    10.0,
+                    Some(vec![10.0]),
+                    true,
+                ),
+                docs.len(),
+            ),
+            PostingIterator::with_query_weight(
+                String::from("b"),
+                1,
+                1,
+                1.0,
+                generate_posting_list_with_freqs(
+                    vec![0, 1, 2],
+                    vec![1, 20, 1],
+                    20.0,
+                    Some(vec![20.0]),
+                    true,
+                ),
+                docs.len(),
+            ),
+        ];
+
+        let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
+        let result = wand
+            .search(
+                &FtsSearchParams::new().with_limit(Some(1)),
+                Arc::new(RowAddrMask::default()),
+                &NoOpMetricsCollector,
+            )
+            .unwrap();
+
+        let addrs = result.into_iter().map(|doc| doc.addr).collect::<Vec<_>>();
+        assert!(matches!(addrs.as_slice(), [CandidateAddr::RowId(1)]));
+    }
+
     #[rstest]
     fn test_wand_batches_lagging_iterators(#[values(false, true)] is_compressed: bool) {
         let mut docs = DocSet::default();
@@ -1919,10 +5894,84 @@ mod tests {
             )
             .unwrap();
 
-        let matched = result.into_iter().map(|doc| doc.row_id).collect::<Vec<_>>();
+        let matched = result
+            .into_iter()
+            .map(|doc| match doc.addr {
+                CandidateAddr::RowId(r) => r,
+                CandidateAddr::Pending(_) => panic!("row_id should be set in this path"),
+            })
+            .collect::<Vec<_>>();
         assert_eq!(matched, vec![2]);
     }
 
+    #[test]
+    fn test_doc_ids_resolves_every_document_a_row_owns() {
+        // A list<string> column indexes each element as its own document, so
+        // one row id owns several doc ids. row 100 -> {0, 1}, row 101 -> {2}.
+        let row_id_col = arrow_array::UInt64Array::from(vec![100_u64, 100, 101]);
+        let num_tokens_col = arrow_array::UInt32Array::from(vec![1_u32, 1, 1]);
+        let docs = DocSet::from_columns(&row_id_col, &num_tokens_col, false, None).unwrap();
+
+        assert_eq!(docs.doc_ids(100).collect::<Vec<_>>(), vec![0, 1]);
+        assert_eq!(docs.doc_ids(101).collect::<Vec<_>>(), vec![2]);
+        assert!(docs.doc_ids(999).next().is_none());
+
+        // legacy shape (row id == doc id) still resolves to a single document.
+        let mut legacy = DocSet::default();
+        legacy.append(7, 1);
+        assert_eq!(legacy.doc_ids(7).collect::<Vec<_>>(), vec![7]);
+        assert!(legacy.doc_ids(8).next().is_none());
+    }
+
+    #[rstest]
+    fn test_flat_search_finds_list_row_with_match_at_non_last_position(
+        #[values(false, true)] is_compressed: bool,
+    ) {
+        // row 100 owns two element-documents (doc 0, doc 1) that share its row
+        // id; row 101 owns doc 2. The query term lives only in doc 0 — the
+        // *non-last* element of row 100. Resolving the row to a single doc id
+        // would evaluate doc 1, miss the term, and drop the row (lancedb#3352).
+        let row_id_col = arrow_array::UInt64Array::from(vec![100_u64, 100, 101]);
+        let num_tokens_col = arrow_array::UInt32Array::from(vec![1_u32, 1, 1]);
+        let docs = DocSet::from_columns(&row_id_col, &num_tokens_col, false, None).unwrap();
+
+        let posting = PostingIterator::with_query_weight(
+            String::from("needle"),
+            0,
+            0,
+            1.0,
+            generate_posting_list(vec![0], 1.0, None, is_compressed),
+            docs.len(),
+        );
+
+        let mut wand = Wand::new(
+            Operator::Or,
+            vec![posting].into_iter(),
+            &docs,
+            InverseDocLengthScorer,
+        );
+        wand.threshold = 0.5;
+
+        let selected = vec![RowAddress::from(100_u64)];
+        let result = wand
+            .flat_search(
+                &FtsSearchParams::default(),
+                Box::new(selected.into_iter()),
+                &NoOpMetricsCollector,
+            )
+            .unwrap();
+
+        // flat_search resolves the prefilter against the DocSet, so the single
+        // match comes back as a concrete RowId(100) rather than a deferred
+        // Pending addr. Asserting on the whole result avoids a never-taken
+        // match arm that would otherwise read as uncovered.
+        let addrs = result.into_iter().map(|doc| doc.addr).collect::<Vec<_>>();
+        assert!(
+            matches!(addrs.as_slice(), [CandidateAddr::RowId(100)]),
+            "expected exactly row 100, got {addrs:?}"
+        );
+    }
+
     #[test]
     fn test_block_max_score_matches_stored_value() {
         let doc_ids = vec![0_u32];
@@ -1935,13 +5984,93 @@ mod tests {
 
         let posting = PostingIterator::new(String::from("test"), 0, 0, posting_list, 1);
 
-        let actual = posting.block_max_score();
+        let actual = posting.block_max_score(&UnitScorer);
         assert!(
             (actual - expected).abs() < 1e-6,
             "block max score should match stored value"
         );
     }
 
+    #[test]
+    fn test_v3_without_impacts_uses_conservative_quantized_score_bound() {
+        let exact_doc_length = 300;
+        let quantized_doc_length = super::super::index::dequantize_doc_length(
+            super::super::index::quantize_doc_length(exact_doc_length),
+        );
+        assert!(quantized_doc_length < exact_doc_length);
+
+        let scorer = Arc::new(MemBM25Scorer::new(100, 1, Default::default()));
+        let stored_exact_score = scorer.doc_weight(1, exact_doc_length);
+        let quantized_score = scorer.doc_weight(1, quantized_doc_length);
+        assert!(quantized_score > stored_exact_score);
+
+        let doc_ids = [0_u32];
+        let frequencies = [1_u32];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(stored_exact_score),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+        )
+        .unwrap();
+        let posting_list = PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            stored_exact_score,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, doc_ids.len());
+        let expected_bound = K1 + 1.0;
+
+        assert_eq!(posting.approximate_upper_bound(), expected_bound);
+        assert_eq!(posting.global_upper_bound(&scorer), expected_bound);
+        assert_eq!(posting.block_max_score(&scorer), expected_bound);
+        assert_eq!(
+            posting.block_max_score_up_to_with_stats(0, &scorer).score,
+            expected_bound
+        );
+        assert!(expected_bound >= quantized_score);
+    }
+
+    #[test]
+    fn test_v3_without_impacts_unknown_scorer_uses_infinite_bound() {
+        let doc_ids = [0_u32];
+        let frequencies = [10_u32];
+        let blocks = compress_posting_list_with_tail_codec_and_block_size(
+            doc_ids.len(),
+            doc_ids.iter(),
+            frequencies.iter(),
+            std::iter::once(10.0),
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+        )
+        .unwrap();
+        let posting_list = PostingList::Compressed(CompressedPostingList::new(
+            blocks,
+            10.0,
+            doc_ids.len() as u32,
+            crate::scalar::inverted::PostingTailCodec::VarintDelta,
+            MAX_POSTING_BLOCK_SIZE,
+            None,
+            None,
+        ));
+        let posting = PostingIterator::new(String::from("term"), 0, 0, posting_list, doc_ids.len());
+
+        assert!(posting.global_upper_bound(&UnitScorer).is_infinite());
+        assert!(posting.block_max_score(&UnitScorer).is_infinite());
+        assert!(
+            posting
+                .block_max_score_up_to_with_stats(0, &UnitScorer)
+                .score
+                .is_infinite()
+        );
+    }
+
     #[rstest]
     fn test_exact_phrase_with_repeated_terms(#[values(false, true)] is_compressed: bool) {
         let mut docs = DocSet::default();
@@ -1990,8 +6119,8 @@ mod tests {
 
         let bm25 = IndexBM25Scorer::new(std::iter::empty());
         let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
-        assert!(wand.check_exact_positions());
-        assert!(wand.check_positions(0));
+        assert!(wand.check_exact_positions().unwrap());
+        assert!(wand.check_positions(0).unwrap());
     }
 
     #[rstest]
@@ -2028,8 +6157,8 @@ mod tests {
 
         let bm25 = IndexBM25Scorer::new(std::iter::empty());
         let wand = Wand::new(Operator::And, postings.into_iter(), &docs, bm25);
-        assert!(wand.check_exact_positions());
-        assert!(wand.check_positions(0));
+        assert!(wand.check_exact_positions().unwrap());
+        assert!(wand.check_positions(0).unwrap());
     }
 
     #[rstest]
@@ -2068,11 +6197,149 @@ mod tests {
         let mut wand = Wand::new(Operator::And, postings.into_iter(), &docs, UnitScorer);
         let first = wand.next().unwrap().unwrap();
         assert_eq!(first.0.doc_id(), 0);
-        assert!(!wand.check_positions(0));
+        assert!(!wand.check_positions(0).unwrap());
 
         wand.threshold = 1.5;
         let second = wand.next().unwrap().unwrap();
         assert_eq!(second.0.doc_id(), 1);
-        assert!(wand.check_positions(0));
+        assert!(wand.check_positions(0).unwrap());
+    }
+
+    /// The bulk conjunction path must return exactly the classic loop's
+    /// results — same docs, freqs, and doc lengths — for both plain AND and
+    /// phrase queries, across multi-block lists with heap/threshold pruning
+    /// in play.
+    #[rstest]
+    #[case::and_k10(false, 0, 10, 3)]
+    #[case::and_k3(false, 0, 3, 3)]
+    #[case::and_two_clauses(false, 0, 10, 2)]
+    #[case::and_four_clauses(false, 0, 10, 4)]
+    #[case::and_five_clauses(false, 0, 10, 5)]
+    #[case::and_six_clauses(false, 0, 10, 6)]
+    #[case::phrase_k10(true, 0, 10, 3)]
+    #[case::phrase_k3(true, 0, 3, 3)]
+    #[case::phrase_slop_three(true, 3, 10, 3)]
+    #[case::phrase_two_clauses(true, 0, 10, 2)]
+    #[case::phrase_four_clauses(true, 0, 10, 4)]
+    #[case::phrase_five_clauses(true, 0, 10, 5)]
+    #[case::phrase_six_clauses(true, 0, 10, 6)]
+    fn test_bulk_and_matches_classic(
+        #[case] phrase: bool,
+        #[case] slop: u32,
+        #[case] limit: usize,
+        #[case] num_clauses: usize,
+    ) {
+        let num_docs = (BLOCK_SIZE * 8 + 37) as u32;
+        let mut docs = DocSet::default();
+        for doc_id in 0..num_docs {
+            docs.append(u64::from(doc_id), 32 + doc_id % 57);
+        }
+
+        // Clauses with different densities; membership comes from a cheap
+        // deterministic mix so docs scatter across blocks. Clause count picks
+        // the dedicated (1-3) or generic (4+) merge kernel.
+        let clause_docs = |modulus: u32, salt: u32| -> Vec<u32> {
+            (0..num_docs)
+                .filter(|doc| (doc.wrapping_mul(2654435761).wrapping_add(salt)) % modulus < 2)
+                .collect()
+        };
+        let clauses = [
+            clause_docs(3, 7),
+            clause_docs(4, 13),
+            clause_docs(5, 29),
+            clause_docs(3, 41),
+            clause_docs(3, 7),
+            clause_docs(4, 13),
+        ][..num_clauses]
+            .to_vec();
+
+        let build_postings = || {
+            clauses
+                .iter()
+                .enumerate()
+                .map(|(term_pos, doc_ids)| {
+                    let list = if phrase {
+                        // Roughly half of each clause's docs put the token at
+                        // position base+term_pos (forming the phrase); the
+                        // rest scatter so the position check has misses.
+                        let positions = doc_ids
+                            .iter()
+                            .map(|&doc| {
+                                if doc % 2 == 0 {
+                                    vec![5 + term_pos as u32, 40 + (doc % 3)]
+                                } else {
+                                    vec![20 + (term_pos as u32) * 4]
+                                }
+                            })
+                            .collect::<Vec<_>>();
+                        generate_posting_list_with_positions(doc_ids.clone(), positions, 8.0, true)
+                    } else {
+                        generate_posting_list(doc_ids.clone(), 8.0, None, true)
+                    };
+                    PostingIterator::with_query_weight(
+                        format!("t{term_pos}"),
+                        term_pos as u32,
+                        term_pos as u32,
+                        1.0 + term_pos as f32 * 0.5,
+                        list,
+                        docs.len(),
+                    )
+                })
+                .collect::<Vec<_>>()
+        };
+
+        let mut params = FtsSearchParams::default().with_limit(Some(limit));
+        if phrase {
+            params.phrase_slop = Some(slop);
+        }
+
+        let normalize = |result: Vec<DocCandidate>| {
+            let mut rows = result
+                .into_iter()
+                .map(|candidate| {
+                    (
+                        candidate.posting_doc_id,
+                        candidate.doc_length,
+                        candidate.freqs,
+                        match candidate.addr {
+                            CandidateAddr::RowId(row_id) => row_id,
+                            CandidateAddr::Pending(doc_id) => u64::from(doc_id),
+                        },
+                    )
+                })
+                .collect::<Vec<_>>();
+            rows.sort_unstable();
+            rows
+        };
+
+        let run = |mode| {
+            let mut wand = Wand::new(
+                Operator::And,
+                build_postings().into_iter(),
+                &docs,
+                UnitScorer,
+            )
+            .with_bulk_and_mode(mode);
+            let rows = normalize(
+                wand.search(
+                    &params,
+                    Arc::new(RowAddrMask::default()),
+                    &NoOpMetricsCollector,
+                )
+                .unwrap(),
+            );
+            let used_bulk = wand.bulk_and_searches > 0;
+            (rows, used_bulk)
+        };
+
+        let (bulk, bulk_used) = run(BulkAndMode::On);
+        let (classic, classic_used) = run(BulkAndMode::Off);
+        let (auto, auto_used) = run(BulkAndMode::Auto);
+        assert!(bulk_used, "on should use bulk conjunction search");
+        assert!(!classic_used, "off should use classic conjunction search");
+        assert_eq!(auto_used, matches!(num_clauses, 2 | 3));
+        assert!(!bulk.is_empty(), "test corpus should produce matches");
+        assert_eq!(bulk, classic);
+        assert_eq!(auto, classic);
     }
 }
diff --git a/rust/lance-index/src/scalar/json.rs b/rust/lance-index/src/scalar/json.rs
index 86c1204e174..09678d6787e 100644
--- a/rust/lance-index/src/scalar/json.rs
+++ b/rust/lance-index/src/scalar/json.rs
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
-    collections::HashMap,
     ops::Bound,
     sync::{Arc, Mutex},
 };
@@ -21,8 +21,8 @@ use datafusion_physical_expr::{
     PhysicalExpr, ScalarFunctionExpr,
     expressions::{Column, Literal},
 };
-use deepsize::DeepSizeOf;
 use futures::StreamExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_datafusion::exec::{LanceExecutionOptions, OneShotExec, get_session_context};
 use lance_datafusion::udf::json::JsonbType;
 use prost::Message;
@@ -33,13 +33,15 @@ use lance_core::{Error, ROW_ID, Result, cache::LanceCache, error::LanceOptionExt
 
 use crate::{
     Index, IndexType,
-    frag_reuse::FragReuseIndex,
     metrics::MetricsCollector,
     registry::IndexPluginRegistry,
     scalar::{
-        AnyQuery, CreatedIndex, IndexStore, ScalarIndex, SearchResult, UpdateCriteria,
+        AnyQuery, CreatedIndex, IndexStore, RowIdRemapper, ScalarIndex, SearchResult,
+        UpdateCriteria,
         expression::{IndexedExpression, ScalarIndexExpr, ScalarIndexSearch, ScalarQueryParser},
-        registry::{ScalarIndexPlugin, TrainingCriteria, TrainingRequest, VALUE_COLUMN_NAME},
+        registry::{
+            BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingRequest, VALUE_COLUMN_NAME,
+        },
     },
 };
 
@@ -61,7 +63,7 @@ impl JsonIndex {
 }
 
 impl DeepSizeOf for JsonIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.target_index.deep_size_of_children(context) + self.path.deep_size_of_children(context)
     }
 }
@@ -76,10 +78,6 @@ impl Index for JsonIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        unimplemented!()
-    }
-
     fn index_type(&self) -> IndexType {
         // TODO: This causes the index to appear as btree in list_indices call.  Need better logic
         // in list_indices to use details instead of index_type.
@@ -118,7 +116,7 @@ impl ScalarIndex for JsonIndex {
 
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         let target_created = self.target_index.remap(mapping, dest_store).await?;
@@ -130,7 +128,7 @@ impl ScalarIndex for JsonIndex {
             index_details: prost_types::Any::from_msg(&json_details)?,
             // TODO: We should store the target index version in the details
             index_version: JSON_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: target_created.files,
         })
     }
 
@@ -152,7 +150,7 @@ impl ScalarIndex for JsonIndex {
             index_details: prost_types::Any::from_msg(&json_details)?,
             // TODO: We should store the target index version in the details
             index_version: JSON_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: target_created.files,
         })
     }
 
@@ -239,12 +237,14 @@ impl JsonQueryParser {
                     index_type,
                     query,
                     needs_recheck,
+                    fragment_bitmap,
                 }) => ScalarIndexExpr::Query(ScalarIndexSearch {
                     column,
                     index_name,
                     index_type,
                     query: Arc::new(JsonQuery::new(query, self.path.clone())),
                     needs_recheck,
+                    fragment_bitmap,
                 }),
                 // This code path should only be hit on leaf expr
                 _ => unreachable!(),
@@ -668,11 +668,7 @@ impl JsonIndexPlugin {
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for JsonIndexPlugin {
-    fn name(&self) -> &str {
-        "Json"
-    }
-
+impl BasicTrainer for JsonIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -690,7 +686,12 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
         let params = serde_json::from_str::<JsonIndexParameters>(params)?;
         let registry = self.registry()?;
         let target_plugin = registry.get_plugin_by_name(&params.target_index_type)?;
-        let target_request = target_plugin.new_training_request(
+        let target_trainer = target_plugin.basic_trainer().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("The '{}' index type does not support basic training, please refer to the index's documentation for more details on how to create this index.", params.target_index_type).into(),
+            )
+        })?;
+        let target_request = target_trainer.new_training_request(
             params.target_index_parameters.as_deref().unwrap_or("{}"),
             &Field::new("", target_type, true),
         )?;
@@ -698,39 +699,6 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
         Ok(Box::new(JsonTrainingRequest::new(params, target_request)))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        // TODO: Need to lookup target plugin via details to figure this out correctly
-        true
-    }
-
-    fn attach_registry(&self, registry: Arc<IndexPluginRegistry>) {
-        let mut reg_ref = self.registry.lock().unwrap();
-        *reg_ref = Some(registry);
-    }
-
-    fn version(&self) -> u32 {
-        JSON_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        // TODO: Allow return Result here
-        let registry = self.registry().unwrap();
-        let json_details =
-            crate::pb::JsonIndexDetails::decode(index_details.value.as_slice()).unwrap();
-        let target_details = json_details.target_details.as_ref().expect_ok().unwrap();
-        let target_plugin = registry.get_plugin_by_details(target_details).unwrap();
-        // TODO: Use something like ${index_name}_${path} for the index name?  Don't have access to path here tho
-        let target_parser = target_plugin.new_query_parser(index_name, index_details)?;
-        Some(Box::new(JsonQueryParser::new(
-            json_details.path.clone(),
-            target_parser,
-        )) as Box<dyn ScalarQueryParser>)
-    }
-
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -757,7 +725,12 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
         let target_plugin = registry.get_plugin_by_name(&request.parameters.target_index_type)?;
 
         // Create a new training request with the inferred type
-        let target_request = target_plugin.new_training_request(
+        let target_trainer = target_plugin.basic_trainer().ok_or_else(|| {
+            Error::invalid_input_source(
+                format!("The '{}' index type does not support basic training, please refer to the index's documentation for more details on how to create this index.", request.parameters.target_index_type).into(),
+            )
+        })?;
+        let target_request = target_trainer.new_training_request(
             request
                 .parameters
                 .target_index_parameters
@@ -766,7 +739,7 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
             &Field::new("", inferred_type, true),
         )?;
 
-        let target_index = target_plugin
+        let target_index = target_trainer
             .train_index(
                 converted_stream,
                 index_store,
@@ -783,15 +756,59 @@ impl ScalarIndexPlugin for JsonIndexPlugin {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&index_details)?,
             index_version: JSON_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files: target_index.files,
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for JsonIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "Json"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        // TODO: Need to lookup target plugin via details to figure this out correctly
+        true
+    }
+
+    fn attach_registry(&self, registry: Arc<IndexPluginRegistry>) {
+        let mut reg_ref = self.registry.lock().unwrap();
+        *reg_ref = Some(registry);
+    }
+
+    fn version(&self) -> u32 {
+        JSON_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        // TODO: Allow return Result here
+        let registry = self.registry().unwrap();
+        let json_details =
+            crate::pb::JsonIndexDetails::decode(index_details.value.as_slice()).unwrap();
+        let target_details = json_details.target_details.as_ref().expect_ok().unwrap();
+        let target_plugin = registry.get_plugin_by_details(target_details).unwrap();
+        // TODO: Use something like ${index_name}_${path} for the index name?  Don't have access to path here tho
+        let target_parser = target_plugin.new_query_parser(index_name, index_details)?;
+        Some(Box::new(JsonQueryParser::new(
+            json_details.path.clone(),
+            target_parser,
+        )) as Box<dyn ScalarQueryParser>)
+    }
 
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         let registry = self.registry().unwrap();
diff --git a/rust/lance-index/src/scalar/label_list.rs b/rust/lance-index/src/scalar/label_list.rs
index d0055e201ac..b96a3ba0346 100644
--- a/rust/lance-index/src/scalar/label_list.rs
+++ b/rust/lance-index/src/scalar/label_list.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
     any::Any,
     collections::HashMap,
@@ -17,28 +18,31 @@ use bytes::Bytes;
 use datafusion::execution::RecordBatchStream;
 use datafusion::physical_plan::{SendableRecordBatchStream, stream::RecordBatchStreamAdapter};
 use datafusion_common::ScalarValue;
-use deepsize::DeepSizeOf;
 use futures::{StreamExt, TryStream, TryStreamExt, stream::BoxStream};
-use lance_arrow::ipc::{read_len_prefixed_bytes_at, write_len_prefixed_bytes};
-use lance_core::cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache};
+use lance_core::cache::{
+    CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache,
+};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::LanceOptionExt;
 use lance_core::{Error, ROW_ID, Result};
 use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps};
 use roaring::RoaringBitmap;
 use tracing::instrument;
 
-use super::{AnyQuery, IndexStore, LabelListQuery, ScalarIndex, bitmap::BitmapIndex};
+use super::{
+    AnyQuery, IndexFile, IndexStore, LabelListQuery, OldIndexDataFilter, ScalarIndex,
+    bitmap::BitmapIndex,
+};
 use super::{BuiltinIndexType, SargableQuery, ScalarIndexParams};
 use super::{MetricsCollector, SearchResult};
-use crate::frag_reuse::FragReuseIndex;
 use crate::pbold;
 use crate::scalar::bitmap::{BitmapIndexPlugin, BitmapIndexState};
 use crate::scalar::expression::{LabelListQueryParser, ScalarQueryParser};
 use crate::scalar::registry::{
-    DefaultTrainingRequest, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
-    VALUE_COLUMN_NAME,
+    BasicTrainer, DefaultTrainingRequest, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria,
+    TrainingOrdering, TrainingRequest, VALUE_COLUMN_NAME, single_flight_open,
 };
-use crate::scalar::{CreatedIndex, UpdateCriteria};
+use crate::scalar::{CreatedIndex, RowIdRemapper, UpdateCriteria};
 use crate::{Index, IndexType};
 
 pub const BITMAP_LOOKUP_NAME: &str = "bitmap_page_lookup.lance";
@@ -89,7 +93,7 @@ impl LabelListIndex {
 
     async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let values_index =
@@ -109,12 +113,6 @@ impl Index for LabelListIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn crate::vector::VectorIndex>> {
-        Err(Error::not_supported_source(
-            "LabeListIndex is not a vector index".into(),
-        ))
-    }
-
     async fn prewarm(&self) -> Result<()> {
         self.values_index.prewarm().await
     }
@@ -215,7 +213,7 @@ impl ScalarIndex for LabelListIndex {
     /// Remap the row ids, creating a new remapped version of this index in `dest_store`
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         let state = self.values_index.load_bitmap_index_state().await?;
@@ -223,12 +221,9 @@ impl ScalarIndex for LabelListIndex {
         let remapped_nulls =
             RowAddrTreeMap::from_iter(self.list_nulls.row_addrs().unwrap().filter_map(|addr| {
                 let addr_as_u64 = u64::from(addr);
-                mapping
-                    .get(&addr_as_u64)
-                    .copied()
-                    .unwrap_or(Some(addr_as_u64))
+                mapping.get(addr_as_u64).unwrap_or(Some(addr_as_u64))
             }));
-        write_label_list_bitmap_index(
+        let file = write_label_list_bitmap_index(
             remapped_state,
             dest_store,
             self.values_index.value_type(),
@@ -240,7 +235,7 @@ impl ScalarIndex for LabelListIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -262,13 +257,15 @@ impl ScalarIndex for LabelListIndex {
         if !new_nulls.is_empty() {
             merged_nulls |= &new_nulls;
         }
-        write_label_list_bitmap_index(merged_state, dest_store, &value_type, &merged_nulls).await?;
+        let file =
+            write_label_list_bitmap_index(merged_state, dest_store, &value_type, &merged_nulls)
+                .await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -443,7 +440,7 @@ fn unnest_chunks(
 
 async fn read_list_nulls(
     store: Arc<dyn IndexStore>,
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
 ) -> Result<RowAddrTreeMap> {
     let reader = store.open_index_file(BITMAP_LOOKUP_NAME).await?;
     if let Some(buffer_idx_str) = reader.schema().metadata.get(LABEL_LIST_NULLS_METADATA_KEY) {
@@ -475,7 +472,7 @@ async fn write_label_list_bitmap_index(
     store: &dyn IndexStore,
     value_type: &DataType,
     list_nulls: &RowAddrTreeMap,
-) -> Result<()> {
+) -> Result<IndexFile> {
     BitmapIndexPlugin::write_bitmap_index_with_extras(
         state,
         store,
@@ -489,6 +486,87 @@ async fn write_label_list_bitmap_index(
     .await
 }
 
+/// Merge multiple LabelList index segments into a single index.
+///
+/// A [`LabelListIndex`] is a [`BitmapIndex`] over the unnested list values plus a
+/// separate `list_nulls` row set. Because distributed segments cover disjoint rows
+/// (distinct fragments), merging is a cheap union of the underlying bitmap states
+/// and of the `list_nulls` sets — no re-scan of source data is required. This
+/// mirrors [`crate::scalar::bitmap::merge_bitmap_indices`] but also carries the
+/// per-segment `list_nulls`. When `old_data_filter` is provided, rows from
+/// retired fragments are removed from both the value bitmaps and `list_nulls`.
+pub async fn merge_label_list_indices(
+    source_indices: &[Arc<LabelListIndex>],
+    dest_store: &dyn IndexStore,
+    old_data_filter: Option<OldIndexDataFilter>,
+    progress: Arc<dyn crate::progress::IndexBuildProgress>,
+) -> Result<CreatedIndex> {
+    if source_indices.is_empty() {
+        return Err(Error::invalid_input(
+            "LabelList segment merge requires at least one source segment".to_string(),
+        ));
+    }
+
+    let value_type = source_indices[0].values_index.value_type().clone();
+    let mut merged_state = HashMap::<ScalarValue, RowAddrTreeMap>::new();
+    let mut merged_nulls = RowAddrTreeMap::new();
+
+    progress
+        .stage_start(
+            "merge_label_list_segments",
+            Some(source_indices.len() as u64),
+            "segments",
+        )
+        .await?;
+    for (idx, source_index) in source_indices.iter().enumerate() {
+        if source_index.values_index.value_type() != &value_type {
+            return Err(Error::invalid_input(format!(
+                "LabelList segment has value type {:?}, expected {:?}",
+                source_index.values_index.value_type(),
+                value_type
+            )));
+        }
+
+        let state = source_index.values_index.load_bitmap_index_state().await?;
+        for (key, mut bitmap) in state {
+            if let Some(filter) = old_data_filter.as_ref() {
+                filter.retain_old_rows(&mut bitmap);
+            }
+            if bitmap.is_empty() {
+                continue;
+            }
+            merged_state
+                .entry(key)
+                .and_modify(|existing| *existing |= &bitmap)
+                .or_insert(bitmap);
+        }
+        let mut list_nulls = source_index.list_nulls.as_ref().clone();
+        if let Some(filter) = old_data_filter.as_ref() {
+            filter.retain_old_rows(&mut list_nulls);
+        }
+        merged_nulls |= &list_nulls;
+        progress
+            .stage_progress("merge_label_list_segments", (idx + 1) as u64)
+            .await?;
+    }
+    progress.stage_complete("merge_label_list_segments").await?;
+
+    progress
+        .stage_start("write_label_list_index", Some(1), "files")
+        .await?;
+    let file =
+        write_label_list_bitmap_index(merged_state, dest_store, &value_type, &merged_nulls).await?;
+    progress.stage_progress("write_label_list_index", 1).await?;
+    progress.stage_complete("write_label_list_index").await?;
+
+    Ok(CreatedIndex {
+        index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
+            .unwrap(),
+        index_version: LABEL_LIST_INDEX_VERSION,
+        files: vec![file],
+    })
+}
+
 /// The serializable state of a [`LabelListIndex`].
 ///
 /// `LabelListIndex` is a thin wrapper around a [`BitmapIndex`] plus a separate
@@ -502,7 +580,7 @@ pub struct LabelListIndexState {
 }
 
 impl DeepSizeOf for LabelListIndexState {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.bitmap_state.deep_size_of_children(context)
             + self.list_nulls.deep_size_of_children(context)
     }
@@ -516,41 +594,56 @@ impl LabelListIndexState {
         })
     }
 
+    fn from_scalar_index(index: &dyn ScalarIndex) -> Result<Self> {
+        let label_list = index
+            .as_any()
+            .downcast_ref::<LabelListIndex>()
+            .ok_or_else(|| {
+                Error::internal(
+                    "LabelListIndexState::from_scalar_index called with a non-label-list index",
+                )
+            })?;
+        Self::from_index(label_list)
+    }
+
     fn into_label_list_index(
         self,
         store: Arc<dyn IndexStore>,
         index_cache: &LanceCache,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Arc<LabelListIndex>> {
         let bitmap = self
             .bitmap_state
-            .into_bitmap_index(store, index_cache, frag_reuse_index)?;
+            .to_bitmap_index(store, index_cache, frag_reuse_index)?;
         Ok(Arc::new(LabelListIndex::new(bitmap, self.list_nulls)))
     }
 }
 
 impl CacheCodecImpl for LabelListIndexState {
+    const TYPE_ID: &'static str = "lance.scalar.LabelListIndexState";
+    const CURRENT_VERSION: u32 = 1;
+
     /// Wire format:
     /// ```text
-    /// [u64 list_nulls_len][list_nulls bytes]
-    /// [bitmap state bytes (self-delimiting)]
+    /// RAW_BLOB : list_nulls (roaring tree map, portable encoding)
+    /// <nested BitmapIndexState body (self-delimiting)>
     /// ```
-    fn serialize(&self, writer: &mut dyn std::io::Write) -> Result<()> {
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         let mut nulls_bytes = Vec::with_capacity(self.list_nulls.serialized_size());
         self.list_nulls.serialize_into(&mut nulls_bytes)?;
-        write_len_prefixed_bytes(writer, &nulls_bytes)?;
-        self.bitmap_state.serialize(writer)?;
+        w.write_raw(&nulls_bytes)?;
+        // The bitmap state writes its own self-delimiting body inline.
+        self.bitmap_state.serialize(w)?;
         Ok(())
     }
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let nulls_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let nulls_bytes = r.read_raw()?;
         let list_nulls = Arc::new(RowAddrTreeMap::deserialize_from(nulls_bytes.as_ref())?);
         // The bitmap state is self-delimiting (length-prefixed null map +
-        // Arrow IPC stream with EOS marker), so we can hand the remaining
-        // tail to it directly.
-        let bitmap_state = BitmapIndexState::deserialize(&data.slice(offset..))?;
+        // Arrow IPC stream with EOS marker); it continues reading the body
+        // from where the null map left off.
+        let bitmap_state = BitmapIndexState::deserialize(r)?;
         Ok(Self {
             bitmap_state,
             list_nulls,
@@ -580,11 +673,7 @@ impl CacheKey for LabelListIndexStateKey {
 pub struct LabelListIndexPlugin;
 
 #[async_trait]
-impl ScalarIndexPlugin for LabelListIndexPlugin {
-    fn name(&self) -> &str {
-        "LabelList"
-    }
-
+impl BasicTrainer for LabelListIndexPlugin {
     fn new_training_request(
         &self,
         _params: &str,
@@ -606,25 +695,6 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
         )))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        true
-    }
-
-    fn version(&self) -> u32 {
-        LABEL_LIST_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        Some(Box::new(LabelListQueryParser::new(
-            index_name,
-            self.name().to_string(),
-        )))
-    }
-
     /// Train a new index
     ///
     /// The provided data must fulfill all the criteria returned by `training_criteria`
@@ -634,15 +704,13 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
         data: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         _request: Box<dyn TrainingRequest>,
-        fragment_ids: Option<Vec<u32>>,
+        // Training over a fragment subset is supported for distributed builds: the
+        // provided `data` stream is already scoped to those fragments, so a partial
+        // index covering exactly those rows is produced. Segments are recombined by
+        // `merge_label_list_indices`.
+        _fragment_ids: Option<Vec<u32>>,
         _progress: Arc<dyn crate::progress::IndexBuildProgress>,
     ) -> Result<CreatedIndex> {
-        if fragment_ids.is_some() {
-            return Err(Error::invalid_input_source(
-                "LabelList index does not support fragment training".into(),
-            ));
-        }
-
         let schema = data.schema();
         let field = schema
             .column_with_name(VALUE_COLUMN_NAME)
@@ -672,21 +740,52 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
         let (state, value_type) =
             BitmapIndexPlugin::build_bitmap_index_state(data, HashMap::new()).await?;
         let list_nulls = list_nulls.lock().unwrap().clone();
-        write_label_list_bitmap_index(state, index_store, &value_type, &list_nulls).await?;
+        let file =
+            write_label_list_bitmap_index(state, index_store, &value_type, &list_nulls).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::LabelListIndexDetails::default())
                 .unwrap(),
             index_version: LABEL_LIST_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for LabelListIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "LabelList"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        true
+    }
+
+    fn version(&self) -> u32 {
+        LABEL_LIST_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        Some(Box::new(LabelListQueryParser::new(
+            index_name,
+            self.name().to_string(),
+        )))
+    }
 
     /// Load an index from storage
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(
@@ -698,7 +797,7 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
     async fn get_from_cache(
         &self,
         index_store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Option<Arc<dyn ScalarIndex>>> {
         let Some(state) = cache.get_with_key(&LabelListIndexStateKey).await else {
@@ -710,18 +809,120 @@ impl ScalarIndexPlugin for LabelListIndexPlugin {
     }
 
     async fn put_in_cache(&self, cache: &LanceCache, index: Arc<dyn ScalarIndex>) -> Result<()> {
-        let label_list = index
-            .as_any()
-            .downcast_ref::<LabelListIndex>()
-            .ok_or_else(|| {
-                Error::internal(
-                    "LabelListIndexPlugin::put_in_cache called with a non-label-list index",
-                )
-            })?;
-        let state = LabelListIndexState::from_index(label_list)?;
+        let state = LabelListIndexState::from_scalar_index(index.as_ref())?;
         cache
             .insert_with_key(&LabelListIndexStateKey, Arc::new(state))
             .await;
         Ok(())
     }
+
+    async fn get_or_insert_in_cache(
+        &self,
+        index_store: Arc<dyn IndexStore>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        cache: &LanceCache,
+        load: ScalarIndexLoad<'_>,
+    ) -> Result<Arc<dyn ScalarIndex>> {
+        single_flight_open(
+            cache,
+            LabelListIndexStateKey,
+            load,
+            LabelListIndexState::from_scalar_index,
+            move |state| {
+                Ok((*state)
+                    .clone()
+                    .into_label_list_index(index_store, cache, frag_reuse_index)?
+                    as Arc<dyn ScalarIndex>)
+            },
+        )
+        .await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use datafusion_common::ScalarValue;
+    use lance_core::cache::CacheCodec;
+    use lance_core::utils::address::RowAddress;
+
+    use super::super::bitmap::BitmapIndexState;
+    use super::super::btree::OrderableScalarValue;
+    use super::*;
+
+    fn sample_state() -> LabelListIndexState {
+        let mut index_map = BTreeMap::new();
+        for k in 0..32i32 {
+            index_map.insert(
+                OrderableScalarValue(ScalarValue::Int32(Some(k))),
+                k as usize,
+            );
+        }
+        let mut bitmap_nulls = RowAddrTreeMap::new();
+        bitmap_nulls.insert(RowAddress::new_from_parts(0, 3).into());
+        let bitmap_state =
+            BitmapIndexState::new_for_test(index_map, bitmap_nulls, DataType::Int32).unwrap();
+
+        let mut list_nulls = RowAddrTreeMap::new();
+        list_nulls.insert(RowAddress::new_from_parts(0, 9).into());
+        LabelListIndexState {
+            bitmap_state,
+            list_nulls: Arc::new(list_nulls),
+        }
+    }
+
+    #[test]
+    fn test_label_list_state_codec_roundtrip() {
+        let state = sample_state();
+        let mut buf = Vec::new();
+        state
+            .serialize(&mut CacheEntryWriter::new(&mut buf))
+            .unwrap();
+        let data = Bytes::from(buf);
+        let mut reader = CacheEntryReader::new(&data, 0, LabelListIndexState::CURRENT_VERSION);
+        let restored = LabelListIndexState::deserialize(&mut reader).unwrap();
+
+        assert_eq!(&*restored.list_nulls, &*state.list_nulls);
+        assert_eq!(
+            restored.bitmap_state.lookup_batch(),
+            state.bitmap_state.lookup_batch()
+        );
+        assert_eq!(
+            restored.bitmap_state.null_map(),
+            state.bitmap_state.null_map()
+        );
+    }
+
+    /// The nested bitmap lookup batch must decode zero-copy through the full
+    /// envelope, proving the leading `list_nulls` RAW_BLOB does not knock the
+    /// nested IPC section off its 64-byte boundary.
+    #[test]
+    fn test_label_list_nested_lookup_is_zero_copy() {
+        const ALIGN: usize = 64;
+        let codec = CacheCodec::from_impl::<LabelListIndexState>();
+        let any: Arc<dyn std::any::Any + Send + Sync> = Arc::new(sample_state());
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+
+        let mut v = vec![0u8; buf.len() + ALIGN];
+        let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+        v[pad..pad + buf.len()].copy_from_slice(&buf);
+        let data = Bytes::from(v).slice(pad..pad + buf.len());
+
+        let restored = codec.deserialize(&data).hit().unwrap();
+        let restored = restored.downcast::<LabelListIndexState>().unwrap();
+
+        let base = data.as_ptr() as usize;
+        let end = base + data.len();
+        for col in restored.bitmap_state.lookup_batch().columns() {
+            for buffer in col.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= base && ptr < end,
+                    "nested bitmap lookup buffer was realigned — misaligned IPC section",
+                );
+            }
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/lance_format.rs b/rust/lance-index/src/scalar/lance_format.rs
index 9fcd888876d..76ebd0cf0da 100644
--- a/rust/lance-index/src/scalar/lance_format.rs
+++ b/rust/lance-index/src/scalar/lance_format.rs
@@ -3,27 +3,24 @@
 
 //! Utilities for serializing and deserializing scalar indices in the lance format
 
-use super::{IndexReader, IndexStore, IndexWriter};
+use super::{IndexFile, IndexReader, IndexStore, IndexWriter};
 use arrow_array::RecordBatch;
 use arrow_schema::Schema;
 use async_trait::async_trait;
 use bytes::Bytes;
-use deepsize::DeepSizeOf;
 use futures::TryStreamExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, cache::LanceCache};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
 use lance_encoding::version::LanceFileVersion;
-use lance_file::previous::{
-    reader::FileReader as PreviousFileReader,
-    writer::{FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider},
-};
-use lance_file::reader::{self as current_reader, FileReaderOptions, ReaderProjection};
+use lance_file::previous::reader::FileReader as PreviousFileReader;
+use lance_file::reader::{FileReader as CurrentFileReader, FileReaderOptions, ReaderProjection};
 use lance_file::writer as current_writer;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
 use lance_io::{ReadBatchParams, object_store::ObjectStore};
 use lance_table::format::SelfDescribingFileReader;
-use lance_table::format::{IndexFile, list_index_files_with_sizes};
+use lance_table::format::list_index_files_with_sizes;
 use object_store::path::Path;
 use std::cmp::min;
 use std::collections::HashMap;
@@ -45,13 +42,18 @@ pub struct LanceIndexStore {
     /// When set, used to avoid HEAD calls when opening files
     file_sizes: HashMap<String, u64>,
     format_version: LanceFileVersion,
+    /// Base I/O priority for all requests this store submits to `scheduler`.
+    io_priority: u64,
 }
 
 impl DeepSizeOf for LanceIndexStore {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // Exclude the shared, session-scoped `metadata_cache` (accounted once by
+        // `Session::deep_size_of_children`): it is not this store's own footprint, and
+        // sizing it here makes every opened index that holds a store report the whole
+        // cache as its own bytes — inflating and N-times double-counting it.
         self.object_store.deep_size_of_children(context)
             + self.index_dir.as_ref().deep_size_of_children(context)
-            + self.metadata_cache.deep_size_of_children(context)
     }
 }
 
@@ -88,6 +90,7 @@ impl LanceIndexStore {
             scheduler,
             file_sizes: HashMap::new(),
             format_version,
+            io_priority: 0,
         }
     }
 
@@ -99,55 +102,79 @@ impl LanceIndexStore {
         self.file_sizes = file_sizes;
         self
     }
-}
 
-#[async_trait]
-impl<M: PreviousManifestProvider + Send + Sync> IndexWriter for PreviousFileWriter<M> {
-    async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64> {
-        let offset = self.tell().await?;
-        self.write(&[batch]).await?;
-        Ok(offset as u64)
+    /// The base I/O priority all this store's requests are submitted at.
+    pub fn io_priority(&self) -> u64 {
+        self.io_priority
     }
 
-    async fn finish(&mut self) -> Result<()> {
-        Self::finish(self).await.map(|_| ())
+    fn index_file_path(&self, name: &str) -> Result<Path> {
+        let relative_path = Path::parse(name).map_err(|err| {
+            Error::invalid_input(format!("invalid index file path {name:?}: {err}"))
+        })?;
+        if self.index_dir.is_root() {
+            return Ok(relative_path);
+        }
+        if relative_path.is_root() {
+            return Ok(self.index_dir.clone());
+        }
+        Path::parse(format!(
+            "{}/{}",
+            self.index_dir.as_ref(),
+            relative_path.as_ref()
+        ))
+        .map_err(|err| Error::invalid_input(format!("invalid index file path {name:?}: {err}")))
     }
+}
 
-    async fn finish_with_metadata(&mut self, metadata: HashMap<String, String>) -> Result<()> {
-        Self::finish_with_metadata(self, &metadata)
-            .await
-            .map(|_| ())
-    }
+struct LanceIndexWriter {
+    path: String,
+    inner: current_writer::FileWriter,
 }
 
 #[async_trait]
-impl IndexWriter for current_writer::FileWriter {
+impl IndexWriter for LanceIndexWriter {
     async fn write_record_batch(&mut self, batch: RecordBatch) -> Result<u64> {
-        let offset = self.tell().await?;
-        self.write_batch(&batch).await?;
+        let offset = self.inner.tell().await?;
+        self.inner.write_batch(&batch).await?;
         Ok(offset)
     }
 
     async fn add_global_buffer(&mut self, data: Bytes) -> Result<u32> {
-        Self::add_global_buffer(self, data).await
+        self.inner.add_global_buffer(data).await
     }
 
-    async fn finish(&mut self) -> Result<()> {
-        Self::finish(self).await.map(|_| ())
+    async fn finish(&mut self) -> Result<IndexFile> {
+        let summary = self.inner.finish().await?;
+        Ok(IndexFile {
+            path: self.path.clone(),
+            size_bytes: summary.size_bytes,
+        })
     }
 
-    async fn finish_with_metadata(&mut self, metadata: HashMap<String, String>) -> Result<()> {
+    async fn finish_with_metadata(
+        &mut self,
+        metadata: HashMap<String, String>,
+    ) -> Result<IndexFile> {
         metadata.into_iter().for_each(|(k, v)| {
-            self.add_schema_metadata(k, v);
+            self.inner.add_schema_metadata(k, v);
         });
-        Self::finish(self).await.map(|_| ())
+        let summary = self.inner.finish().await?;
+        Ok(IndexFile {
+            path: self.path.clone(),
+            size_bytes: summary.size_bytes,
+        })
     }
 }
 
+/// Newtype wrapper to allow implementing IndexReader for PreviousFileReader (a foreign type)
+struct PreviousIndexReader(PreviousFileReader);
+
 #[async_trait]
-impl IndexReader for PreviousFileReader {
+impl IndexReader for PreviousIndexReader {
     async fn read_record_batch(&self, offset: u64, _batch_size: u64) -> Result<RecordBatch> {
-        self.read_batch(offset as i32, ReadBatchParams::RangeFull, self.schema())
+        self.0
+            .read_batch(offset as i32, ReadBatchParams::RangeFull, self.0.schema())
             .await
     }
 
@@ -157,36 +184,39 @@ impl IndexReader for PreviousFileReader {
         projection: Option<&[&str]>,
     ) -> Result<RecordBatch> {
         let projection = match projection {
-            Some(projection) => self.schema().project(projection)?,
-            None => self.schema().clone(),
+            Some(projection) => self.0.schema().project(projection)?,
+            None => self.0.schema().clone(),
         };
-        self.read_range(range, &projection).await
+        self.0.read_range(range, &projection).await
     }
 
     async fn num_batches(&self, _batch_size: u64) -> u32 {
-        self.num_batches() as u32
+        self.0.num_batches() as u32
     }
 
     fn num_rows(&self) -> usize {
-        self.len()
+        self.0.len()
     }
 
     fn schema(&self) -> &lance_core::datatypes::Schema {
-        Self::schema(self)
+        PreviousFileReader::schema(&self.0)
     }
 }
 
+/// Newtype wrapper to allow implementing IndexReader for CurrentFileReader (a foreign type)
+struct CurrentIndexReader(CurrentFileReader);
+
 #[async_trait]
-impl IndexReader for current_reader::FileReader {
+impl IndexReader for CurrentIndexReader {
     async fn read_record_batch(&self, offset: u64, batch_size: u64) -> Result<RecordBatch> {
         let start = offset * batch_size;
         let end = start + batch_size;
-        let end = end.min(self.num_rows());
+        let end = end.min(self.0.num_rows());
         self.read_range(start as usize..end as usize, None).await
     }
 
     async fn read_global_buffer(&self, n: u32) -> Result<Bytes> {
-        Self::read_global_buffer(self, n).await
+        CurrentFileReader::read_global_buffer(&self.0, n).await
     }
 
     async fn read_range(
@@ -196,19 +226,20 @@ impl IndexReader for current_reader::FileReader {
     ) -> Result<RecordBatch> {
         if range.is_empty() {
             return Ok(RecordBatch::new_empty(Arc::new(
-                self.schema().as_ref().into(),
+                self.0.schema().as_ref().into(),
             )));
         }
         let projection = if let Some(projection) = projection {
             ReaderProjection::from_column_names(
-                self.metadata().version(),
-                self.schema(),
+                self.0.metadata().version(),
+                self.0.schema(),
                 projection,
             )?
         } else {
-            ReaderProjection::from_whole_schema(self.schema(), self.metadata().version())
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
         };
         let batches = self
+            .0
             .read_stream_projected(
                 ReadBatchParams::Range(range),
                 u32::MAX,
@@ -223,6 +254,86 @@ impl IndexReader for current_reader::FileReader {
         Ok(batches[0].clone())
     }
 
+    async fn read_ranges(
+        &self,
+        ranges: &[std::ops::Range<usize>],
+        projection: Option<&[&str]>,
+    ) -> Result<RecordBatch> {
+        let empty_batch = || {
+            Ok(RecordBatch::new_empty(Arc::new(
+                self.0.schema().as_ref().into(),
+            )))
+        };
+        if ranges.is_empty() {
+            return empty_batch();
+        }
+        let projection = if let Some(projection) = projection {
+            ReaderProjection::from_column_names(
+                self.0.metadata().version(),
+                self.0.schema(),
+                projection,
+            )?
+        } else {
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
+        };
+        // `DecodeBatchScheduler::schedule_ranges` requires sorted,
+        // non-overlapping ranges; sort internally and permute the
+        // result back to caller order so callers don't have to know.
+        let mut order: Vec<usize> = (0..ranges.len()).collect();
+        order.sort_by_key(|&i| ranges[i].start);
+        let already_sorted = order.iter().enumerate().all(|(i, &j)| i == j);
+        let sorted_ranges: Arc<[std::ops::Range<u64>]> = order
+            .iter()
+            .map(|&i| ranges[i].start as u64..ranges[i].end as u64)
+            .collect();
+        let total_rows: u64 = sorted_ranges.iter().map(|r| r.end - r.start).sum();
+        let batches = self
+            .0
+            .read_stream_projected(
+                ReadBatchParams::Ranges(sorted_ranges),
+                (total_rows as u32).max(1),
+                16,
+                projection,
+                FilterExpression::no_filter(),
+            )
+            .await?
+            .try_collect::<Vec<_>>()
+            .await?;
+        let merged = match batches.len() {
+            0 => return empty_batch(),
+            1 => batches.into_iter().next().unwrap(),
+            _ => {
+                let schema = batches[0].schema();
+                arrow_select::concat::concat_batches(&schema, &batches)?
+            }
+        };
+        if already_sorted {
+            return Ok(merged);
+        }
+        let sorted_sizes: Vec<u32> = order
+            .iter()
+            .map(|&i| (ranges[i].end - ranges[i].start) as u32)
+            .collect();
+        let mut sorted_offsets = Vec::with_capacity(sorted_sizes.len());
+        let mut acc = 0u32;
+        for &s in &sorted_sizes {
+            sorted_offsets.push(acc);
+            acc += s;
+        }
+        let mut sorted_pos = vec![0usize; ranges.len()];
+        for (sp, &oi) in order.iter().enumerate() {
+            sorted_pos[oi] = sp;
+        }
+        let mut take_indices = Vec::with_capacity(total_rows as usize);
+        for &sp in &sorted_pos {
+            for k in 0..sorted_sizes[sp] {
+                take_indices.push(sorted_offsets[sp] + k);
+            }
+        }
+        let take_arr = arrow_array::UInt32Array::from(take_indices);
+        Ok(arrow_select::take::take_record_batch(&merged, &take_arr)?)
+    }
+
     async fn read_range_stream(
         &self,
         range: std::ops::Range<usize>,
@@ -230,41 +341,48 @@ impl IndexReader for current_reader::FileReader {
     ) -> Result<Pin<Box<dyn lance_io::stream::RecordBatchStream>>> {
         if range.is_empty() {
             return Ok(Box::pin(lance_io::stream::RecordBatchStreamAdapter::new(
-                Arc::new(self.schema().as_ref().into()),
+                Arc::new(self.0.schema().as_ref().into()),
                 futures::stream::empty(),
             )));
         }
         let projection = if let Some(projection) = projection {
             ReaderProjection::from_column_names(
-                self.metadata().version(),
-                self.schema(),
+                self.0.metadata().version(),
+                self.0.schema(),
                 projection,
             )?
         } else {
-            ReaderProjection::from_whole_schema(self.schema(), self.metadata().version())
+            ReaderProjection::from_whole_schema(self.0.schema(), self.0.metadata().version())
         };
-        self.read_stream_projected(
-            ReadBatchParams::Range(range),
-            4096,
-            2,
-            projection,
-            FilterExpression::no_filter(),
-        )
-        .await
+        self.0
+            .read_stream_projected(
+                ReadBatchParams::Range(range),
+                4096,
+                2,
+                projection,
+                FilterExpression::no_filter(),
+            )
+            .await
     }
 
     // V2 format has removed the row group concept,
     // so here we assume each batch is with 4096 rows.
     async fn num_batches(&self, batch_size: u64) -> u32 {
-        Self::num_rows(self).div_ceil(batch_size) as u32
+        CurrentFileReader::num_rows(&self.0).div_ceil(batch_size) as u32
     }
 
     fn num_rows(&self) -> usize {
-        Self::num_rows(self) as usize
+        CurrentFileReader::num_rows(&self.0) as usize
     }
 
     fn schema(&self) -> &lance_core::datatypes::Schema {
-        Self::schema(self)
+        CurrentFileReader::schema(&self.0)
+    }
+
+    fn file_size_bytes(&self) -> Option<u64> {
+        // The manifest records each index file's size and passes it to the reader
+        // at open, so it's already in metadata here (no extra I/O).
+        Some(self.0.metadata().file_size())
     }
 }
 
@@ -287,7 +405,7 @@ impl IndexStore for LanceIndexStore {
         name: &str,
         schema: Arc<Schema>,
     ) -> Result<Box<dyn IndexWriter>> {
-        let path = self.index_dir.clone().join(name);
+        let path = self.index_file_path(name)?;
         let schema = schema.as_ref().try_into()?;
         let writer = self.object_store.create(&path).await?;
         let writer = current_writer::FileWriter::try_new(
@@ -298,19 +416,34 @@ impl IndexStore for LanceIndexStore {
                 ..Default::default()
             },
         )?;
-        Ok(Box::new(writer))
+        Ok(Box::new(LanceIndexWriter {
+            path: name.to_string(),
+            inner: writer,
+        }))
+    }
+
+    fn with_io_priority(&self, io_priority: u64) -> Arc<dyn IndexStore> {
+        // The `scheduler` is shared (`Arc`), so this clone is cheap and the new
+        // priority only affects requests this clone submits.
+        Arc::new(Self {
+            io_priority,
+            ..self.clone()
+        })
     }
 
     async fn open_index_file(&self, name: &str) -> Result<Arc<dyn IndexReader>> {
-        let path = self.index_dir.clone().join(name);
+        let path = self.index_file_path(name)?;
         // Use cached file size if available, otherwise unknown (requires HEAD call)
         let cached_size = self
             .file_sizes
             .get(name)
             .map(|&size| CachedFileSize::new(size))
             .unwrap_or_else(CachedFileSize::unknown);
-        let file_scheduler = self.scheduler.open_file(&path, &cached_size).await?;
-        match current_reader::FileReader::try_open(
+        let file_scheduler = self
+            .scheduler
+            .open_file_with_priority(&path, self.io_priority, &cached_size)
+            .await?;
+        match CurrentFileReader::try_open(
             file_scheduler,
             None,
             Arc::<DecoderPlugins>::default(),
@@ -319,18 +452,18 @@ impl IndexStore for LanceIndexStore {
         )
         .await
         {
-            Ok(reader) => Ok(Arc::new(reader)),
+            Ok(reader) => Ok(Arc::new(CurrentIndexReader(reader))),
             Err(e) => {
                 // If the error is a version conflict we can try to read the file with v1 reader
                 if let Error::VersionConflict { .. } = e {
-                    let path = self.index_dir.clone().join(name);
+                    let path = self.index_file_path(name)?;
                     let file_reader = PreviousFileReader::try_new_self_described(
                         &self.object_store,
                         &path,
                         Some(&self.metadata_cache),
                     )
                     .await?;
-                    Ok(Arc::new(file_reader))
+                    Ok(Arc::new(PreviousIndexReader(file_reader)))
                 } else {
                     Err(e)
                 }
@@ -338,8 +471,17 @@ impl IndexStore for LanceIndexStore {
         }
     }
 
-    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<()> {
-        let path = self.index_dir.clone().join(name);
+    async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result<IndexFile> {
+        self.copy_index_file_to(name, name, dest_store).await
+    }
+
+    async fn copy_index_file_to(
+        &self,
+        name: &str,
+        new_name: &str,
+        dest_store: &dyn IndexStore,
+    ) -> Result<IndexFile> {
+        let path = self.index_file_path(name)?;
 
         let other_store = dest_store.as_any().downcast_ref::<Self>();
         match other_store {
@@ -347,13 +489,21 @@ impl IndexStore for LanceIndexStore {
                 // If both this store and the destination are lance stores we can use object_store's copy
                 // This does blindly assume that both stores are using the same underlying object_store
                 // but there is no easy way to verify this and it happens to always be true at the moment
-                let dest_path = dest_store.index_dir.clone().join(name);
-                self.object_store.copy(&path, &dest_path).await
+                let dest_path = dest_store.index_file_path(new_name)?;
+                self.object_store.copy(&path, &dest_path).await?;
+                let size_bytes = match self.file_sizes.get(name) {
+                    Some(size_bytes) => *size_bytes,
+                    None => self.object_store.size(&path).await?,
+                };
+                Ok(IndexFile {
+                    path: new_name.to_string(),
+                    size_bytes,
+                })
             }
             _ => {
                 let reader = self.open_index_file(name).await?;
                 let mut writer = dest_store
-                    .new_index_file(name, Arc::new(reader.schema().into()))
+                    .new_index_file(new_name, Arc::new(reader.schema().into()))
                     .await?;
 
                 for offset in (0..reader.num_rows()).step_by(4096) {
@@ -361,27 +511,40 @@ impl IndexStore for LanceIndexStore {
                     let batch = reader.read_range(offset..next_offset, None).await?;
                     writer.write_record_batch(batch).await?;
                 }
-                writer.finish().await?;
-
-                Ok(())
+                writer.finish().await
             }
         }
     }
 
-    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<()> {
-        let path = self.index_dir.clone().join(name);
-        let new_path = self.index_dir.clone().join(new_name);
+    async fn rename_index_file(&self, name: &str, new_name: &str) -> Result<IndexFile> {
+        let path = self.index_file_path(name)?;
+        let new_path = self.index_file_path(new_name)?;
         self.object_store.copy(&path, &new_path).await?;
-        self.object_store.delete(&path).await
+        self.object_store.delete(&path).await?;
+        let size_bytes = match self.file_sizes.get(name) {
+            Some(size_bytes) => *size_bytes,
+            None => self.object_store.size(&new_path).await?,
+        };
+        Ok(IndexFile {
+            path: new_name.to_string(),
+            size_bytes,
+        })
     }
 
     async fn delete_index_file(&self, name: &str) -> Result<()> {
-        let path = self.index_dir.clone().join(name);
+        let path = self.index_file_path(name)?;
         self.object_store.delete(&path).await
     }
 
     async fn list_files_with_sizes(&self) -> Result<Vec<IndexFile>> {
-        list_index_files_with_sizes(&self.object_store, &self.index_dir).await
+        let files = list_index_files_with_sizes(&self.object_store, &self.index_dir).await?;
+        Ok(files
+            .into_iter()
+            .map(|f| IndexFile {
+                path: f.path,
+                size_bytes: f.size_bytes,
+            })
+            .collect())
     }
 }
 
@@ -395,7 +558,7 @@ mod tests {
     use crate::scalar::bitmap::BitmapIndexPlugin;
     use crate::scalar::btree::{BTreeIndexPlugin, BTreeParameters};
     use crate::scalar::label_list::LabelListIndexPlugin;
-    use crate::scalar::registry::{ScalarIndexPlugin, VALUE_COLUMN_NAME};
+    use crate::scalar::registry::{BasicTrainer, ScalarIndexPlugin, VALUE_COLUMN_NAME};
     use crate::scalar::{
         LabelListQuery, SargableQuery, ScalarIndex, SearchResult,
         bitmap::BitmapIndex,
@@ -416,6 +579,7 @@ mod tests {
     use datafusion_common::ScalarValue;
     use futures::FutureExt;
     use lance_core::ROW_ID;
+    use lance_core::utils::row_addr_remap::RowAddrRemap;
     use lance_core::utils::tempfile::TempDir;
     use lance_datagen::{ArrayGeneratorExt, BatchCount, ByteCount, RowCount, array, gen_batch};
     use lance_select::{RowAddrTreeMap, RowSetOps};
@@ -432,6 +596,46 @@ mod tests {
         Arc::new(LanceIndexStore::new(object_store, test_path, cache))
     }
 
+    #[tokio::test]
+    async fn test_store_deep_size_excludes_metadata_cache() {
+        // The metadata cache is session-scoped and shared; a store must not count
+        // it as its own footprint, so growing the cache must not change store size.
+        struct BlobKey;
+        impl lance_core::cache::CacheKey for BlobKey {
+            type ValueType = Vec<u8>;
+            fn key(&self) -> std::borrow::Cow<'_, str> {
+                std::borrow::Cow::Borrowed("blob")
+            }
+            fn type_name() -> &'static str {
+                "Vec<u8>"
+            }
+        }
+
+        let index_dir = TempDir::default();
+        let test_path = index_dir.obj_path();
+        let (object_store, test_path) = ObjectStore::from_uri(test_path.as_ref())
+            .now_or_never()
+            .unwrap()
+            .unwrap();
+        let cache = Arc::new(lance_core::cache::LanceCache::with_capacity(
+            128 * 1024 * 1024,
+        ));
+        let store = LanceIndexStore::new(object_store, test_path, cache.clone());
+
+        let before = store.deep_size_of();
+        cache
+            .insert_with_key(&BlobKey, Arc::new(vec![0u8; 4 * 1024 * 1024]))
+            .await;
+        // Force moka to commit the write so a cache-inclusive size would grow.
+        let _ = cache.size_bytes().await;
+        let after = store.deep_size_of();
+
+        assert_eq!(
+            before, after,
+            "store deep size must exclude the shared metadata cache"
+        );
+    }
+
     async fn train_index(
         index_store: &Arc<dyn IndexStore>,
         data: impl RecordBatchReader + Send + Sync + 'static,
@@ -479,7 +683,11 @@ mod tests {
             .unwrap();
         let expected = bytes::Bytes::from_static(b"scalar-global-buffer");
         let buffer_idx = writer.add_global_buffer(expected.clone()).await.unwrap();
-        writer.finish().await.unwrap();
+        let write_summary = writer.finish().await.unwrap();
+        let files = index_store.list_files_with_sizes().await.unwrap();
+        assert_eq!(files.len(), 1);
+        assert_eq!(files[0].path, "global-buffer.lance");
+        assert_eq!(write_summary.size_bytes, files[0].size_bytes);
 
         let reader = index_store
             .open_index_file("global-buffer.lance")
@@ -1488,7 +1696,7 @@ mod tests {
         let remapped_dir = TempDir::default();
         let remapped_store = test_store(&remapped_dir);
         index
-            .remap(&mapping, remapped_store.as_ref())
+            .remap(&RowAddrRemap::direct(mapping), remapped_store.as_ref())
             .await
             .unwrap();
         let remapped_index = BitmapIndex::load(remapped_store, None, &LanceCache::no_cache())
diff --git a/rust/lance-index/src/scalar/ngram.rs b/rust/lance-index/src/scalar/ngram.rs
index 5f7ce57da4a..ade22cfc333 100644
--- a/rust/lance-index/src/scalar/ngram.rs
+++ b/rust/lance-index/src/scalar/ngram.rs
@@ -1,38 +1,41 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::any::Any;
 use std::collections::BTreeMap;
 use std::iter::once;
 use std::time::Instant;
-use std::{collections::HashMap, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
 
 use super::lance_format::LanceIndexStore;
 use super::{
-    AnyQuery, BuiltinIndexType, IndexReader, IndexStore, IndexWriter, MetricsCollector,
+    AnyQuery, BuiltinIndexType, IndexFile, IndexReader, IndexStore, IndexWriter, MetricsCollector,
     ScalarIndex, ScalarIndexParams, SearchResult, TextQuery,
 };
-use crate::frag_reuse::FragReuseIndex;
 use crate::metrics::NoOpMetricsCollector;
 use crate::pbold;
 use crate::scalar::expression::{ScalarQueryParser, TextQueryParser};
 use crate::scalar::registry::{
-    DefaultTrainingRequest, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
-    VALUE_COLUMN_NAME,
+    BasicTrainer, DefaultTrainingRequest, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering,
+    TrainingRequest, VALUE_COLUMN_NAME,
 };
-use crate::scalar::{CreatedIndex, UpdateCriteria};
-use crate::vector::VectorIndex;
+use crate::scalar::{CreatedIndex, RowIdRemapper, UpdateCriteria};
 use crate::{Index, IndexType};
 use arrow::array::{AsArray, UInt32Builder};
+use arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer};
 use arrow::datatypes::{UInt32Type, UInt64Type};
 use arrow_array::{BinaryArray, RecordBatch, UInt32Array};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
-use deepsize::DeepSizeOf;
 use futures::{FutureExt, Stream, StreamExt, TryStreamExt, stream};
 use lance_arrow::iter_str_array;
 use lance_core::cache::{CacheKey, LanceCache, WeakLanceCache};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::LanceOptionExt;
 use lance_core::utils::address::RowAddress;
 use lance_core::utils::tempfile::TempDir;
@@ -49,11 +52,20 @@ use roaring::{RoaringBitmap, RoaringTreemap};
 use serde::Serialize;
 use tracing::instrument;
 
+mod ngram_regex;
+pub(crate) use ngram_regex::regex_can_use_index;
+
 const TOKENS_COL: &str = "tokens";
 const POSTING_LIST_COL: &str = "posting_list";
 const POSTINGS_FILENAME: &str = "ngram_postings.lance";
 const NGRAM_INDEX_VERSION: u32 = 0;
 
+/// An i32-offset Binary array can hold at most i32::MAX bytes of values in total,
+/// so a spill state whose serialized posting lists exceed that must be written as
+/// multiple record batches (same approach as the bitmap index).  Leave headroom.
+const MAX_POSTING_LIST_BATCH_BYTES: usize = i32::MAX as usize - 1024 * 1024;
+const POSTING_LIST_STREAM_BATCH_ROWS: usize = 64;
+
 use std::sync::LazyLock;
 
 pub static TOKENS_FIELD: LazyLock<Field> =
@@ -155,7 +167,7 @@ pub struct NGramPostingList {
 }
 
 impl DeepSizeOf for NGramPostingList {
-    fn deep_size_of_children(&self, _: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize {
         self.bitmap.serialized_size()
     }
 }
@@ -181,7 +193,7 @@ impl CacheKey for NGramPostingListKey {
 impl NGramPostingList {
     fn try_from_batch(
         batch: RecordBatch,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     ) -> Result<Self> {
         let bitmap_bytes = batch.column(0).as_binary::<i32>().value(0);
         let mut bitmap = RoaringTreemap::deserialize_from(bitmap_bytes)
@@ -208,12 +220,12 @@ impl NGramPostingList {
 /// Reads on-demand ngram posting lists from storage (and stores them in a cache)
 struct NGramPostingListReader {
     reader: Arc<dyn IndexReader>,
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     index_cache: WeakLanceCache,
 }
 
 impl DeepSizeOf for NGramPostingListReader {
-    fn deep_size_of_children(&self, _: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize {
         0
     }
 }
@@ -285,7 +297,7 @@ impl std::fmt::Debug for NGramIndex {
 }
 
 impl DeepSizeOf for NGramIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.tokens.deep_size_of_children(context)
     }
 }
@@ -293,7 +305,7 @@ impl DeepSizeOf for NGramIndex {
 impl NGramIndex {
     async fn from_store(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Self> {
         let tokens = store.open_index_file(POSTINGS_FILENAME).await?;
@@ -327,49 +339,35 @@ impl NGramIndex {
         })
     }
 
-    fn remap_batch(
+    fn remap_state(
         &self,
-        batch: RecordBatch,
-        mapping: &HashMap<u64, Option<u64>>,
-    ) -> Result<RecordBatch> {
-        let posting_lists_array = batch
-            .column_by_name(POSTING_LIST_COL)
-            .expect_ok()?
-            .as_binary::<i32>();
-
-        let new_posting_lists = posting_lists_array
-            .iter()
+        state: NGramIndexSpillState,
+        mapping: &RowAddrRemap,
+    ) -> Result<Vec<RecordBatch>> {
+        let bitmaps = state
+            .bitmaps
+            .into_iter()
             .map(|posting_list| {
-                let posting_list = posting_list.unwrap();
-                let posting_list = RoaringTreemap::deserialize_from(posting_list)?;
-                let new_posting_list =
-                    RoaringTreemap::from_iter(posting_list.into_iter().filter_map(|row_id| {
-                        match mapping.get(&row_id) {
-                            Some(Some(new_row_id)) => Some(*new_row_id),
-                            Some(None) => None,
-                            None => Some(row_id),
-                        }
-                    }));
-                let mut buf = Vec::with_capacity(new_posting_list.serialized_size());
-                new_posting_list.serialize_into(&mut buf)?;
-                Ok(buf)
+                RoaringTreemap::from_iter(posting_list.into_iter().filter_map(|row_id| {
+                    match mapping.get(row_id) {
+                        Some(Some(new_row_id)) => Some(new_row_id),
+                        Some(None) => None,
+                        None => Some(row_id),
+                    }
+                }))
             })
-            .collect::<Result<Vec<_>>>()?;
-
-        let new_posting_lists_array = BinaryArray::from_iter_values(new_posting_lists);
+            .collect();
 
-        Ok(RecordBatch::try_new(
-            POSTINGS_SCHEMA.clone(),
-            vec![
-                batch.column_by_name(TOKENS_COL).expect_ok()?.clone(),
-                Arc::new(new_posting_lists_array),
-            ],
-        )?)
+        NGramIndexSpillState {
+            tokens: state.tokens,
+            bitmaps,
+        }
+        .try_into_batches()
     }
 
     async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>>
     where
@@ -391,12 +389,6 @@ impl Index for NGramIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Err(Error::invalid_input_source(
-            "NGramIndex is not a vector index".into(),
-        ))
-    }
-
     fn statistics(&self) -> Result<serde_json::Value> {
         let ngram_stats = NGramStatistics {
             num_ngrams: self.tokens.len(),
@@ -476,6 +468,45 @@ impl ScalarIndex for NGramIndex {
                 let row_ids = NGramPostingList::intersect(list_refs);
                 Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids)))
             }
+            TextQuery::Regex(pattern) => {
+                let trigram_query = ngram_regex::regex_to_trigram_query(pattern);
+                match &trigram_query {
+                    // No usable trigram structure (e.g. `a.b`, `.*`): the index
+                    // cannot prune, so every row must be rechecked.
+                    ngram_regex::TrigramQuery::All => {
+                        Ok(SearchResult::at_least(RowAddrTreeMap::new()))
+                    }
+                    // The pattern is provably unsatisfiable.
+                    ngram_regex::TrigramQuery::None => {
+                        Ok(SearchResult::exact(RowAddrTreeMap::new()))
+                    }
+                    _ => {
+                        let mut tokens = HashSet::new();
+                        ngram_regex::collect_tokens(&trigram_query, &mut tokens);
+                        // Fetch the posting list for every trigram the condition
+                        // references; a token absent from the index contributes
+                        // an empty list, which `eval_trigram_query` handles.
+                        let present = tokens.into_iter().filter_map(|token| {
+                            self.tokens.get(&token).map(|offset| (token, *offset))
+                        });
+                        let lists = futures::stream::iter(present.map(|(token, offset)| {
+                            self.list_reader
+                                .ngram_list(offset, metrics)
+                                .map(move |result| result.map(|list| (token, list)))
+                        }))
+                        .buffer_unordered(self.io_parallelism)
+                        .try_collect::<Vec<(u32, Arc<NGramPostingList>)>>()
+                        .await?;
+                        metrics.record_comparisons(lists.len());
+                        let bitmaps: HashMap<u32, RoaringTreemap> = lists
+                            .into_iter()
+                            .map(|(token, list)| (token, list.bitmap.clone()))
+                            .collect();
+                        let row_ids = ngram_regex::eval_trigram_query(&trigram_query, &bitmaps);
+                        Ok(SearchResult::at_most(RowAddrTreeMap::from(row_ids)))
+                    }
+                }
+            }
         }
     }
 
@@ -485,7 +516,7 @@ impl ScalarIndex for NGramIndex {
 
     async fn remap(
         &self,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
         dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         let reader = self.store.open_index_file(POSTINGS_FILENAME).await?;
@@ -493,24 +524,21 @@ impl ScalarIndex for NGramIndex {
             .new_index_file(POSTINGS_FILENAME, POSTINGS_SCHEMA.clone())
             .await?;
 
-        let mut offset = 0;
-        let num_rows = reader.num_rows();
-        const BATCH_SIZE: usize = 64;
-        while offset < num_rows {
-            let batch_size = BATCH_SIZE.min(num_rows - offset);
-            let batch = reader.read_range(offset..offset + batch_size, None).await?;
-            let batch = self.remap_batch(batch, mapping)?;
-            writer.write_record_batch(batch).await?;
-            offset += BATCH_SIZE;
+        let mut spill_stream =
+            NGramIndexBuilder::stream_spill_reader(reader, MAX_POSTING_LIST_BATCH_BYTES)?;
+        while let Some(state) = spill_stream.try_next().await? {
+            for batch in self.remap_state(state, mapping)? {
+                writer.write_record_batch(batch).await?;
+            }
         }
 
-        writer.finish().await?;
+        let file = writer.finish().await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -523,7 +551,7 @@ impl ScalarIndex for NGramIndex {
         let mut builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default())?;
         let spill_files = builder.train(new_data).await?;
 
-        builder
+        let file = builder
             .write_index(dest_store, spill_files, Some(self.store.clone()))
             .await?;
 
@@ -531,7 +559,7 @@ impl ScalarIndex for NGramIndex {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
 
@@ -592,6 +620,85 @@ struct NGramIndexSpillState {
     bitmaps: Vec<RoaringTreemap>,
 }
 
+struct NGramIndexSpillStateBuilder {
+    tokens: UInt32Builder,
+    bitmaps: Vec<RoaringTreemap>,
+    serialized_bytes: usize,
+}
+
+impl NGramIndexSpillStateBuilder {
+    fn new() -> Self {
+        Self {
+            tokens: UInt32Builder::with_capacity(0),
+            bitmaps: Vec::new(),
+            serialized_bytes: 0,
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.bitmaps.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.bitmaps.len()
+    }
+
+    fn push(
+        &mut self,
+        token: u32,
+        bitmap: RoaringTreemap,
+        max_batch_bytes: usize,
+    ) -> Result<Option<NGramIndexSpillState>> {
+        let posting_size = bitmap.serialized_size();
+        if posting_size > max_batch_bytes {
+            return Err(Error::invalid_input(format!(
+                "posting list for ngram token {} serializes to {} bytes, which exceeds the {} bytes that fit in a single binary array",
+                token, posting_size, max_batch_bytes,
+            )));
+        }
+
+        let new_size = self
+            .serialized_bytes
+            .checked_add(posting_size)
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "posting list byte size overflowed while adding ngram token {}",
+                    token
+                ))
+            })?;
+        let full_state = if !self.is_empty() && new_size > max_batch_bytes {
+            Some(self.finish())
+        } else {
+            None
+        };
+
+        self.tokens.append_value(token);
+        self.bitmaps.push(bitmap);
+        self.serialized_bytes = posting_size
+            .checked_add(if full_state.is_some() {
+                0
+            } else {
+                self.serialized_bytes
+            })
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "posting list byte size overflowed while adding ngram token {}",
+                    token
+                ))
+            })?;
+
+        Ok(full_state)
+    }
+
+    fn finish(&mut self) -> NGramIndexSpillState {
+        self.serialized_bytes = 0;
+        NGramIndexSpillState {
+            tokens: std::mem::replace(&mut self.tokens, UInt32Builder::with_capacity(0)).finish(),
+            bitmaps: std::mem::take(&mut self.bitmaps),
+        }
+    }
+}
+
 impl NGramIndexSpillState {
     fn try_from_batch(batch: RecordBatch) -> Result<Self> {
         let tokens = batch
@@ -615,16 +722,63 @@ impl NGramIndexSpillState {
         Ok(Self { tokens, bitmaps })
     }
 
-    fn try_into_batch(self) -> Result<RecordBatch> {
-        let bitmap_array = BinaryArray::from_iter_values(self.bitmaps.into_iter().map(|bitmap| {
-            let mut buf = Vec::with_capacity(bitmap.serialized_size());
-            bitmap.serialize_into(&mut buf).unwrap();
-            buf
-        }));
-        Ok(RecordBatch::try_new(
-            POSTINGS_SCHEMA.clone(),
-            vec![Arc::new(self.tokens), Arc::new(bitmap_array)],
-        )?)
+    fn try_into_batches(self) -> Result<Vec<RecordBatch>> {
+        self.try_into_batches_impl(MAX_POSTING_LIST_BATCH_BYTES)
+    }
+
+    // Split into multiple batches so that the cumulative serialized posting bytes
+    // of each batch stay under `max_batch_bytes`, avoiding i32 offset overflow in
+    // the Binary posting array.  Postings are serialized straight into each batch's
+    // values buffer to avoid a second contiguous copy of multi-GiB payloads.
+    fn try_into_batches_impl(self, max_batch_bytes: usize) -> Result<Vec<RecordBatch>> {
+        debug_assert_eq!(self.tokens.len(), self.bitmaps.len());
+        debug_assert!(max_batch_bytes <= i32::MAX as usize);
+        let make_batch =
+            |tokens: UInt32Array, values: Vec<u8>, offsets: Vec<i32>| -> Result<RecordBatch> {
+                let posting_array = BinaryArray::new(
+                    OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                    Buffer::from_vec(values),
+                    None,
+                );
+                Ok(RecordBatch::try_new(
+                    POSTINGS_SCHEMA.clone(),
+                    vec![Arc::new(tokens), Arc::new(posting_array)],
+                )?)
+            };
+
+        let mut batches = Vec::new();
+        let mut values: Vec<u8> = Vec::new();
+        let mut offsets: Vec<i32> = vec![0];
+        let mut batch_start = 0;
+        for (idx, bitmap) in self.bitmaps.into_iter().enumerate() {
+            let posting_size = bitmap.serialized_size();
+            if posting_size > max_batch_bytes {
+                return Err(Error::invalid_input(format!(
+                    "posting list for ngram token {} serializes to {} bytes, which exceeds the {} bytes that fit in a single binary array",
+                    self.tokens.value(idx),
+                    posting_size,
+                    max_batch_bytes,
+                )));
+            }
+            if values.len() + posting_size > max_batch_bytes {
+                batches.push(make_batch(
+                    self.tokens.slice(batch_start, idx - batch_start),
+                    std::mem::take(&mut values),
+                    std::mem::replace(&mut offsets, vec![0]),
+                )?);
+                batch_start = idx;
+            }
+            bitmap.serialize_into(&mut values)?;
+            offsets.push(values.len() as i32);
+        }
+        if offsets.len() > 1 || batches.is_empty() {
+            batches.push(make_batch(
+                self.tokens.slice(batch_start, offsets.len() - 1),
+                values,
+                offsets,
+            )?);
+        }
+        Ok(batches)
     }
 }
 
@@ -923,33 +1077,67 @@ impl NGramIndexBuilder {
         mut writer: Box<dyn IndexWriter>,
         state: NGramIndexSpillState,
     ) -> Result<()> {
-        writer.write_record_batch(state.try_into_batch()?).await?;
+        Self::write_state(writer.as_mut(), state).await?;
         writer.finish().await?;
 
         Ok(())
     }
 
-    async fn stream_spill_reader(
+    async fn write_state(writer: &mut dyn IndexWriter, state: NGramIndexSpillState) -> Result<()> {
+        for batch in state.try_into_batches()? {
+            writer.write_record_batch(batch).await?;
+        }
+        Ok(())
+    }
+
+    fn stream_spill_reader(
         reader: Arc<dyn IndexReader>,
+        max_batch_bytes: usize,
     ) -> Result<impl Stream<Item = Result<NGramIndexSpillState>>> {
         let num_rows = reader.num_rows();
 
-        Ok(stream::try_unfold(0, move |offset| {
-            let reader = reader.clone();
-            async move {
-                // These are small batches but, in the worst case scenario, each row could
-                // be massive (up to 128MB per row at 1B rows) and we end up breaking memory
-                let batch_size = std::cmp::min(num_rows - offset, 64);
-                if batch_size == 0 {
-                    return Ok(None);
+        Ok(stream::try_unfold(
+            (0, NGramIndexSpillStateBuilder::new()),
+            move |(mut offset, mut builder)| {
+                let reader = reader.clone();
+                async move {
+                    while offset < num_rows {
+                        // A single posting list is already bounded by
+                        // MAX_POSTING_LIST_BATCH_BYTES. Reading one row at a time avoids
+                        // materializing several large postings into the same BinaryArray
+                        // before the byte-bounded writer can split them again.
+                        let batch = reader.read_range(offset..offset + 1, None).await?;
+                        offset += 1;
+
+                        let state = NGramIndexSpillState::try_from_batch(batch)?;
+                        if state.tokens.len() != 1 || state.bitmaps.len() != 1 {
+                            return Err(Error::internal(format!(
+                                "expected one ngram posting row at offset {}, got {} tokens and {} posting lists",
+                                offset - 1,
+                                state.tokens.len(),
+                                state.bitmaps.len(),
+                            )));
+                        }
+                        let token = state.tokens.value(0);
+                        let mut bitmaps = state.bitmaps.into_iter();
+                        let bitmap = bitmaps.next().expect_ok()?;
+                        if let Some(state) = builder.push(token, bitmap, max_batch_bytes)? {
+                            return Ok(Some((state, (offset, builder))));
+                        }
+                        if builder.len() >= POSTING_LIST_STREAM_BATCH_ROWS {
+                            return Ok(Some((builder.finish(), (offset, builder))));
+                        }
+                    }
+
+                    if builder.is_empty() {
+                        Ok(None)
+                    } else {
+                        Ok(Some((builder.finish(), (offset, builder))))
+                    }
                 }
-                let batch = reader.read_range(offset..offset + batch_size, None).await?;
-                let state = NGramIndexSpillState::try_from_batch(batch)?;
-                let new_offset = offset + batch_size;
-                Ok(Some((state, new_offset)))
-            }
-            .boxed()
-        }))
+                .boxed()
+            },
+        ))
     }
 
     async fn stream_spill(
@@ -959,7 +1147,7 @@ impl NGramIndexBuilder {
         let reader = spill_store
             .open_index_file(&Self::spill_filename(id))
             .await?;
-        Self::stream_spill_reader(reader).await
+        Self::stream_spill_reader(reader, MAX_POSTING_LIST_BATCH_BYTES)
     }
 
     fn merge_spill_states(
@@ -1047,7 +1235,7 @@ impl NGramIndexBuilder {
         mut left_stream: impl Stream<Item = Result<NGramIndexSpillState>> + Unpin,
         mut right_stream: impl Stream<Item = Result<NGramIndexSpillState>> + Unpin,
         writer: &mut dyn IndexWriter,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let mut left_state = left_stream.try_next().await?;
         let mut right_state = right_stream.try_next().await?;
 
@@ -1055,21 +1243,21 @@ impl NGramIndexBuilder {
             if left_state.is_none() {
                 // Left is done, full drain right
                 let state = right_state.take().expect_ok()?;
-                writer.write_record_batch(state.try_into_batch()?).await?;
+                Self::write_state(writer, state).await?;
                 while let Some(state) = right_stream.try_next().await? {
-                    writer.write_record_batch(state.try_into_batch()?).await?;
+                    Self::write_state(writer, state).await?;
                 }
             } else if right_state.is_none() {
                 // Right is done, full drain left
                 let state = left_state.take().expect_ok()?;
-                writer.write_record_batch(state.try_into_batch()?).await?;
+                Self::write_state(writer, state).await?;
                 while let Some(state) = left_stream.try_next().await? {
-                    writer.write_record_batch(state.try_into_batch()?).await?;
+                    Self::write_state(writer, state).await?;
                 }
             } else {
                 // There is a batch from both left and right.  Need to merge them
                 let merged = Self::merge_spill_states(&mut left_state, &mut right_state);
-                writer.write_record_batch(merged.try_into_batch()?).await?;
+                Self::write_state(writer, merged).await?;
                 if left_state.is_none() {
                     left_state = left_stream.try_next().await?;
                 }
@@ -1165,7 +1353,7 @@ impl NGramIndexBuilder {
 
         let left_stream = Self::stream_spill(self.spill_store.clone(), new_data_num).await?;
         let old_reader = old_index.open_index_file(POSTINGS_FILENAME).await?;
-        let right_stream = Self::stream_spill_reader(old_reader).await?;
+        let right_stream = Self::stream_spill_reader(old_reader, MAX_POSTING_LIST_BATCH_BYTES)?;
 
         Self::merge_spill_streams(left_stream, right_stream, writer.as_mut()).await?;
 
@@ -1181,7 +1369,7 @@ impl NGramIndexBuilder {
         store: &dyn IndexStore,
         spill_files: Vec<usize>,
         old_index: Option<Arc<dyn IndexStore>>,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let mut writer = store
             .new_index_file(POSTINGS_FILENAME, POSTINGS_SCHEMA.clone())
             .await?;
@@ -1189,15 +1377,14 @@ impl NGramIndexBuilder {
         if spill_files.is_empty() {
             if let Some(old_index) = old_index {
                 // An update with no new data, just copy the old index to the new store
-                old_index.copy_index_file(POSTINGS_FILENAME, store).await?;
+                return old_index.copy_index_file(POSTINGS_FILENAME, store).await;
             } else {
                 // Training an index with no data, make an empty index
                 let mut writer = store
                     .new_index_file(POSTINGS_FILENAME, POSTINGS_SCHEMA.clone())
                     .await?;
-                writer.finish().await?;
+                return writer.finish().await;
             }
-            return Ok(());
         }
 
         let mut index_to_copy = self.merge_spills(spill_files).await?;
@@ -1211,14 +1398,9 @@ impl NGramIndexBuilder {
             .open_index_file(&Self::spill_filename(index_to_copy))
             .await?;
 
-        let num_rows = reader.num_rows();
-        let mut offset = 0;
-
-        while offset < num_rows {
-            let batch_size = std::cmp::min(num_rows - offset, 64);
-            let batch = reader.read_range(offset..offset + batch_size, None).await?;
-            writer.write_record_batch(batch).await?;
-            offset += batch_size;
+        let mut spill_stream = Self::stream_spill_reader(reader, MAX_POSTING_LIST_BATCH_BYTES)?;
+        while let Some(state) = spill_stream.try_next().await? {
+            Self::write_state(writer.as_mut(), state).await?;
         }
 
         writer.finish().await
@@ -1232,7 +1414,7 @@ impl NGramIndexPlugin {
     pub async fn train_ngram_index(
         batches_source: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let mut builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default())?;
 
         let spill_files = builder.train(batches_source).await?;
@@ -1242,11 +1424,7 @@ impl NGramIndexPlugin {
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for NGramIndexPlugin {
-    fn name(&self) -> &str {
-        "NGram"
-    }
-
+impl BasicTrainer for NGramIndexPlugin {
     fn new_training_request(
         &self,
         _params: &str,
@@ -1264,26 +1442,6 @@ impl ScalarIndexPlugin for NGramIndexPlugin {
         )))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        false
-    }
-
-    fn version(&self) -> u32 {
-        NGRAM_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        Some(Box::new(TextQueryParser::new(
-            index_name,
-            self.name().to_string(),
-            true,
-        )))
-    }
-
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -1298,20 +1456,54 @@ impl ScalarIndexPlugin for NGramIndexPlugin {
             ));
         }
 
-        Self::train_ngram_index(data, index_store).await?;
+        let file = Self::train_ngram_index(data, index_store).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::NGramIndexDetails::default())
                 .unwrap(),
             index_version: NGRAM_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files: vec![file],
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for NGramIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "NGram"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        false
+    }
+
+    fn version(&self) -> u32 {
+        NGRAM_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        Some(Box::new(TextQueryParser::new(
+            index_name,
+            self.name().to_string(),
+            // needs_recheck: ngram results are an inexact candidate superset.
+            true,
+            // supports_regex: the ngram index can answer regex queries.
+            true,
+        )))
+    }
 
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(NGramIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
@@ -1320,34 +1512,85 @@ impl ScalarIndexPlugin for NGramIndexPlugin {
 
 #[cfg(test)]
 mod tests {
+    use lance_core::utils::row_addr_remap::RowAddrRemap;
+    use rstest::rstest;
     use std::{
         collections::{HashMap, HashSet},
         sync::Arc,
     };
 
-    use arrow::datatypes::UInt64Type;
-    use arrow_array::{Array, RecordBatch, StringArray, UInt64Array};
+    use arrow::array::AsArray;
+    use arrow::datatypes::{UInt32Type, UInt64Type};
+    use arrow_array::{Array, RecordBatch, StringArray, UInt32Array, UInt64Array};
     use arrow_schema::{DataType, Field, Schema};
+    use async_trait::async_trait;
     use datafusion::{
         execution::SendableRecordBatchStream, physical_plan::stream::RecordBatchStreamAdapter,
     };
     use datafusion_common::DataFusionError;
     use futures::{TryStreamExt, stream};
     use itertools::Itertools;
-    use lance_core::{ROW_ID, cache::LanceCache, utils::tempfile::TempDir};
+    use lance_core::{Error, ROW_ID, Result, cache::LanceCache, utils::tempfile::TempDir};
     use lance_datagen::{BatchCount, ByteCount, RowCount};
     use lance_io::object_store::ObjectStore;
     use lance_select::RowAddrTreeMap;
     use lance_tokenizer::TextAnalyzer;
+    use roaring::RoaringTreemap;
 
     use crate::scalar::{
-        ScalarIndex, SearchResult, TextQuery,
+        IndexReader, IndexStore, ScalarIndex, SearchResult, TextQuery,
         lance_format::LanceIndexStore,
         ngram::{NGramIndex, NGramIndexBuilder, NGramIndexBuilderOptions},
     };
     use crate::{metrics::NoOpMetricsCollector, scalar::registry::VALUE_COLUMN_NAME};
 
-    use super::{NGRAM_TOKENIZER, ngram_to_token, tokenize_visitor};
+    use super::{
+        NGRAM_TOKENIZER, NGramIndexSpillState, POSTINGS_FILENAME, POSTINGS_SCHEMA, ngram_to_token,
+        tokenize_visitor,
+    };
+
+    struct MaxReadRangeReader {
+        inner: Arc<dyn IndexReader>,
+        max_rows: usize,
+    }
+
+    #[async_trait]
+    impl IndexReader for MaxReadRangeReader {
+        async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result<RecordBatch> {
+            self.inner.read_record_batch(n, batch_size).await
+        }
+
+        async fn read_range(
+            &self,
+            range: std::ops::Range<usize>,
+            projection: Option<&[&str]>,
+        ) -> Result<RecordBatch> {
+            let rows = range.end - range.start;
+            if rows > self.max_rows {
+                return Err(Error::invalid_input(format!(
+                    "read_range requested {} rows, max is {}",
+                    rows, self.max_rows,
+                )));
+            }
+            self.inner.read_range(range, projection).await
+        }
+
+        async fn num_batches(&self, batch_size: u64) -> u32 {
+            self.inner.num_batches(batch_size).await
+        }
+
+        fn num_rows(&self) -> usize {
+            self.inner.num_rows()
+        }
+
+        fn schema(&self) -> &lance_core::datatypes::Schema {
+            self.inner.schema()
+        }
+
+        fn file_size_bytes(&self) -> Option<u64> {
+            self.inner.file_size_bytes()
+        }
+    }
 
     fn collect_tokens(analyzer: &TextAnalyzer, text: &str) -> Vec<String> {
         let mut tokens = Vec::with_capacity(text.len() * 3);
@@ -1539,6 +1782,107 @@ mod tests {
         assert_eq!(expected, res);
     }
 
+    #[test_log::test(tokio::test)]
+    async fn test_ngram_regex_search() {
+        // Same corpus as test_basic_ngram_index.
+        let data = StringArray::from_iter_values([
+            "cat",         // 0
+            "dog",         // 1
+            "cat dog",     // 2
+            "dog cat",     // 3
+            "elephant",    // 4
+            "mouse",       // 5
+            "rhino",       // 6
+            "giraffe",     // 7
+            "rhinos nose", // 8
+        ]);
+        let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false),
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]));
+        let data =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap();
+        let data = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            stream::once(std::future::ready(Ok(data))),
+        ));
+
+        let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap();
+        let (index, _tmpdir) = do_train(builder, data).await;
+
+        async fn search(index: &NGramIndex, pattern: &str) -> SearchResult {
+            index
+                .search(
+                    &TextQuery::Regex(pattern.to_string()),
+                    &NoOpMetricsCollector,
+                )
+                .await
+                .unwrap()
+        }
+
+        // A plain literal yields the same candidates as contains("cat").
+        assert_eq!(
+            search(&index, "cat").await,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([0, 2, 3]))
+        );
+
+        // Alternation -> union of each branch's rows.
+        assert_eq!(
+            search(&index, "(cat|dog)").await,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 2, 3]))
+        );
+
+        // AND across `.*`: must contain both the `rhino` and `nose` trigrams, so
+        // row 6 ("rhino") is correctly excluded and only row 8 survives.
+        assert_eq!(
+            search(&index, "rhino.*nose").await,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([8]))
+        );
+
+        // No derivable trigram -> recheck everything.
+        assert_eq!(
+            search(&index, "a.b").await,
+            SearchResult::at_least(RowAddrTreeMap::new())
+        );
+
+        // A trigram that is absent from the index -> empty candidate set.
+        assert_eq!(
+            search(&index, "zzz").await,
+            SearchResult::at_most(RowAddrTreeMap::new())
+        );
+    }
+
+    #[test_log::test(tokio::test)]
+    async fn test_ngram_regex_search_nulls() {
+        // Rows: cat(0), dog(1), NULL(2), NULL(3), cat dog(4).
+        let data = simple_data_with_nulls();
+        let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap();
+        let (index, _tmpdir) = do_train(builder, data).await;
+
+        // The NULL rows (2, 3) must never appear in the candidate set.
+        let res = index
+            .search(&TextQuery::Regex("cat".to_string()), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert_eq!(
+            res,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([0, 4]))
+        );
+
+        let res = index
+            .search(
+                &TextQuery::Regex("(cat|dog)".to_string()),
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+        assert_eq!(
+            res,
+            SearchResult::at_most(RowAddrTreeMap::from_iter([0, 1, 4]))
+        );
+    }
+
     fn test_data_schema() -> Arc<Schema> {
         Arc::new(Schema::new(vec![
             Field::new(VALUE_COLUMN_NAME, DataType::Utf8, true),
@@ -1650,7 +1994,10 @@ mod tests {
         ));
 
         let remapping = HashMap::from([(2, Some(100)), (3, None), (4, Some(101))]);
-        index.remap(&remapping, test_store.as_ref()).await.unwrap();
+        index
+            .remap(&RowAddrRemap::direct(remapping), test_store.as_ref())
+            .await
+            .unwrap();
 
         let index = NGramIndex::from_store(test_store, None, &LanceCache::no_cache())
             .await
@@ -1662,6 +2009,61 @@ mod tests {
         assert_eq!(null_posting_list, vec![100]);
     }
 
+    // Like `test_ngram_index_remap` but covering both RowAddrRemap modes: rows
+    // 0..4 of frag 0 are rewritten into frag 10; row 4 is deleted.
+    fn ngram_remap_compact() -> RowAddrRemap {
+        use lance_core::utils::row_addr_remap::GroupInput;
+        use roaring::RoaringTreemap;
+        RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter(0u64..4),
+            old_frag_ids: vec![0],
+            new_frags: vec![(10, 4)],
+        }])
+        .unwrap()
+    }
+
+    fn ngram_remap_explicit() -> RowAddrRemap {
+        RowAddrRemap::direct(
+            (0u64..4)
+                .map(|i| (i, Some((10u64 << 32) | i)))
+                .chain(std::iter::once((4u64, None)))
+                .collect(),
+        )
+    }
+
+    #[rstest]
+    #[case(ngram_remap_compact())]
+    #[case(ngram_remap_explicit())]
+    #[tokio::test]
+    async fn test_ngram_index_remap_compact(#[case] remap: RowAddrRemap) {
+        let data = simple_data_with_nulls();
+        let builder = NGramIndexBuilder::try_new(NGramIndexBuilderOptions::default()).unwrap();
+        let (index, _tmpdir) = do_train(builder, data).await;
+
+        let row_ids = row_ids_in_index(&index).await;
+        assert_eq!(row_ids, vec![0, 1, 2, 3, 4]);
+
+        let new_tmpdir = Arc::new(TempDir::default());
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            new_tmpdir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        index.remap(&remap, test_store.as_ref()).await.unwrap();
+
+        let index = NGramIndex::from_store(test_store, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+        let addr = |offset: u64| (10u64 << 32) | offset;
+        let row_ids = row_ids_in_index(&index).await;
+        assert_eq!(row_ids, vec![addr(0), addr(1), addr(2), addr(3)]);
+
+        // rows 2 and 3 are the null docs; both are rewritten into frag 10.
+        let null_posting_list = get_null_posting_list(&index).await;
+        assert_eq!(null_posting_list, vec![addr(2), addr(3)]);
+    }
+
     #[test_log::test(tokio::test)]
     async fn test_ngram_index_merge() {
         let data = simple_data_with_nulls();
@@ -1733,4 +2135,145 @@ mod tests {
 
         assert_eq!(index.tokens.len(), 29012);
     }
+
+    #[test]
+    fn test_spill_state_chunks_by_byte_size() {
+        let bitmaps = (0..8u64)
+            .map(|i| RoaringTreemap::from_iter(0..(i + 1) * 100))
+            .collect::<Vec<_>>();
+        let tokens = UInt32Array::from_iter_values(0..8);
+        let state = NGramIndexSpillState {
+            tokens: tokens.clone(),
+            bitmaps: bitmaps.clone(),
+        };
+
+        // Small enough that several splits are required, large enough that some
+        // batches hold more than one posting
+        let max_batch_bytes = bitmaps.iter().map(|b| b.serialized_size()).max().unwrap() * 2;
+        let batches = state.try_into_batches_impl(max_batch_bytes).unwrap();
+        assert!(batches.len() > 1);
+
+        // Token order and posting contents survive the chunking
+        let mut row = 0;
+        for batch in &batches {
+            let batch_tokens = batch["tokens"].as_primitive::<UInt32Type>();
+            let batch_postings = batch["posting_list"].as_binary::<i32>();
+            let mut batch_bytes = 0;
+            for i in 0..batch.num_rows() {
+                assert_eq!(batch_tokens.value(i), tokens.value(row));
+                let posting = batch_postings.value(i);
+                batch_bytes += posting.len();
+                assert_eq!(
+                    RoaringTreemap::deserialize_from(posting).unwrap(),
+                    bitmaps[row]
+                );
+                row += 1;
+            }
+            assert!(batch_bytes <= max_batch_bytes || batch.num_rows() == 1);
+        }
+        assert_eq!(row, 8);
+    }
+
+    #[test_log::test(tokio::test)]
+    async fn test_spill_reader_does_not_materialize_multirow_posting_batches() {
+        let bitmaps = (0..8u64)
+            .map(|i| RoaringTreemap::from_iter(0..(i + 1) * 100))
+            .collect::<Vec<_>>();
+        let tokens = UInt32Array::from_iter_values(0..8);
+        let state = NGramIndexSpillState {
+            tokens: tokens.clone(),
+            bitmaps: bitmaps.clone(),
+        };
+        let max_batch_bytes = bitmaps.iter().map(|b| b.serialized_size()).max().unwrap() * 2;
+
+        let tmpdir = Arc::new(TempDir::default());
+        let store = LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.obj_path(),
+            Arc::new(LanceCache::no_cache()),
+        );
+        let mut writer = store
+            .new_index_file(POSTINGS_FILENAME, POSTINGS_SCHEMA.clone())
+            .await
+            .unwrap();
+        for batch in state.try_into_batches().unwrap() {
+            writer.write_record_batch(batch).await.unwrap();
+        }
+        writer.finish().await.unwrap();
+
+        let reader = store.open_index_file(POSTINGS_FILENAME).await.unwrap();
+        let reader = Arc::new(MaxReadRangeReader {
+            inner: reader,
+            max_rows: 1,
+        });
+        let states = NGramIndexBuilder::stream_spill_reader(reader, max_batch_bytes)
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert!(states.len() > 1);
+
+        let mut row = 0;
+        for state in states {
+            let batch_bytes = state
+                .bitmaps
+                .iter()
+                .map(RoaringTreemap::serialized_size)
+                .sum::<usize>();
+            assert!(batch_bytes <= max_batch_bytes || state.bitmaps.len() == 1);
+            for (token, bitmap) in state.tokens.values().iter().zip(state.bitmaps) {
+                assert_eq!(*token, tokens.value(row));
+                assert_eq!(bitmap, bitmaps[row]);
+                row += 1;
+            }
+        }
+        assert_eq!(row, 8);
+    }
+
+    #[test]
+    fn test_spill_state_rejects_oversized_posting() {
+        let bitmap = RoaringTreemap::from_iter(0..1000u64);
+        let too_small = bitmap.serialized_size() - 1;
+        let state = NGramIndexSpillState {
+            tokens: UInt32Array::from_iter_values([42]),
+            bitmaps: vec![bitmap],
+        };
+        let err = state.try_into_batches_impl(too_small).unwrap_err();
+        assert!(err.to_string().contains("token 42"), "{}", err);
+    }
+
+    #[test]
+    fn test_empty_spill_state_yields_one_empty_batch() {
+        let state = NGramIndexSpillState {
+            tokens: UInt32Array::from_iter_values([]),
+            bitmaps: vec![],
+        };
+        let batches = state.try_into_batches().unwrap();
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].num_rows(), 0);
+    }
+
+    // Reproduces https://linear.app/lancedb/issue/ENT-874: serialized posting lists
+    // totalling more than i32::MAX bytes used to panic with "byte array offset
+    // overflow" when packed into a single Binary array.
+    #[test]
+    #[ignore = "needs ~8 GiB of RAM and a couple of minutes; run manually"]
+    fn test_spill_state_over_i32_max_bytes() {
+        // Every 16th value keeps each container an array container (4096 entries,
+        // 2 bytes per value, immune to run compression), so the treemap serializes
+        // to ~450 MiB.  Six copies exceed i32::MAX total bytes.
+        let bitmap = RoaringTreemap::from_sorted_iter((0..225_000_000u64).map(|v| v * 16)).unwrap();
+        assert!(bitmap.serialized_size() > 400 * 1024 * 1024);
+        let bitmaps = vec![bitmap; 6];
+        let tokens = UInt32Array::from_iter_values(0..6);
+        let state = NGramIndexSpillState { tokens, bitmaps };
+
+        let batches = state.try_into_batches().unwrap();
+        assert!(batches.len() > 1);
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 6);
+        for batch in &batches {
+            let postings = batch["posting_list"].as_binary::<i32>();
+            assert!(postings.value_data().len() <= i32::MAX as usize);
+        }
+    }
 }
diff --git a/rust/lance-index/src/scalar/ngram/ngram_regex.rs b/rust/lance-index/src/scalar/ngram/ngram_regex.rs
new file mode 100644
index 00000000000..ee67c479a71
--- /dev/null
+++ b/rust/lance-index/src/scalar/ngram/ngram_regex.rs
@@ -0,0 +1,673 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Deriving a trigram pre-filter from a regular expression.
+//!
+//! This is the query-side counterpart of the ngram index that lets us
+//! accelerate `regexp_like` / `regexp_match` predicates the same way the index
+//! already accelerates `contains`. The idea (the same one Postgres `pg_trgm`
+//! and Russ Cox's Google Code Search use) is to derive, from the regex, a
+//! boolean condition over trigram presence that is *necessary* for any string
+//! to match, evaluate it against the inverted index, and let the scan recheck
+//! the true regex on the surviving rows.
+//!
+//! The derived condition is a [`TrigramQuery`] -- an AND/OR tree of trigram
+//! tokens. `AND` maps onto posting-list intersection and `OR` onto union, which
+//! is exactly the set algebra the ngram index is built for.
+//!
+//! # Soundness
+//!
+//! The single invariant that matters is that the condition must never require a
+//! trigram that a matching string could lack -- otherwise we would drop real
+//! matches (a false negative, far worse than a false positive, which the recheck
+//! removes). Everything here is therefore a conservative *over*-approximation:
+//! when in doubt we emit [`TrigramQuery::All`] ("no constraint, recheck
+//! everything"). Concretely:
+//!
+//! * Every trigram requirement is produced by [`trigrams_of_string`], which runs
+//!   the *same* tokenizer the index was built with, so a string shorter than a
+//!   trigram (or with no alphanumeric run) contributes no requirement.
+//! * Character classes and case-insensitive folds are treated as a single
+//!   unknown character (`All`), because the index's normalization does not agree
+//!   with Unicode case folding (e.g. `(?i)c` also matches `ℂ`, which the index
+//!   does not fold to `c`). Literal runs -- the common case -- are fully used.
+//! * When the exact / prefix / suffix string sets grow past a bound we first fold
+//!   their trigrams into the running condition and only then drop the strings, so
+//!   collapsing precision never removes a necessary trigram.
+
+use std::collections::{BTreeSet, HashMap, HashSet};
+
+use regex_syntax::hir::{Class, Hir, HirKind};
+use roaring::RoaringTreemap;
+
+use super::{NGRAM_N, NGRAM_TOKENIZER, ngram_to_token, tokenize_visitor};
+
+/// Maximum number of strings kept in an `exact` / `prefix` / `suffix` set before
+/// it is folded into the trigram condition and dropped.
+const MAX_SET_SIZE: usize = 16;
+/// Maximum length (in characters) of a string kept in a set. Longer strings are
+/// trimmed to a sound shorter affix.
+const MAX_STRING_LEN: usize = 32;
+
+/// A boolean condition over trigram presence that is *necessary* for a regex to
+/// match. `All` means "no constraint" and `None` means "unsatisfiable"; by
+/// construction these only ever appear at the root of the tree.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum TrigramQuery {
+    /// No constraint: every row is a candidate (the scan must recheck all rows).
+    All,
+    /// Unsatisfiable: no row can match.
+    None,
+    /// The given trigram token must be present.
+    Trigram(u32),
+    /// Every child condition must hold (posting-list intersection).
+    And(Vec<Self>),
+    /// At least one child condition must hold (posting-list union).
+    Or(Vec<Self>),
+}
+
+impl TrigramQuery {
+    /// Build an `AND` of conditions, applying identity (`All`), absorbing
+    /// (`None`), flattening, sorting and de-duplication so the result is
+    /// canonical and free of nested `All`/`None`.
+    fn and(items: Vec<Self>) -> Self {
+        let mut flat = Vec::with_capacity(items.len());
+        for item in items {
+            match item {
+                Self::All => {}                               // identity
+                Self::None => return Self::None,              // absorbing
+                Self::And(children) => flat.extend(children), // flatten
+                other => flat.push(other),
+            }
+        }
+        flat.sort();
+        flat.dedup();
+        match flat.len() {
+            0 => Self::All,
+            1 => flat.pop().unwrap(),
+            _ => Self::And(flat),
+        }
+    }
+
+    /// Build an `OR` of conditions, applying absorbing (`All`), identity
+    /// (`None`), flattening, sorting and de-duplication.
+    fn or(items: Vec<Self>) -> Self {
+        let mut flat = Vec::with_capacity(items.len());
+        for item in items {
+            match item {
+                Self::All => return Self::All,               // absorbing
+                Self::None => {}                             // identity
+                Self::Or(children) => flat.extend(children), // flatten
+                other => flat.push(other),
+            }
+        }
+        flat.sort();
+        flat.dedup();
+        match flat.len() {
+            0 => Self::None,
+            1 => flat.pop().unwrap(),
+            _ => Self::Or(flat),
+        }
+    }
+}
+
+/// Information about the set of strings a sub-expression can match, used to
+/// build a necessary trigram condition bottom-up. For every string `s` the
+/// sub-expression matches: `s` is in `exact` (when it is `Some`), `s` starts
+/// with some member of `prefix` and ends with some member of `suffix`, and `s`
+/// satisfies `match_q`.
+struct RegexInfo {
+    /// Whether the sub-expression can match the empty string.
+    emptyable: bool,
+    /// The complete set of strings the sub-expression matches, or `None` if that
+    /// set is unbounded / unknown.
+    exact: Option<BTreeSet<String>>,
+    /// Strings that every match must start with (empty = unknown).
+    prefix: BTreeSet<String>,
+    /// Strings that every match must end with (empty = unknown).
+    suffix: BTreeSet<String>,
+    /// A necessary trigram condition for the sub-expression.
+    match_q: TrigramQuery,
+}
+
+impl RegexInfo {
+    /// The empty string (also used for zero-width anchors): matches only `""`.
+    fn empty_string() -> Self {
+        let empty = BTreeSet::from([String::new()]);
+        Self {
+            emptyable: true,
+            exact: Some(empty.clone()),
+            prefix: empty.clone(),
+            suffix: empty,
+            match_q: TrigramQuery::All,
+        }
+    }
+
+    /// A fixed literal string.
+    fn literal(s: &str) -> Self {
+        let set = BTreeSet::from([s.to_string()]);
+        Self {
+            emptyable: s.is_empty(),
+            exact: Some(set.clone()),
+            prefix: set.clone(),
+            suffix: set,
+            match_q: trigrams_of_string(s),
+        }
+    }
+
+    /// A single unknown character (a character class we cannot pin down).
+    fn any_char() -> Self {
+        Self {
+            emptyable: false,
+            exact: None,
+            prefix: BTreeSet::new(),
+            suffix: BTreeSet::new(),
+            match_q: TrigramQuery::All,
+        }
+    }
+
+    /// Enforce the size/length bounds, folding any information about to be
+    /// discarded into `match_q` first so that precision loss never drops a
+    /// necessary trigram. Idempotent.
+    fn bound(&mut self) {
+        let oversized_exact = self.exact.as_ref().is_some_and(|exact| {
+            exact.len() > MAX_SET_SIZE || exact.iter().any(|s| s.chars().count() > MAX_STRING_LEN)
+        });
+        if oversized_exact {
+            let exact = self.exact.take().expect("checked above");
+            self.fold_into_match(&exact);
+        }
+
+        self.prefix = self
+            .prefix
+            .iter()
+            .map(|s| leading(s, MAX_STRING_LEN))
+            .collect();
+        if self.prefix.len() > MAX_SET_SIZE {
+            let prefix = std::mem::take(&mut self.prefix);
+            self.fold_into_match(&prefix);
+        }
+
+        self.suffix = self
+            .suffix
+            .iter()
+            .map(|s| trailing(s, MAX_STRING_LEN))
+            .collect();
+        if self.suffix.len() > MAX_SET_SIZE {
+            let suffix = std::mem::take(&mut self.suffix);
+            self.fold_into_match(&suffix);
+        }
+    }
+
+    /// AND the trigrams of `set` (a complete set of possible affixes/strings)
+    /// into `match_q`. Sound because the set is exhaustive for its role.
+    fn fold_into_match(&mut self, set: &BTreeSet<String>) {
+        let folded = trigrams_of_set(set.iter());
+        let current = std::mem::replace(&mut self.match_q, TrigramQuery::All);
+        self.match_q = TrigramQuery::and(vec![current, folded]);
+    }
+}
+
+/// AND together the trigrams of `s`. Reuses the index's own tokenizer so the
+/// tokens are normalized (lowercase, ASCII-folded, alphanumeric-bounded)
+/// exactly as they were stored. Returns `All` if `s` yields no trigram (too
+/// short, or no run of three alphanumeric characters).
+fn trigrams_of_string(s: &str) -> TrigramQuery {
+    let mut tokens = Vec::new();
+    tokenize_visitor(&NGRAM_TOKENIZER, s, |ngram| {
+        tokens.push(TrigramQuery::Trigram(ngram_to_token(ngram, NGRAM_N)));
+    });
+    TrigramQuery::and(tokens)
+}
+
+/// OR together the trigram conditions of each string in `set`. An empty set
+/// means "unknown" and yields `All` (no constraint); if any member yields `All`
+/// the whole OR is `All`.
+fn trigrams_of_set<'a>(set: impl IntoIterator<Item = &'a String>) -> TrigramQuery {
+    let queries: Vec<_> = set.into_iter().map(|s| trigrams_of_string(s)).collect();
+    if queries.is_empty() {
+        return TrigramQuery::All;
+    }
+    TrigramQuery::or(queries)
+}
+
+/// Concatenate every string in `a` with every string in `b`.
+fn cross_concat(a: &BTreeSet<String>, b: &BTreeSet<String>) -> BTreeSet<String> {
+    let mut out = BTreeSet::new();
+    for x in a {
+        for y in b {
+            out.insert(format!("{x}{y}"));
+        }
+    }
+    out
+}
+
+/// The first `n` characters of `s` (a sound shorter prefix).
+fn leading(s: &str, n: usize) -> String {
+    s.chars().take(n).collect()
+}
+
+/// The last `n` characters of `s` (a sound shorter suffix).
+fn trailing(s: &str, n: usize) -> String {
+    let count = s.chars().count();
+    s.chars().skip(count.saturating_sub(n)).collect()
+}
+
+/// If `class` matches exactly one scalar value, return that character.
+fn singleton_char(class: &Class) -> Option<char> {
+    match class {
+        Class::Unicode(u) => {
+            let ranges = u.ranges();
+            match ranges {
+                [r] if r.start() == r.end() => Some(r.start()),
+                _ => None,
+            }
+        }
+        Class::Bytes(b) => {
+            let ranges = b.ranges();
+            match ranges {
+                [r] if r.start() == r.end() && r.start() < 0x80 => Some(r.start() as char),
+                _ => None,
+            }
+        }
+    }
+}
+
+/// Compute the [`RegexInfo`] for `hir` bottom-up.
+fn analyze(hir: &Hir) -> RegexInfo {
+    let mut info = match hir.kind() {
+        // Zero-width: the empty match. Anchors (^, $, \b) carry no trigram.
+        HirKind::Empty | HirKind::Look(_) => RegexInfo::empty_string(),
+        HirKind::Literal(lit) => match std::str::from_utf8(&lit.0) {
+            Ok(s) => RegexInfo::literal(s),
+            // A literal that is not valid UTF-8 cannot be reasoned about here.
+            Err(_) => RegexInfo::any_char(),
+        },
+        HirKind::Class(class) => match singleton_char(class) {
+            Some(ch) => RegexInfo::literal(ch.encode_utf8(&mut [0u8; 4])),
+            None => RegexInfo::any_char(),
+        },
+        HirKind::Repetition(rep) => {
+            let inner = analyze(&rep.sub);
+            let at_least_one = rep.min >= 1;
+            RegexInfo {
+                emptyable: !at_least_one || inner.emptyable,
+                // We do not unroll bounded repetitions, so the matched set is
+                // unbounded as far as we are concerned.
+                exact: None,
+                prefix: if at_least_one {
+                    inner.prefix.clone()
+                } else {
+                    BTreeSet::new()
+                },
+                suffix: if at_least_one {
+                    inner.suffix.clone()
+                } else {
+                    BTreeSet::new()
+                },
+                // Only a required occurrence (min >= 1) contributes; the single
+                // inner match is necessary, never multiplied.
+                match_q: if at_least_one {
+                    inner.match_q
+                } else {
+                    TrigramQuery::All
+                },
+            }
+        }
+        HirKind::Capture(cap) => analyze(&cap.sub),
+        HirKind::Concat(subs) => analyze_concat(subs),
+        HirKind::Alternation(subs) => analyze_alternation(subs),
+    };
+    info.bound();
+    info
+}
+
+fn analyze_concat(subs: &[Hir]) -> RegexInfo {
+    let mut acc = RegexInfo::empty_string();
+    for sub in subs {
+        acc = concat_info(acc, analyze(sub));
+    }
+    acc
+}
+
+/// Combine two adjacent sub-expressions. This is the subtle part: it recovers
+/// trigrams that straddle the junction via the cross product of `acc.suffix` and
+/// `next.prefix`.
+fn concat_info(acc: RegexInfo, next: RegexInfo) -> RegexInfo {
+    let emptyable = acc.emptyable && next.emptyable;
+
+    // Trigrams spanning the junction (computed from the pre-merge affixes).
+    let boundary = if acc.suffix.is_empty() || next.prefix.is_empty() {
+        TrigramQuery::All
+    } else {
+        trigrams_of_set(cross_concat(&acc.suffix, &next.prefix).iter())
+    };
+
+    // exact = acc.exact x next.exact, only while both are finite and small.
+    let exact = match (&acc.exact, &next.exact) {
+        (Some(a), Some(b)) if a.len().saturating_mul(b.len()) <= MAX_SET_SIZE => {
+            Some(cross_concat(a, b))
+        }
+        _ => None,
+    };
+
+    // A match starts with acc's full string (when known) then next's prefix,
+    // otherwise with acc's own prefix.
+    let prefix = match &acc.exact {
+        Some(a) if !next.prefix.is_empty() => cross_concat(a, &next.prefix),
+        Some(a) => a.clone(),
+        None => acc.prefix.clone(),
+    };
+
+    // Mirror image for the suffix (driven by the right side).
+    let suffix = match &next.exact {
+        Some(b) if !acc.suffix.is_empty() => cross_concat(&acc.suffix, b),
+        Some(b) => b.clone(),
+        None => next.suffix.clone(),
+    };
+
+    let match_q = TrigramQuery::and(vec![acc.match_q, next.match_q, boundary]);
+
+    let mut info = RegexInfo {
+        emptyable,
+        exact,
+        prefix,
+        suffix,
+        match_q,
+    };
+    info.bound();
+    info
+}
+
+fn analyze_alternation(subs: &[Hir]) -> RegexInfo {
+    let infos: Vec<RegexInfo> = subs.iter().map(analyze).collect();
+
+    let emptyable = infos.iter().any(|i| i.emptyable);
+
+    let exact = if infos.iter().all(|i| i.exact.is_some()) {
+        Some(
+            infos
+                .iter()
+                .flat_map(|i| i.exact.as_ref().unwrap().iter().cloned())
+                .collect(),
+        )
+    } else {
+        None
+    };
+
+    // A common prefix exists only if every branch contributes one.
+    let prefix = if infos.iter().all(|i| !i.prefix.is_empty()) {
+        infos
+            .iter()
+            .flat_map(|i| i.prefix.iter().cloned())
+            .collect()
+    } else {
+        BTreeSet::new()
+    };
+    let suffix = if infos.iter().all(|i| !i.suffix.is_empty()) {
+        infos
+            .iter()
+            .flat_map(|i| i.suffix.iter().cloned())
+            .collect()
+    } else {
+        BTreeSet::new()
+    };
+
+    let match_q = TrigramQuery::or(infos.into_iter().map(|i| i.match_q).collect());
+
+    RegexInfo {
+        emptyable,
+        exact,
+        prefix,
+        suffix,
+        match_q,
+    }
+}
+
+/// Derive a necessary trigram condition from a regular expression pattern.
+///
+/// Returns [`TrigramQuery::All`] when no useful condition can be derived (an
+/// unparsable pattern, or one with no trigram-able literal structure such as
+/// `a.b` or `.*`); callers must treat that as "recheck everything".
+pub fn regex_to_trigram_query(pattern: &str) -> TrigramQuery {
+    // An unparsable pattern cannot be accelerated; rechecking is still safe.
+    let Ok(hir) = regex_syntax::parse(pattern) else {
+        return TrigramQuery::All;
+    };
+    let info = analyze(&hir);
+
+    let mut conditions = vec![info.match_q];
+    if let Some(exact) = &info.exact {
+        if exact.is_empty() {
+            // The expression matches nothing.
+            return TrigramQuery::None;
+        }
+        conditions.push(trigrams_of_set(exact.iter()));
+    }
+    conditions.push(trigrams_of_set(info.prefix.iter()));
+    conditions.push(trigrams_of_set(info.suffix.iter()));
+    TrigramQuery::and(conditions)
+}
+
+/// Whether a regular expression yields any trigram condition the index can use
+/// to prune candidates. When it does not (e.g. `a.b`, `.*`, or a case-insensitive
+/// pattern), callers should leave the predicate to a full scan rather than route
+/// it to the index, which would otherwise have to ask the scan to recheck every
+/// row -- a path the index result type (`AtLeast`) does not support.
+pub fn regex_can_use_index(pattern: &str) -> bool {
+    regex_to_trigram_query(pattern) != TrigramQuery::All
+}
+
+/// Collect the distinct trigram tokens referenced anywhere in the tree.
+pub fn collect_tokens(query: &TrigramQuery, out: &mut HashSet<u32>) {
+    match query {
+        TrigramQuery::Trigram(token) => {
+            out.insert(*token);
+        }
+        TrigramQuery::And(items) | TrigramQuery::Or(items) => {
+            for item in items {
+                collect_tokens(item, out);
+            }
+        }
+        TrigramQuery::All | TrigramQuery::None => {}
+    }
+}
+
+/// Evaluate the tree against a map of `trigram token -> posting list`. A token
+/// missing from the map contributes an empty set (sound: a required trigram that
+/// is absent everywhere yields no rows; an absent OR branch contributes
+/// nothing). `All` / `None` are handled by the caller before evaluation.
+pub fn eval_trigram_query(
+    query: &TrigramQuery,
+    bitmaps: &HashMap<u32, RoaringTreemap>,
+) -> RoaringTreemap {
+    match query {
+        TrigramQuery::Trigram(token) => bitmaps.get(token).cloned().unwrap_or_default(),
+        TrigramQuery::And(items) => {
+            let mut iter = items.iter();
+            let mut acc = match iter.next() {
+                Some(first) => eval_trigram_query(first, bitmaps),
+                None => return RoaringTreemap::new(),
+            };
+            for item in iter {
+                if acc.is_empty() {
+                    break;
+                }
+                acc &= &eval_trigram_query(item, bitmaps);
+            }
+            acc
+        }
+        TrigramQuery::Or(items) => {
+            let mut acc = RoaringTreemap::new();
+            for item in items {
+                acc |= &eval_trigram_query(item, bitmaps);
+            }
+            acc
+        }
+        TrigramQuery::All | TrigramQuery::None => RoaringTreemap::new(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A single trigram condition, hashed the same way the index hashes it.
+    fn tri(trigram: &str) -> TrigramQuery {
+        TrigramQuery::Trigram(ngram_to_token(trigram, NGRAM_N))
+    }
+
+    fn q(pattern: &str) -> TrigramQuery {
+        regex_to_trigram_query(pattern)
+    }
+
+    #[test]
+    fn test_single_literal_trigram() {
+        assert_eq!(q("foo"), tri("foo"));
+    }
+
+    #[test]
+    fn test_multi_trigram_literal() {
+        assert_eq!(
+            q("foobar"),
+            TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")])
+        );
+    }
+
+    #[test]
+    fn test_wildcard_splits_into_and() {
+        // `.*` breaks the literal run; both sides are required.
+        assert_eq!(
+            q("foo.*bar"),
+            TrigramQuery::and(vec![tri("foo"), tri("bar")])
+        );
+    }
+
+    #[test]
+    fn test_alternation_is_or() {
+        assert_eq!(
+            q("(cat|dog)"),
+            TrigramQuery::or(vec![tri("cat"), tri("dog")])
+        );
+    }
+
+    #[test]
+    fn test_anchors_are_transparent() {
+        assert_eq!(
+            q("^rhino"),
+            TrigramQuery::and(vec![tri("rhi"), tri("hin"), tri("ino")])
+        );
+        assert_eq!(q("nose$"), TrigramQuery::and(vec![tri("nos"), tri("ose")]));
+    }
+
+    #[test]
+    fn test_boundary_trigram_recovered_across_groups() {
+        // A capturing group is not merged into the adjacent literals, so this
+        // exercises the suffix x prefix cross product that recovers the `foo`
+        // trigram straddling the `(o)` group boundary in "foobar".
+        assert_eq!(
+            q("fo(o)bar"), // spellchecker:disable-line
+            TrigramQuery::and(vec![tri("foo"), tri("oob"), tri("oba"), tri("bar")])
+        );
+    }
+
+    #[test]
+    fn test_no_trigram_yields_all() {
+        // No run of three literal characters anywhere.
+        assert_eq!(q("a.b"), TrigramQuery::All);
+        assert_eq!(q(".*"), TrigramQuery::All);
+        // Every alternation branch is shorter than a trigram, so we must not
+        // require either two-character branch as a (non-existent) trigram.
+        assert_eq!(q("fo|ba"), TrigramQuery::All); // spellchecker:disable-line
+    }
+
+    #[test]
+    fn test_case_insensitive_not_accelerated() {
+        // Unicode case folding (e.g. `(?i)c` also matches U+2102) does not agree
+        // with the index's normalization, so case-insensitive patterns are left
+        // unaccelerated (correct via recheck) rather than risk a false negative.
+        assert_eq!(q("(?i)Cat"), TrigramQuery::All);
+    }
+
+    #[test]
+    fn test_unparsable_pattern_yields_all() {
+        assert_eq!(q("("), TrigramQuery::All);
+    }
+
+    #[test]
+    fn test_large_alternation_stays_bounded() {
+        // More than MAX_SET_SIZE branches: must still produce a sound OR without
+        // panicking or exploding.
+        let pattern = (0..40)
+            .map(|i| format!("aa{i:02}zz"))
+            .collect::<Vec<_>>()
+            .join("|");
+        let result = q(&pattern);
+        // Each branch shares the trigram `aa0`/`aa1`/... and `zz`-ish endings;
+        // the important property is that it is a sound non-empty condition.
+        assert_ne!(result, TrigramQuery::None);
+    }
+
+    #[test]
+    fn test_plus_requires_inner() {
+        // `(abc)+` must contain at least one `abc`.
+        assert_eq!(q("(abc)+"), tri("abc"));
+    }
+
+    #[test]
+    fn test_optional_group_is_not_required() {
+        // `(foo)?bar` -> foo optional, bar required.
+        assert_eq!(q("(foo)?bar"), tri("bar"));
+    }
+
+    #[test]
+    fn test_eval_and_or_with_missing_tokens() {
+        let foo = ngram_to_token("foo", NGRAM_N);
+        let bar = ngram_to_token("bar", NGRAM_N);
+        let mut bitmaps = HashMap::new();
+        bitmaps.insert(foo, RoaringTreemap::from_iter([1u64, 2, 3]));
+        bitmaps.insert(bar, RoaringTreemap::from_iter([2u64, 3, 4]));
+        // `baz` is absent from the index.
+
+        // AND intersects.
+        let and = TrigramQuery::and(vec![tri("foo"), tri("bar")]);
+        assert_eq!(
+            eval_trigram_query(&and, &bitmaps),
+            RoaringTreemap::from_iter([2u64, 3])
+        );
+
+        // OR unions.
+        let or = TrigramQuery::or(vec![tri("foo"), tri("bar")]);
+        assert_eq!(
+            eval_trigram_query(&or, &bitmaps),
+            RoaringTreemap::from_iter([1u64, 2, 3, 4])
+        );
+
+        // A missing token is empty: it zeroes an AND but is harmless in an OR.
+        let and_missing = TrigramQuery::and(vec![tri("foo"), tri("baz")]);
+        assert!(eval_trigram_query(&and_missing, &bitmaps).is_empty());
+        let or_missing = TrigramQuery::or(vec![tri("foo"), tri("baz")]);
+        assert_eq!(
+            eval_trigram_query(&or_missing, &bitmaps),
+            RoaringTreemap::from_iter([1u64, 2, 3])
+        );
+    }
+
+    #[test]
+    fn test_collect_tokens() {
+        let query = TrigramQuery::and(vec![
+            tri("foo"),
+            TrigramQuery::or(vec![tri("bar"), tri("baz")]),
+        ]);
+        let mut tokens = HashSet::new();
+        collect_tokens(&query, &mut tokens);
+        assert_eq!(
+            tokens,
+            HashSet::from([
+                ngram_to_token("foo", NGRAM_N),
+                ngram_to_token("bar", NGRAM_N),
+                ngram_to_token("baz", NGRAM_N),
+            ])
+        );
+    }
+}
diff --git a/rust/lance-index/src/scalar/registry.rs b/rust/lance-index/src/scalar/registry.rs
index 0add98d8ab3..fb7a460e9ab 100644
--- a/rust/lance-index/src/scalar/registry.rs
+++ b/rust/lance-index/src/scalar/registry.rs
@@ -7,66 +7,28 @@ use std::sync::Arc;
 use arrow_schema::Field;
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
+use futures::future::BoxFuture;
 use lance_core::{
     Result,
-    cache::{LanceCache, UnsizedCacheKey},
+    cache::{CacheKey, LanceCache, UnsizedCacheKey},
+    deepsize::DeepSizeOf,
 };
 
 use crate::progress::IndexBuildProgress;
 use crate::registry::IndexPluginRegistry;
-use crate::{
-    frag_reuse::FragReuseIndex,
-    scalar::{CreatedIndex, IndexStore, ScalarIndex, expression::ScalarQueryParser},
-};
+use crate::scalar::RowIdRemapper;
+use crate::scalar::{CreatedIndex, IndexStore, ScalarIndex, expression::ScalarQueryParser};
+// Re-export training types that were previously defined here
+pub use crate::scalar::{TrainingCriteria, TrainingOrdering};
 
 pub const VALUE_COLUMN_NAME: &str = "value";
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum TrainingOrdering {
-    /// The input will arrive sorted by the value column in ascending order
-    Values,
-    /// The input will arrive sorted by the address column in ascending order
-    Addresses,
-    /// The input will arrive in an arbitrary order
-    None,
-}
-
-#[derive(Debug, Clone)]
-pub struct TrainingCriteria {
-    pub ordering: TrainingOrdering,
-    pub needs_row_ids: bool,
-    pub needs_row_addrs: bool,
-}
-
-impl TrainingCriteria {
-    pub fn new(ordering: TrainingOrdering) -> Self {
-        Self {
-            ordering,
-            needs_row_ids: false,
-            needs_row_addrs: false,
-        }
-    }
-
-    pub fn with_row_id(mut self) -> Self {
-        self.needs_row_ids = true;
-        self
-    }
-
-    pub fn with_row_addr(mut self) -> Self {
-        self.needs_row_addrs = true;
-        self
-    }
-}
-
-/// A trait that describes what criteria is needed to train an index
-///
-/// The training process has two steps.  First, the parameters are given to the
-/// plugin and it creates a TrainingRequest.  Then, the caller prepares the training
-/// data and calls train_index.
+/// A trait object for plugin-specific training parameters and data requirements.
 ///
-/// The call to train_index will include the training request.  This allows the plugin
-/// to stash any deserialized parameter info in the request and fetch it later during
-/// training by downcasting to the appropriate type.
+/// Returned by [`BasicTrainer::new_training_request`]. The caller uses
+/// [`criteria`](Self::criteria) to prepare the training data stream, then passes
+/// the request back to [`BasicTrainer::train_index`], which may downcast
+/// it to the plugin-specific concrete type to recover parsed parameters.
 pub trait TrainingRequest: std::any::Any + Send + Sync {
     fn as_any(&self) -> &dyn std::any::Any;
     fn criteria(&self) -> &TrainingCriteria;
@@ -93,26 +55,35 @@ impl TrainingRequest for DefaultTrainingRequest {
     }
 }
 
-/// A trait for scalar index plugins
+/// Implemented by indexes that can train on a stream of column data.
+///
+/// The training process has two stages. In the first stage, the caller provides
+/// index parameters and receives a [`TrainingRequest`] that describes what criteria
+/// the training data must satisfy (e.g. sort order, row-ID availability). In the
+/// second stage, the caller prepares the data accordingly and calls
+/// [`train_index`](Self::train_index).
+///
+/// Any scalar index plugin that builds from a column data stream should implement
+/// this trait.
 #[async_trait]
-pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
-    /// Creates a new training request from the given parameters
+pub trait BasicTrainer: Send + Sync {
+    /// Creates a new training request from the given parameters.
     ///
-    /// This training request specifies the criteria that the data must satisfy to train the index.
-    /// For example, does the index require the input data to be sorted?
+    /// The returned request specifies the criteria the training data must satisfy.
+    /// It is the caller's responsibility to prepare data that meets those criteria
+    /// before calling [`train_index`](Self::train_index).
     fn new_training_request(&self, params: &str, field: &Field)
     -> Result<Box<dyn TrainingRequest>>;
 
-    /// Train a new index
+    /// Train a new index from a prepared data stream.
     ///
-    /// The provided data must fulfill all the criteria returned by `training_criteria`.
-    /// It is the caller's responsibility to ensure this.
+    /// The provided data must fulfill all the criteria returned by
+    /// [`new_training_request`](Self::new_training_request). It is the caller's
+    /// responsibility to ensure this.
     ///
-    /// Returns index details that describe the index.  These details can potentially be
-    /// useful for planning (although this will currently require inside information on
-    /// the index type) and they will need to be provided when loading the index.
-    ///
-    /// It is the caller's responsibility to store these details somewhere.
+    /// Returns index details describing the index. These details may be useful for
+    /// planning and must be provided when loading the index. It is the caller's
+    /// responsibility to store them.
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -121,6 +92,36 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
         fragment_ids: Option<Vec<u32>>,
         progress: Arc<dyn IndexBuildProgress>,
     ) -> Result<CreatedIndex>;
+}
+
+/// A trait for scalar index plugins
+#[async_trait]
+pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
+    /// Returns this plugin's [`BasicTrainer`] implementation, if any.
+    ///
+    /// Training an index can be a complex process.  For example, a btree index might
+    /// be trained using a shuffler from a distributed OLAP system such as
+    /// Spark or Ray.  A vector index can be trained by sampling the column to create
+    /// a kmeans model and then streaming the vectors to assign partitions.  Encapsulating
+    /// the entire set of possible approaches is beyond what this trait can model.
+    /// This is especially true because this is a low-level crate with no concept of a table
+    /// or a dataset.
+    ///
+    /// However, in many cases, an index can be trained on a (potentially sorted) stream
+    /// of column data.  There is also significant utility in being able to provide users
+    /// with a simple generic "create an index" API.
+    ///
+    /// This method is a compromise.  Indexes that support training on a stream of column
+    /// data should override this to return `Some(self)`.  Indexes that need their own
+    /// individualized training approaches should return `None` and provide their own
+    /// methods for training.
+    ///
+    /// An index can take both approaches.  Providing a simple (but maybe less
+    /// efficient) stream-based trainer while also providing more specialized index
+    /// creation methods elsewhere.
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        None
+    }
 
     /// A short name for the index
     ///
@@ -158,7 +159,7 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
         &self,
         index_store: Arc<dyn IndexStore>,
         index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>>;
 
@@ -170,14 +171,14 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
     /// The default implementation reads an in-memory `Arc<dyn ScalarIndex>` entry.
     /// Plugins whose index has a serializable representation should override this
     /// (together with [`put_in_cache`](Self::put_in_cache)) to store that
-    /// representation under a sized [`CacheKey`](lance_core::cache::CacheKey) with
+    /// representation under a sized [`CacheKey`] with
     /// a codec, and reconstruct the index here. `index_store` and
     /// `frag_reuse_index` are provided so the override can rebuild the index
     /// without re-reading metadata.
     async fn get_from_cache(
         &self,
         _index_store: Arc<dyn IndexStore>,
-        _frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        _frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Option<Arc<dyn ScalarIndex>>> {
         Ok(cache.get_unsized_with_key(&ScalarIndexCacheKey).await)
@@ -196,6 +197,31 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
         Ok(())
     }
 
+    /// Open an index through the cache, awaiting `load` only on a miss.
+    ///
+    /// The default read-through / write-back over
+    /// [`get_from_cache`](Self::get_from_cache) and
+    /// [`put_in_cache`](Self::put_in_cache) does not coalesce concurrent cold
+    /// opens; plugins with a serializable form should override with
+    /// [`single_flight_open`] so one shared load populates the sized state key.
+    async fn get_or_insert_in_cache(
+        &self,
+        index_store: Arc<dyn IndexStore>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
+        cache: &LanceCache,
+        load: ScalarIndexLoad<'_>,
+    ) -> Result<Arc<dyn ScalarIndex>> {
+        if let Some(index) = self
+            .get_from_cache(index_store, frag_reuse_index, cache)
+            .await?
+        {
+            return Ok(index);
+        }
+        let index = load.await?;
+        self.put_in_cache(cache, index.clone()).await?;
+        Ok(index)
+    }
+
     /// Optional hook allowing a plugin to provide statistics without loading the index.
     async fn load_statistics(
         &self,
@@ -219,6 +245,48 @@ pub trait ScalarIndexPlugin: Send + Sync + std::fmt::Debug {
     }
 }
 
+/// A boxed, `Send` future performing the storage-level load of a scalar index
+/// (compat checks, `load_index`, metrics).
+///
+/// Passed to [`ScalarIndexPlugin::get_or_insert_in_cache`], which awaits it at
+/// most once on a cache miss and drops it un-awaited on a warm hit.
+pub type ScalarIndexLoad<'a> = BoxFuture<'a, Result<Arc<dyn ScalarIndex>>>;
+
+/// Single-flight open helper for plugins with a serializable form.
+///
+/// Concurrent cold opens of the same index coalesce onto one `load`: it runs
+/// once, `to_state` converts the opened index to its sized state, and the state
+/// is cached under `state_key` (persisted via the key's
+/// [`CacheCodec`](lance_core::cache::CacheCodec)). Every caller — warm hits
+/// included — then rebuilds the index from the shared state via `from_state`, an
+/// IO-free reconstruct.
+///
+/// `to_state` / `from_state` mirror the plugin's
+/// [`put_in_cache`](ScalarIndexPlugin::put_in_cache) /
+/// [`get_from_cache`](ScalarIndexPlugin::get_from_cache), keeping the state
+/// representation defined in one place.
+pub async fn single_flight_open<K, ToState, FromState>(
+    cache: &LanceCache,
+    state_key: K,
+    load: ScalarIndexLoad<'_>,
+    to_state: ToState,
+    from_state: FromState,
+) -> Result<Arc<dyn ScalarIndex>>
+where
+    K: CacheKey + Send,
+    K::ValueType: DeepSizeOf + Send + Sync + 'static,
+    ToState: FnOnce(&dyn ScalarIndex) -> Result<K::ValueType> + Send,
+    FromState: FnOnce(Arc<K::ValueType>) -> Result<Arc<dyn ScalarIndex>> + Send,
+{
+    let state = cache
+        .get_or_insert_with_key(state_key, move || async move {
+            let index = load.await?;
+            to_state(index.as_ref())
+        })
+        .await?;
+    from_state(state)
+}
+
 /// In-memory cache key for a whole `Arc<dyn ScalarIndex>`.
 ///
 /// Used by the default [`ScalarIndexPlugin::get_from_cache`] /
diff --git a/rust/lance-index/src/scalar/rtree.rs b/rust/lance-index/src/scalar/rtree.rs
index 246c41ba006..0d6b54a02ca 100644
--- a/rust/lance-index/src/scalar/rtree.rs
+++ b/rust/lance-index/src/scalar/rtree.rs
@@ -1,19 +1,17 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use crate::frag_reuse::FragReuseIndex;
 use crate::metrics::{MetricsCollector, NoOpMetricsCollector};
 use crate::scalar::expression::{GeoQueryParser, ScalarQueryParser};
 use crate::scalar::lance_format::LanceIndexStore;
 use crate::scalar::registry::{
-    ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
+    BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
 };
 use crate::scalar::rtree::sort::Sorter;
 use crate::scalar::{
-    AnyQuery, BuiltinIndexType, CreatedIndex, GeoQuery, IndexReader, IndexStore, IndexWriter,
-    ScalarIndex, ScalarIndexParams, SearchResult, UpdateCriteria,
+    AnyQuery, BuiltinIndexType, CreatedIndex, GeoQuery, IndexFile, IndexReader, IndexStore,
+    IndexWriter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, UpdateCriteria,
 };
-use crate::vector::VectorIndex;
 use crate::{Index, IndexType, pb};
 use arrow_array::UInt32Array;
 use arrow_array::cast::AsArray;
@@ -24,7 +22,6 @@ use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion_common::DataFusionError;
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
 use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, stream};
 use geoarrow_array::array::{RectArray, from_arrow_array};
@@ -33,7 +30,9 @@ use geoarrow_array::{GeoArrowArray, GeoArrowArrayAccessor, IntoArrow};
 use geoarrow_schema::{Dimension, RectType};
 use lance_arrow::RecordBatchExt;
 use lance_core::cache::{CacheKey, LanceCache, WeakLanceCache};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::address::RowAddress;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use lance_core::utils::tempfile::TempDir;
 use lance_core::{Error, ROW_ID, Result};
 use lance_datafusion::chunker::chunk_concat_stream;
@@ -235,8 +234,8 @@ pub enum RTreeCacheKey {
 pub struct RTreeCacheValue(Arc<RecordBatch>);
 
 impl DeepSizeOf for RTreeCacheValue {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
-        self.0.get_array_memory_size()
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.0.deep_size_of_children(context)
     }
 }
 
@@ -259,7 +258,7 @@ impl CacheKey for RTreeCacheKey {
 pub struct RTreeIndex {
     pub(crate) metadata: Arc<RTreeMetadata>,
     store: Arc<dyn IndexStore>,
-    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
     index_cache: WeakLanceCache,
     pages_reader: Arc<dyn IndexReader>,
     nulls_reader: Arc<dyn IndexReader>,
@@ -277,7 +276,7 @@ impl std::fmt::Debug for RTreeIndex {
 impl RTreeIndex {
     pub async fn load(
         store: Arc<dyn IndexStore>,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>> {
         let pages_reader = store.open_index_file(RTREE_PAGES_NAME).await?;
@@ -430,7 +429,7 @@ impl RTreeIndex {
 }
 
 impl DeepSizeOf for RTreeIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let mut total_size = 0;
 
         total_size += self.store.deep_size_of_children(context);
@@ -449,12 +448,6 @@ impl Index for RTreeIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Err(Error::not_supported_source(
-            "RTreeIndex is not vector index".into(),
-        ))
-    }
-
     fn statistics(&self) -> Result<serde_json::Value> {
         serde_json::to_value(self.metadata.clone())
             .map_err(|e| Error::internal(format!("Error serializing statistics: {}", e)))
@@ -551,7 +544,7 @@ impl ScalarIndex for RTreeIndex {
 
     async fn remap(
         &self,
-        _mapping: &HashMap<u64, Option<u64>>,
+        _mapping: &RowAddrRemap,
         _dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         Err(Error::invalid_input_source(
@@ -593,7 +586,7 @@ impl ScalarIndex for RTreeIndex {
             num_items: self.metadata.num_items + stats.num_items,
         };
 
-        RTreeIndexPlugin::train_rtree_index(
+        let files = RTreeIndexPlugin::train_rtree_index(
             merged_bbox_data,
             merge_stats,
             self.metadata.page_size,
@@ -604,7 +597,7 @@ impl ScalarIndex for RTreeIndex {
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::RTreeIndexDetails::default())?,
             index_version: RTREE_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -830,7 +823,7 @@ impl RTreeIndexPlugin {
         total_bbox: BoundingBox,
         store: &dyn IndexStore,
         page_size: u32,
-    ) -> Result<()> {
+    ) -> Result<IndexFile> {
         let mut page_idx: u64 = 0;
         let mut writer = store
             .new_index_file(RTREE_PAGES_NAME, RTREE_PAGE_SCHEMA.clone())
@@ -868,12 +861,13 @@ impl RTreeIndexPlugin {
             .finish_with_metadata(
                 RTreeMetadata::new(page_size, page_idx, num_items, total_bbox).into_map(),
             )
-            .await?;
-
-        Ok(())
+            .await
     }
 
-    pub async fn write_nulls(store: &dyn IndexStore, null_map: RowAddrTreeMap) -> Result<()> {
+    pub async fn write_nulls(
+        store: &dyn IndexStore,
+        null_map: RowAddrTreeMap,
+    ) -> Result<IndexFile> {
         let mut writer = store
             .new_index_file(RTREE_NULLS_NAME, RTREE_NULLS_SCHEMA.clone())
             .await?;
@@ -893,12 +887,12 @@ impl RTreeIndexPlugin {
         stats: BboxStreamStats,
         page_size: u32,
         store: &dyn IndexStore,
-    ) -> Result<()> {
+    ) -> Result<Vec<IndexFile>> {
         // new sorted stream
         let sorter = HilbertSorter::new(stats.total_bbox);
         let sorted_data = sorter.sort(bbox_data).await?;
 
-        Self::write_index(
+        let page_file = Self::write_index(
             sorted_data,
             stats.num_items,
             stats.total_bbox,
@@ -907,18 +901,14 @@ impl RTreeIndexPlugin {
         )
         .await?;
 
-        Self::write_nulls(store, stats.null_map).await?;
+        let nulls_file = Self::write_nulls(store, stats.null_map).await?;
 
-        Ok(())
+        Ok(vec![page_file, nulls_file])
     }
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for RTreeIndexPlugin {
-    fn name(&self) -> &str {
-        "RTree"
-    }
-
+impl BasicTrainer for RTreeIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -964,14 +954,25 @@ impl ScalarIndexPlugin for RTreeIndexPlugin {
             Self::process_and_analyze_bbox_stream(bbox_data, page_size, spill_store.clone())
                 .await?;
 
-        Self::train_rtree_index(bbox_data, stats, page_size, index_store).await?;
+        let files = Self::train_rtree_index(bbox_data, stats, page_size, index_store).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pb::RTreeIndexDetails::default())?,
             index_version: RTREE_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files,
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for RTreeIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "RTree"
+    }
 
     fn provides_exact_answer(&self) -> bool {
         false
@@ -996,7 +997,7 @@ impl ScalarIndexPlugin for RTreeIndexPlugin {
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(RTreeIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
diff --git a/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs b/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
index e6c10a20575..9816bfebd51 100644
--- a/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
+++ b/rust/lance-index/src/scalar/rtree/sort/hilbert_sort.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use crate::Result;
 use crate::scalar::rtree::sort::Sorter;
 use arrow_array::{ArrayRef, UInt32Array};
 use arrow_schema::{ArrowError, DataType as ArrowDataType, Field as ArrowField, Field};
@@ -19,9 +18,9 @@ use datafusion_physical_expr::expressions::Column as DFColumn;
 use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};
 use geoarrow_array::array::from_arrow_array;
 use geoarrow_array::{GeoArrowArray, GeoArrowArrayAccessor};
+use lance_core::Result;
 use lance_datafusion::exec::{LanceExecutionOptions, OneShotExec, execute_plan};
 use lance_geo::bbox::{BoundingBox, bounding_box};
-use std::any::Any;
 use std::sync::Arc;
 
 const HILBERT_FIELD_NAME: &str = "_hilbert";
@@ -149,10 +148,6 @@ impl HilbertUDF {
 }
 
 impl ScalarUDFImpl for HilbertUDF {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn name(&self) -> &str {
         HILBERT_UDF_NAME
     }
diff --git a/rust/lance-index/src/scalar/zoned.rs b/rust/lance-index/src/scalar/zoned.rs
index 7ceed851bae..9c090f5e1d8 100644
--- a/rust/lance-index/src/scalar/zoned.rs
+++ b/rust/lance-index/src/scalar/zoned.rs
@@ -92,13 +92,14 @@ where
     pub async fn train(
         mut self,
         stream: SendableRecordBatchStream,
-    ) -> Result<Vec<P::ZoneStatistics>> {
+    ) -> Result<(Vec<P::ZoneStatistics>, RowAddrTreeMap)> {
         let zone_size = usize::try_from(self.zone_capacity).map_err(|_| {
             Error::invalid_input("zone capacity does not fit into usize on this platform")
         })?;
 
         let mut batches = chunk_concat_stream(stream, zone_size);
         let mut zones = Vec::new();
+        let mut null_rows = RowAddrTreeMap::new();
         let mut current_fragment_id: Option<u64> = None;
         let mut current_zone_len: usize = 0;
         let mut zone_start_offset: Option<u64> = None;
@@ -155,6 +156,16 @@ where
                 self.processor
                     .process_chunk(&values.slice(batch_offset, take))?;
 
+                // Record exact row addresses for null values in this chunk.
+                let chunk = values.slice(batch_offset, take);
+                if chunk.null_count() > 0 {
+                    for i in 0..take {
+                        if values.is_null(batch_offset + i) {
+                            null_rows.insert(row_addr_col.value(batch_offset + i));
+                        }
+                    }
+                }
+
                 // Track the first and last row offsets to handle non-contiguous offsets
                 // after deletions. Zone length (offset span) is computed as (last - first + 1),
                 // not the actual row count.
@@ -200,7 +211,7 @@ where
             }
         }
 
-        Ok(zones)
+        Ok((zones, null_rows))
     }
 
     /// Flushes a non-empty zone and resets the processor state.
@@ -266,21 +277,21 @@ where
 }
 
 /// Helper that retrains zones from `stream` and appends them to the existing
-/// statistics. Useful for index update paths that need to merge new fragments
-/// into an existing zone list.
+/// statistics. Returns the combined zone list and the null-row bitmap for the
+/// new data only — callers are responsible for merging with any existing bitmap.
 pub async fn rebuild_zones<P>(
     existing: &[P::ZoneStatistics],
     trainer: ZoneTrainer<P>,
     stream: SendableRecordBatchStream,
-) -> Result<Vec<P::ZoneStatistics>>
+) -> Result<(Vec<P::ZoneStatistics>, RowAddrTreeMap)>
 where
     P: ZoneProcessor,
     P::ZoneStatistics: Clone,
 {
     let mut combined = existing.to_vec();
-    let mut new_zones = trainer.train(stream).await?;
+    let (mut new_zones, null_rows) = trainer.train(stream).await?;
     combined.append(&mut new_zones);
-    Ok(combined)
+    Ok((combined, null_rows))
 }
 
 #[cfg(test)]
@@ -362,7 +373,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones: offsets [0..=3], [4..=7], [8..=9]
         assert_eq!(stats.len(), 3);
@@ -393,7 +404,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Two zones, one per fragment (capacity=10 is large enough)
         assert_eq!(stats.len(), 2);
@@ -447,7 +458,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone containing the 3 valid rows (empty batches skipped)
         assert_eq!(stats.len(), 1);
@@ -469,7 +480,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 1).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones, one per row (capacity=1)
         assert_eq!(stats.len(), 3);
@@ -494,7 +505,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10000).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone containing all 100 rows (capacity is large enough)
         assert_eq!(stats.len(), 1);
@@ -530,7 +541,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Two zones: first 4 rows, then remaining 2 rows
         assert_eq!(stats.len(), 2);
@@ -561,7 +572,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 3).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Three zones: frag 0 full zone, frag 0 partial (flushed at boundary), frag 1
         assert_eq!(stats.len(), 3);
@@ -602,7 +613,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 4).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Should create 2 zones (capacity=4):
         // Zone 0: rows at offsets [0, 1, 5, 7] (4 rows)
@@ -637,7 +648,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // One zone with 3 rows, but offset span [0..=200] so length=201 due to large gaps
         assert_eq!(stats.len(), 1);
@@ -663,7 +674,7 @@ mod tests {
 
         let processor = MockProcessor::new();
         let trainer = ZoneTrainer::new(processor, 10).unwrap();
-        let stats = trainer.train(stream).await.unwrap();
+        let (stats, _) = trainer.train(stream).await.unwrap();
 
         // Should create 3 zones (one per fragment)
         assert_eq!(stats.len(), 3);
@@ -810,7 +821,7 @@ mod tests {
         ));
 
         let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap();
-        let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap();
+        let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap();
         // Existing zone should remain unchanged and new stats appended afterwards
         assert_eq!(rebuilt.len(), 2);
         assert_eq!(rebuilt[0].sum, 50);
@@ -840,7 +851,7 @@ mod tests {
         ));
 
         let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap();
-        let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap();
+        let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap();
         // Existing zone plus two new fragments should yield three total zones
         assert_eq!(rebuilt.len(), 3);
         assert_eq!(rebuilt[0].bound.fragment_id, 0);
diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs
index 64601a64f96..65d58b49f81 100644
--- a/rust/lance-index/src/scalar/zonemap.rs
+++ b/rust/lance-index/src/scalar/zonemap.rs
@@ -12,19 +12,20 @@
 //! false positives that require rechecking.
 //!
 //!
-use crate::Any;
 use crate::pbold;
 use crate::scalar::expression::{SargableQueryParser, ScalarQueryParser};
 use crate::scalar::registry::{
-    ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
+    BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest,
 };
 use crate::scalar::{
-    BuiltinIndexType, CreatedIndex, SargableQuery, ScalarIndexParams, UpdateCriteria,
+    BuiltinIndexType, CreatedIndex, IndexFile, SargableQuery, ScalarIndexParams, UpdateCriteria,
     compute_next_prefix,
 };
 use lance_arrow_stats::StatisticsAccumulator;
 use lance_core::cache::{LanceCache, WeakLanceCache};
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use serde::{Deserialize, Serialize};
+use std::any::Any;
 use std::sync::LazyLock;
 
 use arrow_array::{
@@ -33,16 +34,16 @@ use arrow_array::{
 use arrow_schema::{DataType, Field};
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion_common::ScalarValue;
+use lance_select::RowAddrTreeMap;
 use std::{collections::HashMap, sync::Arc};
 
 use super::{AnyQuery, IndexStore, MetricsCollector, ScalarIndex, SearchResult};
-use crate::scalar::FragReuseIndex;
-use crate::vector::VectorIndex;
+use crate::scalar::RowIdRemapper;
 use crate::{Index, IndexType};
 use async_trait::async_trait;
-use deepsize::DeepSizeOf;
 use lance_core::Error;
 use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use roaring::RoaringBitmap;
 
 use super::zoned::{ZoneBound, ZoneProcessor, ZoneTrainer, rebuild_zones, search_zones};
@@ -50,6 +51,7 @@ const ROWS_PER_ZONE_DEFAULT: u64 = 8192; // 1 zone every two batches
 
 const ZONEMAP_FILENAME: &str = "zonemap.lance";
 const ZONEMAP_SIZE_META_KEY: &str = "rows_per_zone";
+const NULL_BITMAP_META_KEY: &str = "null_bitmap";
 const ZONEMAP_INDEX_VERSION: u32 = 0;
 
 /// Basic stats about zonemap index
@@ -66,7 +68,7 @@ struct ZoneMapStatistics {
 }
 
 impl DeepSizeOf for ZoneMapStatistics {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         // Estimate sizes for ScalarValue
         let min_size = self.min.size() - std::mem::size_of::<ScalarValue>();
         let max_size = self.max.size() - std::mem::size_of::<ScalarValue>();
@@ -108,8 +110,11 @@ pub struct ZoneMapIndex {
     // The maximum rows per zone provided by user
     rows_per_zone: u64,
     store: Arc<dyn IndexStore>,
-    fri: Option<Arc<FragReuseIndex>>,
+    fri: Option<Arc<dyn RowIdRemapper>>,
     index_cache: WeakLanceCache,
+    // Exact set of null row addresses across all zones; None when loaded from an
+    // older index that did not persist this bitmap.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl std::fmt::Debug for ZoneMapIndex {
@@ -126,8 +131,8 @@ impl std::fmt::Debug for ZoneMapIndex {
 }
 
 impl DeepSizeOf for ZoneMapIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.zones.deep_size_of_children(context)
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context)
     }
 }
 
@@ -151,26 +156,53 @@ impl ZoneMapIndex {
         Self::zone_has_finite_min(zone) && !(zone.max.is_null() || Self::scalar_is_nan(&zone.max))
     }
 
-    fn finite_value_may_be_in_zone(value: &ScalarValue, zone: &ZoneMapStatistics) -> bool {
-        if !Self::zone_has_finite_min(zone) || value < &zone.min {
-            return false;
-        }
-
-        if Self::scalar_is_nan(&zone.max) {
-            // A NaN max means this zone had both NaNs and finite values.  The
-            // finite max is not persisted, so keep the zone as a false positive
-            // instead of using total ordering to prune it.
-            return true;
+    /// Global `[min, max]` folded across one or more ZoneMap segments (the
+    /// disjoint per-column segments of a multi-segment index), without a scan.
+    ///
+    /// `None` when no zone has a finite bound, or when any zone's `max` is NaN:
+    /// `ScalarValue`'s total order ranks NaN above every finite value, so a
+    /// NaN-bearing zone hides its true finite max and no sound upper bound exists
+    /// without a scan — folding only the finite maxes would yield a *subset* that
+    /// prunes live rows. Folding raw zones (not each segment's `value_range`)
+    /// keeps this exact across segments.
+    ///
+    /// Otherwise the range is a superset of the segments' live values,
+    /// conservative under deletion vectors: safe to prune with, not guaranteed
+    /// tight. The caller must ensure the segments jointly cover every live
+    /// fragment.
+    pub fn value_range_over<'a>(
+        segments: impl IntoIterator<Item = &'a Self>,
+    ) -> Option<(ScalarValue, ScalarValue)> {
+        let mut min: Option<&ScalarValue> = None;
+        let mut max: Option<&ScalarValue> = None;
+        for zone in segments.into_iter().flat_map(|seg| seg.zones.iter()) {
+            if Self::scalar_is_nan(&zone.max) {
+                return None;
+            }
+            if Self::scalar_is_finite_bound(&zone.min)
+                && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt()))
+            {
+                min = Some(&zone.min);
+            }
+            if Self::scalar_is_finite_bound(&zone.max)
+                && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt()))
+            {
+                max = Some(&zone.max);
+            }
         }
+        Some((min?.clone(), max?.clone()))
+    }
 
-        !zone.max.is_null() && value <= &zone.max
+    /// A scalar usable as a global-range bound: non-null and, for floats, non-NaN.
+    fn scalar_is_finite_bound(v: &ScalarValue) -> bool {
+        !v.is_null() && !Self::scalar_is_nan(v)
     }
 
     /// Evaluates whether a zone could potentially contain values matching the query.
     ///
-    /// NaN query values use the explicit `nan_count`.  When the stored max is
-    /// NaN we do not treat it as a finite upper bound; that representation means
-    /// the zone had finite values plus NaNs, and the finite max was not persisted.
+    /// NaN query values use the explicit `nan_count`. For finite query values,
+    /// `ScalarValue` total ordering keeps finite values below a stored NaN max,
+    /// so zones with finite values plus NaNs remain conservative false positives.
     fn evaluate_zone_against_query(
         &self,
         zone: &ZoneMapStatistics,
@@ -206,7 +238,7 @@ impl ZoneMapIndex {
                     return Ok(false);
                 }
 
-                Ok(Self::finite_value_may_be_in_zone(target, zone))
+                Ok(target >= &zone.min && target <= &zone.max)
             }
             SargableQuery::Range(start, end) => {
                 // Zone overlaps with query range if there's any intersection between
@@ -233,10 +265,8 @@ impl ZoneMapIndex {
                                     return Ok(zone.nan_count > 0);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(zone.nan_count > 0);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(zone.nan_count > 0);
                             }
                             _ => {}
                         }
@@ -263,10 +293,8 @@ impl ZoneMapIndex {
                                     return Ok(false); // Nothing is greater than NaN
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(false); // Nothing is greater than NaN
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(false); // Nothing is greater than NaN
                             }
                             _ => {}
                         }
@@ -290,10 +318,8 @@ impl ZoneMapIndex {
                                     return Ok(zone.nan_count > 0 || zone_min <= e);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(zone.nan_count > 0 || zone_min <= e);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(zone.nan_count > 0 || zone_min <= e);
                             }
                             _ => {}
                         }
@@ -313,10 +339,8 @@ impl ZoneMapIndex {
                                     return Ok(true);
                                 }
                             }
-                            ScalarValue::Float64(Some(f)) => {
-                                if f.is_nan() {
-                                    return Ok(true);
-                                }
+                            ScalarValue::Float64(Some(f)) if f.is_nan() => {
+                                return Ok(true);
                             }
                             _ => {}
                         }
@@ -336,22 +360,28 @@ impl ZoneMapIndex {
                             ScalarValue::Float16(Some(f)) => {
                                 if f.is_nan() {
                                     zone.nan_count > 0
+                                } else if !Self::zone_has_finite_min(zone) {
+                                    false
                                 } else {
-                                    Self::finite_value_may_be_in_zone(value, zone)
+                                    value >= &zone.min && value <= &zone.max
                                 }
                             }
                             ScalarValue::Float32(Some(f)) => {
                                 if f.is_nan() {
                                     zone.nan_count > 0
+                                } else if !Self::zone_has_finite_min(zone) {
+                                    false
                                 } else {
-                                    Self::finite_value_may_be_in_zone(value, zone)
+                                    value >= &zone.min && value <= &zone.max
                                 }
                             }
                             ScalarValue::Float64(Some(f)) => {
                                 if f.is_nan() {
                                     zone.nan_count > 0
+                                } else if !Self::zone_has_finite_min(zone) {
+                                    false
                                 } else {
-                                    Self::finite_value_may_be_in_zone(value, zone)
+                                    value >= &zone.min && value <= &zone.max
                                 }
                             }
                             _ => {
@@ -419,7 +449,7 @@ impl ZoneMapIndex {
     /// Load the scalar index from storage
     async fn load(
         store: Arc<dyn IndexStore>,
-        fri: Option<Arc<FragReuseIndex>>,
+        fri: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
     ) -> Result<Arc<Self>>
     where
@@ -436,21 +466,34 @@ impl ZoneMapIndex {
             .get(ZONEMAP_SIZE_META_KEY)
             .and_then(|bs| bs.parse().ok())
             .unwrap_or(ROWS_PER_ZONE_DEFAULT);
+
+        let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) {
+            let idx = idx_str.parse::<u32>().map_err(|e| {
+                Error::invalid_input(format!("invalid null bitmap buffer index: {e}"))
+            })?;
+            let bytes = index_file.read_global_buffer(idx).await?;
+            Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?)
+        } else {
+            None
+        };
+
         Ok(Arc::new(Self::try_from_serialized(
             zone_maps,
             store,
             fri,
             index_cache,
             rows_per_zone,
+            null_rows,
         )?))
     }
 
     fn try_from_serialized(
         data: RecordBatch,
         store: Arc<dyn IndexStore>,
-        fri: Option<Arc<FragReuseIndex>>,
+        fri: Option<Arc<dyn RowIdRemapper>>,
         index_cache: &LanceCache,
         rows_per_zone: u64,
+        null_rows: Option<RowAddrTreeMap>,
     ) -> Result<Self> {
         // The RecordBatch should have columns: min, max, null_count
         let min_col = data
@@ -512,6 +555,7 @@ impl ZoneMapIndex {
                 store,
                 fri,
                 index_cache: WeakLanceCache::from(index_cache),
+                null_rows,
             });
         }
 
@@ -543,6 +587,7 @@ impl ZoneMapIndex {
             store,
             fri,
             index_cache: WeakLanceCache::from(index_cache),
+            null_rows,
         })
     }
 }
@@ -557,12 +602,6 @@ impl Index for ZoneMapIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Err(Error::invalid_input_source(
-            "ZoneMapIndex is not a vector index".into(),
-        ))
-    }
-
     async fn prewarm(&self) -> Result<()> {
         // Not much to prewarm
         Ok(())
@@ -599,11 +638,21 @@ impl ScalarIndex for ZoneMapIndex {
         metrics: &dyn MetricsCollector,
     ) -> Result<SearchResult> {
         let query = query.as_any().downcast_ref::<SargableQuery>().unwrap();
+        if let SargableQuery::IsNull() = query
+            && let Some(null_rows) = &self.null_rows
+        {
+            return Ok(SearchResult::exact(null_rows.clone()));
+        }
+
         search_zones(&self.zones, metrics, |zone| {
             self.evaluate_zone_against_query(zone, query)
         })
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        true
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
@@ -611,7 +660,7 @@ impl ScalarIndex for ZoneMapIndex {
     /// Remap the row ids, creating a new remapped version of this index in `dest_store`
     async fn remap(
         &self,
-        _mapping: &HashMap<u64, Option<u64>>,
+        _mapping: &RowAddrRemap,
         _dest_store: &dyn IndexStore,
     ) -> Result<CreatedIndex> {
         Err(Error::invalid_input_source(
@@ -633,19 +682,30 @@ impl ScalarIndex for ZoneMapIndex {
         let options = ZoneMapIndexBuilderParams::new(self.rows_per_zone);
         let processor = ZoneMapProcessor::new(value_type.clone())?;
         let trainer = ZoneTrainer::new(processor, self.rows_per_zone)?;
-        let updated_zones = rebuild_zones(&self.zones, trainer, new_data).await?;
+        let (updated_zones, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?;
+
+        // Merge existing and new null rows.  If the existing index had no null bitmap
+        // (legacy format — null positions unknown), preserve that None: updating cannot
+        // recover the missing information, and claiming the result has zero nulls would
+        // be a false negative.  Only a full retrain produces a fresh, complete bitmap.
+        let merged_null_rows = self.null_rows.as_ref().map(|existing| {
+            let mut merged = existing.clone();
+            merged |= &new_null_rows;
+            merged
+        });
 
         // Serialize the combined zones back into the index file
         let mut builder = ZoneMapIndexBuilder::try_new(options, self.data_type.clone())?;
         builder.options.rows_per_zone = self.rows_per_zone;
         builder.maps = updated_zones;
-        builder.write_index(dest_store).await?;
+        builder.null_rows = merged_null_rows;
+        let files = builder.write_index(dest_store).await?;
 
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
                 .unwrap(),
             index_version: ZONEMAP_INDEX_VERSION,
-            files: Some(dest_store.list_files_with_sizes().await?),
+            files,
         })
     }
 
@@ -659,6 +719,76 @@ impl ScalarIndex for ZoneMapIndex {
         let params = serde_json::to_value(ZoneMapIndexBuilderParams::new(self.rows_per_zone))?;
         Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(&params))
     }
+
+    /// Single-segment `[min, max]` folded from this index's zones; see
+    /// [`value_range_over`](Self::value_range_over) for the full contract.
+    fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> {
+        Self::value_range_over([self])
+    }
+}
+
+/// Merge caller-selected ZoneMap segments into one self-contained segment.
+pub async fn merge_zonemap_indices(
+    source_indices: &[&ZoneMapIndex],
+    dest_store: &dyn IndexStore,
+    fragment_filter: &RoaringBitmap,
+) -> Result<CreatedIndex> {
+    let first = source_indices.first().ok_or_else(|| {
+        Error::invalid_input("merge_zonemap_indices requires at least one source index")
+    })?;
+    let rows_per_zone = first.rows_per_zone;
+    let data_type = first.data_type.clone();
+
+    let mut zones = Vec::new();
+    let mut merged_null_rows = RowAddrTreeMap::new();
+    let mut any_missing_bitmap = false;
+    for source in source_indices {
+        if source.rows_per_zone != rows_per_zone {
+            return Err(Error::invalid_input(format!(
+                "cannot merge ZoneMap segments with different rows_per_zone values: {} and {}",
+                rows_per_zone, source.rows_per_zone
+            )));
+        }
+        if source.data_type != data_type {
+            return Err(Error::invalid_input(format!(
+                "cannot merge ZoneMap segments with different value types: {:?} and {:?}",
+                data_type, source.data_type
+            )));
+        }
+        zones.extend(
+            source
+                .zones
+                .iter()
+                .filter(|zone| {
+                    u32::try_from(zone.bound.fragment_id)
+                        .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id))
+                })
+                .cloned(),
+        );
+        match &source.null_rows {
+            Some(null_rows) => {
+                let mut filtered = null_rows.clone();
+                filtered.retain_fragments(fragment_filter.iter());
+                merged_null_rows |= &filtered;
+            }
+            None => any_missing_bitmap = true,
+        }
+    }
+    zones.sort_by_key(|zone| (zone.bound.fragment_id, zone.bound.start));
+
+    let mut builder =
+        ZoneMapIndexBuilder::try_new(ZoneMapIndexBuilderParams::new(rows_per_zone), data_type)?;
+    builder.maps = zones;
+    if !any_missing_bitmap {
+        builder.null_rows = Some(merged_null_rows);
+    }
+    let files = builder.write_index(dest_store).await?;
+
+    Ok(CreatedIndex {
+        index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()).unwrap(),
+        index_version: ZONEMAP_INDEX_VERSION,
+        files,
+    })
 }
 
 fn default_rows_per_zone() -> u64 {
@@ -702,6 +832,10 @@ pub struct ZoneMapIndexBuilder {
 
     items_type: DataType,
     maps: Vec<ZoneMapStatistics>,
+    // None means "legacy index — null positions unknown"; Some means a complete bitmap.
+    // write_index omits the null-bitmap global buffer when this is None, preserving the
+    // legacy format so that downstream searches remain conservative.
+    null_rows: Option<RowAddrTreeMap>,
 }
 
 impl ZoneMapIndexBuilder {
@@ -710,6 +844,7 @@ impl ZoneMapIndexBuilder {
             options,
             items_type,
             maps: Vec::new(),
+            null_rows: None,
         })
     }
 
@@ -719,7 +854,9 @@ impl ZoneMapIndexBuilder {
     pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> {
         let processor = ZoneMapProcessor::new(self.items_type.clone())?;
         let trainer = ZoneTrainer::new(processor, self.options.rows_per_zone)?;
-        self.maps = trainer.train(batches_source).await?;
+        let (maps, null_rows) = trainer.train(batches_source).await?;
+        self.maps = maps;
+        self.null_rows = Some(null_rows);
         Ok(())
     }
 
@@ -772,7 +909,7 @@ impl ZoneMapIndexBuilder {
         Ok(RecordBatch::try_new(schema, columns)?)
     }
 
-    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<()> {
+    pub async fn write_index(self, index_store: &dyn IndexStore) -> Result<Vec<IndexFile>> {
         let record_batch = self.zonemap_stats_as_batch()?;
 
         let mut file_schema = record_batch.schema().as_ref().clone();
@@ -785,8 +922,24 @@ impl ZoneMapIndexBuilder {
             .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema))
             .await?;
         index_file.write_record_batch(record_batch).await?;
-        index_file.finish().await?;
-        Ok(())
+
+        let zonemap_file = if let Some(null_rows) = self.null_rows {
+            let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size());
+            null_rows.serialize_into(&mut null_bitmap_bytes)?;
+            let null_bitmap_idx = index_file
+                .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes))
+                .await?;
+            index_file
+                .finish_with_metadata(HashMap::from([(
+                    NULL_BITMAP_META_KEY.to_string(),
+                    null_bitmap_idx.to_string(),
+                )]))
+                .await?
+        } else {
+            index_file.finish_with_metadata(HashMap::new()).await?
+        };
+
+        Ok(vec![zonemap_file])
     }
 }
 
@@ -891,16 +1044,14 @@ impl ZoneMapIndexPlugin {
         batches_source: SendableRecordBatchStream,
         index_store: &dyn IndexStore,
         options: Option<ZoneMapIndexBuilderParams>,
-    ) -> Result<()> {
-        // train_zonemap_index: calling scan_aligned_chunks
+    ) -> Result<Vec<IndexFile>> {
         let value_type = batches_source.schema().field(0).data_type().clone();
 
         let mut builder = ZoneMapIndexBuilder::try_new(options.unwrap_or_default(), value_type)?;
 
         builder.train(batches_source).await?;
 
-        builder.write_index(index_store).await?;
-        Ok(())
+        builder.write_index(index_store).await
     }
 }
 
@@ -928,11 +1079,7 @@ impl TrainingRequest for ZoneMapIndexTrainingRequest {
 }
 
 #[async_trait]
-impl ScalarIndexPlugin for ZoneMapIndexPlugin {
-    fn name(&self) -> &str {
-        "ZoneMap"
-    }
-
+impl BasicTrainer for ZoneMapIndexPlugin {
     fn new_training_request(
         &self,
         params: &str,
@@ -949,26 +1096,6 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin {
         Ok(Box::new(ZoneMapIndexTrainingRequest::new(params)))
     }
 
-    fn provides_exact_answer(&self) -> bool {
-        false
-    }
-
-    fn version(&self) -> u32 {
-        ZONEMAP_INDEX_VERSION
-    }
-
-    fn new_query_parser(
-        &self,
-        index_name: String,
-        _index_details: &prost_types::Any,
-    ) -> Option<Box<dyn ScalarQueryParser>> {
-        Some(Box::new(SargableQueryParser::new(
-            index_name,
-            self.name().to_string(),
-            true,
-        )))
-    }
-
     async fn train_index(
         &self,
         data: SendableRecordBatchStream,
@@ -984,20 +1111,51 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin {
                     "must provide training request created by new_training_request".into(),
                 )
             })?;
-        Self::train_zonemap_index(data, index_store, Some(request.params)).await?;
+        let files = Self::train_zonemap_index(data, index_store, Some(request.params)).await?;
         Ok(CreatedIndex {
             index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default())
                 .unwrap(),
             index_version: ZONEMAP_INDEX_VERSION,
-            files: Some(index_store.list_files_with_sizes().await?),
+            files,
         })
     }
+}
+
+#[async_trait]
+impl ScalarIndexPlugin for ZoneMapIndexPlugin {
+    fn basic_trainer(&self) -> Option<&dyn BasicTrainer> {
+        Some(self)
+    }
+
+    fn name(&self) -> &str {
+        "ZoneMap"
+    }
+
+    fn provides_exact_answer(&self) -> bool {
+        false
+    }
+
+    fn version(&self) -> u32 {
+        ZONEMAP_INDEX_VERSION
+    }
+
+    fn new_query_parser(
+        &self,
+        index_name: String,
+        _index_details: &prost_types::Any,
+    ) -> Option<Box<dyn ScalarQueryParser>> {
+        Some(Box::new(SargableQueryParser::new(
+            index_name,
+            self.name().to_string(),
+            true,
+        )))
+    }
 
     async fn load_index(
         &self,
         index_store: Arc<dyn IndexStore>,
         _index_details: &prost_types::Any,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+        frag_reuse_index: Option<Arc<dyn RowIdRemapper>>,
         cache: &LanceCache,
     ) -> Result<Arc<dyn ScalarIndex>> {
         Ok(ZoneMapIndex::load(index_store, frag_reuse_index, cache).await? as Arc<dyn ScalarIndex>)
@@ -1012,8 +1170,8 @@ mod tests {
 
     use crate::scalar::zoned::ZoneBound;
     use crate::scalar::zonemap::{ZoneMapIndexPlugin, ZoneMapStatistics};
-    use arrow::datatypes::Float32Type;
-    use arrow_array::{Array, RecordBatch, UInt64Array, record_batch};
+    use arrow::datatypes::{ArrowPrimitiveType, Float32Type, Int64Type};
+    use arrow_array::{Array, PrimitiveArray, RecordBatch, UInt64Array, record_batch};
     use arrow_schema::{DataType, Field, Schema};
     use datafusion::execution::SendableRecordBatchStream;
     use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
@@ -1028,13 +1186,14 @@ mod tests {
     use lance_datagen::ArrayGeneratorExt;
     use lance_datagen::{BatchCount, RowCount, array};
     use lance_io::object_store::ObjectStore;
-    use lance_select::{NullableRowAddrSet, RowAddrTreeMap};
+    use lance_select::RowAddrTreeMap;
 
     use crate::scalar::{
         SargableQuery, ScalarIndex, SearchResult,
         lance_format::LanceIndexStore,
         zonemap::{
             ZONEMAP_FILENAME, ZONEMAP_SIZE_META_KEY, ZoneMapIndex, ZoneMapIndexBuilderParams,
+            merge_zonemap_indices,
         },
     };
 
@@ -1065,6 +1224,124 @@ mod tests {
         Box::pin(RecordBatchStreamAdapter::new(schema, stream))
     }
 
+    /// Build a single-column ZoneMap of primitive type `T` from `fragments`
+    /// (one batch -> one fragment), with small zones, then load it back.
+    async fn train_and_load<T: ArrowPrimitiveType>(
+        fragments: Vec<Vec<Option<T::Native>>>,
+    ) -> Arc<ZoneMapIndex>
+    where
+        PrimitiveArray<T>: From<Vec<Option<T::Native>>>,
+    {
+        let tmpdir = TempObjDir::default();
+        let test_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            VALUE_COLUMN_NAME,
+            T::DATA_TYPE,
+            true,
+        )]));
+        let batches: Vec<RecordBatch> = fragments
+            .into_iter()
+            .map(|vals| {
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![Arc::new(PrimitiveArray::<T>::from(vals))],
+                )
+                .unwrap()
+            })
+            .collect();
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema.clone(),
+            stream::iter(batches.into_iter().map(Ok)),
+        ));
+        let stream = add_row_addr(stream);
+
+        ZoneMapIndexPlugin::train_zonemap_index(
+            stream,
+            test_store.as_ref(),
+            Some(ZoneMapIndexBuilderParams::new(2)),
+        )
+        .await
+        .unwrap();
+
+        ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .expect("Failed to load ZoneMapIndex")
+    }
+
+    #[tokio::test]
+    async fn test_value_range_spans_fragments() {
+        // Two fragments, multiple zones each; global min/max straddle both.
+        let index = train_and_load::<Int64Type>(vec![
+            vec![Some(10), Some(50), Some(30)],
+            vec![Some(5), Some(99), Some(42)],
+        ])
+        .await;
+        assert_eq!(
+            index.value_range(),
+            Some((ScalarValue::Int64(Some(5)), ScalarValue::Int64(Some(99))))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_value_range_all_null_is_none() {
+        let index = train_and_load::<Int64Type>(vec![vec![None, None, None]]).await;
+        assert_eq!(index.value_range(), None);
+    }
+
+    #[tokio::test]
+    async fn test_value_range_nan_max_is_none() {
+        // Zones of size 2: [1.0, 2.0] then [100.0, NaN]. The NaN-bearing zone hides
+        // its finite max (100.0), so the only sound answer is None.
+        let index = train_and_load::<Float32Type>(vec![vec![
+            Some(1.0),
+            Some(2.0),
+            Some(100.0),
+            Some(f32::NAN),
+        ]])
+        .await;
+        assert_eq!(index.value_range(), None);
+    }
+
+    #[tokio::test]
+    async fn test_value_range_over_folds_segments() {
+        // Two disjoint segments of one logical index; the global range straddles
+        // both (min and max from `b`), proving the fold spans segments.
+        let a = train_and_load::<Int64Type>(vec![vec![Some(5), Some(9)]]).await;
+        let b = train_and_load::<Int64Type>(vec![vec![Some(1), Some(20)]]).await;
+        assert_eq!(
+            ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]),
+            Some((ScalarValue::Int64(Some(1)), ScalarValue::Int64(Some(20))))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_value_range_over_nan_in_any_segment_is_none() {
+        // NaN in one segment hides that segment's finite max; the cross-segment
+        // fold must bail to None just as the single-segment path does.
+        let a = train_and_load::<Float32Type>(vec![vec![Some(1.0), Some(2.0)]]).await;
+        let b = train_and_load::<Float32Type>(vec![vec![Some(100.0), Some(f32::NAN)]]).await;
+        assert_eq!(
+            ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn test_value_range_over_skips_all_null_segment() {
+        // An all-null segment yields no finite zone; folding it with a finite
+        // segment returns the finite segment's range (null contributes nothing).
+        let a = train_and_load::<Int64Type>(vec![vec![None, None]]).await;
+        let b = train_and_load::<Int64Type>(vec![vec![Some(3), Some(7)]]).await;
+        assert_eq!(
+            ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]),
+            Some((ScalarValue::Int64(Some(3)), ScalarValue::Int64(Some(7))))
+        );
+    }
+
     #[tokio::test]
     async fn test_empty_zonemap_index() {
         let tmpdir = TempObjDir::default();
@@ -1389,6 +1666,17 @@ mod tests {
             );
         }
 
+        let zone = &index.zones[0];
+        assert!(matches!(
+            zone.max,
+            ScalarValue::Float32(Some(value)) if value.is_nan()
+        ));
+        let finite_target = ScalarValue::Float32(Some(1000.0));
+        assert!(
+            finite_target >= zone.min && finite_target <= zone.max,
+            "ScalarValue total ordering keeps finite values below NaN max"
+        );
+
         // Test search for NaN values using Equals with NaN
         let query = SargableQuery::Equals(ScalarValue::Float32(Some(f32::NAN)));
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
@@ -1458,7 +1746,7 @@ mod tests {
         // Test IsNull query (should match nothing since there are no null values)
         let query = SargableQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty()));
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
 
         // Test range queries with NaN bounds
         // Range with NaN as start bound (included)
@@ -1501,7 +1789,7 @@ mod tests {
         );
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
         // Should match nothing since nothing is greater than NaN
-        assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty()));
+        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new()));
 
         // Test IsIn query with mixed float types (Float16, Float32, Float64)
         let query = SargableQuery::IsIn(vec![
@@ -1685,7 +1973,7 @@ mod tests {
         // 8. IsNull query (no nulls in data, should match nothing)
         let query = SargableQuery::IsNull();
         let result = index.search(&query, &NoOpMetricsCollector).await.unwrap();
-        assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new()));
+        assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new()));
         // 9. IsIn query: [0, 100, 101, 50]
         let query = SargableQuery::IsIn(vec![
             ScalarValue::Int32(Some(0)),
@@ -2394,6 +2682,7 @@ mod tests {
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix query for "foo"
@@ -2466,6 +2755,7 @@ mod tests {
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix "test"
@@ -2533,6 +2823,7 @@ mod tests {
             store: test_store,
             fri: None,
             index_cache: WeakLanceCache::from(&LanceCache::no_cache()),
+            null_rows: None,
         };
 
         // Test LikePrefix with LargeUtf8
@@ -2585,4 +2876,204 @@ mod tests {
         // All max characters
         assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None);
     }
+
+    // When merging zone map segments, if ANY source segment has null_rows = None
+    // (legacy — null positions unknown), the merged result must also be None.
+    // The bug: any_null_bitmap is set to true as soon as one source has Some(...),
+    // and the None sources are silently skipped.  The merged index then has
+    // null_rows = Some(partial_bitmap), so an IsNull search returns exact results
+    // that only cover the modern segment's nulls — a false negative for the legacy
+    // segment whose null positions were never tracked.
+    #[tokio::test]
+    async fn test_merge_with_legacy_none_segment_not_treated_as_no_nulls() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        use arrow_array::{Int32Array, UInt32Array};
+
+        // Index A: fragment 0, modern — has a complete null bitmap with 2 known null rows.
+        let schema_a = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch_a = RecordBatch::try_new(
+            schema_a,
+            vec![
+                Arc::new(Int32Array::from(vec![Some(1i32)])) as _,
+                Arc::new(Int32Array::from(vec![Some(5i32)])) as _,
+                Arc::new(UInt32Array::from(vec![2u32])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![10u64])) as _,
+            ],
+        )
+        .unwrap();
+        let mut modern_null_rows = RowAddrTreeMap::new();
+        modern_null_rows.insert(3); // frag 0 row 3
+        modern_null_rows.insert(7); // frag 0 row 7
+        let cache = LanceCache::no_cache();
+        let index_a = Arc::new(
+            ZoneMapIndex::try_from_serialized(
+                batch_a,
+                store.clone(),
+                None,
+                &cache,
+                10,
+                Some(modern_null_rows), // modern: complete bitmap
+            )
+            .unwrap(),
+        );
+
+        // Index B: fragment 1, legacy — null_rows = None despite null_count = 3.
+        let schema_b = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch_b = RecordBatch::try_new(
+            schema_b,
+            vec![
+                Arc::new(Int32Array::from(vec![Some(10i32)])) as _,
+                Arc::new(Int32Array::from(vec![Some(20i32)])) as _,
+                Arc::new(UInt32Array::from(vec![3u32])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![1u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![10u64])) as _,
+            ],
+        )
+        .unwrap();
+        let index_b = Arc::new(
+            ZoneMapIndex::try_from_serialized(
+                batch_b,
+                store.clone(),
+                None,
+                &cache,
+                10,
+                None, // legacy: null positions unknown
+            )
+            .unwrap(),
+        );
+
+        let dest_tmpdir = TempObjDir::default();
+        let dest_store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            dest_tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        let all_frags = RoaringBitmap::from_iter([0u32, 1]);
+        merge_zonemap_indices(
+            &[index_a.as_ref(), index_b.as_ref()],
+            dest_store.as_ref(),
+            &all_frags,
+        )
+        .await
+        .unwrap();
+
+        let merged = ZoneMapIndex::load(dest_store.clone(), None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        // Index B had null_rows = None, so the merged index cannot know all null positions.
+        // IsNull must NOT return exact — that would be a false negative for fragment 1's nulls.
+        let result = merged
+            .search(&SargableQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+
+        // With the bug: any_null_bitmap=true (from A) → null_rows=Some(A's bitmap only)
+        //              → IsNull returns exact, missing B's unknown nulls ← FALSE NEGATIVE
+        // With the fix: any_null_bitmap=false (because B is None) → null_rows=None
+        //              → IsNull falls through to zone scan → AtMost
+        assert!(
+            !result.is_exact(),
+            "IsNull on a merged index where one source had null_rows=None must not return \
+             exact; the legacy segment had null_count=3 so its nulls exist at unknown positions"
+        );
+    }
+
+    // Writes a zonemap file in the legacy format (no null bitmap global buffer),
+    // simulating an index created before the null bitmap feature was added.
+    async fn write_legacy_zonemap(store: &dyn IndexStore, null_count: u32) {
+        use arrow_array::{Int32Array, UInt32Array};
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("min", DataType::Int32, true),
+            Field::new("max", DataType::Int32, true),
+            Field::new("null_count", DataType::UInt32, false),
+            Field::new("nan_count", DataType::UInt32, false),
+            Field::new("fragment_id", DataType::UInt64, false),
+            Field::new("zone_start", DataType::UInt64, false),
+            Field::new("zone_length", DataType::UInt64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![Some(0)])) as _,
+                Arc::new(Int32Array::from(vec![Some(99)])) as _,
+                Arc::new(UInt32Array::from(vec![null_count])) as _,
+                Arc::new(UInt32Array::from(vec![0u32])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![0u64])) as _,
+                Arc::new(UInt64Array::from(vec![100u64])) as _,
+            ],
+        )
+        .unwrap();
+        let mut file_schema = schema.as_ref().clone();
+        file_schema
+            .metadata
+            .insert(ZONEMAP_SIZE_META_KEY.to_string(), "8192".to_string());
+        let mut writer = store
+            .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema))
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_legacy_zonemap_no_null_bitmap() {
+        let tmpdir = TempObjDir::default();
+        let store = Arc::new(LanceIndexStore::new(
+            Arc::new(ObjectStore::local()),
+            tmpdir.clone(),
+            Arc::new(LanceCache::no_cache()),
+        ));
+
+        // Write a legacy index with one zone that has nulls but no null bitmap.
+        write_legacy_zonemap(store.as_ref(), 10).await;
+
+        let index = ZoneMapIndex::load(store, None, &LanceCache::no_cache())
+            .await
+            .expect("failed to load legacy zonemap");
+
+        assert!(
+            index.null_rows.is_none(),
+            "legacy index should have no null bitmap"
+        );
+
+        // IS NULL should fall back to the zone-scan path and return AtMost, not Exact.
+        let result = index
+            .search(&SargableQuery::IsNull(), &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        assert!(
+            !result.is_exact(),
+            "IS NULL on a legacy index should not be exact"
+        );
+    }
 }
diff --git a/rust/lance-index/src/vector.rs b/rust/lance-index/src/vector.rs
index a6ff3eb02d5..76b9ccc7a51 100644
--- a/rust/lance-index/src/vector.rs
+++ b/rust/lance-index/src/vector.rs
@@ -4,18 +4,19 @@
 //! Vector Index
 //!
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::any::Any;
 use std::fmt::Debug;
-use std::{collections::HashMap, sync::Arc};
+use std::sync::Arc;
 
 use arrow_array::{ArrayRef, Float32Array, RecordBatch, UInt32Array};
 use arrow_schema::Field;
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use deepsize::DeepSizeOf;
 use futures::stream;
 use ivf::storage::IvfModel;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID_FIELD, Result};
 use lance_io::traits::Reader;
 use lance_linalg::distance::DistanceType;
@@ -82,6 +83,23 @@ pub static CENTROID_DIST_FIELD: LazyLock<arrow_schema::Field> = LazyLock::new(||
 
 pub const DEFAULT_QUERY_PARALLELISM: i32 = 0;
 
+/// Controls the speed / accuracy tradeoff for approximate vector search.
+///
+/// This currently only affects RQ-quantized vector indexes, such as IVF_RQ.
+/// Other index types ignore this setting.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum ApproxMode {
+    /// Prefer lower query latency, which can reduce recall.
+    Fast,
+
+    /// Use the default balance between query latency and recall.
+    #[default]
+    Normal,
+
+    /// Prefer higher recall, which can increase query latency.
+    Accurate,
+}
+
 /// Query parameters for the vector indices
 
 #[derive(Debug, Clone)]
@@ -141,6 +159,12 @@ pub struct Query {
     /// the distance between the query and the centroid
     /// this is only used for IVF index with Rabit quantization
     pub dist_q_c: f32,
+
+    /// Controls the speed / accuracy tradeoff for approximate vector search.
+    ///
+    /// This currently only affects RQ-quantized vector indexes, such as IVF_RQ.
+    /// Other index types ignore this setting.
+    pub approx_mode: ApproxMode,
 }
 
 impl From<pb::VectorMetricType> for DistanceType {
@@ -385,7 +409,7 @@ pub trait VectorIndex: Send + Sync + std::fmt::Debug + Index {
     ///
     /// If an old row id is not in the mapping then it should be
     /// left alone.
-    async fn remap(&mut self, mapping: &HashMap<u64, Option<u64>>) -> Result<()>;
+    async fn remap(&mut self, mapping: &RowAddrRemap) -> Result<()>;
 
     /// The metric type of this vector index.
     fn metric_type(&self) -> DistanceType;
@@ -396,6 +420,14 @@ pub trait VectorIndex: Send + Sync + std::fmt::Debug + Index {
 
     /// the index type of this vector index.
     fn sub_index_type(&self) -> (SubIndexType, QuantizationType);
+
+    /// The cumulative I/O performed while opening this index (file footers, IVF
+    /// centroids, quantization metadata).  This is a one-time cost; it is
+    /// reported once, on the query that actually opens the index, and is `None`
+    /// for index implementations that do not track it.
+    fn open_io_stats(&self) -> Option<lance_io::scheduler::ScanStats> {
+        None
+    }
 }
 
 // it can be an IVF index or a partition of IVF index
diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs
index 1df04d4b134..7a47fa88d54 100644
--- a/rust/lance-index/src/vector/bq.rs
+++ b/rust/lance-index/src/vector/bq.rs
@@ -14,15 +14,19 @@ use lance_core::{Error, Result};
 use num_traits::Float;
 use serde::{Deserialize, Serialize};
 
+use crate::vector::bq::storage::RabitQuantizationMetadata;
 use crate::vector::quantizer::QuantizerBuildParams;
 
 pub mod builder;
+pub(crate) mod dist_table_quant;
+pub mod ex_dot;
+pub mod prune;
 pub mod rotation;
 pub mod storage;
 pub mod transform;
 
 pub const RABIT_MIN_NUM_BITS: u8 = 1;
-pub const RABIT_MAX_NUM_BITS: u8 = 8;
+pub const RABIT_MAX_NUM_BITS: u8 = 9;
 pub const RABIT_BINARY_NUM_BITS: u8 = 1;
 
 #[derive(Clone, Default)]
@@ -104,10 +108,16 @@ impl FromStr for RQRotationType {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug)]
 pub struct RQBuildParams {
     pub num_bits: u8,
     pub rotation_type: RQRotationType,
+    /// Optional pre-built rotation to reuse instead of generating a fresh random one.
+    ///
+    /// Distributed `IVF_RQ` builds mint one rotation and broadcast it so every segment
+    /// rotates vectors identically. This is transient build-time state and is never
+    /// persisted to the `RabitQuantization` params proto.
+    pub rotation: Option<RabitQuantizationMetadata>,
 }
 
 pub fn validate_rq_num_bits(num_bits: u8) -> Result<()> {
@@ -121,14 +131,7 @@ pub fn validate_rq_num_bits(num_bits: u8) -> Result<()> {
 }
 
 pub fn validate_supported_rq_num_bits(num_bits: u8) -> Result<()> {
-    validate_rq_num_bits(num_bits)?;
-    if num_bits != RABIT_BINARY_NUM_BITS {
-        return Err(Error::not_supported(format!(
-            "IVF_RQ num_bits={} is not supported yet; only num_bits=1 is supported",
-            num_bits
-        )));
-    }
-    Ok(())
+    validate_rq_num_bits(num_bits)
 }
 
 pub fn rabit_ex_bits(num_bits: u8) -> Result<u8> {
@@ -155,6 +158,7 @@ impl RQBuildParams {
         Self {
             num_bits,
             rotation_type: RQRotationType::default(),
+            rotation: None,
         }
     }
 
@@ -162,6 +166,7 @@ impl RQBuildParams {
         Self {
             num_bits,
             rotation_type,
+            rotation: None,
         }
     }
 }
@@ -190,6 +195,7 @@ impl Default for RQBuildParams {
         Self {
             num_bits: 1,
             rotation_type: RQRotationType::default(),
+            rotation: None,
         }
     }
 }
@@ -234,7 +240,7 @@ mod tests {
     #[test]
     fn test_rabit_num_bits_validation() {
         validate_rq_num_bits(1).unwrap();
-        validate_rq_num_bits(8).unwrap();
+        validate_rq_num_bits(9).unwrap();
 
         let err = validate_rq_num_bits(0).unwrap_err();
         assert!(
@@ -243,31 +249,28 @@ mod tests {
             err
         );
 
-        let err = validate_rq_num_bits(9).unwrap_err();
+        let err = validate_rq_num_bits(10).unwrap_err();
         assert!(
             err.to_string().contains("IVF_RQ num_bits must be in"),
             "{}",
             err
         );
 
-        let err = validate_supported_rq_num_bits(2).unwrap_err();
-        assert!(
-            err.to_string().contains("only num_bits=1 is supported"),
-            "{}",
-            err
-        );
+        validate_supported_rq_num_bits(1).unwrap();
+        validate_supported_rq_num_bits(9).unwrap();
     }
 
     #[test]
     fn test_rabit_split_code_byte_sizing() {
         assert_eq!(rabit_ex_bits(1).unwrap(), 0);
-        assert_eq!(rabit_ex_bits(8).unwrap(), 7);
+        assert_eq!(rabit_ex_bits(9).unwrap(), 8);
 
         assert_eq!(rabit_binary_code_bytes(128), 16);
         assert_eq!(rabit_binary_code_bytes(129), 17);
 
         assert_eq!(rabit_ex_code_bytes(128, 0).unwrap(), 0);
         assert_eq!(rabit_ex_code_bytes(128, 3).unwrap(), 48);
+        assert_eq!(rabit_ex_code_bytes(128, 8).unwrap(), 128);
         assert_eq!(rabit_ex_code_bytes(129, 3).unwrap(), 49);
     }
 }
diff --git a/rust/lance-index/src/vector/bq/builder.rs b/rust/lance-index/src/vector/bq/builder.rs
index f98c370aefc..9eb7fc76903 100644
--- a/rust/lance-index/src/vector/bq/builder.rs
+++ b/rust/lance-index/src/vector/bq/builder.rs
@@ -8,8 +8,8 @@ use arrow::datatypes::{Float16Type, Float32Type, Float64Type};
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, UInt8Array};
 use arrow_schema::{DataType, Field};
 use bitvec::prelude::{BitVec, Lsb0};
-use deepsize::DeepSizeOf;
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, FloatType};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use ndarray::{Axis, ShapeBuilder, s};
 use num_traits::{AsPrimitive, FromPrimitive};
@@ -18,13 +18,16 @@ use rayon::prelude::*;
 
 use crate::vector::bq::storage::{
     RABIT_CODE_COLUMN, RABIT_METADATA_KEY, RabitQuantizationMetadata, RabitQuantizationStorage,
-    rabit_binary_code_field,
+    RabitQueryEstimator, rabit_binary_code_field, rabit_ex_code_field,
+};
+use crate::vector::bq::transform::{
+    ADD_FACTORS_FIELD, ERROR_FACTORS_FIELD, EX_ADD_FACTORS_FIELD, EX_SCALE_FACTORS_FIELD,
+    SCALE_FACTORS_FIELD,
 };
-use crate::vector::bq::transform::{ADD_FACTORS_FIELD, SCALE_FACTORS_FIELD};
 use crate::vector::bq::{
-    RQBuildParams, RQRotationType, rabit_binary_code_bytes,
-    rotation::{apply_fast_rotation, random_fast_rotation_signs},
-    validate_supported_rq_num_bits,
+    RQBuildParams, RQRotationType, rabit_binary_code_bytes, rabit_ex_bits,
+    rotation::{apply_fast_rotation, fast_rotation_signs_len, random_fast_rotation_signs},
+    validate_rq_num_bits,
 };
 use crate::vector::quantizer::{Quantization, Quantizer, QuantizerBuildParams};
 
@@ -57,6 +60,14 @@ pub struct RabitQuantizer {
     metadata: RabitQuantizationMetadata,
 }
 
+pub(crate) struct RabitQuantizedBatch {
+    pub binary_codes: ArrayRef,
+    pub ex_codes: Option<ArrayRef>,
+    pub ex_res_dot_dists: Option<Vec<f32>>,
+    pub rotated_residuals: Option<Vec<f32>>,
+    pub ex_code_values: Option<Vec<u8>>,
+}
+
 #[inline]
 fn pack_sign_bits(codes: &mut [u8], rotated: &[f32]) {
     codes.fill(0);
@@ -67,6 +78,117 @@ fn pack_sign_bits(codes: &mut [u8], rotated: &[f32]) {
     }
 }
 
+const EX_QUANTIZATION_EPSILON: f32 = 1.0e-5;
+const EX_TIGHT_START: [f32; 9] = [0.0, 0.15, 0.20, 0.52, 0.59, 0.71, 0.75, 0.77, 0.81];
+
+fn best_ex_rescale_factor(abs_normalized: &[f32], ex_bits: u8) -> f32 {
+    let max_value = abs_normalized
+        .iter()
+        .copied()
+        .filter(|value| value.is_finite())
+        .fold(0.0f32, f32::max);
+    if max_value <= 0.0 {
+        return 0.0;
+    }
+
+    let max_code = (1usize << ex_bits) - 1;
+    let t_end = ((max_code + 10) as f32) / max_value;
+    let t_start = t_end * EX_TIGHT_START[ex_bits as usize];
+
+    let mut current_codes = Vec::with_capacity(abs_normalized.len());
+    let mut squared_denominator = abs_normalized.len() as f32 * 0.25;
+    let mut numerator = 0.0f32;
+    let mut thresholds = Vec::with_capacity(abs_normalized.len() * max_code);
+
+    for (idx, &value) in abs_normalized.iter().enumerate() {
+        if value <= 0.0 || !value.is_finite() {
+            current_codes.push(0usize);
+            continue;
+        }
+
+        let current = ((t_start * value) + EX_QUANTIZATION_EPSILON)
+            .floor()
+            .clamp(0.0, max_code as f32) as usize;
+        current_codes.push(current);
+        squared_denominator += (current * current + current) as f32;
+        numerator += (current as f32 + 0.5) * value;
+
+        let mut next = current + 1;
+        while next <= max_code {
+            let threshold = next as f32 / value;
+            if threshold < t_end {
+                thresholds.push((threshold, idx));
+            }
+            next += 1;
+        }
+    }
+
+    thresholds.sort_unstable_by(|(left, _), (right, _)| left.total_cmp(right));
+
+    let mut best_inner_product = numerator / squared_denominator.sqrt();
+    let mut best_t = t_start;
+    for (threshold, idx) in thresholds {
+        current_codes[idx] += 1;
+        let updated = current_codes[idx];
+        squared_denominator += (2 * updated) as f32;
+        numerator += abs_normalized[idx];
+
+        let current_inner_product = numerator / squared_denominator.sqrt();
+        if current_inner_product > best_inner_product {
+            best_inner_product = current_inner_product;
+            best_t = threshold;
+        }
+    }
+
+    best_t
+}
+
+fn quantize_ex_code(
+    rotated: &[f32],
+    ex_bits: u8,
+    ex_code_dst: &mut [u8],
+    ex_code_values_dst: &mut [u8],
+) -> f32 {
+    debug_assert_eq!(rotated.len(), ex_code_values_dst.len());
+    let norm_squared = rotated.iter().map(|value| value * value).sum::<f32>();
+    if norm_squared <= f32::EPSILON || !norm_squared.is_finite() {
+        ex_code_dst.fill(0);
+        ex_code_values_dst.fill(0);
+        return 0.0;
+    }
+
+    let norm = norm_squared.sqrt();
+    let abs_normalized = rotated
+        .iter()
+        .map(|value| value.abs() / norm)
+        .collect::<Vec<_>>();
+    let t = best_ex_rescale_factor(&abs_normalized, ex_bits);
+    let max_code = ((1u16 << ex_bits) - 1) as u8;
+    let mask = max_code;
+    let code_bias = -((1u32 << ex_bits) as f32 - 0.5);
+    let mut residual_dot_code = 0.0f32;
+
+    for ((&value, &abs_value), ex_code_value) in rotated
+        .iter()
+        .zip(abs_normalized.iter())
+        .zip(ex_code_values_dst.iter_mut())
+    {
+        let mut ex_code = ((t * abs_value) + EX_QUANTIZATION_EPSILON)
+            .floor()
+            .clamp(0.0, max_code as f32) as u8;
+        if value.is_sign_negative() {
+            ex_code = (!ex_code) & mask;
+        }
+        let sign_code = u8::from(value.is_sign_positive());
+        let full_code = ((sign_code as u32) << ex_bits) + ex_code as u32;
+        residual_dot_code += value * (full_code as f32 + code_bias);
+        *ex_code_value = ex_code;
+    }
+
+    crate::vector::bq::ex_dot::pack_blocked_row(ex_code_values_dst, ex_bits, ex_code_dst);
+    residual_dot_code
+}
+
 impl RabitQuantizer {
     pub fn new<T: ArrowFloatType>(num_bits: u8, dim: i32) -> Self {
         Self::new_with_rotation::<T>(num_bits, dim, RQRotationType::default())
@@ -100,6 +222,7 @@ impl RabitQuantizer {
                     code_dim: code_dim as u32,
                     num_bits,
                     packed: false,
+                    query_estimator: RabitQueryEstimator::RawQuery,
                 }
             }
             RQRotationType::Fast => RabitQuantizationMetadata {
@@ -110,6 +233,7 @@ impl RabitQuantizer {
                 code_dim: code_dim as u32,
                 num_bits,
                 packed: false,
+                query_estimator: RabitQueryEstimator::RawQuery,
             },
         };
         Self { metadata }
@@ -123,6 +247,69 @@ impl RabitQuantizer {
         self.metadata.rotation_type
     }
 
+    pub fn metadata_ref(&self) -> &RabitQuantizationMetadata {
+        &self.metadata
+    }
+
+    fn from_supplied_rotation(params: &RQBuildParams, dim: usize) -> Result<Option<Self>> {
+        let Some(metadata) = params.rotation.as_ref() else {
+            return Ok(None);
+        };
+
+        if metadata.num_bits != params.num_bits {
+            return Err(Error::invalid_input(format!(
+                "rabitq_model num_bits={} does not match requested num_bits={}",
+                metadata.num_bits, params.num_bits
+            )));
+        }
+
+        let rotated_dim = metadata.rotated_dim();
+        if rotated_dim != dim {
+            return Err(Error::invalid_input(format!(
+                "rabitq_model dimension={} does not match vector dimension={}",
+                rotated_dim, dim
+            )));
+        }
+
+        match metadata.rotation_type {
+            RQRotationType::Fast => {
+                let signs = metadata.fast_rotation_signs.as_ref().ok_or_else(|| {
+                    Error::invalid_input(
+                        "rabitq_model fast rotation is missing fast_rotation_signs".to_string(),
+                    )
+                })?;
+                let expected_len = fast_rotation_signs_len(dim);
+                if signs.len() != expected_len {
+                    return Err(Error::invalid_input(format!(
+                        "rabitq_model fast_rotation_signs length={} does not match expected length={} for dimension={}",
+                        signs.len(),
+                        expected_len,
+                        dim
+                    )));
+                }
+            }
+            RQRotationType::Matrix => {
+                let rotate_mat = metadata.rotate_mat.as_ref().ok_or_else(|| {
+                    Error::invalid_input(
+                        "rabitq_model matrix rotation is missing rotate_mat".to_string(),
+                    )
+                })?;
+                if rotate_mat.len() != dim || rotate_mat.value_length() != dim as i32 {
+                    return Err(Error::invalid_input(format!(
+                        "rabitq_model matrix rotation shape=({}, {}) does not match vector dimension={}",
+                        rotate_mat.len(),
+                        rotate_mat.value_length(),
+                        dim
+                    )));
+                }
+            }
+        }
+
+        Ok(Some(Self {
+            metadata: metadata.clone(),
+        }))
+    }
+
     #[inline]
     fn fast_rotation_signs(&self) -> &[u8] {
         self.metadata
@@ -188,6 +375,60 @@ impl RabitQuantizer {
         }
     }
 
+    pub fn rotate_fsl_to_f32(&self, vectors: &FixedSizeListArray) -> Result<Vec<f32>> {
+        match vectors.value_type() {
+            DataType::Float16 => self.rotate_fsl_to_f32_typed::<Float16Type>(vectors),
+            DataType::Float32 => self.rotate_fsl_to_f32_typed::<Float32Type>(vectors),
+            DataType::Float64 => self.rotate_fsl_to_f32_typed::<Float64Type>(vectors),
+            value_type => Err(Error::invalid_input(format!(
+                "Unsupported data type: {:?}",
+                value_type
+            ))),
+        }
+    }
+
+    fn rotate_fsl_to_f32_typed<T: ArrowFloatType>(
+        &self,
+        vectors: &FixedSizeListArray,
+    ) -> Result<Vec<f32>>
+    where
+        T::Native: AsPrimitive<f32> + Sync,
+    {
+        let dim = self.dim();
+        if vectors.value_length() as usize != dim {
+            return Err(Error::invalid_input(format!(
+                "Vector dimension mismatch: {} != {}",
+                vectors.value_length(),
+                dim
+            )));
+        }
+        let values = vectors
+            .values()
+            .as_any()
+            .downcast_ref::<T::ArrayType>()
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Vector values have unexpected data type: {}",
+                    vectors.value_type()
+                ))
+            })?
+            .as_slice();
+        let vec_mat = ndarray::ArrayView2::from_shape((vectors.len(), dim), values)
+            .map_err(|e| Error::invalid_input(e.to_string()))?;
+        let rotated = self.rotate_vectors::<T>(vec_mat.t());
+        let code_dim = self.code_dim();
+        let mut row_major = vec![0.0f32; vectors.len() * code_dim];
+        for row_idx in 0..vectors.len() {
+            for (dst, value) in row_major[row_idx * code_dim..(row_idx + 1) * code_dim]
+                .iter_mut()
+                .zip(rotated.column(row_idx).iter())
+            {
+                *dst = *value;
+            }
+        }
+        Ok(row_major)
+    }
+
     pub fn dim(&self) -> usize {
         self.code_dim()
     }
@@ -209,7 +450,7 @@ impl RabitQuantizer {
             )));
         }
 
-        let sqrt_dim = (dim as f32 * self.metadata.num_bits as f32).sqrt();
+        let sqrt_dim = (dim as f32).sqrt();
         let values = residual_vectors
             .values()
             .as_any()
@@ -308,6 +549,125 @@ impl RabitQuantizer {
             }
         }
     }
+
+    pub(crate) fn quantize_split(
+        &self,
+        vectors: &FixedSizeListArray,
+    ) -> Result<RabitQuantizedBatch> {
+        match vectors.value_type() {
+            DataType::Float16 => self.transform_split::<Float16Type>(vectors),
+            DataType::Float32 => self.transform_split::<Float32Type>(vectors),
+            DataType::Float64 => self.transform_split::<Float64Type>(vectors),
+            value_type => Err(Error::invalid_input(format!(
+                "Unsupported data type: {:?}",
+                value_type
+            ))),
+        }
+    }
+
+    fn transform_split<T: ArrowFloatType>(
+        &self,
+        residual_vectors: &FixedSizeListArray,
+    ) -> Result<RabitQuantizedBatch>
+    where
+        T::Native: AsPrimitive<f32> + Sync,
+    {
+        let ex_bits = rabit_ex_bits(self.metadata.num_bits)?;
+        let n = residual_vectors.len();
+        let dim = self.dim();
+        debug_assert_eq!(residual_vectors.values().len(), n * dim);
+        let values = residual_vectors
+            .values()
+            .as_any()
+            .downcast_ref::<T::ArrayType>()
+            .unwrap()
+            .as_slice();
+        let code_dim = self.code_dim();
+        let code_bytes = rabit_binary_code_bytes(code_dim);
+        let ex_code_bytes = if ex_bits == 0 {
+            0
+        } else {
+            crate::vector::bq::ex_dot::blocked_ex_code_bytes(code_dim, ex_bits)
+        };
+
+        let mut encoded_codes = vec![0u8; n * code_bytes];
+        let mut encoded_ex_codes = (ex_bits != 0).then(|| vec![0u8; n * ex_code_bytes]);
+        let mut ex_res_dot_dists = (ex_bits != 0).then(|| vec![0.0f32; n]);
+        let mut rotated_residuals = vec![0.0f32; n * code_dim];
+        let mut ex_code_values = (ex_bits != 0).then(|| vec![0u8; n * code_dim]);
+
+        match self.rotation_type() {
+            RQRotationType::Matrix => {
+                let vectors = ndarray::ArrayView2::from_shape((n, dim), values)
+                    .map_err(|e| Error::invalid_input(e.to_string()))?;
+                let vectors = vectors.t();
+                let rotated_vectors = self.rotate_vectors::<T>(vectors);
+
+                encoded_codes
+                    .chunks_mut(code_bytes)
+                    .zip(rotated_residuals.chunks_mut(code_dim))
+                    .enumerate()
+                    .for_each(|(row_idx, (code_dst, rotated_dst))| {
+                        for (dst, value) in rotated_dst
+                            .iter_mut()
+                            .zip(rotated_vectors.column(row_idx).iter())
+                        {
+                            *dst = *value;
+                        }
+                        pack_sign_bits(code_dst, rotated_dst);
+                    });
+            }
+            RQRotationType::Fast => {
+                let signs = self.fast_rotation_signs();
+                encoded_codes
+                    .par_chunks_mut(code_bytes)
+                    .zip(rotated_residuals.par_chunks_mut(code_dim))
+                    .zip(values.par_chunks_exact(dim))
+                    .for_each(|((code_dst, rotated_dst), input)| {
+                        apply_fast_rotation(input, rotated_dst, signs);
+                        pack_sign_bits(code_dst, rotated_dst);
+                    });
+            }
+        }
+
+        if ex_bits != 0 {
+            let encoded_ex_codes = encoded_ex_codes
+                .as_mut()
+                .expect("ex-code buffer should exist for multi-bit RQ");
+            let ex_res_dot_dists = ex_res_dot_dists
+                .as_mut()
+                .expect("ex dot buffer should exist for multi-bit RQ");
+            let ex_code_values = ex_code_values
+                .as_mut()
+                .expect("ex-code value buffer should exist for multi-bit RQ");
+            encoded_ex_codes
+                .par_chunks_mut(ex_code_bytes)
+                .zip(ex_code_values.par_chunks_mut(code_dim))
+                .zip(ex_res_dot_dists.par_iter_mut())
+                .zip(rotated_residuals.par_chunks(code_dim))
+                .for_each(|(((ex_dst, ex_values_dst), ex_dot_dst), rotated)| {
+                    *ex_dot_dst = quantize_ex_code(rotated, ex_bits, ex_dst, ex_values_dst);
+                });
+        }
+
+        let binary_codes = UInt8Array::from(encoded_codes);
+        let ex_codes = encoded_ex_codes.map(UInt8Array::from);
+        Ok(RabitQuantizedBatch {
+            binary_codes: Arc::new(FixedSizeListArray::try_new_from_values(
+                binary_codes,
+                code_bytes as i32,
+            )?),
+            ex_codes: ex_codes
+                .map(|ex_codes| {
+                    FixedSizeListArray::try_new_from_values(ex_codes, ex_code_bytes as i32)
+                        .map(|array| Arc::new(array) as ArrayRef)
+                })
+                .transpose()?,
+            ex_res_dot_dists,
+            rotated_residuals: Some(rotated_residuals),
+            ex_code_values,
+        })
+    }
 }
 
 impl Quantization for RabitQuantizer {
@@ -320,7 +680,7 @@ impl Quantization for RabitQuantizer {
         _: lance_linalg::distance::DistanceType,
         params: &Self::BuildParams,
     ) -> Result<Self> {
-        validate_supported_rq_num_bits(params.num_bits)?;
+        validate_rq_num_bits(params.num_bits)?;
 
         let dim = data.as_fixed_size_list().value_length() as usize;
         if !dim.is_multiple_of(u8::BITS as usize) {
@@ -328,6 +688,9 @@ impl Quantization for RabitQuantizer {
                 "vector dimension must be divisible by 8 for IVF_RQ",
             ));
         }
+        if let Some(q) = Self::from_supplied_rotation(params, dim)? {
+            return Ok(q);
+        }
 
         let q = match data.as_fixed_size_list().value_type() {
             DataType::Float16 => Self::new_with_rotation::<Float16Type>(
@@ -413,7 +776,7 @@ impl Quantization for RabitQuantizer {
         metadata: &Self::Metadata,
         _: lance_linalg::distance::DistanceType,
     ) -> Result<Quantizer> {
-        validate_supported_rq_num_bits(metadata.num_bits)?;
+        validate_rq_num_bits(metadata.num_bits)?;
         Ok(Quantizer::Rabit(Self {
             metadata: metadata.clone(),
         }))
@@ -424,7 +787,18 @@ impl Quantization for RabitQuantizer {
     }
 
     fn extra_fields(&self) -> Vec<Field> {
-        vec![ADD_FACTORS_FIELD.clone(), SCALE_FACTORS_FIELD.clone()]
+        let mut fields = vec![ADD_FACTORS_FIELD.clone(), SCALE_FACTORS_FIELD.clone()];
+        if self.metadata.query_estimator == RabitQueryEstimator::RawQuery {
+            fields.push(ERROR_FACTORS_FIELD.clone());
+        }
+        if let Some(ex_code_field) = rabit_ex_code_field(self.code_dim(), self.metadata.num_bits)
+            .expect("RabitQ num_bits should be validated")
+        {
+            fields.push(ex_code_field);
+            fields.push(EX_ADD_FACTORS_FIELD.clone());
+            fields.push(EX_SCALE_FACTORS_FIELD.clone());
+        }
+        fields
     }
 }
 
@@ -516,6 +890,8 @@ mod tests {
     use lance_linalg::distance::DistanceType;
     use rstest::rstest;
 
+    use crate::vector::bq::storage::RABIT_BLOCKED_EX_CODE_COLUMN;
+
     #[rstest]
     #[case(8)]
     #[case(16)]
@@ -579,6 +955,36 @@ mod tests {
         assert_eq!(*code_bytes, 16);
     }
 
+    #[test]
+    fn test_rabit_quantizer_extra_fields_include_raw_query_error_factor() {
+        let q = RabitQuantizer::new_with_rotation::<Float32Type>(1, 128, RQRotationType::Fast);
+        let fields = q.extra_fields();
+        assert!(
+            fields
+                .iter()
+                .any(|field| field.name() == ERROR_FACTORS_FIELD.name())
+        );
+        assert!(
+            !fields
+                .iter()
+                .any(|field| field.name() == RABIT_BLOCKED_EX_CODE_COLUMN)
+        );
+
+        let q = RabitQuantizer::new_with_rotation::<Float32Type>(3, 128, RQRotationType::Fast);
+        let fields = q.extra_fields();
+        for expected in [
+            ERROR_FACTORS_FIELD.name().as_str(),
+            RABIT_BLOCKED_EX_CODE_COLUMN,
+            EX_ADD_FACTORS_FIELD.name().as_str(),
+            EX_SCALE_FACTORS_FIELD.name().as_str(),
+        ] {
+            assert!(
+                fields.iter().any(|field| field.name().as_str() == expected),
+                "missing {expected}"
+            );
+        }
+    }
+
     #[test]
     fn test_rabit_quantizer_requires_dim_divisible_by_8() {
         let vectors = Float32Array::from(vec![0.0f32; 4 * 30]);
@@ -595,28 +1001,96 @@ mod tests {
     }
 
     #[test]
-    fn test_rabit_quantizer_rejects_unsupported_num_bits() {
+    fn test_rabit_quantizer_reuses_supplied_rotation() {
         let vectors = Float32Array::from(vec![0.0f32; 4 * 32]);
         let fsl = FixedSizeListArray::try_new_from_values(vectors, 32).unwrap();
+        let supplied =
+            RabitQuantizer::new_with_rotation::<Float32Type>(3, 32, RQRotationType::Fast)
+                .metadata(None);
+        let supplied_signs = supplied.fast_rotation_signs.clone();
+
+        let mut params = RQBuildParams::with_rotation_type(3, RQRotationType::Fast);
+        params.rotation = Some(supplied);
+
+        let quantizer = RabitQuantizer::build(&fsl, DistanceType::L2, &params).unwrap();
+        let metadata = quantizer.metadata_ref();
+        assert_eq!(metadata.num_bits, 3);
+        assert_eq!(metadata.rotation_type, RQRotationType::Fast);
+        assert_eq!(metadata.fast_rotation_signs, supplied_signs);
+    }
 
-        let err =
-            RabitQuantizer::build(&fsl, DistanceType::L2, &RQBuildParams::new(0)).unwrap_err();
+    #[test]
+    fn test_rabit_quantizer_validates_supplied_rotation() {
+        let vectors = Float32Array::from(vec![0.0f32; 4 * 32]);
+        let fsl = FixedSizeListArray::try_new_from_values(vectors, 32).unwrap();
+        let supplied =
+            RabitQuantizer::new_with_rotation::<Float32Type>(3, 32, RQRotationType::Fast)
+                .metadata(None);
+
+        let mut wrong_num_bits = supplied.clone();
+        wrong_num_bits.num_bits = 1;
+        let mut params = RQBuildParams::with_rotation_type(3, RQRotationType::Fast);
+        params.rotation = Some(wrong_num_bits);
+        let err = RabitQuantizer::build(&fsl, DistanceType::L2, &params).unwrap_err();
         assert!(
-            err.to_string().contains("IVF_RQ num_bits must be in"),
-            "{}",
-            err
+            err.to_string()
+                .contains("does not match requested num_bits")
         );
 
+        let mut wrong_dim = supplied.clone();
+        wrong_dim.code_dim = 64;
+        let mut params = RQBuildParams::with_rotation_type(3, RQRotationType::Fast);
+        params.rotation = Some(wrong_dim);
+        let err = RabitQuantizer::build(&fsl, DistanceType::L2, &params).unwrap_err();
+        assert!(err.to_string().contains("does not match vector dimension"));
+
+        let mut wrong_sign_len = supplied;
+        wrong_sign_len.fast_rotation_signs.as_mut().unwrap().pop();
+        let mut params = RQBuildParams::with_rotation_type(3, RQRotationType::Fast);
+        params.rotation = Some(wrong_sign_len);
+        let err = RabitQuantizer::build(&fsl, DistanceType::L2, &params).unwrap_err();
+        assert!(err.to_string().contains("fast_rotation_signs length"));
+    }
+
+    #[test]
+    fn test_rabit_quantizer_accepts_multi_bit_range() {
+        let vectors = Float32Array::from(vec![0.0f32; 4 * 32]);
+        let fsl = FixedSizeListArray::try_new_from_values(vectors, 32).unwrap();
+
         let err =
-            RabitQuantizer::build(&fsl, DistanceType::L2, &RQBuildParams::new(2)).unwrap_err();
+            RabitQuantizer::build(&fsl, DistanceType::L2, &RQBuildParams::new(0)).unwrap_err();
         assert!(
-            err.to_string().contains("only num_bits=1 is supported"),
+            err.to_string().contains("IVF_RQ num_bits must be in"),
             "{}",
             err
         );
 
+        for rotation_type in [RQRotationType::Fast, RQRotationType::Matrix] {
+            let quantizer = RabitQuantizer::build(
+                &fsl,
+                DistanceType::L2,
+                &RQBuildParams::with_rotation_type(9, rotation_type),
+            )
+            .unwrap();
+            let quantized = quantizer.quantize_split(&fsl).unwrap();
+            assert!(quantized.ex_codes.is_some());
+            assert_eq!(
+                quantized.binary_codes.as_fixed_size_list().value_length(),
+                4
+            );
+            assert_eq!(
+                quantized
+                    .ex_codes
+                    .unwrap()
+                    .as_fixed_size_list()
+                    .value_length(),
+                // dim=32 is padded to one 64-dim block at ex_bits=8.
+                64
+            );
+        }
+
         let err =
-            RabitQuantizer::build(&fsl, DistanceType::L2, &RQBuildParams::new(9)).unwrap_err();
+            RabitQuantizer::build(&fsl, DistanceType::L2, &RQBuildParams::new(10)).unwrap_err();
         assert!(
             err.to_string().contains("IVF_RQ num_bits must be in"),
             "{}",
diff --git a/rust/lance-index/src/vector/bq/dist_table_quant.rs b/rust/lance-index/src/vector/bq/dist_table_quant.rs
new file mode 100644
index 00000000000..22196f06edb
--- /dev/null
+++ b/rust/lance-index/src/vector/bq/dist_table_quant.rs
@@ -0,0 +1,935 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! SIMD kernels for quantizing the RaBitQ FastScan distance table.
+//!
+//! Once per (query, probed partition) the `dim * 4`-entry `f32` distance
+//! table is quantized into `u8` (fast/normal approx modes) or `u16`
+//! (accurate mode) FastScan LUT entries: a min/max pass over the table
+//! followed by an affine quantize-and-narrow pass. Both passes are branchy
+//! in scalar form, so they get the same runtime-dispatch treatment as
+//! [`super::ex_dot`]: explicit AVX-512/AVX2 kernels on x86_64 and a portable
+//! fold elsewhere that LLVM auto-vectorizes (NEON is part of the aarch64
+//! baseline).
+//!
+//! Table values are sums of rotated-query components: always finite, never
+//! NaN, so lanewise IEEE `min`/`max` matches `total_cmp` ordering. The only
+//! divergence is the sign of zero, which callers cannot observe: `d - qmin`
+//! and the `qmin == qmax` early-out are arithmetically identical either way.
+//!
+//! Quantization rounds half-to-even so that the scalar fallback and the SIMD
+//! kernels agree bit-exactly. All paths round with fixed-mode rounding,
+//! independent of the dynamic MXCSR rounding mode native code may have
+//! installed: the SIMD kernels use the converts' static rounding and the
+//! scalar path (also the SIMD tails) rounds via `f32::floor` rather than
+//! `f32::round_ties_even`, which can lower to an MXCSR-honoring instruction on
+//! x86. Relative to the pre-SIMD implementation (`f32::round`,
+//! half-away-from-zero) this can move a LUT entry by 1 on exact .5 ties, which
+//! is within the table's inherent quantization error.
+
+use std::mem::MaybeUninit;
+use std::sync::LazyLock;
+
+use super::storage::SEGMENT_NUM_CODES;
+
+type MinMaxFn = fn(&[f32]) -> (f32, f32);
+type QuantizeU8Fn = fn(&[f32], f32, f32, &mut [MaybeUninit<u8>]);
+type QuantizeU16Fn = fn(&[f32], f32, f32, &mut [MaybeUninit<u16>]);
+
+/// How the caller reconstructs binary inner-product distances from the
+/// FastScan accumulator sums computed against the quantized LUT.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub enum DistTableDequant {
+    /// Reconstruct each distance with the affine map
+    /// `q_sum * (qmax - qmin) / SCALE + num_tables * qmin`. Returned whenever
+    /// that map is finite, including a zero/sub-resolution range — then the
+    /// LUT is zeroed and every distance collapses to the constant
+    /// `num_tables * qmin`.
+    Affine { qmin: f32, qmax: f32 },
+    /// `num_tables * {qmin, qmax, qmax - qmin}` overflowed f32, so the affine
+    /// reconstruction would yield NaN/inf. The LUT is zeroed; the caller must
+    /// compute exact distances directly from the f32 table.
+    Exact,
+}
+
+/// Quantize `dist_table` into `u8` FastScan LUT entries in the caller-owned
+/// scratch buffer, returning how the caller must dequantize the FastScan
+/// sums (see [`DistTableDequant`]). `dist_table` must be non-empty and all
+/// values finite.
+pub fn quantize_dist_table_into(
+    dist_table: &[f32],
+    quantized_dist_table: &mut Vec<u8>,
+) -> DistTableDequant {
+    debug_assert!(!dist_table.is_empty(), "dist table must be non-empty");
+    let (qmin, qmax) = min_max(dist_table);
+    if dequant_overflows(dist_table.len(), qmin, qmax) {
+        // The caller's affine reconstruction would be non-finite; it computes
+        // exact distances and ignores the LUT, but keep the buffer valid.
+        quantized_dist_table.clear();
+        quantized_dist_table.resize(dist_table.len(), 0);
+        return DistTableDequant::Exact;
+    }
+    let factor = u8::MAX as f32 / (qmax - qmin);
+    if !factor.is_finite() {
+        // Zero or sub-u8-resolution range (e.g. an all-zeros query): the LUT
+        // carries no information, but the finite affine map sends every sum
+        // to the constant `num_tables * qmin`.
+        quantized_dist_table.clear();
+        quantized_dist_table.resize(dist_table.len(), 0);
+        return DistTableDequant::Affine { qmin, qmax };
+    }
+    quantized_dist_table.clear();
+    quantized_dist_table.reserve(dist_table.len());
+    quantize_u8(
+        dist_table,
+        qmin,
+        factor,
+        &mut quantized_dist_table.spare_capacity_mut()[..dist_table.len()],
+    );
+    // SAFETY: the kernel initialized every element in the reserved range.
+    unsafe {
+        quantized_dist_table.set_len(dist_table.len());
+    }
+    DistTableDequant::Affine { qmin, qmax }
+}
+
+/// `u16` variant of [`quantize_dist_table_into`] for the accurate approx
+/// mode.
+pub fn quantize_dist_table_u16_into(
+    dist_table: &[f32],
+    quantized_dist_table: &mut Vec<u16>,
+) -> DistTableDequant {
+    debug_assert!(!dist_table.is_empty(), "dist table must be non-empty");
+    let (qmin, qmax) = min_max(dist_table);
+    if dequant_overflows(dist_table.len(), qmin, qmax) {
+        quantized_dist_table.clear();
+        quantized_dist_table.resize(dist_table.len(), 0);
+        return DistTableDequant::Exact;
+    }
+    let factor = u16::MAX as f32 / (qmax - qmin);
+    if !factor.is_finite() {
+        quantized_dist_table.clear();
+        quantized_dist_table.resize(dist_table.len(), 0);
+        return DistTableDequant::Affine { qmin, qmax };
+    }
+    quantized_dist_table.clear();
+    quantized_dist_table.reserve(dist_table.len());
+    quantize_u16(
+        dist_table,
+        qmin,
+        factor,
+        &mut quantized_dist_table.spare_capacity_mut()[..dist_table.len()],
+    );
+    // SAFETY: the kernel initialized every element in the reserved range.
+    unsafe {
+        quantized_dist_table.set_len(dist_table.len());
+    }
+    DistTableDequant::Affine { qmin, qmax }
+}
+
+/// Whether the caller's affine dequantization
+/// `q_sum * (qmax - qmin) / SCALE + num_tables * qmin` would overflow `f32`
+/// for some row. Each row's reconstructed binary IP lies in
+/// `[num_tables * qmin, num_tables * qmax]` and its quantized term is at most
+/// `num_tables * (qmax - qmin)`, so if any of those is non-finite the table
+/// must fall back to exact distances. The bound is scale-independent — the
+/// `1 / SCALE` factor and the `q_sum <= num_tables * SCALE` range cancel.
+/// Real dist tables are bounded sums of rotated-query components and never
+/// approach this; the guard exists so a pathological query degrades to exact
+/// distances instead of producing NaN.
+fn dequant_overflows(table_len: usize, qmin: f32, qmax: f32) -> bool {
+    let num_tables = (table_len / SEGMENT_NUM_CODES) as f32;
+    !(num_tables * qmin).is_finite()
+        || !(num_tables * qmax).is_finite()
+        || !(num_tables * (qmax - qmin)).is_finite()
+}
+
+fn min_max(values: &[f32]) -> (f32, f32) {
+    static KERNEL: LazyLock<MinMaxFn> = LazyLock::new(select_min_max);
+    KERNEL(values)
+}
+
+fn quantize_u8(values: &[f32], qmin: f32, factor: f32, out: &mut [MaybeUninit<u8>]) {
+    static KERNEL: LazyLock<QuantizeU8Fn> = LazyLock::new(select_quantize_u8);
+    KERNEL(values, qmin, factor, out)
+}
+
+fn quantize_u16(values: &[f32], qmin: f32, factor: f32, out: &mut [MaybeUninit<u16>]) {
+    static KERNEL: LazyLock<QuantizeU16Fn> = LazyLock::new(select_quantize_u16);
+    KERNEL(values, qmin, factor, out)
+}
+
+fn select_min_max() -> MinMaxFn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::arch::is_x86_feature_detected!("avx512f") {
+            return x86::min_max_avx512_dispatch;
+        }
+        if std::arch::is_x86_feature_detected!("avx2") {
+            return x86::min_max_avx2_dispatch;
+        }
+    }
+    min_max_fold
+}
+
+fn select_quantize_u8() -> QuantizeU8Fn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::arch::is_x86_feature_detected!("avx512f") {
+            return x86::quantize_u8_avx512_dispatch;
+        }
+        if std::arch::is_x86_feature_detected!("avx2") {
+            return x86::quantize_u8_avx2_dispatch;
+        }
+    }
+    quantize_u8_scalar
+}
+
+fn select_quantize_u16() -> QuantizeU16Fn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::arch::is_x86_feature_detected!("avx512f") {
+            return x86::quantize_u16_avx512_dispatch;
+        }
+        if std::arch::is_x86_feature_detected!("avx2") {
+            return x86::quantize_u16_avx2_dispatch;
+        }
+    }
+    quantize_u16_scalar
+}
+
+const FOLD_LANES: usize = 16;
+
+/// Portable 16-lane min/max fold; the scalar fallback and the aarch64 path.
+/// The `if` comparisons (rather than `f32::min`/`max`, which carry NaN
+/// bookkeeping) lower to lanewise min/max instructions on targets with
+/// baseline SIMD.
+fn min_max_fold(values: &[f32]) -> (f32, f32) {
+    let mut mins = [f32::INFINITY; FOLD_LANES];
+    let mut maxs = [f32::NEG_INFINITY; FOLD_LANES];
+    let mut chunks = values.chunks_exact(FOLD_LANES);
+    for chunk in &mut chunks {
+        let chunk: &[f32; FOLD_LANES] = chunk.try_into().expect("chunks_exact length");
+        for (i, &v) in chunk.iter().enumerate() {
+            mins[i] = if v < mins[i] { v } else { mins[i] };
+            maxs[i] = if v > maxs[i] { v } else { maxs[i] };
+        }
+    }
+    let mut min = f32::INFINITY;
+    let mut max = f32::NEG_INFINITY;
+    for v in mins {
+        min = if v < min { v } else { min };
+    }
+    for v in maxs {
+        max = if v > max { v } else { max };
+    }
+    for &v in chunks.remainder() {
+        min = if v < min { v } else { min };
+        max = if v > max { v } else { max };
+    }
+    (min, max)
+}
+
+/// Round `x` to the nearest integer, ties to even — the same rule the SIMD
+/// converts use — with fixed-mode operations only, so the result never
+/// depends on the dynamic rounding mode native code may have installed.
+///
+/// On x86, `f32::round_ties_even` can lower to an MXCSR-honoring instruction
+/// (outside an SSE4.1 context), so nearest-even is built from `f32::floor`,
+/// which is always fixed-mode. `x` is a non-negative quantization product, so
+/// only the upward tie case is reachable, but the form is correct for any
+/// finite `x` whose floor fits in `i64`. Elsewhere (e.g. aarch64) the standard
+/// `round_ties_even` is already a fixed-mode instruction (`frintn`) that the
+/// quantize loop — which has no dedicated SIMD kernel there — vectorizes, so
+/// it is kept.
+#[inline(always)]
+fn round_ties_even_fixed(x: f32) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        let lower = x.floor();
+        let frac = x - lower;
+        let round_up = frac > 0.5 || (frac == 0.5 && (lower as i64 & 1) != 0);
+        lower + f32::from(round_up)
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        x.round_ties_even()
+    }
+}
+
+fn quantize_u8_scalar(values: &[f32], qmin: f32, factor: f32, out: &mut [MaybeUninit<u8>]) {
+    debug_assert_eq!(values.len(), out.len());
+    for (quantized, &d) in out.iter_mut().zip(values) {
+        quantized.write(round_ties_even_fixed((d - qmin) * factor) as u8);
+    }
+}
+
+fn quantize_u16_scalar(values: &[f32], qmin: f32, factor: f32, out: &mut [MaybeUninit<u16>]) {
+    debug_assert_eq!(values.len(), out.len());
+    for (quantized, &d) in out.iter_mut().zip(values) {
+        quantized.write(round_ties_even_fixed((d - qmin) * factor) as u16);
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+    use std::mem::MaybeUninit;
+
+    use super::{quantize_u8_scalar, quantize_u16_scalar};
+
+    pub(super) fn min_max_avx512_dispatch(values: &[f32]) -> (f32, f32) {
+        // SAFETY: only selected when AVX-512F was detected.
+        unsafe { min_max_avx512(values) }
+    }
+
+    #[target_feature(enable = "avx512f")]
+    unsafe fn min_max_avx512(values: &[f32]) -> (f32, f32) {
+        // Two accumulators per direction break the lanewise min/max latency
+        // chain; they are reduced once at the end.
+        let mut min0 = _mm512_set1_ps(f32::INFINITY);
+        let mut min1 = min0;
+        let mut max0 = _mm512_set1_ps(f32::NEG_INFINITY);
+        let mut max1 = max0;
+        let mut chunks = values.chunks_exact(32);
+        for chunk in &mut chunks {
+            // SAFETY: the chunk holds 32 consecutive floats.
+            let (v0, v1) = unsafe {
+                (
+                    _mm512_loadu_ps(chunk.as_ptr()),
+                    _mm512_loadu_ps(chunk.as_ptr().add(16)),
+                )
+            };
+            min0 = _mm512_min_ps(min0, v0);
+            max0 = _mm512_max_ps(max0, v0);
+            min1 = _mm512_min_ps(min1, v1);
+            max1 = _mm512_max_ps(max1, v1);
+        }
+        let mut min = _mm512_reduce_min_ps(_mm512_min_ps(min0, min1));
+        let mut max = _mm512_reduce_max_ps(_mm512_max_ps(max0, max1));
+        for &v in chunks.remainder() {
+            min = if v < min { v } else { min };
+            max = if v > max { v } else { max };
+        }
+        (min, max)
+    }
+
+    pub(super) fn min_max_avx2_dispatch(values: &[f32]) -> (f32, f32) {
+        // SAFETY: only selected when AVX2 was detected.
+        unsafe { min_max_avx2(values) }
+    }
+
+    #[target_feature(enable = "avx2")]
+    unsafe fn min_max_avx2(values: &[f32]) -> (f32, f32) {
+        let mut min0 = _mm256_set1_ps(f32::INFINITY);
+        let mut min1 = min0;
+        let mut max0 = _mm256_set1_ps(f32::NEG_INFINITY);
+        let mut max1 = max0;
+        let mut chunks = values.chunks_exact(16);
+        for chunk in &mut chunks {
+            // SAFETY: the chunk holds 16 consecutive floats.
+            let (v0, v1) = unsafe {
+                (
+                    _mm256_loadu_ps(chunk.as_ptr()),
+                    _mm256_loadu_ps(chunk.as_ptr().add(8)),
+                )
+            };
+            min0 = _mm256_min_ps(min0, v0);
+            max0 = _mm256_max_ps(max0, v0);
+            min1 = _mm256_min_ps(min1, v1);
+            max1 = _mm256_max_ps(max1, v1);
+        }
+        let mut min = reduce_min_avx2(_mm256_min_ps(min0, min1));
+        let mut max = reduce_max_avx2(_mm256_max_ps(max0, max1));
+        for &v in chunks.remainder() {
+            min = if v < min { v } else { min };
+            max = if v > max { v } else { max };
+        }
+        (min, max)
+    }
+
+    #[inline]
+    #[target_feature(enable = "avx2")]
+    fn reduce_min_avx2(v: __m256) -> f32 {
+        let halves = _mm_min_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps::<1>(v));
+        let pairs = _mm_min_ps(halves, _mm_movehl_ps(halves, halves));
+        let single = _mm_min_ss(pairs, _mm_shuffle_ps::<0b01>(pairs, pairs));
+        _mm_cvtss_f32(single)
+    }
+
+    #[inline]
+    #[target_feature(enable = "avx2")]
+    fn reduce_max_avx2(v: __m256) -> f32 {
+        let halves = _mm_max_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps::<1>(v));
+        let pairs = _mm_max_ps(halves, _mm_movehl_ps(halves, halves));
+        let single = _mm_max_ss(pairs, _mm_shuffle_ps::<0b01>(pairs, pairs));
+        _mm_cvtss_f32(single)
+    }
+
+    /// Load 16 floats and affine-quantize them into `i32` lanes, rounding to
+    /// nearest-even with static rounding (`_MM_FROUND_TO_NEAREST_INT`) so the
+    /// result does not depend on the dynamic MXCSR rounding mode and matches
+    /// the scalar [`super::round_ties_even_fixed`].
+    #[inline]
+    #[target_feature(enable = "avx512f")]
+    unsafe fn quantize16_epi32(src: *const f32, min: __m512, factor: __m512) -> __m512i {
+        // SAFETY: the caller guarantees 16 floats are readable at `src`.
+        let v = unsafe { _mm512_loadu_ps(src) };
+        let scaled = _mm512_mul_ps(_mm512_sub_ps(v, min), factor);
+        _mm512_cvt_roundps_epi32::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(scaled)
+    }
+
+    pub(super) fn quantize_u8_avx512_dispatch(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u8>],
+    ) {
+        // SAFETY: only selected when AVX-512F was detected.
+        unsafe { quantize_u8_avx512(values, qmin, factor, out) }
+    }
+
+    #[target_feature(enable = "avx512f")]
+    unsafe fn quantize_u8_avx512(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u8>],
+    ) {
+        debug_assert_eq!(values.len(), out.len());
+        let min = _mm512_set1_ps(qmin);
+        let factor_v = _mm512_set1_ps(factor);
+        let full = values.len() - values.len() % 16;
+        let src = values.as_ptr();
+        let dst = out.as_mut_ptr().cast::<u8>();
+        for i in (0..full).step_by(16) {
+            // SAFETY: `i + 16 <= values.len() == out.len()`.
+            unsafe {
+                let q = quantize16_epi32(src.add(i), min, factor_v);
+                // Unsigned-saturating i32 -> u8 narrow: lanes are in
+                // [0, 255] plus float epsilon, which saturation clips.
+                _mm_storeu_si128(dst.add(i).cast(), _mm512_cvtusepi32_epi8(q));
+            }
+        }
+        quantize_u8_scalar(&values[full..], qmin, factor, &mut out[full..]);
+    }
+
+    pub(super) fn quantize_u16_avx512_dispatch(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u16>],
+    ) {
+        // SAFETY: only selected when AVX-512F was detected.
+        unsafe { quantize_u16_avx512(values, qmin, factor, out) }
+    }
+
+    #[target_feature(enable = "avx512f")]
+    unsafe fn quantize_u16_avx512(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u16>],
+    ) {
+        debug_assert_eq!(values.len(), out.len());
+        let min = _mm512_set1_ps(qmin);
+        let factor_v = _mm512_set1_ps(factor);
+        let full = values.len() - values.len() % 16;
+        let src = values.as_ptr();
+        let dst = out.as_mut_ptr().cast::<u16>();
+        for i in (0..full).step_by(16) {
+            // SAFETY: `i + 16 <= values.len() == out.len()`.
+            unsafe {
+                let q = quantize16_epi32(src.add(i), min, factor_v);
+                _mm256_storeu_si256(dst.add(i).cast(), _mm512_cvtusepi32_epi16(q));
+            }
+        }
+        quantize_u16_scalar(&values[full..], qmin, factor, &mut out[full..]);
+    }
+
+    /// Load 8 floats and affine-quantize them into `i32` lanes. AVX2 has no
+    /// embedded-rounding convert, so round to nearest-even explicitly with
+    /// `_mm256_round_ps` (which ignores MXCSR); the subsequent convert then
+    /// sees an integral value, so its dynamic rounding mode cannot change the
+    /// result, keeping it bit-identical to the scalar
+    /// [`super::round_ties_even_fixed`].
+    #[inline]
+    #[target_feature(enable = "avx2")]
+    unsafe fn quantize8_epi32(src: *const f32, min: __m256, factor: __m256) -> __m256i {
+        // SAFETY: the caller guarantees 8 floats are readable at `src`.
+        let v = unsafe { _mm256_loadu_ps(src) };
+        let scaled = _mm256_mul_ps(_mm256_sub_ps(v, min), factor);
+        let rounded = _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(scaled);
+        _mm256_cvtps_epi32(rounded)
+    }
+
+    pub(super) fn quantize_u8_avx2_dispatch(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u8>],
+    ) {
+        // SAFETY: only selected when AVX2 was detected.
+        unsafe { quantize_u8_avx2(values, qmin, factor, out) }
+    }
+
+    #[target_feature(enable = "avx2")]
+    unsafe fn quantize_u8_avx2(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u8>],
+    ) {
+        debug_assert_eq!(values.len(), out.len());
+        let min = _mm256_set1_ps(qmin);
+        let factor_v = _mm256_set1_ps(factor);
+        // The 32->16 and 16->8 packs interleave the two 128-bit lanes; this
+        // permutation of 32-bit groups restores natural order.
+        let restore = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
+        let full = values.len() - values.len() % 32;
+        let src = values.as_ptr();
+        let dst = out.as_mut_ptr().cast::<u8>();
+        for i in (0..full).step_by(32) {
+            // SAFETY: `i + 32 <= values.len() == out.len()`.
+            unsafe {
+                let q0 = quantize8_epi32(src.add(i), min, factor_v);
+                let q1 = quantize8_epi32(src.add(i + 8), min, factor_v);
+                let q2 = quantize8_epi32(src.add(i + 16), min, factor_v);
+                let q3 = quantize8_epi32(src.add(i + 24), min, factor_v);
+                // Unsigned-saturating i32 -> u16 -> u8 narrows: lanes are in
+                // [0, 255] plus float epsilon, which saturation clips.
+                let lo = _mm256_packus_epi32(q0, q1);
+                let hi = _mm256_packus_epi32(q2, q3);
+                let bytes = _mm256_permutevar8x32_epi32(_mm256_packus_epi16(lo, hi), restore);
+                _mm256_storeu_si256(dst.add(i).cast(), bytes);
+            }
+        }
+        quantize_u8_scalar(&values[full..], qmin, factor, &mut out[full..]);
+    }
+
+    pub(super) fn quantize_u16_avx2_dispatch(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u16>],
+    ) {
+        // SAFETY: only selected when AVX2 was detected.
+        unsafe { quantize_u16_avx2(values, qmin, factor, out) }
+    }
+
+    #[target_feature(enable = "avx2")]
+    unsafe fn quantize_u16_avx2(
+        values: &[f32],
+        qmin: f32,
+        factor: f32,
+        out: &mut [MaybeUninit<u16>],
+    ) {
+        debug_assert_eq!(values.len(), out.len());
+        let min = _mm256_set1_ps(qmin);
+        let factor_v = _mm256_set1_ps(factor);
+        let full = values.len() - values.len() % 16;
+        let src = values.as_ptr();
+        let dst = out.as_mut_ptr().cast::<u16>();
+        for i in (0..full).step_by(16) {
+            // SAFETY: `i + 16 <= values.len() == out.len()`.
+            unsafe {
+                let q0 = quantize8_epi32(src.add(i), min, factor_v);
+                let q1 = quantize8_epi32(src.add(i + 8), min, factor_v);
+                // The pack interleaves the 128-bit lanes as
+                // [q0_lo, q1_lo, q0_hi, q1_hi]; the 64-bit-lane permute
+                // restores [q0_lo, q0_hi, q1_lo, q1_hi].
+                let packed = _mm256_packus_epi32(q0, q1);
+                let words = _mm256_permute4x64_epi64::<0b11_01_10_00>(packed);
+                _mm256_storeu_si256(dst.add(i).cast(), words);
+            }
+        }
+        quantize_u16_scalar(&values[full..], qmin, factor, &mut out[full..]);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::rngs::SmallRng;
+    use rand::{Rng, SeedableRng};
+    use rstest::rstest;
+
+    /// Straightforward scalar reference implementing the documented
+    /// semantics: `total_cmp` min/max plus nearest-even rounding.
+    fn reference_min_max(values: &[f32]) -> (f32, f32) {
+        let min = values
+            .iter()
+            .cloned()
+            .min_by(|a, b| a.total_cmp(b))
+            .unwrap();
+        let max = values
+            .iter()
+            .cloned()
+            .max_by(|a, b| a.total_cmp(b))
+            .unwrap();
+        (min, max)
+    }
+
+    fn reference_u8(values: &[f32]) -> (DistTableDequant, Vec<u8>) {
+        let (qmin, qmax) = reference_min_max(values);
+        if dequant_overflows(values.len(), qmin, qmax) {
+            return (DistTableDequant::Exact, vec![0; values.len()]);
+        }
+        let factor = u8::MAX as f32 / (qmax - qmin);
+        if !factor.is_finite() {
+            return (
+                DistTableDequant::Affine { qmin, qmax },
+                vec![0; values.len()],
+            );
+        }
+        let quantized = values
+            .iter()
+            .map(|&d| ((d - qmin) * factor).round_ties_even() as u8)
+            .collect();
+        (DistTableDequant::Affine { qmin, qmax }, quantized)
+    }
+
+    fn reference_u16(values: &[f32]) -> (DistTableDequant, Vec<u16>) {
+        let (qmin, qmax) = reference_min_max(values);
+        if dequant_overflows(values.len(), qmin, qmax) {
+            return (DistTableDequant::Exact, vec![0; values.len()]);
+        }
+        let factor = u16::MAX as f32 / (qmax - qmin);
+        if !factor.is_finite() {
+            return (
+                DistTableDequant::Affine { qmin, qmax },
+                vec![0; values.len()],
+            );
+        }
+        let quantized = values
+            .iter()
+            .map(|&d| ((d - qmin) * factor).round_ties_even() as u16)
+            .collect();
+        (DistTableDequant::Affine { qmin, qmax }, quantized)
+    }
+
+    fn available_kernels() -> Vec<(&'static str, MinMaxFn, QuantizeU8Fn, QuantizeU16Fn)> {
+        // `mut` is only exercised on x86_64 where extra kernels may be pushed.
+        #[allow(unused_mut)]
+        let mut kernels = vec![(
+            "scalar",
+            min_max_fold as MinMaxFn,
+            quantize_u8_scalar as QuantizeU8Fn,
+            quantize_u16_scalar as QuantizeU16Fn,
+        )];
+        #[cfg(target_arch = "x86_64")]
+        {
+            if std::arch::is_x86_feature_detected!("avx2") {
+                kernels.push((
+                    "avx2",
+                    x86::min_max_avx2_dispatch,
+                    x86::quantize_u8_avx2_dispatch,
+                    x86::quantize_u16_avx2_dispatch,
+                ));
+            }
+            if std::arch::is_x86_feature_detected!("avx512f") {
+                kernels.push((
+                    "avx512",
+                    x86::min_max_avx512_dispatch,
+                    x86::quantize_u8_avx512_dispatch,
+                    x86::quantize_u16_avx512_dispatch,
+                ));
+            }
+        }
+        kernels
+    }
+
+    /// Every available kernel must agree bit-exactly with the reference on
+    /// the given input.
+    fn check_against_reference(values: &[f32]) {
+        let (expected_dequant_u8, expected_u8) = reference_u8(values);
+        let (expected_dequant_u16, expected_u16) = reference_u16(values);
+        let (expected_min, expected_max) = reference_min_max(values);
+
+        for (name, min_max_fn, quantize_u8_fn, quantize_u16_fn) in available_kernels() {
+            let (qmin, qmax) = min_max_fn(values);
+            assert_eq!(
+                (qmin, qmax),
+                (expected_min, expected_max),
+                "kernel={name} len={}",
+                values.len()
+            );
+
+            // The quantize kernels are only invoked on the populated path, so
+            // mirror that guard before exercising them directly.
+            let overflows = dequant_overflows(values.len(), qmin, qmax);
+            let factor_u8 = u8::MAX as f32 / (qmax - qmin);
+            if !overflows && factor_u8.is_finite() {
+                let mut out_u8 = Vec::with_capacity(values.len());
+                quantize_u8_fn(
+                    values,
+                    qmin,
+                    factor_u8,
+                    &mut out_u8.spare_capacity_mut()[..values.len()],
+                );
+                // SAFETY: the kernel initialized every element.
+                unsafe { out_u8.set_len(values.len()) };
+                assert_eq!(out_u8, expected_u8, "kernel={name} len={}", values.len());
+            }
+
+            let factor_u16 = u16::MAX as f32 / (qmax - qmin);
+            if !overflows && factor_u16.is_finite() {
+                let mut out_u16 = Vec::with_capacity(values.len());
+                quantize_u16_fn(
+                    values,
+                    qmin,
+                    factor_u16,
+                    &mut out_u16.spare_capacity_mut()[..values.len()],
+                );
+                // SAFETY: the kernel initialized every element.
+                unsafe { out_u16.set_len(values.len()) };
+                assert_eq!(out_u16, expected_u16, "kernel={name} len={}", values.len());
+            }
+        }
+
+        // The public entry points exercise the dispatched kernels, the
+        // dequantization classification, and the scratch-buffer handling.
+        let mut out_u8 = Vec::new();
+        assert_eq!(
+            quantize_dist_table_into(values, &mut out_u8),
+            expected_dequant_u8,
+            "len={}",
+            values.len()
+        );
+        assert_eq!(out_u8, expected_u8, "len={}", values.len());
+        let mut out_u16 = Vec::new();
+        assert_eq!(
+            quantize_dist_table_u16_into(values, &mut out_u16),
+            expected_dequant_u16,
+            "len={}",
+            values.len()
+        );
+        assert_eq!(out_u16, expected_u16, "len={}", values.len());
+    }
+
+    #[rstest]
+    fn test_quantize_matches_reference(
+        #[values(1, 2, 15, 16, 17, 31, 32, 33, 63, 64, 100, 6144, 6160)] len: usize,
+        #[values(1.0, 1e-3, 1e4)] scale: f32,
+    ) {
+        let mut rng = SmallRng::seed_from_u64(42 + len as u64);
+        let values = (0..len)
+            .map(|_| rng.random_range(-scale..scale))
+            .collect::<Vec<_>>();
+        check_against_reference(&values);
+    }
+
+    /// Integer tables with range 510 (resp. 131070) make `factor` exactly
+    /// 0.5, so odd values land on exact .5 ties; all kernels must round them
+    /// to even and agree with each other.
+    #[test]
+    fn test_exact_half_ties_round_to_even() {
+        let values = (0..=510).map(|v| v as f32).collect::<Vec<_>>();
+        check_against_reference(&values);
+        let mut quantized = Vec::new();
+        assert_eq!(
+            quantize_dist_table_into(&values, &mut quantized),
+            DistTableDequant::Affine {
+                qmin: 0.0,
+                qmax: 510.0
+            }
+        );
+        // Spot-check nearest-even: 0.5 -> 0, 1.5 -> 2, 127.5 -> 128,
+        // 254.5 -> 254.
+        assert_eq!(&quantized[..6], &[0, 0, 1, 2, 2, 2]);
+        assert_eq!(quantized[255], 128);
+        assert_eq!(quantized[509], 254);
+        assert_eq!(quantized[510], 255);
+
+        // Integers up to 131070 are exactly representable in f32.
+        let values = (0..=510).map(|v| (v * 257) as f32).collect::<Vec<_>>();
+        check_against_reference(&values);
+        let mut quantized = Vec::new();
+        assert_eq!(
+            quantize_dist_table_u16_into(&values, &mut quantized),
+            DistTableDequant::Affine {
+                qmin: 0.0,
+                qmax: 131070.0
+            }
+        );
+        // value * 0.5 = 128.5 -> 128, 385.5 -> 386 under nearest-even.
+        assert_eq!(&quantized[..4], &[0, 128, 257, 386]);
+        assert_eq!(quantized[510], u16::MAX);
+    }
+
+    #[test]
+    fn test_negative_and_mixed_sign_values() {
+        let mut rng = SmallRng::seed_from_u64(7);
+        let values = (0..1000)
+            .map(|_| rng.random_range(-100.0f32..-1.0))
+            .collect::<Vec<_>>();
+        check_against_reference(&values);
+        let values = (0..999)
+            .map(|i| (i as f32 - 499.5) * 0.75)
+            .collect::<Vec<_>>();
+        check_against_reference(&values);
+    }
+
+    #[rstest]
+    fn test_all_equal_input_zeroes_table(#[values(0.0, -7.25, 3.5)] value: f32) {
+        let values = vec![value; 100];
+        check_against_reference(&values);
+        // Zero range: a zeroed LUT plus the finite affine map (every sum maps
+        // to `num_tables * value`).
+        let expected = DistTableDequant::Affine {
+            qmin: value,
+            qmax: value,
+        };
+        let mut quantized = vec![1u8; 5];
+        assert_eq!(quantize_dist_table_into(&values, &mut quantized), expected);
+        assert_eq!(quantized, vec![0; 100]);
+        let mut quantized = vec![1u16; 5];
+        assert_eq!(
+            quantize_dist_table_u16_into(&values, &mut quantized),
+            expected
+        );
+        assert_eq!(quantized, vec![0; 100]);
+    }
+
+    /// A finite sub-resolution range zeroes the LUT but still dequantizes
+    /// with the finite affine map (`Affine`), whereas a range whose
+    /// `num_tables`-scaled reconstruction overflows must signal `Exact` so the
+    /// caller computes exact distances instead of `0 * inf = NaN`.
+    #[test]
+    fn test_degenerate_range_classification() {
+        // factor = 255 / 1e-38 overflows to +inf, but the reconstruction
+        // (num_tables * {0, 1e-38}) stays finite -> Affine, zeroed LUT.
+        let mut tiny_range = vec![0.0f32; 32];
+        tiny_range[1] = 1e-38;
+        // num_tables * (2e38 - (-2e38)) overflows f32 -> Exact.
+        let mut huge_range = vec![0.0f32; 32];
+        huge_range[0] = -2e38;
+        huge_range[1] = 2e38;
+        // factor = 65535 / 1e-35 overflows only in the u16 variant; the u8
+        // variant still quantizes normally.
+        let mut u16_only = vec![0.0f32; 32];
+        u16_only[1] = 1e-35;
+
+        for values in [&tiny_range, &huge_range, &u16_only] {
+            check_against_reference(values);
+        }
+        let mut quantized_u8 = Vec::new();
+        assert_eq!(
+            quantize_dist_table_into(&tiny_range, &mut quantized_u8),
+            DistTableDequant::Affine {
+                qmin: 0.0,
+                qmax: 1e-38
+            }
+        );
+        assert_eq!(quantized_u8, vec![0; 32]);
+        assert_eq!(
+            quantize_dist_table_into(&huge_range, &mut quantized_u8),
+            DistTableDequant::Exact
+        );
+        assert_eq!(quantized_u8, vec![0; 32]);
+        let mut quantized_u16 = Vec::new();
+        assert_eq!(
+            quantize_dist_table_u16_into(&u16_only, &mut quantized_u16),
+            DistTableDequant::Affine {
+                qmin: 0.0,
+                qmax: 1e-35
+            }
+        );
+        assert_eq!(quantized_u16, vec![0; 32]);
+        assert_eq!(
+            quantize_dist_table_into(&u16_only, &mut quantized_u8),
+            DistTableDequant::Affine {
+                qmin: 0.0,
+                qmax: 1e-35
+            }
+        );
+        assert_eq!(quantized_u8[1], u8::MAX);
+    }
+
+    /// `-0.0 == 0.0` must keep taking the zero-range path (zeroed LUT,
+    /// `Affine`) even though SIMD min/max may pick either sign for the
+    /// extremes.
+    #[test]
+    fn test_signed_zero_mix_zeroes_table() {
+        let mut values = vec![0.0f32; 64];
+        values.iter_mut().step_by(2).for_each(|v| *v = -0.0);
+        let mut quantized = Vec::new();
+        match quantize_dist_table_into(&values, &mut quantized) {
+            DistTableDequant::Affine { qmin, qmax } => assert_eq!(qmin, qmax),
+            other => panic!("expected Affine, got {other:?}"),
+        }
+        assert_eq!(quantized, vec![0; 64]);
+    }
+
+    /// Every quantizer — scalar, AVX2, AVX-512, including the SIMD kernels'
+    /// scalar tails — must round with fixed nearest-even, independent of the
+    /// dynamic MXCSR rounding mode. Run each with MXCSR forced to
+    /// round-toward-zero and require it still matches the nearest-even
+    /// reference (computed under the default mode). `factor == 0.5` puts odd
+    /// integers on exact .5 ties, where truncation (1.5 -> 1) and nearest-even
+    /// (1.5 -> 2) disagree, so a path that honored MXCSR would fail. The
+    /// length (511) is deliberately not a multiple of the SIMD step so the
+    /// kernels' scalar tails are exercised too.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    #[allow(deprecated)] // _mm_getcsr/_mm_setcsr: no stable non-asm replacement.
+    fn test_quantize_rounding_ignores_mxcsr() {
+        use std::arch::x86_64::{_MM_ROUND_MASK, _MM_ROUND_TOWARD_ZERO, _mm_getcsr, _mm_setcsr};
+
+        let values = (0..=510).map(|v| v as f32).collect::<Vec<_>>();
+        // Computed under the default (nearest-even) rounding mode.
+        let (_, expected_u8) = reference_u8(&values);
+        let (_, expected_u16) = reference_u16(&values);
+        let factor_u8 = u8::MAX as f32 / 510.0;
+        let factor_u16 = u16::MAX as f32 / 510.0;
+
+        for (name, _, quantize_u8_fn, quantize_u16_fn) in available_kernels() {
+            let mut out_u8 = Vec::with_capacity(values.len());
+            let mut out_u16 = Vec::with_capacity(values.len());
+            // SAFETY: SSE is baseline on x86_64. MXCSR is restored before any
+            // assertion so a failure cannot leak the truncating mode.
+            let saved = unsafe { _mm_getcsr() };
+            unsafe {
+                _mm_setcsr((saved & !_MM_ROUND_MASK) | _MM_ROUND_TOWARD_ZERO);
+                quantize_u8_fn(
+                    &values,
+                    0.0,
+                    factor_u8,
+                    &mut out_u8.spare_capacity_mut()[..values.len()],
+                );
+                quantize_u16_fn(
+                    &values,
+                    0.0,
+                    factor_u16,
+                    &mut out_u16.spare_capacity_mut()[..values.len()],
+                );
+                _mm_setcsr(saved);
+                out_u8.set_len(values.len());
+                out_u16.set_len(values.len());
+            }
+            assert_eq!(out_u8, expected_u8, "kernel={name} under truncating MXCSR");
+            assert_eq!(
+                out_u16, expected_u16,
+                "kernel={name} under truncating MXCSR"
+            );
+        }
+    }
+
+    /// The scratch buffer must be fully overwritten across reuses with
+    /// different lengths.
+    #[test]
+    fn test_scratch_buffer_reuse() {
+        let mut rng = SmallRng::seed_from_u64(11);
+        let mut scratch_u8 = vec![7u8; 500];
+        let mut scratch_u16 = vec![7u16; 500];
+        for len in [48, 512, 16] {
+            let values = (0..len)
+                .map(|_| rng.random_range(-1.0f32..1.0))
+                .collect::<Vec<_>>();
+            quantize_dist_table_into(&values, &mut scratch_u8);
+            assert_eq!(scratch_u8, reference_u8(&values).1);
+            quantize_dist_table_u16_into(&values, &mut scratch_u16);
+            assert_eq!(scratch_u16, reference_u16(&values).1);
+        }
+    }
+}
diff --git a/rust/lance-index/src/vector/bq/ex_dot.rs b/rust/lance-index/src/vector/bq/ex_dot.rs
new file mode 100644
index 00000000000..1aeb83ba40c
--- /dev/null
+++ b/rust/lance-index/src/vector/bq/ex_dot.rs
@@ -0,0 +1,1078 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Inner-product kernels between an `f32` query and bit-packed RaBitQ ex-codes.
+//!
+//! Multi-bit RaBitQ reranking reduces to `sum_d query[d] * ex_code[d]`, where
+//! `ex_code[d]` is an unsigned `ex_bits`-wide integer. Materializing a
+//! `dim * 2^ex_bits` lookup table and gathering one entry per dimension is
+//! cache-hostile (the table is 1MiB for `ex_bits=8`, `dim=1024`); these kernels
+//! instead unpack the codes with shifts and masks and FMA them against the
+//! query directly, following the kernel design of the RaBitQ reference library
+//! (<https://github.com/VectorDB-NTU/RaBitQ-Library>, Apache-2.0).
+//!
+//! Codes are stored in the *blocked* layout: dims are grouped into 64-dim
+//! blocks (the last block zero-padded) and bit-interleaved within each block
+//! so that the SIMD unpack emits codes in natural dim order:
+//!
+//! ```text
+//! per 64-dim block (T = ex_bits - 1, the top bit; "run k" = dims 16k..16k+16):
+//! 1 bit:  [8B]  bit i of the LE word = dim i
+//! 2 bits: [16B] byte b = dims {b, b+16, b+32, b+48} at bit pairs 0/2/4/6
+//! 3 bits: [16B 2-bit plane as above][8B top-bit plane]
+//! 4 bits: [32B] byte 8j+b = dim 16j+b (low nibble) | dim 16j+8+b (high nibble)
+//! 5 bits: [32B 4-bit plane: byte b = dims b|b+16; byte 16+b = dims b+32|b+48]
+//!         [8B top-bit plane]
+//! 6 bits: [48B] byte 16k+b = dim 16k+b (6 low bits) | bits 2k..2k+2 of
+//!         dim 48+b (2 high bits)
+//! 7 bits: [48B as 6 bits][8B top-bit plane]
+//! 8 bits: [64B] identity
+//! top-bit plane: top bit of dim 16k+b at bit 8*(b%8) + 2k + b/8 of a LE u64
+//! ```
+//!
+//! Because unpack order is natural, the kernels read the rotated query
+//! directly; it only needs zero-padding ([`pad_query_into`]) when the rotated
+//! dim is not a multiple of 64. Legacy indexes store ex codes sequentially
+//! (LSB-first bit stream) and are repacked once at load time
+//! ([`repack_sequential_row`]); for `ex_bits` ∈ {1, 8} the two layouts agree
+//! (modulo trailing padding, which the kernels tolerate) and rows are used as
+//! stored.
+
+use std::sync::LazyLock;
+
+/// Dims are packed in blocks of this size; the query is zero-padded to a
+/// whole number of blocks when the rotated dim is not already a multiple.
+pub const EX_DOT_BLOCK_DIMS: usize = 64;
+
+/// `f32` length of the query consumed by the kernels.
+pub fn padded_query_len(dim: usize) -> usize {
+    dim.next_multiple_of(EX_DOT_BLOCK_DIMS)
+}
+
+/// Whether the legacy sequential layout of a row already matches the blocked
+/// layout (modulo trailing zero padding, which the kernels tolerate), so
+/// legacy rows can be consumed without repacking.
+pub fn sequential_matches_blocked(ex_bits: u8) -> bool {
+    matches!(ex_bits, 1 | 8)
+}
+
+/// Bytes per row of the blocked ex-code layout.
+pub fn blocked_ex_code_bytes(dim: usize, ex_bits: u8) -> usize {
+    debug_assert!((1..=8).contains(&ex_bits));
+    padded_query_len(dim) * ex_bits as usize / 8
+}
+
+/// Dimensions per unpacking group for the given code width.
+fn group_dims(ex_bits: u8) -> usize {
+    match ex_bits {
+        1 | 4 | 8 => 16,
+        _ => EX_DOT_BLOCK_DIMS,
+    }
+}
+
+fn group_bytes(ex_bits: u8) -> usize {
+    group_dims(ex_bits) * ex_bits as usize / 8
+}
+
+/// Extract the `ex_bits`-wide code of `dim_idx` from a sequentially bit-packed
+/// row (LSB-first, codes may straddle byte boundaries).
+#[inline]
+pub fn packed_ex_code_value(row_codes: &[u8], dim_idx: usize, ex_bits: u8) -> u8 {
+    debug_assert!(ex_bits > 0);
+    let bit_offset = dim_idx * ex_bits as usize;
+    let byte_idx = bit_offset / u8::BITS as usize;
+    let bit_shift = bit_offset % u8::BITS as usize;
+    let bits = row_codes[byte_idx] as u16
+        | row_codes
+            .get(byte_idx + 1)
+            .map(|byte| (*byte as u16) << u8::BITS)
+            .unwrap_or_default();
+    let mask = (1u16 << ex_bits) - 1;
+    ((bits >> bit_shift) & mask) as u8
+}
+
+/// Zero-pad the rotated query to a whole number of 64-dim blocks. Only needed
+/// when `dim` is not a multiple of [`EX_DOT_BLOCK_DIMS`]; aligned queries are
+/// passed to the kernels as-is.
+pub fn pad_query_into(rotated_query: &[f32], out: &mut [f32]) {
+    debug_assert_eq!(out.len(), padded_query_len(rotated_query.len()));
+    out[..rotated_query.len()].copy_from_slice(rotated_query);
+    out[rotated_query.len()..].fill(0.0);
+}
+
+/// Pack the top bit of each of 64 codes into a `u64` so kernels can position
+/// it with two shifts per 16-code run: the top bit of dim `16k + b` is stored
+/// at bit `8 * (b % 8) + 2k + b / 8`.
+fn pack_top_plane(block_values: &[u8; 64], top_bit: u8) -> u64 {
+    let mut plane = 0u64;
+    for k in 0..4 {
+        for b in 0..16 {
+            let bit = (block_values[16 * k + b] >> top_bit) & 1;
+            plane |= (bit as u64) << (8 * (b % 8) + 2 * k + b / 8);
+        }
+    }
+    plane
+}
+
+/// Shift `plane` so that its bit `8j + from_bit` lands at bit `8j + to_bit`.
+#[inline(always)]
+fn shift_plane(plane: u64, from_bit: usize, to_bit: usize) -> u64 {
+    if from_bit >= to_bit {
+        plane >> (from_bit - to_bit)
+    } else {
+        plane << (to_bit - from_bit)
+    }
+}
+
+/// Pack one block of 64 code values (natural dim order) into the blocked
+/// layout described in the module docs.
+fn pack_block(ex_bits: u8, block_values: &[u8; 64], out: &mut [u8]) {
+    let v = block_values;
+    match ex_bits {
+        1 => {
+            for (b, byte) in out[..8].iter_mut().enumerate() {
+                *byte = (0..8).fold(0, |acc, t| acc | ((v[8 * b + t] & 1) << t));
+            }
+        }
+        2 | 3 => {
+            for b in 0..16 {
+                out[b] = (v[b] & 0b11)
+                    | ((v[16 + b] & 0b11) << 2)
+                    | ((v[32 + b] & 0b11) << 4)
+                    | ((v[48 + b] & 0b11) << 6);
+            }
+            if ex_bits == 3 {
+                out[16..24].copy_from_slice(&pack_top_plane(v, 2).to_le_bytes());
+            }
+        }
+        4 => {
+            for unit in 0..4 {
+                for b in 0..8 {
+                    out[8 * unit + b] =
+                        (v[16 * unit + b] & 0x0f) | ((v[16 * unit + 8 + b] & 0x0f) << 4);
+                }
+            }
+        }
+        5 => {
+            for b in 0..16 {
+                out[b] = (v[b] & 0x0f) | ((v[16 + b] & 0x0f) << 4);
+                out[16 + b] = (v[32 + b] & 0x0f) | ((v[48 + b] & 0x0f) << 4);
+            }
+            out[32..40].copy_from_slice(&pack_top_plane(v, 4).to_le_bytes());
+        }
+        6 | 7 => {
+            // Runs 0..3 keep their 6 low bits in place; the fourth run's dims
+            // are split into three 2-bit pieces stored in the runs' top bits.
+            for k in 0..3 {
+                for b in 0..16 {
+                    out[16 * k + b] =
+                        (v[16 * k + b] & 0x3f) | (((v[48 + b] >> (2 * k)) & 0b11) << 6);
+                }
+            }
+            if ex_bits == 7 {
+                out[48..56].copy_from_slice(&pack_top_plane(v, 6).to_le_bytes());
+            }
+        }
+        8 => out[..64].copy_from_slice(v),
+        _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+    }
+}
+
+/// Pack one row of unpacked code values (one `u8` per dim) into the blocked
+/// layout; the writer path. `out` must have [`blocked_ex_code_bytes`] bytes.
+pub fn pack_blocked_row(values: &[u8], ex_bits: u8, out: &mut [u8]) {
+    debug_assert_eq!(out.len(), blocked_ex_code_bytes(values.len(), ex_bits));
+    let block_bytes = EX_DOT_BLOCK_DIMS * ex_bits as usize / 8;
+    let mut block_values = [0u8; 64];
+    for (block, out) in out.chunks_exact_mut(block_bytes).enumerate() {
+        let base = block * EX_DOT_BLOCK_DIMS;
+        let count = EX_DOT_BLOCK_DIMS.min(values.len() - base);
+        block_values[..count].copy_from_slice(&values[base..base + count]);
+        block_values[count..].fill(0);
+        pack_block(ex_bits, &block_values, out);
+    }
+}
+
+/// Repack one legacy sequentially bit-packed row into the blocked layout.
+/// `out` must have [`blocked_ex_code_bytes`] bytes.
+pub fn repack_sequential_row(seq_row: &[u8], dim: usize, ex_bits: u8, out: &mut [u8]) {
+    debug_assert_eq!(out.len(), blocked_ex_code_bytes(dim, ex_bits));
+    let block_bytes = EX_DOT_BLOCK_DIMS * ex_bits as usize / 8;
+    let mut block_values = [0u8; 64];
+    for (block, out) in out.chunks_exact_mut(block_bytes).enumerate() {
+        block_values.fill(0);
+        let base = block * EX_DOT_BLOCK_DIMS;
+        let count = EX_DOT_BLOCK_DIMS.min(dim.saturating_sub(base));
+        for (i, value) in block_values[..count].iter_mut().enumerate() {
+            *value = packed_ex_code_value(seq_row, base + i, ex_bits);
+        }
+        pack_block(ex_bits, &block_values, out);
+    }
+}
+
+/// Unpack one code group into per-dim values (natural dim order). Reference
+/// implementation for the SIMD unpackers; also the scalar fallback.
+fn unpack_group(ex_bits: u8, group_codes: &[u8], out: &mut [u8; 64]) {
+    debug_assert_eq!(group_codes.len(), group_bytes(ex_bits));
+    match ex_bits {
+        1 => {
+            for (i, value) in out[..16].iter_mut().enumerate() {
+                *value = (group_codes[i / 8] >> (i % 8)) & 1;
+            }
+        }
+        2 => {
+            for k in 0..4 {
+                for b in 0..16 {
+                    out[16 * k + b] = (group_codes[b] >> (2 * k)) & 0b11;
+                }
+            }
+        }
+        3 => {
+            let plane = u64::from_le_bytes(group_codes[16..24].try_into().unwrap());
+            for k in 0..4 {
+                for b in 0..16 {
+                    let top = (plane >> (8 * (b % 8) + 2 * k + b / 8)) & 1;
+                    out[16 * k + b] = ((group_codes[b] >> (2 * k)) & 0b11) | ((top as u8) << 2);
+                }
+            }
+        }
+        4 => {
+            for b in 0..8 {
+                out[b] = group_codes[b] & 0x0f;
+                out[8 + b] = group_codes[b] >> 4;
+            }
+        }
+        5 => {
+            let plane = u64::from_le_bytes(group_codes[32..40].try_into().unwrap());
+            for k in 0..4 {
+                for b in 0..16 {
+                    let nibble = (group_codes[16 * (k / 2) + b] >> (4 * (k % 2))) & 0x0f;
+                    let top = (plane >> (8 * (b % 8) + 2 * k + b / 8)) & 1;
+                    out[16 * k + b] = nibble | ((top as u8) << 4);
+                }
+            }
+        }
+        6 | 7 => {
+            for k in 0..3 {
+                for b in 0..16 {
+                    out[16 * k + b] = group_codes[16 * k + b] & 0x3f;
+                }
+            }
+            for b in 0..16 {
+                out[48 + b] = (group_codes[b] >> 6)
+                    | ((group_codes[16 + b] >> 6) << 2)
+                    | ((group_codes[32 + b] >> 6) << 4);
+            }
+            if ex_bits == 7 {
+                let plane = u64::from_le_bytes(group_codes[48..56].try_into().unwrap());
+                for k in 0..4 {
+                    for b in 0..16 {
+                        let top = (plane >> (8 * (b % 8) + 2 * k + b / 8)) & 1;
+                        out[16 * k + b] |= (top as u8) << 6;
+                    }
+                }
+            }
+        }
+        8 => out[..16].copy_from_slice(group_codes),
+        _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+    }
+}
+
+/// `sum_d query[d] * code[d]` for one row of blocked-layout codes.
+///
+/// The query must cover a whole number of 64-dim blocks (the rotated query
+/// as-is for aligned dims, otherwise zero-padded via [`pad_query_into`]);
+/// `codes` is the blocked row slice. Rows shorter than the padded query
+/// length are treated as zero-padded.
+pub type ExDotFn = fn(&[f32], &[u8]) -> f32;
+
+/// Resolve the dot kernel for `ex_bits` once; the result can be cached by the
+/// caller for per-candidate use.
+pub fn ex_dot_kernel(ex_bits: u8) -> ExDotFn {
+    debug_assert!((1..=8).contains(&ex_bits));
+    static KERNELS: LazyLock<[ExDotFn; 8]> =
+        LazyLock::new(|| std::array::from_fn(|i| select_ex_dot_kernel(i as u8 + 1)));
+    KERNELS[usize::from(ex_bits) - 1]
+}
+
+fn select_ex_dot_kernel(ex_bits: u8) -> ExDotFn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::arch::is_x86_feature_detected!("avx512f") {
+            return x86::avx512_kernel(ex_bits);
+        }
+        if std::arch::is_x86_feature_detected!("avx2") && std::arch::is_x86_feature_detected!("fma")
+        {
+            return x86::avx2_kernel(ex_bits);
+        }
+    }
+    #[cfg(target_arch = "aarch64")]
+    {
+        // NEON is part of the aarch64 baseline.
+        return neon::kernel(ex_bits);
+    }
+    #[allow(unreachable_code)]
+    scalar_kernel(ex_bits)
+}
+
+fn scalar_kernel(ex_bits: u8) -> ExDotFn {
+    match ex_bits {
+        1 => ex_dot_scalar::<1>,
+        2 => ex_dot_scalar::<2>,
+        3 => ex_dot_scalar::<3>,
+        4 => ex_dot_scalar::<4>,
+        5 => ex_dot_scalar::<5>,
+        6 => ex_dot_scalar::<6>,
+        7 => ex_dot_scalar::<7>,
+        8 => ex_dot_scalar::<8>,
+        _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+    }
+}
+
+fn ex_dot_scalar<const EX_BITS: u8>(ex_query: &[f32], codes: &[u8]) -> f32 {
+    let group_dims = group_dims(EX_BITS);
+    let bytes_per_group = group_bytes(EX_BITS);
+    debug_assert_eq!(ex_query.len() % EX_DOT_BLOCK_DIMS, 0);
+    debug_assert!(codes.len() * u8::BITS as usize <= ex_query.len() * EX_BITS as usize);
+
+    let mut sum = 0.0f32;
+    let mut unpacked = [0u8; 64];
+    let mut padded = [0u8; 56];
+    for (group, query) in ex_query.chunks_exact(group_dims).enumerate() {
+        let start = group * bytes_per_group;
+        if start >= codes.len() {
+            // The remaining query lanes are zero padding.
+            break;
+        }
+        let group_codes = if start + bytes_per_group <= codes.len() {
+            &codes[start..start + bytes_per_group]
+        } else {
+            let avail = codes.len() - start;
+            padded[..bytes_per_group].fill(0);
+            padded[..avail].copy_from_slice(&codes[start..]);
+            &padded[..bytes_per_group]
+        };
+        unpack_group(EX_BITS, group_codes, &mut unpacked);
+        for (q, &code) in query.iter().zip(unpacked[..group_dims].iter()) {
+            sum += q * code as f32;
+        }
+    }
+    sum
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use super::ExDotFn;
+    use std::arch::x86_64::*;
+
+    pub(super) fn avx2_kernel(ex_bits: u8) -> ExDotFn {
+        match ex_bits {
+            1 => dot_u1_avx2_dispatch,
+            2 => dot_u2_avx2_dispatch,
+            3 => dot_u3_avx2_dispatch,
+            4 => dot_u4_avx2_dispatch,
+            5 => dot_u5_avx2_dispatch,
+            6 => dot_u6_avx2_dispatch,
+            7 => dot_u7_avx2_dispatch,
+            8 => dot_u8_avx2_dispatch,
+            _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+        }
+    }
+
+    pub(super) fn avx512_kernel(ex_bits: u8) -> ExDotFn {
+        match ex_bits {
+            1 => dot_u1_avx512_dispatch,
+            2 => dot_u2_avx512_dispatch,
+            3 => dot_u3_avx512_dispatch,
+            4 => dot_u4_avx512_dispatch,
+            5 => dot_u5_avx512_dispatch,
+            6 => dot_u6_avx512_dispatch,
+            7 => dot_u7_avx512_dispatch,
+            8 => dot_u8_avx512_dispatch,
+            _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+        }
+    }
+
+    /// Broadcast a byte to the 8 bytes of a `u64`.
+    #[inline(always)]
+    fn splat_byte(byte: u8) -> u64 {
+        byte as u64 * 0x0101_0101_0101_0101
+    }
+
+    // Unpack helpers. They read exactly one group of code bytes and return
+    // runs of 16 codes matching the kernel-order query. Only SSE2 (baseline on
+    // x86_64) is required.
+
+    /// 16 1-bit codes from 2 bytes: compare each replicated byte against
+    /// per-lane bit masks to turn set bits into 0/1 bytes.
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u1(ptr: *const u8) -> [__m128i; 1] {
+        let (b0, b1) = unsafe { (ptr.read(), ptr.add(1).read()) };
+        let bytes = _mm_set_epi64x(splat_byte(b1) as i64, splat_byte(b0) as i64);
+        let bit_select = _mm_set1_epi64x(0x8040_2010_0804_0201u64 as i64);
+        let selected = _mm_cmpeq_epi8(_mm_and_si128(bytes, bit_select), bit_select);
+        [_mm_and_si128(selected, _mm_set1_epi8(1))]
+    }
+
+    /// 64 2-bit codes from 16 bytes: byte b holds dims 4b..4b+3 at bit pairs.
+    /// The 16-bit shifts drag bits across byte boundaries, which the per-byte
+    /// mask removes.
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u2(ptr: *const u8) -> [__m128i; 4] {
+        let raw = unsafe { _mm_loadu_si128(ptr as *const __m128i) };
+        let mask = _mm_set1_epi8(0b11);
+        [
+            _mm_and_si128(raw, mask),
+            _mm_and_si128(_mm_srli_epi16::<2>(raw), mask),
+            _mm_and_si128(_mm_srli_epi16::<4>(raw), mask),
+            _mm_and_si128(_mm_srli_epi16::<6>(raw), mask),
+        ]
+    }
+
+    /// Position the top-bit plane (see [`super::pack_top_plane`]) of run `k`
+    /// at `top_bit` within each byte.
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    fn top_plane_run(plane: u64, k: usize, top_bit: usize) -> __m128i {
+        let lo = super::shift_plane(plane, 2 * k, top_bit);
+        let hi = super::shift_plane(plane, 2 * k + 1, top_bit);
+        _mm_and_si128(
+            _mm_set_epi64x(hi as i64, lo as i64),
+            _mm_set1_epi8(1 << top_bit),
+        )
+    }
+
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u3(ptr: *const u8) -> [__m128i; 4] {
+        let mut runs = unsafe { unpack_u2(ptr) };
+        let plane = unsafe { (ptr.add(16) as *const u64).read_unaligned() };
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = _mm_or_si128(*run, top_plane_run(plane, k, 2));
+        }
+        runs
+    }
+
+    /// 16 4-bit codes from 8 bytes: low nibbles are the even dims, high
+    /// nibbles the odd dims.
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u4(ptr: *const u8) -> [__m128i; 1] {
+        let word = unsafe { (ptr as *const u64).read_unaligned() };
+        let mask = 0x0f0f_0f0f_0f0f_0f0fu64;
+        [_mm_set_epi64x(
+            ((word >> 4) & mask) as i64,
+            (word & mask) as i64,
+        )]
+    }
+
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u5(ptr: *const u8) -> [__m128i; 4] {
+        let blk0 = unsafe { _mm_loadu_si128(ptr as *const __m128i) };
+        let blk1 = unsafe { _mm_loadu_si128(ptr.add(16) as *const __m128i) };
+        let plane = unsafe { (ptr.add(32) as *const u64).read_unaligned() };
+        let mask = _mm_set1_epi8(0x0f);
+        let mut runs = [
+            _mm_and_si128(blk0, mask),
+            _mm_and_si128(_mm_srli_epi16::<4>(blk0), mask),
+            _mm_and_si128(blk1, mask),
+            _mm_and_si128(_mm_srli_epi16::<4>(blk1), mask),
+        ];
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = _mm_or_si128(*run, top_plane_run(plane, k, 4));
+        }
+        runs
+    }
+
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u6(ptr: *const u8) -> [__m128i; 4] {
+        let blk0 = unsafe { _mm_loadu_si128(ptr as *const __m128i) };
+        let blk1 = unsafe { _mm_loadu_si128(ptr.add(16) as *const __m128i) };
+        let blk2 = unsafe { _mm_loadu_si128(ptr.add(32) as *const __m128i) };
+        let mask6 = _mm_set1_epi8(0x3f);
+        let mask2 = _mm_set1_epi8(0b1100_0000u8 as i8);
+        let stolen = _mm_or_si128(
+            _mm_or_si128(
+                _mm_srli_epi16::<6>(_mm_and_si128(blk0, mask2)),
+                _mm_srli_epi16::<4>(_mm_and_si128(blk1, mask2)),
+            ),
+            _mm_srli_epi16::<2>(_mm_and_si128(blk2, mask2)),
+        );
+        [
+            _mm_and_si128(blk0, mask6),
+            _mm_and_si128(blk1, mask6),
+            _mm_and_si128(blk2, mask6),
+            stolen,
+        ]
+    }
+
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u7(ptr: *const u8) -> [__m128i; 4] {
+        let mut runs = unsafe { unpack_u6(ptr) };
+        let plane = unsafe { (ptr.add(48) as *const u64).read_unaligned() };
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = _mm_or_si128(*run, top_plane_run(plane, k, 6));
+        }
+        runs
+    }
+
+    #[inline]
+    #[target_feature(enable = "sse2")]
+    unsafe fn unpack_u8x16(ptr: *const u8) -> [__m128i; 1] {
+        [unsafe { _mm_loadu_si128(ptr as *const __m128i) }]
+    }
+
+    /// FMA 16 code bytes against 16 query floats (AVX2: two 8-float halves).
+    #[inline]
+    #[target_feature(enable = "avx2", enable = "fma")]
+    unsafe fn fma16_avx2(codes: __m128i, query: *const f32, acc: &mut [__m256; 2]) {
+        let lo = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(codes));
+        acc[0] = _mm256_fmadd_ps(lo, unsafe { _mm256_loadu_ps(query) }, acc[0]);
+        let hi = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(_mm_srli_si128::<8>(codes)));
+        acc[1] = _mm256_fmadd_ps(hi, unsafe { _mm256_loadu_ps(query.add(8)) }, acc[1]);
+    }
+
+    #[inline]
+    #[target_feature(enable = "avx2")]
+    unsafe fn reduce_add_avx2(acc: [__m256; 2]) -> f32 {
+        let v = _mm256_add_ps(acc[0], acc[1]);
+        let halves = _mm_add_ps(_mm256_castps256_ps128(v), _mm256_extractf128_ps::<1>(v));
+        let pairs = _mm_add_ps(halves, _mm_movehl_ps(halves, halves));
+        let total = _mm_add_ss(pairs, _mm_shuffle_ps::<0b01>(pairs, pairs));
+        _mm_cvtss_f32(total)
+    }
+
+    /// FMA 16 code bytes against 16 query floats (AVX-512: one 16-float lane).
+    #[inline]
+    #[target_feature(enable = "avx512f")]
+    unsafe fn fma16_avx512(codes: __m128i, query: *const f32, acc: &mut __m512) {
+        let values = _mm512_cvtepi32_ps(_mm512_cvtepu8_epi32(codes));
+        *acc = _mm512_fmadd_ps(values, unsafe { _mm512_loadu_ps(query) }, *acc);
+    }
+
+    macro_rules! x86_dot_kernel {
+        ($name:ident, $dispatch:ident, $unpack:ident, $ex_bits:expr, $runs:expr) => {
+            #[target_feature(enable = "avx2", enable = "fma")]
+            unsafe fn $name(ex_query: &[f32], codes: &[u8]) -> f32 {
+                const GROUP_DIMS: usize = if $runs == 1 { 16 } else { 64 };
+                const GROUP_BYTES: usize = GROUP_DIMS * $ex_bits / 8;
+                debug_assert_eq!(ex_query.len() % super::EX_DOT_BLOCK_DIMS, 0);
+                debug_assert!(codes.len() * 8 <= ex_query.len() * $ex_bits);
+
+                let groups = ex_query.len() / GROUP_DIMS;
+                let full_groups = (codes.len() / GROUP_BYTES).min(groups);
+                // Two accumulators per run position break the FMA latency
+                // chain; they are summed once at the end.
+                let mut acc = [_mm256_setzero_ps(); 2];
+                for group in 0..full_groups {
+                    // SAFETY: `group < full_groups` keeps both the code group
+                    // and the query run in bounds.
+                    let runs = unsafe { $unpack(codes.as_ptr().add(group * GROUP_BYTES)) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_avx2(
+                                codes16,
+                                ex_query.as_ptr().add(group * GROUP_DIMS + run * 16),
+                                &mut acc,
+                            )
+                        };
+                    }
+                }
+                let consumed = full_groups * GROUP_BYTES;
+                if consumed < codes.len() && full_groups < groups {
+                    // Zero-pad the final partial code group on the stack.
+                    let mut padded = [0u8; GROUP_BYTES];
+                    padded[..codes.len() - consumed].copy_from_slice(&codes[consumed..]);
+                    let runs = unsafe { $unpack(padded.as_ptr()) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_avx2(
+                                codes16,
+                                ex_query.as_ptr().add(full_groups * GROUP_DIMS + run * 16),
+                                &mut acc,
+                            )
+                        };
+                    }
+                }
+                unsafe { reduce_add_avx2(acc) }
+            }
+
+            fn $dispatch(ex_query: &[f32], codes: &[u8]) -> f32 {
+                // SAFETY: only selected when AVX2 and FMA were detected.
+                unsafe { $name(ex_query, codes) }
+            }
+        };
+    }
+
+    macro_rules! x86_dot_kernel_avx512 {
+        ($name:ident, $dispatch:ident, $unpack:ident, $ex_bits:expr, $runs:expr) => {
+            #[target_feature(enable = "avx512f")]
+            unsafe fn $name(ex_query: &[f32], codes: &[u8]) -> f32 {
+                const GROUP_DIMS: usize = if $runs == 1 { 16 } else { 64 };
+                const GROUP_BYTES: usize = GROUP_DIMS * $ex_bits / 8;
+                debug_assert_eq!(ex_query.len() % super::EX_DOT_BLOCK_DIMS, 0);
+                debug_assert!(codes.len() * 8 <= ex_query.len() * $ex_bits);
+
+                let groups = ex_query.len() / GROUP_DIMS;
+                let full_groups = (codes.len() / GROUP_BYTES).min(groups);
+                // Alternating by group as well as run keeps two independent
+                // FMA chains even for the single-run widths.
+                let mut acc = [_mm512_setzero_ps(); 2];
+                for group in 0..full_groups {
+                    // SAFETY: `group < full_groups` keeps both the code group
+                    // and the query run in bounds.
+                    let runs = unsafe { $unpack(codes.as_ptr().add(group * GROUP_BYTES)) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_avx512(
+                                codes16,
+                                ex_query.as_ptr().add(group * GROUP_DIMS + run * 16),
+                                &mut acc[(group + run) % 2],
+                            )
+                        };
+                    }
+                }
+                let consumed = full_groups * GROUP_BYTES;
+                if consumed < codes.len() && full_groups < groups {
+                    let mut padded = [0u8; GROUP_BYTES];
+                    padded[..codes.len() - consumed].copy_from_slice(&codes[consumed..]);
+                    let runs = unsafe { $unpack(padded.as_ptr()) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_avx512(
+                                codes16,
+                                ex_query.as_ptr().add(full_groups * GROUP_DIMS + run * 16),
+                                &mut acc[(full_groups + run) % 2],
+                            )
+                        };
+                    }
+                }
+                _mm512_reduce_add_ps(_mm512_add_ps(acc[0], acc[1]))
+            }
+
+            fn $dispatch(ex_query: &[f32], codes: &[u8]) -> f32 {
+                // SAFETY: only selected when AVX-512F was detected.
+                unsafe { $name(ex_query, codes) }
+            }
+        };
+    }
+
+    x86_dot_kernel!(dot_u1_avx2, dot_u1_avx2_dispatch, unpack_u1, 1, 1);
+    x86_dot_kernel!(dot_u2_avx2, dot_u2_avx2_dispatch, unpack_u2, 2, 4);
+    x86_dot_kernel!(dot_u3_avx2, dot_u3_avx2_dispatch, unpack_u3, 3, 4);
+    x86_dot_kernel!(dot_u4_avx2, dot_u4_avx2_dispatch, unpack_u4, 4, 1);
+    x86_dot_kernel!(dot_u5_avx2, dot_u5_avx2_dispatch, unpack_u5, 5, 4);
+    x86_dot_kernel!(dot_u6_avx2, dot_u6_avx2_dispatch, unpack_u6, 6, 4);
+    x86_dot_kernel!(dot_u7_avx2, dot_u7_avx2_dispatch, unpack_u7, 7, 4);
+    x86_dot_kernel!(dot_u8_avx2, dot_u8_avx2_dispatch, unpack_u8x16, 8, 1);
+
+    x86_dot_kernel_avx512!(dot_u1_avx512, dot_u1_avx512_dispatch, unpack_u1, 1, 1);
+    x86_dot_kernel_avx512!(dot_u2_avx512, dot_u2_avx512_dispatch, unpack_u2, 2, 4);
+    x86_dot_kernel_avx512!(dot_u3_avx512, dot_u3_avx512_dispatch, unpack_u3, 3, 4);
+    x86_dot_kernel_avx512!(dot_u4_avx512, dot_u4_avx512_dispatch, unpack_u4, 4, 1);
+    x86_dot_kernel_avx512!(dot_u5_avx512, dot_u5_avx512_dispatch, unpack_u5, 5, 4);
+    x86_dot_kernel_avx512!(dot_u6_avx512, dot_u6_avx512_dispatch, unpack_u6, 6, 4);
+    x86_dot_kernel_avx512!(dot_u7_avx512, dot_u7_avx512_dispatch, unpack_u7, 7, 4);
+    x86_dot_kernel_avx512!(dot_u8_avx512, dot_u8_avx512_dispatch, unpack_u8x16, 8, 1);
+}
+
+#[cfg(target_arch = "aarch64")]
+mod neon {
+    use super::ExDotFn;
+    use std::arch::aarch64::*;
+
+    pub(super) fn kernel(ex_bits: u8) -> ExDotFn {
+        match ex_bits {
+            1 => dot_u1_neon_dispatch,
+            2 => dot_u2_neon_dispatch,
+            3 => dot_u3_neon_dispatch,
+            4 => dot_u4_neon_dispatch,
+            5 => dot_u5_neon_dispatch,
+            6 => dot_u6_neon_dispatch,
+            7 => dot_u7_neon_dispatch,
+            8 => dot_u8_neon_dispatch,
+            _ => unreachable!("invalid RabitQ ex_bits={ex_bits}"),
+        }
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u1(ptr: *const u8) -> [uint8x16_t; 1] {
+        let (b0, b1) = unsafe { (ptr.read(), ptr.add(1).read()) };
+        let bytes = vcombine_u8(vdup_n_u8(b0), vdup_n_u8(b1));
+        let bit_select = vreinterpretq_u8_u64(vdupq_n_u64(0x8040_2010_0804_0201));
+        [vandq_u8(vtstq_u8(bytes, bit_select), vdupq_n_u8(1))]
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u2(ptr: *const u8) -> [uint8x16_t; 4] {
+        let raw = unsafe { vld1q_u8(ptr) };
+        let mask = vdupq_n_u8(0b11);
+        [
+            vandq_u8(raw, mask),
+            vandq_u8(vshrq_n_u8::<2>(raw), mask),
+            vandq_u8(vshrq_n_u8::<4>(raw), mask),
+            vshrq_n_u8::<6>(raw),
+        ]
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    fn top_plane_run(plane: u64, k: usize, top_bit: usize) -> uint8x16_t {
+        let lo = super::shift_plane(plane, 2 * k, top_bit);
+        let hi = super::shift_plane(plane, 2 * k + 1, top_bit);
+        vandq_u8(
+            vreinterpretq_u8_u64(vcombine_u64(vcreate_u64(lo), vcreate_u64(hi))),
+            vdupq_n_u8(1 << top_bit),
+        )
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u3(ptr: *const u8) -> [uint8x16_t; 4] {
+        let mut runs = unsafe { unpack_u2(ptr) };
+        let plane = unsafe { (ptr.add(16) as *const u64).read_unaligned() };
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = vorrq_u8(*run, top_plane_run(plane, k, 2));
+        }
+        runs
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u4(ptr: *const u8) -> [uint8x16_t; 1] {
+        let word = unsafe { (ptr as *const u64).read_unaligned() };
+        let mask = 0x0f0f_0f0f_0f0f_0f0fu64;
+        [vreinterpretq_u8_u64(vcombine_u64(
+            vcreate_u64(word & mask),
+            vcreate_u64((word >> 4) & mask),
+        ))]
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u5(ptr: *const u8) -> [uint8x16_t; 4] {
+        let blk0 = unsafe { vld1q_u8(ptr) };
+        let blk1 = unsafe { vld1q_u8(ptr.add(16)) };
+        let plane = unsafe { (ptr.add(32) as *const u64).read_unaligned() };
+        let mask = vdupq_n_u8(0x0f);
+        let mut runs = [
+            vandq_u8(blk0, mask),
+            vshrq_n_u8::<4>(blk0),
+            vandq_u8(blk1, mask),
+            vshrq_n_u8::<4>(blk1),
+        ];
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = vorrq_u8(*run, top_plane_run(plane, k, 4));
+        }
+        runs
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u6(ptr: *const u8) -> [uint8x16_t; 4] {
+        let blk0 = unsafe { vld1q_u8(ptr) };
+        let blk1 = unsafe { vld1q_u8(ptr.add(16)) };
+        let blk2 = unsafe { vld1q_u8(ptr.add(32)) };
+        let mask6 = vdupq_n_u8(0x3f);
+        let stolen = vorrq_u8(
+            vorrq_u8(
+                vshrq_n_u8::<6>(blk0),
+                vshlq_n_u8::<2>(vshrq_n_u8::<6>(blk1)),
+            ),
+            vshlq_n_u8::<4>(vshrq_n_u8::<6>(blk2)),
+        );
+        [
+            vandq_u8(blk0, mask6),
+            vandq_u8(blk1, mask6),
+            vandq_u8(blk2, mask6),
+            stolen,
+        ]
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u7(ptr: *const u8) -> [uint8x16_t; 4] {
+        let mut runs = unsafe { unpack_u6(ptr) };
+        let plane = unsafe { (ptr.add(48) as *const u64).read_unaligned() };
+        for (k, run) in runs.iter_mut().enumerate() {
+            *run = vorrq_u8(*run, top_plane_run(plane, k, 6));
+        }
+        runs
+    }
+
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn unpack_u8x16(ptr: *const u8) -> [uint8x16_t; 1] {
+        [unsafe { vld1q_u8(ptr) }]
+    }
+
+    /// FMA 16 code bytes against 16 query floats over four 4-float lanes.
+    #[inline]
+    #[target_feature(enable = "neon")]
+    unsafe fn fma16_neon(codes: uint8x16_t, query: *const f32, acc: &mut [float32x4_t; 4]) {
+        let lo = vmovl_u8(vget_low_u8(codes));
+        let hi = vmovl_u8(vget_high_u8(codes));
+        let c0 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(lo)));
+        let c1 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(lo)));
+        let c2 = vcvtq_f32_u32(vmovl_u16(vget_low_u16(hi)));
+        let c3 = vcvtq_f32_u32(vmovl_u16(vget_high_u16(hi)));
+        unsafe {
+            acc[0] = vfmaq_f32(acc[0], c0, vld1q_f32(query));
+            acc[1] = vfmaq_f32(acc[1], c1, vld1q_f32(query.add(4)));
+            acc[2] = vfmaq_f32(acc[2], c2, vld1q_f32(query.add(8)));
+            acc[3] = vfmaq_f32(acc[3], c3, vld1q_f32(query.add(12)));
+        }
+    }
+
+    macro_rules! neon_dot_kernel {
+        ($name:ident, $dispatch:ident, $unpack:ident, $ex_bits:expr, $runs:expr) => {
+            #[target_feature(enable = "neon")]
+            unsafe fn $name(ex_query: &[f32], codes: &[u8]) -> f32 {
+                const GROUP_DIMS: usize = if $runs == 1 { 16 } else { 64 };
+                const GROUP_BYTES: usize = GROUP_DIMS * $ex_bits / 8;
+                debug_assert_eq!(ex_query.len() % super::EX_DOT_BLOCK_DIMS, 0);
+                debug_assert!(codes.len() * 8 <= ex_query.len() * $ex_bits);
+
+                let groups = ex_query.len() / GROUP_DIMS;
+                let full_groups = (codes.len() / GROUP_BYTES).min(groups);
+                let mut acc = [vdupq_n_f32(0.0); 4];
+                for group in 0..full_groups {
+                    // SAFETY: `group < full_groups` keeps both the code group
+                    // and the query run in bounds.
+                    let runs = unsafe { $unpack(codes.as_ptr().add(group * GROUP_BYTES)) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_neon(
+                                codes16,
+                                ex_query.as_ptr().add(group * GROUP_DIMS + run * 16),
+                                &mut acc,
+                            )
+                        };
+                    }
+                }
+                let consumed = full_groups * GROUP_BYTES;
+                if consumed < codes.len() && full_groups < groups {
+                    // Zero-pad the final partial code group on the stack.
+                    let mut padded = [0u8; GROUP_BYTES];
+                    padded[..codes.len() - consumed].copy_from_slice(&codes[consumed..]);
+                    let runs = unsafe { $unpack(padded.as_ptr()) };
+                    for (run, codes16) in runs.into_iter().enumerate() {
+                        unsafe {
+                            fma16_neon(
+                                codes16,
+                                ex_query.as_ptr().add(full_groups * GROUP_DIMS + run * 16),
+                                &mut acc,
+                            )
+                        };
+                    }
+                }
+                vaddvq_f32(vaddq_f32(
+                    vaddq_f32(acc[0], acc[1]),
+                    vaddq_f32(acc[2], acc[3]),
+                ))
+            }
+
+            fn $dispatch(ex_query: &[f32], codes: &[u8]) -> f32 {
+                // SAFETY: NEON is part of the aarch64 baseline.
+                unsafe { $name(ex_query, codes) }
+            }
+        };
+    }
+
+    neon_dot_kernel!(dot_u1_neon, dot_u1_neon_dispatch, unpack_u1, 1, 1);
+    neon_dot_kernel!(dot_u2_neon, dot_u2_neon_dispatch, unpack_u2, 2, 4);
+    neon_dot_kernel!(dot_u3_neon, dot_u3_neon_dispatch, unpack_u3, 3, 4);
+    neon_dot_kernel!(dot_u4_neon, dot_u4_neon_dispatch, unpack_u4, 4, 1);
+    neon_dot_kernel!(dot_u5_neon, dot_u5_neon_dispatch, unpack_u5, 5, 4);
+    neon_dot_kernel!(dot_u6_neon, dot_u6_neon_dispatch, unpack_u6, 6, 4);
+    neon_dot_kernel!(dot_u7_neon, dot_u7_neon_dispatch, unpack_u7, 7, 4);
+    neon_dot_kernel!(dot_u8_neon, dot_u8_neon_dispatch, unpack_u8x16, 8, 1);
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::rngs::SmallRng;
+    use rand::{Rng, SeedableRng};
+    use rstest::rstest;
+
+    /// Bit-pack code values sequentially (LSB-first), the on-disk ex-code layout.
+    fn pack_sequential(values: &[u8], ex_bits: u8) -> Vec<u8> {
+        let mut out = vec![0u8; (values.len() * ex_bits as usize).div_ceil(8)];
+        for (dim, &value) in values.iter().enumerate() {
+            let bit_offset = dim * ex_bits as usize;
+            let bits = (value as u16) << (bit_offset % 8);
+            out[bit_offset / 8] |= bits as u8;
+            if bits >> 8 != 0 {
+                out[bit_offset / 8 + 1] |= (bits >> 8) as u8;
+            }
+        }
+        out
+    }
+
+    fn kernel_codes(values: &[u8], dim: usize, ex_bits: u8) -> Vec<u8> {
+        debug_assert_eq!(values.len(), dim);
+        let mut out = vec![0u8; blocked_ex_code_bytes(dim, ex_bits)];
+        pack_blocked_row(values, ex_bits, &mut out);
+        out
+    }
+
+    fn available_kernels(ex_bits: u8) -> Vec<(&'static str, ExDotFn)> {
+        // `mut` is only exercised on x86_64 where extra kernels may be pushed.
+        #[allow(unused_mut)]
+        let mut kernels = vec![
+            ("scalar", scalar_kernel(ex_bits)),
+            ("dispatched", ex_dot_kernel(ex_bits)),
+        ];
+        #[cfg(target_arch = "x86_64")]
+        {
+            if std::arch::is_x86_feature_detected!("avx2")
+                && std::arch::is_x86_feature_detected!("fma")
+            {
+                kernels.push(("avx2", x86::avx2_kernel(ex_bits)));
+            }
+            if std::arch::is_x86_feature_detected!("avx512f") {
+                kernels.push(("avx512", x86::avx512_kernel(ex_bits)));
+            }
+        }
+        kernels
+    }
+
+    #[rstest]
+    fn test_ex_dot_matches_reference(
+        #[values(1, 2, 3, 4, 5, 6, 7, 8)] ex_bits: u8,
+        #[values(7, 16, 60, 64, 100, 128, 1024, 1536, 2048)] dim: usize,
+    ) {
+        let mut rng = SmallRng::seed_from_u64(42 + ex_bits as u64 * 1000 + dim as u64);
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        let values = (0..dim)
+            .map(|_| rng.random_range(0..=max_code))
+            .collect::<Vec<_>>();
+        let query = (0..dim)
+            .map(|_| rng.random_range(-1.0f32..1.0))
+            .collect::<Vec<_>>();
+
+        let expected = query
+            .iter()
+            .zip(values.iter())
+            .map(|(q, &c)| *q as f64 * c as f64)
+            .sum::<f64>();
+
+        let codes = kernel_codes(&values, dim, ex_bits);
+        let mut ex_query = vec![0.0; padded_query_len(dim)];
+        pad_query_into(&query, &mut ex_query);
+
+        let tolerance = 1e-3 * expected.abs().max(1.0);
+        for (name, kernel) in available_kernels(ex_bits) {
+            let actual = kernel(&ex_query, &codes) as f64;
+            assert!(
+                (actual - expected).abs() <= tolerance,
+                "ex_bits={ex_bits} dim={dim} kernel={name}: {actual} != {expected}"
+            );
+        }
+    }
+
+    #[rstest]
+    fn test_unpack_group_roundtrip(#[values(1, 2, 3, 4, 5, 6, 7, 8)] ex_bits: u8) {
+        let mut rng = SmallRng::seed_from_u64(7 + ex_bits as u64);
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        let values = (0..EX_DOT_BLOCK_DIMS)
+            .map(|_| rng.random_range(0..=max_code))
+            .collect::<Vec<_>>();
+        let codes = kernel_codes(&values, EX_DOT_BLOCK_DIMS, ex_bits);
+
+        // Unpacking each kernel group must reproduce the values in natural
+        // dim order.
+        let dims = group_dims(ex_bits);
+        let bytes = group_bytes(ex_bits);
+        let mut unpacked = [0u8; 64];
+        for group in 0..EX_DOT_BLOCK_DIMS / dims {
+            unpack_group(
+                ex_bits,
+                &codes[group * bytes..(group + 1) * bytes],
+                &mut unpacked,
+            );
+            assert_eq!(
+                &unpacked[..dims],
+                &values[group * dims..(group + 1) * dims],
+                "ex_bits={ex_bits} group={group}"
+            );
+        }
+    }
+
+    /// The legacy sequential rows must repack into exactly what the writer
+    /// produces from the unpacked values.
+    #[rstest]
+    fn test_repack_sequential_matches_blocked(
+        #[values(1, 2, 3, 4, 5, 6, 7, 8)] ex_bits: u8,
+        #[values(7, 64, 100, 1536)] dim: usize,
+    ) {
+        let mut rng = SmallRng::seed_from_u64(11 + ex_bits as u64 * 100 + dim as u64);
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        let values = (0..dim)
+            .map(|_| rng.random_range(0..=max_code))
+            .collect::<Vec<_>>();
+        let seq = pack_sequential(&values, ex_bits);
+
+        let mut repacked = vec![0u8; blocked_ex_code_bytes(dim, ex_bits)];
+        repack_sequential_row(&seq, dim, ex_bits, &mut repacked);
+        assert_eq!(repacked, kernel_codes(&values, dim, ex_bits));
+
+        // For the widths where the sequential layout is already blocked
+        // (modulo trailing padding), the raw row must be a prefix.
+        if sequential_matches_blocked(ex_bits) {
+            assert_eq!(&repacked[..seq.len()], &seq);
+            assert!(repacked[seq.len()..].iter().all(|&byte| byte == 0));
+        }
+    }
+
+    /// Dense dim sweep for the bit-plane widths: every tail shape within the
+    /// 64-dim kernel group, plus multi-group sizes.
+    #[rstest]
+    fn test_ex_dot_plane_widths_dense_dims(#[values(3, 5)] ex_bits: u8) {
+        let mut rng = SmallRng::seed_from_u64(97 + ex_bits as u64);
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        for dim in (1..=160).chain([255, 256, 1000, 1536, 2048]) {
+            let values = (0..dim)
+                .map(|_| rng.random_range(0..=max_code))
+                .collect::<Vec<_>>();
+            let query = (0..dim)
+                .map(|_| rng.random_range(-1.0f32..1.0))
+                .collect::<Vec<_>>();
+            let expected = query
+                .iter()
+                .zip(values.iter())
+                .map(|(q, &c)| *q as f64 * c as f64)
+                .sum::<f64>();
+
+            let codes = kernel_codes(&values, dim, ex_bits);
+            let mut ex_query = vec![0.0; padded_query_len(dim)];
+            pad_query_into(&query, &mut ex_query);
+            let tolerance = 1e-3 * expected.abs().max(1.0);
+            for (name, kernel) in available_kernels(ex_bits) {
+                let actual = kernel(&ex_query, &codes) as f64;
+                assert!(
+                    (actual - expected).abs() <= tolerance,
+                    "ex_bits={ex_bits} dim={dim} kernel={name}: {actual} != {expected}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_pad_query_pads_with_zeros() {
+        let query = vec![1.0f32; 100];
+        let mut padded = vec![f32::NAN; padded_query_len(query.len())];
+        pad_query_into(&query, &mut padded);
+        assert_eq!(padded.len(), 128);
+        assert_eq!(&padded[..100], &query[..]);
+        assert!(padded[100..].iter().all(|&value| value == 0.0));
+    }
+}
diff --git a/rust/lance-index/src/vector/bq/prune.rs b/rust/lance-index/src/vector/bq/prune.rs
new file mode 100644
index 00000000000..e67ab6642b8
--- /dev/null
+++ b/rust/lance-index/src/vector/bq/prune.rs
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! SIMD kernels for the RaBitQ top-k lower-bound pruning scan.
+//!
+//! Multi-bit IVF_RQ search gates the exact ex-code rerank with a per-row
+//! distance lower bound: after the binary FastScan fills the per-row binary
+//! inner products, every row of the partition is classified against the query
+//! upper bound and the current top-k heap threshold, and only the survivors
+//! (typically well under 1%) are reranked. The classification is the per-row
+//! formula of `RabitDistCalculator::raw_query_lower_bound`:
+//!
+//! ```text
+//! lower_bound = (binary_ip - 0.5 * sum_q) * scale_factor
+//!             + add_factor + query_factor
+//!             - error_factor * query_error
+//! ```
+//!
+//! These kernels evaluate the formula and both comparisons for
+//! [`PRUNE_LANES`] rows at a time, returning bit masks instead of values so
+//! the caller can skip whole groups (the overwhelmingly common case) and run
+//! the existing scalar rerank only for the surviving lanes.
+//!
+//! Correctness contract:
+//!
+//! - The lower bound is computed with exactly the operation order of the
+//!   scalar helper — multiplies and adds, never FMA. A fused multiply-add
+//!   rounds differently, which could prune a row the scalar code would have
+//!   kept; with bit-identical lower bounds the masks reproduce the scalar
+//!   `>=` decisions exactly, keeping heap contents and prune-stats counters
+//!   unchanged.
+//! - Comparisons use ordered-quiet GE predicates (`_CMP_GE_OQ`), matching
+//!   scalar `>=`: a NaN lower bound is never pruned and falls through to the
+//!   exact rerank.
+//! - The heap threshold may be a stale snapshot (it only ever tightens); the
+//!   caller re-checks surviving lanes against live values, so a stale
+//!   threshold can only over-select survivors, never wrongly prune.
+
+use std::sync::LazyLock;
+
+/// Rows classified per kernel invocation.
+pub const PRUNE_LANES: usize = 16;
+
+/// Per-query constants of the lower-bound formula, mirroring
+/// `RabitDistCalculator::raw_query_lower_bound` term by term.
+#[derive(Debug, Clone, Copy)]
+pub struct LowerBoundTerms {
+    /// `0.5 * sum_q`, subtracted from the binary inner product.
+    pub half_sum_q: f32,
+    pub query_factor: f32,
+    pub query_error: f32,
+}
+
+/// Classify [`PRUNE_LANES`] rows against the pruning bounds.
+///
+/// Arguments are the per-row binary inner products, scale factors, add
+/// factors, and error factors, followed by the formula constants, the query
+/// upper bound, and the heap threshold (`None` while the heap is not full,
+/// which disables the heap mask).
+///
+/// Returns `(pruned_upper_bound, pruned_heap)` masks: bit `i` of
+/// `pruned_upper_bound` is set when `lower_bound[i] >= upper_bound`, and bit
+/// `i` of `pruned_heap` is set when the row is not already pruned by the
+/// upper bound and `lower_bound[i] >= heap_threshold`. Surviving rows are the
+/// zero bits of the OR of both masks.
+pub type PruneMaskFn = fn(
+    &[f32; PRUNE_LANES],
+    &[f32; PRUNE_LANES],
+    &[f32; PRUNE_LANES],
+    &[f32; PRUNE_LANES],
+    LowerBoundTerms,
+    f32,
+    Option<f32>,
+) -> (u16, u16);
+
+/// Resolve the prune-mask kernel for the running CPU once; the result can be
+/// cached by the caller for per-partition use.
+pub fn prune_mask_kernel() -> PruneMaskFn {
+    static KERNEL: LazyLock<PruneMaskFn> = LazyLock::new(select_prune_mask_kernel);
+    *KERNEL
+}
+
+fn select_prune_mask_kernel() -> PruneMaskFn {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if std::arch::is_x86_feature_detected!("avx512f") {
+            return x86::prune_masks_avx512_dispatch;
+        }
+        if std::arch::is_x86_feature_detected!("avx2") {
+            return x86::prune_masks_avx2_dispatch;
+        }
+    }
+    // On aarch64 the plain 16-wide loop auto-vectorizes to NEON (part of the
+    // baseline), so no dedicated kernel is needed.
+    prune_masks_portable
+}
+
+/// Portable implementation; also the reference for the SIMD kernels.
+fn prune_masks_portable(
+    dists: &[f32; PRUNE_LANES],
+    scale_factors: &[f32; PRUNE_LANES],
+    add_factors: &[f32; PRUNE_LANES],
+    error_factors: &[f32; PRUNE_LANES],
+    terms: LowerBoundTerms,
+    upper_bound: f32,
+    heap_threshold: Option<f32>,
+) -> (u16, u16) {
+    let mut lower_bounds = [0.0f32; PRUNE_LANES];
+    for lane in 0..PRUNE_LANES {
+        lower_bounds[lane] = ((dists[lane] - terms.half_sum_q) * scale_factors[lane]
+            + add_factors[lane]
+            + terms.query_factor)
+            - error_factors[lane] * terms.query_error;
+    }
+    let mut pruned_upper_bound = 0u16;
+    for (lane, lower_bound) in lower_bounds.iter().enumerate() {
+        pruned_upper_bound |= u16::from(*lower_bound >= upper_bound) << lane;
+    }
+    let mut pruned_heap = 0u16;
+    if let Some(threshold) = heap_threshold {
+        for (lane, lower_bound) in lower_bounds.iter().enumerate() {
+            pruned_heap |= u16::from(*lower_bound >= threshold) << lane;
+        }
+        pruned_heap &= !pruned_upper_bound;
+    }
+    (pruned_upper_bound, pruned_heap)
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use super::{LowerBoundTerms, PRUNE_LANES};
+    use std::arch::x86_64::*;
+
+    /// Lower bounds for 8 lanes with the scalar operation order (no FMA).
+    #[inline]
+    #[target_feature(enable = "avx")]
+    fn lower_bounds_avx(
+        dists: __m256,
+        scale_factors: __m256,
+        add_factors: __m256,
+        error_factors: __m256,
+        half_sum_q: __m256,
+        query_factor: __m256,
+        query_error: __m256,
+    ) -> __m256 {
+        let binary_distance = _mm256_add_ps(
+            _mm256_add_ps(
+                _mm256_mul_ps(_mm256_sub_ps(dists, half_sum_q), scale_factors),
+                add_factors,
+            ),
+            query_factor,
+        );
+        _mm256_sub_ps(binary_distance, _mm256_mul_ps(error_factors, query_error))
+    }
+
+    #[inline]
+    #[target_feature(enable = "avx")]
+    fn ge_mask_avx(lower_bounds_lo: __m256, lower_bounds_hi: __m256, bound: f32) -> u16 {
+        let bound = _mm256_set1_ps(bound);
+        let lo = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_lo, bound));
+        let hi = _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(lower_bounds_hi, bound));
+        (lo | (hi << 8)) as u16
+    }
+
+    #[target_feature(enable = "avx2")]
+    unsafe fn prune_masks_avx2(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+    ) -> (u16, u16) {
+        let half_sum_q = _mm256_set1_ps(terms.half_sum_q);
+        let query_factor = _mm256_set1_ps(terms.query_factor);
+        let query_error = _mm256_set1_ps(terms.query_error);
+        // SAFETY: the array references guarantee 16 readable floats each.
+        let lower_bounds_lo = unsafe {
+            lower_bounds_avx(
+                _mm256_loadu_ps(dists.as_ptr()),
+                _mm256_loadu_ps(scale_factors.as_ptr()),
+                _mm256_loadu_ps(add_factors.as_ptr()),
+                _mm256_loadu_ps(error_factors.as_ptr()),
+                half_sum_q,
+                query_factor,
+                query_error,
+            )
+        };
+        let lower_bounds_hi = unsafe {
+            lower_bounds_avx(
+                _mm256_loadu_ps(dists.as_ptr().add(8)),
+                _mm256_loadu_ps(scale_factors.as_ptr().add(8)),
+                _mm256_loadu_ps(add_factors.as_ptr().add(8)),
+                _mm256_loadu_ps(error_factors.as_ptr().add(8)),
+                half_sum_q,
+                query_factor,
+                query_error,
+            )
+        };
+        let pruned_upper_bound = ge_mask_avx(lower_bounds_lo, lower_bounds_hi, upper_bound);
+        let pruned_heap = match heap_threshold {
+            Some(threshold) => {
+                ge_mask_avx(lower_bounds_lo, lower_bounds_hi, threshold) & !pruned_upper_bound
+            }
+            None => 0,
+        };
+        (pruned_upper_bound, pruned_heap)
+    }
+
+    pub(super) fn prune_masks_avx2_dispatch(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+    ) -> (u16, u16) {
+        // SAFETY: only selected when AVX2 was detected.
+        unsafe {
+            prune_masks_avx2(
+                dists,
+                scale_factors,
+                add_factors,
+                error_factors,
+                terms,
+                upper_bound,
+                heap_threshold,
+            )
+        }
+    }
+
+    #[target_feature(enable = "avx512f")]
+    unsafe fn prune_masks_avx512(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+    ) -> (u16, u16) {
+        // SAFETY: the array references guarantee 16 readable floats each.
+        let (dists, scale_factors, add_factors, error_factors) = unsafe {
+            (
+                _mm512_loadu_ps(dists.as_ptr()),
+                _mm512_loadu_ps(scale_factors.as_ptr()),
+                _mm512_loadu_ps(add_factors.as_ptr()),
+                _mm512_loadu_ps(error_factors.as_ptr()),
+            )
+        };
+        let binary_distance = _mm512_add_ps(
+            _mm512_add_ps(
+                _mm512_mul_ps(
+                    _mm512_sub_ps(dists, _mm512_set1_ps(terms.half_sum_q)),
+                    scale_factors,
+                ),
+                add_factors,
+            ),
+            _mm512_set1_ps(terms.query_factor),
+        );
+        let lower_bounds = _mm512_sub_ps(
+            binary_distance,
+            _mm512_mul_ps(error_factors, _mm512_set1_ps(terms.query_error)),
+        );
+        let pruned_upper_bound =
+            _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(upper_bound));
+        let pruned_heap = match heap_threshold {
+            Some(threshold) => {
+                _mm512_cmp_ps_mask::<_CMP_GE_OQ>(lower_bounds, _mm512_set1_ps(threshold))
+                    & !pruned_upper_bound
+            }
+            None => 0,
+        };
+        (pruned_upper_bound, pruned_heap)
+    }
+
+    pub(super) fn prune_masks_avx512_dispatch(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+    ) -> (u16, u16) {
+        // SAFETY: only selected when AVX-512F was detected.
+        unsafe {
+            prune_masks_avx512(
+                dists,
+                scale_factors,
+                add_factors,
+                error_factors,
+                terms,
+                upper_bound,
+                heap_threshold,
+            )
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::rngs::SmallRng;
+    use rand::{Rng, SeedableRng};
+
+    fn available_kernels() -> Vec<(&'static str, PruneMaskFn)> {
+        // `mut` is only exercised on x86_64 where extra kernels may be pushed.
+        #[allow(unused_mut)]
+        let mut kernels = vec![
+            ("portable", prune_masks_portable as PruneMaskFn),
+            ("dispatched", prune_mask_kernel()),
+        ];
+        #[cfg(target_arch = "x86_64")]
+        {
+            if std::arch::is_x86_feature_detected!("avx2") {
+                kernels.push(("avx2", x86::prune_masks_avx2_dispatch));
+            }
+            if std::arch::is_x86_feature_detected!("avx512f") {
+                kernels.push(("avx512", x86::prune_masks_avx512_dispatch));
+            }
+        }
+        kernels
+    }
+
+    /// Per-lane reference mirroring `raw_query_lower_bound` and the scalar
+    /// pruning checks of the top-k scan.
+    fn reference_masks(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+    ) -> (u16, u16) {
+        let mut pruned_upper_bound = 0u16;
+        let mut pruned_heap = 0u16;
+        for lane in 0..PRUNE_LANES {
+            let lower_bound = (dists[lane] - terms.half_sum_q) * scale_factors[lane]
+                + add_factors[lane]
+                + terms.query_factor
+                - error_factors[lane] * terms.query_error;
+            if lower_bound >= upper_bound {
+                pruned_upper_bound |= 1 << lane;
+            } else if heap_threshold.is_some_and(|threshold| lower_bound >= threshold) {
+                pruned_heap |= 1 << lane;
+            }
+        }
+        (pruned_upper_bound, pruned_heap)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn assert_kernels_match_reference(
+        dists: &[f32; PRUNE_LANES],
+        scale_factors: &[f32; PRUNE_LANES],
+        add_factors: &[f32; PRUNE_LANES],
+        error_factors: &[f32; PRUNE_LANES],
+        terms: LowerBoundTerms,
+        upper_bound: f32,
+        heap_threshold: Option<f32>,
+        case: &str,
+    ) {
+        let expected = reference_masks(
+            dists,
+            scale_factors,
+            add_factors,
+            error_factors,
+            terms,
+            upper_bound,
+            heap_threshold,
+        );
+        for (name, kernel) in available_kernels() {
+            let actual = kernel(
+                dists,
+                scale_factors,
+                add_factors,
+                error_factors,
+                terms,
+                upper_bound,
+                heap_threshold,
+            );
+            assert_eq!(
+                actual, expected,
+                "kernel={name} case={case}: masks {actual:04x?} != {expected:04x?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_prune_masks_match_reference_on_random_inputs() {
+        let mut rng = SmallRng::seed_from_u64(42);
+        for round in 0..200 {
+            let mut dists = [0.0f32; PRUNE_LANES];
+            let mut scale_factors = [0.0f32; PRUNE_LANES];
+            let mut add_factors = [0.0f32; PRUNE_LANES];
+            let mut error_factors = [0.0f32; PRUNE_LANES];
+            for lane in 0..PRUNE_LANES {
+                dists[lane] = rng.random_range(-100.0f32..100.0);
+                scale_factors[lane] = rng.random_range(-2.0f32..2.0);
+                add_factors[lane] = rng.random_range(-10.0f32..10.0);
+                error_factors[lane] = rng.random_range(0.0f32..5.0);
+            }
+            let terms = LowerBoundTerms {
+                half_sum_q: rng.random_range(-50.0f32..50.0),
+                query_factor: rng.random_range(-10.0f32..10.0),
+                query_error: rng.random_range(0.0f32..2.0),
+            };
+            let upper_bound = rng.random_range(-50.0f32..50.0);
+            let heap_threshold = if round % 3 == 0 {
+                None
+            } else {
+                Some(rng.random_range(-50.0f32..50.0))
+            };
+            assert_kernels_match_reference(
+                &dists,
+                &scale_factors,
+                &add_factors,
+                &error_factors,
+                terms,
+                upper_bound,
+                heap_threshold,
+                &format!("random round {round}"),
+            );
+        }
+    }
+
+    #[test]
+    fn test_prune_masks_exact_boundaries() {
+        // With scale=1, err=0, half_sum_q=0, query_factor=0 the lower bound
+        // is the input itself, so bounds can be placed exactly on lanes.
+        let dists: [f32; PRUNE_LANES] = std::array::from_fn(|lane| lane as f32);
+        let scale_factors = [1.0f32; PRUNE_LANES];
+        let add_factors = [0.0f32; PRUNE_LANES];
+        let error_factors = [0.0f32; PRUNE_LANES];
+        let terms = LowerBoundTerms {
+            half_sum_q: 0.0,
+            query_factor: 0.0,
+            query_error: 1.0,
+        };
+        // Equality must prune (scalar uses `>=`): lanes 3.. hit the upper
+        // bound, lanes 1..3 hit only the heap threshold.
+        let (pruned_upper_bound, pruned_heap) = prune_masks_portable(
+            &dists,
+            &scale_factors,
+            &add_factors,
+            &error_factors,
+            terms,
+            3.0,
+            Some(1.0),
+        );
+        assert_eq!(pruned_upper_bound, 0xfff8);
+        assert_eq!(pruned_heap, 0x0006);
+        assert_kernels_match_reference(
+            &dists,
+            &scale_factors,
+            &add_factors,
+            &error_factors,
+            terms,
+            3.0,
+            Some(1.0),
+            "exact boundaries",
+        );
+        // No heap threshold: only the upper-bound mask is set.
+        assert_kernels_match_reference(
+            &dists,
+            &scale_factors,
+            &add_factors,
+            &error_factors,
+            terms,
+            3.0,
+            None,
+            "no heap threshold",
+        );
+    }
+
+    #[test]
+    fn test_prune_masks_nan_and_infinity_semantics() {
+        let mut dists = [0.0f32; PRUNE_LANES];
+        dists[0] = f32::NAN;
+        dists[1] = f32::INFINITY;
+        dists[2] = f32::NEG_INFINITY;
+        dists[3] = 1.0;
+        let mut scale_factors = [1.0f32; PRUNE_LANES];
+        scale_factors[4] = f32::NAN;
+        let add_factors = [0.0f32; PRUNE_LANES];
+        let mut error_factors = [0.0f32; PRUNE_LANES];
+        error_factors[5] = f32::INFINITY;
+        let terms = LowerBoundTerms {
+            half_sum_q: 0.0,
+            query_factor: 0.0,
+            query_error: 1.0,
+        };
+        for (upper_bound, heap_threshold) in [
+            (0.5, Some(0.0)),
+            (f32::INFINITY, Some(f32::NEG_INFINITY)),
+            (f32::NAN, Some(f32::NAN)),
+            (0.5, None),
+        ] {
+            assert_kernels_match_reference(
+                &dists,
+                &scale_factors,
+                &add_factors,
+                &error_factors,
+                terms,
+                upper_bound,
+                heap_threshold,
+                &format!("special values ub={upper_bound} thr={heap_threshold:?}"),
+            );
+        }
+        // NaN lower bounds (lane 0 via a NaN binary inner product, lane 4 via
+        // a NaN scale factor) must never be pruned by either mask.
+        let (pruned_upper_bound, pruned_heap) = prune_masks_portable(
+            &dists,
+            &scale_factors,
+            &add_factors,
+            &error_factors,
+            terms,
+            0.5,
+            Some(0.0),
+        );
+        assert_eq!(pruned_upper_bound & 0b1_0001, 0);
+        assert_eq!(pruned_heap & 0b1_0001, 0);
+    }
+}
diff --git a/rust/lance-index/src/vector/bq/rotation.rs b/rust/lance-index/src/vector/bq/rotation.rs
index 4f4895ac198..de8c8acccb3 100644
--- a/rust/lance-index/src/vector/bq/rotation.rs
+++ b/rust/lance-index/src/vector/bq/rotation.rs
@@ -138,9 +138,13 @@ fn sign_bytes_per_round(dim: usize) -> usize {
     dim.div_ceil(8)
 }
 
+pub(crate) fn fast_rotation_signs_len(dim: usize) -> usize {
+    FAST_ROTATION_ROUNDS * sign_bytes_per_round(dim)
+}
+
 pub fn random_fast_rotation_signs(dim: usize) -> Vec<u8> {
     // Each round needs one random sign bit per dimension.
-    let mut signs = vec![0u8; FAST_ROTATION_ROUNDS * sign_bytes_per_round(dim)];
+    let mut signs = vec![0u8; fast_rotation_signs_len(dim)];
     rand::rng().fill_bytes(&mut signs);
     signs
 }
diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs
index ca66a19d858..72fa8d4b056 100644
--- a/rust/lance-index/src/vector/bq/storage.rs
+++ b/rust/lance-index/src/vector/bq/storage.rs
@@ -1,10 +1,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::borrow::Cow;
-use std::collections::HashMap;
+use std::collections::{BinaryHeap, HashMap};
 use std::ops::Sub;
-use std::sync::Arc;
+use std::sync::{
+    Arc, OnceLock,
+    atomic::{AtomicU64, Ordering},
+};
 
 use arrow::array::AsArray;
 use arrow::datatypes::{Float16Type, Float32Type, Float64Type, UInt8Type, UInt64Type};
@@ -14,12 +18,12 @@ use arrow_array::{
 use arrow_schema::{DataType, Field, SchemaRef};
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
-use deepsize::DeepSizeOf;
-use itertools::Itertools;
+use itertools::{Itertools, izip};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, RecordBatchExt};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
-use lance_linalg::distance::{DistanceType, Dot};
+use lance_linalg::distance::{DistanceType, Dot, dot, l2::l2};
 use lance_linalg::simd::{
     self,
     dist_table::{BATCH_SIZE, PERM0, PERM0_INVERSE},
@@ -37,21 +41,185 @@ use serde::{Deserialize, Serialize};
 
 use crate::frag_reuse::FragReuseIndex;
 use crate::pb;
+use crate::vector::ApproxMode;
+use crate::vector::bq::dist_table_quant::{
+    DistTableDequant, quantize_dist_table_into, quantize_dist_table_u16_into,
+};
+use crate::vector::bq::ex_dot::{
+    EX_DOT_BLOCK_DIMS, ExDotFn, blocked_ex_code_bytes, ex_dot_kernel, pad_query_into,
+    padded_query_len, repack_sequential_row, sequential_matches_blocked,
+};
+use crate::vector::bq::prune::{LowerBoundTerms, PRUNE_LANES, prune_mask_kernel};
 use crate::vector::bq::rotation::{apply_fast_rotation, apply_fast_rotation_in_place};
-use crate::vector::bq::transform::{ADD_FACTORS_COLUMN, SCALE_FACTORS_COLUMN};
+use crate::vector::bq::transform::{
+    ADD_FACTORS_COLUMN, ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN,
+    SCALE_FACTORS_COLUMN,
+};
 use crate::vector::bq::{
     RQRotationType, rabit_binary_code_bytes, rabit_ex_bits, rabit_ex_code_bytes,
-    validate_supported_rq_num_bits,
+    validate_rq_num_bits,
 };
+use crate::vector::graph::{OrderedFloat, OrderedNode};
 use crate::vector::pq::storage::transpose;
 use crate::vector::quantizer::{QuantizerMetadata, QuantizerStorage};
-use crate::vector::storage::{DistCalculator, QueryResidual, VectorStore};
+use crate::vector::storage::{
+    DistCalculator, DistanceCalculatorOptions, QueryResidual, RabitRawQueryContext, VectorStore,
+};
 
 pub const RABIT_METADATA_KEY: &str = "lance:rabit";
 pub const RABIT_CODE_COLUMN: &str = "_rabit_codes";
-pub const RABIT_EX_CODE_COLUMN: &str = "_rabit_ex_codes";
+/// Legacy ex-code column: sequential LSB-first bit stream per row. Read-only;
+/// rows are repacked into the blocked layout at load time.
+pub const RABIT_EX_CODE_COLUMN: &str = "__ex_codes";
+/// Ex-code column in the blocked layout consumed by the ex-dot kernels (see
+/// `ex_dot` module docs). Indexes written with this column cannot be read by
+/// older versions, which fail with a missing-column error instead of
+/// misinterpreting the bytes.
+pub const RABIT_BLOCKED_EX_CODE_COLUMN: &str = "__blocked_ex_codes";
 pub const SEGMENT_LENGTH: usize = 4;
 pub const SEGMENT_NUM_CODES: usize = 1 << SEGMENT_LENGTH;
+const RABIT_PRUNE_STATS_ENV: &str = "LANCE_RQ_PRUNE_STATS";
+const RABIT_PRUNE_STATS_INTERVAL_ENV: &str = "LANCE_RQ_PRUNE_STATS_INTERVAL";
+const DEFAULT_RABIT_PRUNE_STATS_INTERVAL: u64 = 1024;
+
+#[derive(Default)]
+struct RabitPruneStats {
+    calls: AtomicU64,
+    candidates: AtomicU64,
+    pruned_upper_bound: AtomicU64,
+    pruned_heap: AtomicU64,
+    exact: AtomicU64,
+    exact_rejected: AtomicU64,
+}
+
+#[derive(Default)]
+struct RabitPruneBypassStats {
+    calls: AtomicU64,
+}
+
+static RABIT_PRUNE_STATS: OnceLock<RabitPruneStats> = OnceLock::new();
+static RABIT_PRUNE_BYPASS_STATS: OnceLock<RabitPruneBypassStats> = OnceLock::new();
+static RABIT_PRUNE_STATS_ENABLED: OnceLock<bool> = OnceLock::new();
+static RABIT_PRUNE_STATS_INTERVAL: OnceLock<u64> = OnceLock::new();
+
+fn rabit_prune_stats_enabled() -> bool {
+    *RABIT_PRUNE_STATS_ENABLED.get_or_init(|| match std::env::var(RABIT_PRUNE_STATS_ENV) {
+        Ok(value) => {
+            let value = value.to_ascii_lowercase();
+            !matches!(value.as_str(), "" | "0" | "false" | "off" | "no")
+        }
+        Err(_) => false,
+    })
+}
+
+fn rabit_prune_stats_interval() -> u64 {
+    *RABIT_PRUNE_STATS_INTERVAL.get_or_init(|| {
+        std::env::var(RABIT_PRUNE_STATS_INTERVAL_ENV)
+            .ok()
+            .and_then(|value| value.parse::<u64>().ok())
+            .filter(|interval| *interval > 0)
+            .unwrap_or(DEFAULT_RABIT_PRUNE_STATS_INTERVAL)
+    })
+}
+
+fn ratio(numerator: u64, denominator: u64) -> f64 {
+    if denominator == 0 {
+        0.0
+    } else {
+        numerator as f64 / denominator as f64
+    }
+}
+
+fn emit_rabit_prune_stats(message: &str) {
+    log::warn!(
+        target: "lance_index::vector::bq::prune_stats",
+        "{}",
+        message
+    );
+}
+
+/// Per-scan tallies of the raw-query lower-bound gating, reported through
+/// `record_rabit_prune_stats`.
+#[derive(Default)]
+struct RabitPruneCounters {
+    candidates: usize,
+    pruned_upper_bound: usize,
+    pruned_heap: usize,
+    exact: usize,
+    exact_rejected: usize,
+}
+
+fn record_rabit_prune_stats(counters: &RabitPruneCounters) {
+    if !rabit_prune_stats_enabled() {
+        return;
+    }
+    let RabitPruneCounters {
+        candidates,
+        pruned_upper_bound,
+        pruned_heap,
+        exact,
+        exact_rejected,
+    } = *counters;
+
+    let stats = RABIT_PRUNE_STATS.get_or_init(RabitPruneStats::default);
+    let calls = stats.calls.fetch_add(1, Ordering::Relaxed) + 1;
+    let candidates = stats
+        .candidates
+        .fetch_add(candidates as u64, Ordering::Relaxed)
+        + candidates as u64;
+    let pruned_upper_bound = stats
+        .pruned_upper_bound
+        .fetch_add(pruned_upper_bound as u64, Ordering::Relaxed)
+        + pruned_upper_bound as u64;
+    let pruned_heap = stats
+        .pruned_heap
+        .fetch_add(pruned_heap as u64, Ordering::Relaxed)
+        + pruned_heap as u64;
+    let exact = stats.exact.fetch_add(exact as u64, Ordering::Relaxed) + exact as u64;
+    let exact_rejected = stats
+        .exact_rejected
+        .fetch_add(exact_rejected as u64, Ordering::Relaxed)
+        + exact_rejected as u64;
+    let interval = rabit_prune_stats_interval();
+    if calls.is_multiple_of(interval) {
+        let pruned = pruned_upper_bound + pruned_heap;
+        emit_rabit_prune_stats(&format!(
+            "ivf_rq_prune_stats calls={} candidates={} pruned={} pruned_upper_bound={} pruned_heap={} prune_ratio={:.6} exact={} exact_ratio={:.6} exact_rejected={} exact_reject_ratio={:.6}",
+            calls,
+            candidates,
+            pruned,
+            pruned_upper_bound,
+            pruned_heap,
+            ratio(pruned, candidates),
+            exact,
+            ratio(exact, candidates),
+            exact_rejected,
+            ratio(exact_rejected, exact),
+        ));
+    }
+}
+
+fn record_rabit_prune_bypass(reason: &'static str) {
+    if !rabit_prune_stats_enabled() {
+        return;
+    }
+
+    let stats = RABIT_PRUNE_BYPASS_STATS.get_or_init(RabitPruneBypassStats::default);
+    let calls = stats.calls.fetch_add(1, Ordering::Relaxed) + 1;
+    if calls.is_multiple_of(rabit_prune_stats_interval()) {
+        emit_rabit_prune_stats(&format!(
+            "ivf_rq_prune_stats_bypass calls={} reason={}",
+            calls, reason
+        ));
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum RabitQueryEstimator {
+    ResidualQuery,
+    RawQuery,
+}
 
 pub fn rabit_binary_code_field(rotated_dim: usize) -> Field {
     Field::new(
@@ -70,10 +238,10 @@ pub fn rabit_ex_code_field(rotated_dim: usize, num_bits: u8) -> Result<Option<Fi
         return Ok(None);
     }
     Ok(Some(Field::new(
-        RABIT_EX_CODE_COLUMN,
+        RABIT_BLOCKED_EX_CODE_COLUMN,
         DataType::FixedSizeList(
             Arc::new(Field::new("item", DataType::UInt8, true)),
-            rabit_ex_code_bytes(rotated_dim, ex_bits)? as i32,
+            blocked_ex_code_bytes(rotated_dim, ex_bits) as i32,
         ),
         true,
     )))
@@ -96,6 +264,8 @@ pub struct RabitQuantizationMetadata {
     pub code_dim: u32,
     pub num_bits: u8,
     pub packed: bool,
+    #[serde(default = "default_query_estimator_compat")]
+    pub query_estimator: RabitQueryEstimator,
 }
 
 impl RabitQuantizationMetadata {
@@ -120,6 +290,11 @@ fn default_rotation_type_compat() -> RQRotationType {
     RQRotationType::Matrix
 }
 
+fn default_query_estimator_compat() -> RabitQueryEstimator {
+    // Released IVF_RQ indexes predate this marker and used residual queries.
+    RabitQueryEstimator::ResidualQuery
+}
+
 impl RabitQuantizationMetadata {
     fn code_dim(&self) -> usize {
         self.rotated_dim()
@@ -196,13 +371,44 @@ impl RabitQuantizationMetadata {
             }
         }
     }
+
+    pub fn prepare_raw_query_context(&self, query: &dyn Array) -> Result<RabitRawQueryContext> {
+        validate_rq_num_bits(self.num_bits)?;
+        let code_dim = self.code_dim();
+        let ex_bits = rabit_ex_bits(self.num_bits)?;
+        let dist_table_len = code_dim * 4;
+
+        let mut rotated_query = vec![0.0; code_dim];
+        self.rotate_vector_with_residual_into(query, None, &mut rotated_query);
+
+        let mut dist_table = vec![0.0; dist_table_len];
+        build_dist_table_direct_into::<Float32Type>(&rotated_query, &mut dist_table);
+
+        // The kernels consume the rotated query directly; a zero-padded copy
+        // is only needed when the rotated dim is not block-aligned.
+        let mut ex_query = Vec::new();
+        if ex_bits > 0 && !code_dim.is_multiple_of(EX_DOT_BLOCK_DIMS) {
+            ex_query.resize(padded_query_len(code_dim), 0.0);
+            pad_query_into(&rotated_query, &mut ex_query);
+        }
+
+        let sum_q = rotated_query.iter().copied().sum();
+        Ok(RabitRawQueryContext {
+            code_dim,
+            ex_bits,
+            rotated_query,
+            dist_table,
+            ex_query,
+            sum_q,
+        })
+    }
 }
 
 impl DeepSizeOf for RabitQuantizationMetadata {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.rotate_mat
             .as_ref()
-            .map(|inv_p| inv_p.get_array_memory_size())
+            .map(|inv_p| (inv_p as &dyn arrow_array::Array).deep_size_of_children(context))
             .unwrap_or(0)
             + self
                 .fast_rotation_signs
@@ -283,11 +489,26 @@ pub struct RabitQuantizationStorage {
     codes: FixedSizeListArray,
     add_factors: Float32Array,
     scale_factors: Float32Array,
+    error_factors: Option<Float32Array>,
+    // ex codes in the blocked kernel layout; always aliases the batch column
+    // (legacy sequential batches are normalized at load, replacing the
+    // sequential column with the repacked one, so rewrites emit the blocked
+    // format).
+    ex_codes: Option<FixedSizeListArray>,
+    packed_ex_codes: Option<FixedSizeListArray>,
+    ex_add_factors: Option<Float32Array>,
+    ex_scale_factors: Option<Float32Array>,
 }
 
 impl DeepSizeOf for RabitQuantizationStorage {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.metadata.deep_size_of_children(context) + self.batch.get_array_memory_size()
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.metadata.deep_size_of_children(context)
+            + self.batch.deep_size_of_children(context)
+            + self
+                .packed_ex_codes
+                .as_ref()
+                .map(|codes| (codes as &dyn Array).deep_size_of_children(context))
+                .unwrap_or_default()
     }
 }
 
@@ -296,7 +517,7 @@ impl RabitQuantizationStorage {
         self.metadata.code_dim()
     }
 
-    fn query_factor(&self, dist_q_c: f32) -> f32 {
+    fn residual_query_factor(&self, dist_q_c: f32) -> f32 {
         match self.distance_type {
             DistanceType::L2 => dist_q_c,
             DistanceType::Cosine | DistanceType::Dot => dist_q_c - 1.0,
@@ -307,22 +528,114 @@ impl RabitQuantizationStorage {
         }
     }
 
+    fn raw_query_factor(
+        &self,
+        dist_q_c: f32,
+        rotated_query: &[f32],
+        rotated_centroid: Option<&[f32]>,
+    ) -> f32 {
+        match self.distance_type {
+            DistanceType::L2 => dist_q_c,
+            DistanceType::Dot => rotated_centroid
+                .map(|centroid| -dot(rotated_query, centroid))
+                .unwrap_or(dist_q_c - 1.0),
+            DistanceType::Cosine => dist_q_c - 1.0,
+            _ => unimplemented!(
+                "RabitQ does not support distance type: {}",
+                self.distance_type
+            ),
+        }
+    }
+
+    fn raw_query_error(
+        &self,
+        dist_q_c: f32,
+        rotated_query: &[f32],
+        rotated_centroid: Option<&[f32]>,
+    ) -> f32 {
+        match self.distance_type {
+            DistanceType::L2 => dist_q_c.max(0.0).sqrt(),
+            DistanceType::Dot => rotated_centroid
+                .map(|centroid| l2(rotated_query, centroid).sqrt())
+                .unwrap_or_else(|| dist_q_c.max(0.0).sqrt()),
+            DistanceType::Cosine => dist_q_c.max(0.0).sqrt(),
+            _ => unimplemented!(
+                "RabitQ does not support distance type: {}",
+                self.distance_type
+            ),
+        }
+    }
+
+    fn uses_raw_query_lower_bound_gating(&self) -> bool {
+        self.metadata.query_estimator == RabitQueryEstimator::RawQuery
+            && self.metadata.num_bits > 1
+            && self.error_factors.is_some()
+    }
+
+    fn raw_query_error_for_gating(
+        &self,
+        dist_q_c: f32,
+        rotated_query: &[f32],
+        rotated_centroid: Option<&[f32]>,
+    ) -> f32 {
+        if self.uses_raw_query_lower_bound_gating() {
+            self.raw_query_error(dist_q_c, rotated_query, rotated_centroid)
+        } else {
+            0.0
+        }
+    }
+
     fn distance_calculator_from_parts<'a>(
         &'a self,
-        dim: usize,
-        dist_q_c: f32,
-        dist_table: Cow<'a, [f32]>,
-        sum_q: f32,
+        parts: RabitDistCalculatorParts<'a>,
     ) -> RabitDistCalculator<'a> {
+        let RabitDistCalculatorParts {
+            dim,
+            dist_table,
+            ex_query,
+            sum_q,
+            query_factor,
+            query_error,
+            approx_mode,
+        } = parts;
+        let ex_code_len = self
+            .ex_codes
+            .as_ref()
+            .map(|codes| codes.value_length() as usize)
+            .unwrap_or_default();
+        let ex_codes = self
+            .ex_codes
+            .as_ref()
+            .map(|codes| codes.values().as_primitive::<UInt8Type>().values().as_ref());
+        let packed_ex_codes = self
+            .packed_ex_codes
+            .as_ref()
+            .map(|codes| codes.values().as_primitive::<UInt8Type>().values().as_ref());
         RabitDistCalculator::new(
             dim,
             self.metadata.num_bits,
+            self.metadata.query_estimator,
             dist_table,
+            ex_query,
             sum_q,
             self.codes.values().as_primitive::<UInt8Type>().values(),
+            ex_codes,
+            ex_code_len,
             self.add_factors.values(),
             self.scale_factors.values(),
-            self.query_factor(dist_q_c),
+            self.error_factors
+                .as_ref()
+                .map(|factors| factors.values().as_ref()),
+            self.ex_add_factors
+                .as_ref()
+                .map(|factors| factors.values().as_ref()),
+            self.ex_scale_factors
+                .as_ref()
+                .map(|factors| factors.values().as_ref()),
+            packed_ex_codes,
+            query_factor,
+            query_error,
+            approx_mode,
         )
     }
 
@@ -489,17 +802,68 @@ fn copy_subtract_f32(lhs: &[f32], rhs: &[f32], output: &mut [f32]) {
     }
 }
 
+struct RabitDistCalculatorParts<'a> {
+    dim: usize,
+    dist_table: Cow<'a, [f32]>,
+    ex_query: Cow<'a, [f32]>,
+    sum_q: f32,
+    query_factor: f32,
+    query_error: f32,
+    approx_mode: ApproxMode,
+}
+
+/// Loop-invariant inputs of the raw-query multi-bit top-k scans: the row
+/// count, the resolved ex-code state for exact reranking, and the query
+/// bounds.
+struct RawQueryTopkContext<'a> {
+    n: usize,
+    k: usize,
+    ex_bits: u8,
+    ex_codes: &'a [u8],
+    ex_add_factors: &'a [f32],
+    ex_scale_factors: &'a [f32],
+    query_lower_bound: f32,
+    query_upper_bound: f32,
+}
+
+/// Pick the query slice the ex-dot kernels consume: the rotated query itself
+/// when the dim is block-aligned, otherwise a zero-padded copy.
+fn kernel_query<'a>(rotated_query: &'a [f32], padded: &'a [f32]) -> &'a [f32] {
+    if rotated_query.len().is_multiple_of(EX_DOT_BLOCK_DIMS) {
+        rotated_query
+    } else {
+        padded
+    }
+}
+
 pub struct RabitDistCalculator<'a> {
     dim: usize,
+    num_bits: u8,
+    query_estimator: RabitQueryEstimator,
     // n * d / 8 binary-code bytes
     codes: &'a [u8],
+    // per-row ex codes in the blocked kernel layout
+    ex_codes: Option<&'a [u8]>,
+    // bytes per ex-code row; legacy rows for layout-compatible widths may be
+    // shorter than the blocked size, which the kernels treat as zero padding
+    ex_code_len: usize,
     // this is a flattened 2D array of size d/4 * 16,
     // we split the query codes into d/4 chunks, each chunk is with 4 elements,
     // then dist_table[i][j] is the distance between the i-th query code and the code j
     dist_table: Cow<'a, [f32]>,
+    // the rotated query, zero-padded to a 64-dim multiple when needed; also
+    // the source for the FastScan ex LUT on the legacy bypass path
+    ex_query: Cow<'a, [f32]>,
+    ex_dot: Option<ExDotFn>,
     add_factors: &'a [f32],
     scale_factors: &'a [f32],
+    error_factors: Option<&'a [f32]>,
+    ex_add_factors: Option<&'a [f32]>,
+    ex_scale_factors: Option<&'a [f32]>,
+    packed_ex_codes: Option<&'a [u8]>,
     query_factor: f32,
+    query_error: f32,
+    approx_mode: ApproxMode,
 
     sum_q: f32,
     sqrt_d: f32,
@@ -510,156 +874,107 @@ impl<'a> RabitDistCalculator<'a> {
     pub fn new(
         dim: usize,
         num_bits: u8,
+        query_estimator: RabitQueryEstimator,
         dist_table: Cow<'a, [f32]>,
+        ex_query: Cow<'a, [f32]>,
         sum_q: f32,
         codes: &'a [u8],
+        ex_codes: Option<&'a [u8]>,
+        ex_code_len: usize,
         add_factors: &'a [f32],
         scale_factors: &'a [f32],
+        error_factors: Option<&'a [f32]>,
+        ex_add_factors: Option<&'a [f32]>,
+        ex_scale_factors: Option<&'a [f32]>,
+        packed_ex_codes: Option<&'a [u8]>,
         query_factor: f32,
+        query_error: f32,
+        approx_mode: ApproxMode,
     ) -> Self {
+        let ex_dot = (num_bits > 1).then(|| ex_dot_kernel(num_bits - 1));
         Self {
             dim,
+            num_bits,
+            query_estimator,
             codes,
+            ex_codes,
+            ex_code_len,
             dist_table,
+            ex_query,
+            ex_dot,
             add_factors,
             scale_factors,
+            error_factors,
+            ex_add_factors,
+            ex_scale_factors,
+            packed_ex_codes,
             query_factor,
+            query_error,
+            approx_mode,
             sqrt_d: (dim as f32 * num_bits as f32).sqrt(),
             sum_q,
         }
     }
-}
-
-#[inline]
-fn lowbit(x: usize) -> usize {
-    1 << x.trailing_zeros()
-}
-
-#[inline]
-pub fn build_dist_table_direct<T: ArrowFloatType>(qc: &[T::Native]) -> Vec<f32>
-where
-    T::Native: AsPrimitive<f32>,
-{
-    // every 4 bits (SEGMENT_LENGTH) is a segment, and we need to compute the distance between the segment and all the codes
-    // so there are dim/4 segments, and the number of codes is 16 (2^{SEGMENT_LENGTH}),
-    // so we have dim/4 * 16 = dim * 4 elements in the dist_table
-    let mut dist_table = vec![0.0; qc.len() * 4];
-    build_dist_table_direct_into::<T>(qc, &mut dist_table);
-    dist_table
-}
-
-fn build_dist_table_direct_into<T: ArrowFloatType>(qc: &[T::Native], dist_table: &mut [f32])
-where
-    T::Native: AsPrimitive<f32>,
-{
-    debug_assert_eq!(dist_table.len(), qc.len() * 4);
-    qc.chunks_exact(SEGMENT_LENGTH)
-        .zip(dist_table.chunks_exact_mut(SEGMENT_NUM_CODES))
-        .for_each(|(sub_vec, dist_table)| {
-            dist_table[0] = 0.0;
-            build_dist_table_for_subvec::<T>(sub_vec, dist_table);
-        });
-}
-
-#[inline(always)]
-fn build_dist_table_for_subvec<T: ArrowFloatType>(sub_vec: &[T::Native], dist_table: &mut [f32])
-where
-    T::Native: AsPrimitive<f32>,
-{
-    // skip 0 because it's always 0
-    (1..SEGMENT_NUM_CODES).for_each(|j| {
-        // this is a little bit tricky,
-        // j represents a subset of 4 bits, that if the i-th bit of `j` is 1,
-        // then we need to add the distance of the i-th dim of the segment.
-        // but we don't need to check all bits of `j`,
-        // because `j` = `j - lowbit(j)` + `lowbit(j)`,
-        // where `j-lowbit(j)` is less than `j`,
-        // which means dist_table[j-lowbit(j)] is already computed,
-        // and we can use it to compute dist_table[j]
-        // for example, if j = 0b1010, then j - lowbit(j) = 0b1000,
-        // and dist_table[0b1000] is already computed,
-        // so dist_table[0b1010] = dist_table[0b1000] + sub_vec[LOWBIT_IDX[0b1010]];
-        // where lowbit(0b1010) = 0b10, LOWBIT_IDX[0b1010] = LOWBIT_IDX[0b10] = 1.
-        dist_table[j] = dist_table[j - lowbit(j)] + sub_vec[LOWBIT_IDX[j]].as_();
-    })
-}
-
-// Quantize the distance table into a caller-owned buffer.
-#[inline]
-fn quantize_dist_table_into(dist_table: &[f32], quantized_dist_table: &mut Vec<u8>) -> (f32, f32) {
-    let (qmin, qmax) = dist_table
-        .iter()
-        .cloned()
-        .minmax_by(|a, b| a.total_cmp(b))
-        .into_option()
-        .unwrap();
-    // this happens if the query is all zeros
-    if qmin == qmax {
-        quantized_dist_table.clear();
-        quantized_dist_table.resize(dist_table.len(), 0);
-        return (qmin, qmax);
-    }
-    let factor = 255.0 / (qmax - qmin);
-    quantized_dist_table.clear();
-    quantized_dist_table.reserve(dist_table.len());
-    let spare = quantized_dist_table.spare_capacity_mut();
-    for (quantized, &d) in spare[..dist_table.len()].iter_mut().zip(dist_table.iter()) {
-        quantized.write(((d - qmin) * factor).round() as u8);
-    }
-    // SAFETY: every element in the reserved range was initialized in the loop above.
-    unsafe {
-        quantized_dist_table.set_len(dist_table.len());
-    }
-
-    (qmin, qmax)
-}
-
-impl DistCalculator for RabitDistCalculator<'_> {
-    #[inline(always)]
-    fn distance(&self, id: u32) -> f32 {
-        let id = id as usize;
-        let code_len = rabit_binary_code_bytes(self.dim);
-        let num_vectors = self.codes.len() / code_len;
-        let dist =
-            compute_single_rq_distance(self.codes, id, num_vectors, code_len, &self.dist_table);
 
-        // distance between quantized vector and query vector
-        let dist_vq_qr = (2.0 * dist - self.sum_q) / self.sqrt_d;
-        dist_vq_qr * self.scale_factors[id] + self.add_factors[id] + self.query_factor
+    /// `sum_d query[d] * ex_code[d]` for the candidate's packed ex codes.
+    #[inline]
+    fn ex_code_dot(&self, ex_codes: &[u8], id: usize) -> f32 {
+        let ex_dot = self
+            .ex_dot
+            .expect("raw-query multi-bit RQ requires an ex-dot kernel");
+        ex_dot(
+            self.ex_query.as_ref(),
+            &ex_codes[id * self.ex_code_len..(id + 1) * self.ex_code_len],
+        )
     }
 
-    #[inline(always)]
-    fn distance_all(&self, _: usize) -> Vec<f32> {
-        let mut dists = Vec::new();
-        let mut quantized_dists = Vec::new();
-        let mut quantized_dists_table = Vec::new();
-        self.distance_all_with_scratch(
-            0,
-            &mut dists,
-            &mut quantized_dists,
-            &mut quantized_dists_table,
-        );
-        dists
+    /// Fill `dists[0..n]` with exact per-row binary distances computed
+    /// directly from the f32 dist table — the fallback when the quantized
+    /// reconstruction scale would be non-finite ([`DistTableDequant::Exact`]).
+    #[allow(clippy::uninit_vec)]
+    fn fill_exact_binary_distances(&self, n: usize, code_len: usize, dists: &mut Vec<f32>) {
+        dists.clear();
+        dists.reserve(n);
+        // SAFETY: the loop initializes every element in [0, n).
+        unsafe {
+            dists.set_len(n);
+        }
+        dists.iter_mut().enumerate().for_each(|(id, dist)| {
+            *dist = compute_single_rq_distance(self.codes, id, n, code_len, &self.dist_table);
+        });
     }
 
-    #[inline(always)]
     #[allow(clippy::uninit_vec)]
-    fn distance_all_with_scratch(
+    fn binary_distances_with_scratch(
         &self,
-        _: usize,
+        n: usize,
+        code_len: usize,
         dists: &mut Vec<f32>,
         quantized_dists: &mut Vec<u16>,
         quantized_dists_table: &mut Vec<u8>,
-    ) {
-        let code_len = rabit_binary_code_bytes(self.dim);
-        let n = self.codes.len() / code_len;
-        if n == 0 {
-            dists.clear();
-            quantized_dists.clear();
-            return;
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) -> usize {
+        if self.approx_mode == ApproxMode::Accurate {
+            return self.binary_distances_hacc_with_scratch(
+                n,
+                code_len,
+                dists,
+                quantized_dists,
+                quantized_dists_table,
+                hacc_quantized_dists,
+            );
         }
 
-        let (qmin, qmax) = quantize_dist_table_into(&self.dist_table, quantized_dists_table);
+        let (qmin, qmax) = match quantize_dist_table_into(&self.dist_table, quantized_dists_table) {
+            DistTableDequant::Affine { qmin, qmax } => (qmin, qmax),
+            DistTableDequant::Exact => {
+                // The affine reconstruction would be non-finite; compute every
+                // binary distance exactly and report no SIMD rows so the
+                // ex-rerank caller takes the per-row path for all of them.
+                self.fill_exact_binary_distances(n, code_len, dists);
+                return 0;
+            }
+        };
         let remainder = n % BATCH_SIZE;
         let simd_len = n - remainder;
         quantized_dists.clear();
@@ -677,7 +992,7 @@ impl DistCalculator for RabitDistCalculator<'_> {
         );
 
         let range = (qmax - qmin) / 255.0;
-        let num_tables = quantized_dists_table.len() / 16;
+        let num_tables = quantized_dists_table.len() / SEGMENT_NUM_CODES;
         let sum_min = num_tables as f32 * qmin;
         dists.clear();
         dists.reserve(n);
@@ -690,529 +1005,2458 @@ impl DistCalculator for RabitDistCalculator<'_> {
         simd_dists
             .iter_mut()
             .zip(quantized_dists.iter())
-            .enumerate()
-            .for_each(|(id, (dist, q_dist))| {
-                let dist_vq = (*q_dist as f32) * range + sum_min;
-                let dist_vq_qr = (2.0 * dist_vq - self.sum_q) / self.sqrt_d;
-                *dist =
-                    dist_vq_qr * self.scale_factors[id] + self.add_factors[id] + self.query_factor;
+            .for_each(|(dist, q_dist)| {
+                *dist = (*q_dist as f32) * range + sum_min;
             });
 
         remainder_dists
             .iter_mut()
             .enumerate()
             .for_each(|(id, dist)| {
-                *dist = self.distance((simd_len + id) as u32);
+                *dist = compute_single_rq_distance(
+                    self.codes,
+                    simd_len + id,
+                    n,
+                    code_len,
+                    &self.dist_table,
+                );
             });
+        simd_len
     }
-}
 
-impl VectorStore for RabitQuantizationStorage {
-    type DistanceCalculator<'a> = RabitDistCalculator<'a>;
+    #[allow(clippy::uninit_vec)]
+    fn binary_distances_hacc_with_scratch(
+        &self,
+        n: usize,
+        code_len: usize,
+        dists: &mut Vec<f32>,
+        quantized_dist_table: &mut Vec<u16>,
+        hacc_dist_table: &mut Vec<u8>,
+        quantized_dists: &mut Vec<u32>,
+    ) -> usize {
+        let (qmin, qmax) =
+            match quantize_dist_table_u16_into(&self.dist_table, quantized_dist_table) {
+                DistTableDequant::Affine { qmin, qmax } => (qmin, qmax),
+                DistTableDequant::Exact => {
+                    // See binary_distances_with_scratch: non-finite affine
+                    // scale falls back to exact per-row distances.
+                    self.fill_exact_binary_distances(n, code_len, dists);
+                    return 0;
+                }
+            };
+        simd::dist_table::transfer_4bit_dist_table_u16(quantized_dist_table, hacc_dist_table);
+        let remainder = n % BATCH_SIZE;
+        let simd_len = n - remainder;
+        quantized_dists.clear();
+        quantized_dists.reserve(simd_len);
+        // SAFETY: sum_4bit_hacc_dist_table overwrites each element in the batch range.
+        unsafe {
+            quantized_dists.set_len(simd_len);
+        }
+        simd::dist_table::sum_4bit_hacc_dist_table(
+            simd_len,
+            code_len,
+            self.codes,
+            hacc_dist_table,
+            quantized_dists,
+        );
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
+        let range = (qmax - qmin) / u16::MAX as f32;
+        let num_tables = quantized_dist_table.len() / SEGMENT_NUM_CODES;
+        let sum_min = num_tables as f32 * qmin;
+        dists.clear();
+        dists.reserve(n);
+        // SAFETY: the batch section writes [0, simd_len), and the
+        // remainder section writes [simd_len, n).
+        unsafe {
+            dists.set_len(n);
+        }
+        let (simd_dists, remainder_dists) = dists.split_at_mut(simd_len);
+        simd_dists
+            .iter_mut()
+            .zip(quantized_dists.iter())
+            .for_each(|(dist, q_dist)| {
+                *dist = (*q_dist as f32) * range + sum_min;
+            });
 
-    fn schema(&self) -> &SchemaRef {
-        self.batch.schema_ref()
-    }
-
-    fn to_batches(&self) -> Result<impl Iterator<Item = RecordBatch> + Send> {
-        Ok(std::iter::once(self.batch.clone()))
-    }
-
-    fn append_batch(&self, _batch: RecordBatch, _vector_column: &str) -> Result<Self> {
-        unimplemented!("RabitQ does not support append_batch")
-    }
-
-    fn len(&self) -> usize {
-        self.batch.num_rows()
-    }
-
-    fn row_id(&self, id: u32) -> u64 {
-        self.row_ids.value(id as usize)
-    }
-
-    fn row_ids(&self) -> impl Iterator<Item = &u64> {
-        self.row_ids.values().iter()
+        remainder_dists
+            .iter_mut()
+            .enumerate()
+            .for_each(|(id, dist)| {
+                *dist = compute_single_rq_distance(
+                    self.codes,
+                    simd_len + id,
+                    n,
+                    code_len,
+                    &self.dist_table,
+                );
+            });
+        simd_len
     }
 
-    fn distance_type(&self) -> DistanceType {
-        self.distance_type
+    #[inline]
+    fn binary_distance_factor_params(&self) -> (f32, f32) {
+        match self.query_estimator {
+            RabitQueryEstimator::ResidualQuery => (2.0 / self.sqrt_d, -self.sum_q / self.sqrt_d),
+            RabitQueryEstimator::RawQuery => (1.0, -0.5 * self.sum_q),
+        }
     }
 
-    // qr = (q-c)
-    #[inline(never)]
-    fn dist_calculator(&self, qr: Arc<dyn Array>, dist_q_c: f32) -> Self::DistanceCalculator<'_> {
-        let code_dim = self.code_dim();
-        let rotated_qr = self.rotate_query_vector(code_dim, &qr);
-        let dist_table = build_dist_table_direct::<Float32Type>(&rotated_qr);
-        let sum_q = rotated_qr.into_iter().sum();
-
-        self.distance_calculator_from_parts(code_dim, dist_q_c, Cow::Owned(dist_table), sum_q)
+    #[allow(clippy::uninit_vec)]
+    fn one_bit_distances_with_scratch(
+        &self,
+        n: usize,
+        code_len: usize,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        self.binary_distances_with_scratch(
+            n,
+            code_len,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        );
+        let (binary_distance_multiplier, binary_distance_offset) =
+            self.binary_distance_factor_params();
+        dists.iter_mut().enumerate().for_each(|(id, dist)| {
+            let binary_dist = *dist;
+            *dist = (binary_dist * binary_distance_multiplier + binary_distance_offset)
+                * self.scale_factors[id]
+                + self.add_factors[id]
+                + self.query_factor;
+        });
     }
 
-    // qr = (q-c)
-    #[inline(never)]
-    fn dist_calculator_with_scratch<'a>(
-        &'a self,
-        qr: Arc<dyn Array>,
-        dist_q_c: f32,
-        residual: Option<QueryResidual<'_>>,
-        f32_scratch: &'a mut Vec<f32>,
-    ) -> Self::DistanceCalculator<'a> {
-        let code_dim = self.code_dim();
-        let dist_table_len = code_dim * 4;
-        f32_scratch.resize(code_dim + dist_table_len, 0.0);
-
-        let sum_q = {
-            let (rotated_qr, dist_table) = f32_scratch.split_at_mut(code_dim);
-            match residual {
-                Some(QueryResidual::Centroid(residual_centroid)) => {
-                    self.rotate_query_vector_into(
-                        code_dim,
-                        &qr,
-                        Some(residual_centroid),
-                        rotated_qr,
+    #[allow(clippy::uninit_vec)]
+    fn apply_raw_query_multi_bit_distances(
+        &self,
+        simd_len: usize,
+        dists: &mut [f32],
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+    ) {
+        let ex_bits = self.num_bits - 1;
+        let ex_codes = self
+            .ex_codes
+            .expect("raw-query multi-bit RQ requires ex codes");
+        let ex_add_factors = self
+            .ex_add_factors
+            .expect("raw-query multi-bit RQ requires ex add factors");
+        let ex_scale_factors = self
+            .ex_scale_factors
+            .expect("raw-query multi-bit RQ requires ex scale factors");
+        let code_scale = (1u32 << ex_bits) as f32;
+        let code_bias = -(code_scale - 0.5);
+
+        let fastscan_len = if simd_len > 0 && supports_ex_fastscan(ex_bits) {
+            self.packed_ex_codes
+                .map(|packed_ex_codes| {
+                    let fastscan_len = simd_len;
+                    let fastscan_code_len = self.ex_code_len;
+                    let (qmin, qmax, quantization_max) = quantize_ex_fastscan_dist_table_into(
+                        ex_bits,
+                        self.ex_code_len,
+                        self.ex_query.as_ref(),
+                        quantized_dists_table,
                     );
-                }
-                None => {
-                    self.rotate_query_vector_into(code_dim, &qr, None, rotated_qr);
-                }
-            }
-            build_dist_table_direct_into::<Float32Type>(rotated_qr, dist_table);
-            rotated_qr.iter().copied().sum()
+                    quantized_dists.clear();
+                    quantized_dists.reserve(fastscan_len);
+                    // SAFETY: sum_4bit_dist_table overwrites each element in the SIMD batch range.
+                    unsafe {
+                        quantized_dists.set_len(fastscan_len);
+                    }
+                    simd::dist_table::sum_4bit_dist_table(
+                        fastscan_len,
+                        fastscan_code_len,
+                        packed_ex_codes,
+                        quantized_dists_table,
+                        quantized_dists,
+                    );
+
+                    let range = (qmax - qmin) / quantization_max;
+                    let num_tables = quantized_dists_table.len() / SEGMENT_NUM_CODES;
+                    let sum_min = num_tables as f32 * qmin;
+                    dists
+                        .iter_mut()
+                        .take(fastscan_len)
+                        .zip(quantized_dists.iter())
+                        .enumerate()
+                        .for_each(|(id, (dist, q_ex_dist))| {
+                            let ex_dist = (*q_ex_dist as f32) * range + sum_min;
+                            let full_dot = code_scale * *dist + ex_dist + code_bias * self.sum_q;
+                            *dist = full_dot * ex_scale_factors[id]
+                                + ex_add_factors[id]
+                                + self.query_factor;
+                        });
+                    fastscan_len
+                })
+                .unwrap_or_default()
+        } else {
+            0
         };
 
-        self.distance_calculator_from_parts(
-            code_dim,
-            dist_q_c,
-            Cow::Borrowed(&f32_scratch[code_dim..code_dim + dist_table_len]),
-            sum_q,
-        )
+        dists
+            .iter_mut()
+            .enumerate()
+            .skip(fastscan_len)
+            .for_each(|(id, dist)| {
+                let ex_dist = self.ex_code_dot(ex_codes, id);
+                let full_dot = code_scale * *dist + ex_dist + code_bias * self.sum_q;
+                *dist = full_dot * ex_scale_factors[id] + ex_add_factors[id] + self.query_factor;
+            });
     }
 
-    // TODO: implement this
-    // This method is required for HNSW, we can't support HNSW_RABIT before this is implemented
-    fn dist_calculator_from_id(&self, _: u32) -> Self::DistanceCalculator<'_> {
-        unimplemented!("RabitQ does not support dist_calculator_from_id")
+    #[inline]
+    fn raw_query_binary_distance(&self, id: usize, binary_ip: f32) -> f32 {
+        (binary_ip - 0.5 * self.sum_q) * self.scale_factors[id]
+            + self.add_factors[id]
+            + self.query_factor
     }
-}
 
-const LOWBIT_IDX: [usize; 16] = {
-    let mut array = [0; 16];
-    let mut i = 1;
-    while i < 16 {
-        array[i] = i.trailing_zeros() as usize;
-        i += 1;
+    #[inline]
+    fn raw_query_lower_bound(&self, id: usize, binary_ip: f32) -> Option<f32> {
+        let error_factors = self.error_factors?;
+        Some(self.raw_query_binary_distance(id, binary_ip) - error_factors[id] * self.query_error)
     }
-    array
-};
 
-fn get_column(
-    quantization_code: &[u8],
-    code_len: usize,
-    row: usize,
-    col_idx: usize,
-    codes: &mut [u8; 32],
-) {
-    for (i, code) in codes.iter_mut().enumerate() {
-        let vec_idx = row + i;
-        *code = quantization_code[vec_idx * code_len + col_idx];
+    #[inline]
+    #[allow(clippy::too_many_arguments)]
+    fn raw_query_multi_bit_exact_distance(
+        &self,
+        id: usize,
+        binary_ip: f32,
+        ex_bits: u8,
+        ex_codes: &[u8],
+        ex_add_factors: &[f32],
+        ex_scale_factors: &[f32],
+    ) -> f32 {
+        let ex_dist = self.ex_code_dot(ex_codes, id);
+        let code_bias = -((1u32 << ex_bits) as f32 - 0.5);
+        let full_dot = (1u32 << ex_bits) as f32 * binary_ip + ex_dist + code_bias * self.sum_q;
+        full_dot * ex_scale_factors[id] + ex_add_factors[id] + self.query_factor
     }
-}
-
-pub fn pack_codes(codes: &FixedSizeListArray) -> FixedSizeListArray {
-    let code_len = codes.value_length() as usize;
 
-    // round up num of vectors to multiple of batch size (32)
-    let num_blocks = codes.len() / BATCH_SIZE;
-    let num_packed_vectors = num_blocks * BATCH_SIZE;
+    /// Compute the binary inner products into `dists` and resolve the inputs
+    /// shared by the raw-query multi-bit top-k scans. Returns `None` when the
+    /// partition has no rows.
+    #[allow(clippy::too_many_arguments)]
+    fn raw_query_multi_bit_topk_context(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) -> Option<RawQueryTopkContext<'_>> {
+        let code_len = rabit_binary_code_bytes(self.dim);
+        let n = self.codes.len() / code_len;
+        if n == 0 {
+            dists.clear();
+            quantized_dists.clear();
+            hacc_quantized_dists.clear();
+            return None;
+        }
 
-    // calculate total size for packed blocks
-    // we pack each 32 vectors into a block, each block contains 2 codes (1byte) of each vector
-    // so every 32 vectors would produce code_len blocks
-    // the low 16 bytes of each block is the codes for the low 4 bits of each vector
-    // the high 16 bytes of each block is the codes for the high 4 bits of each vector
-    let mut blocks = vec![0u8; codes.values().len()];
+        self.binary_distances_with_scratch(
+            n,
+            code_len,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        );
 
-    let codes_values = codes
-        .slice(0, num_packed_vectors)
-        .values()
-        .as_primitive::<UInt8Type>()
-        .clone();
-    let codes_values = codes_values.values();
+        Some(RawQueryTopkContext {
+            n,
+            k,
+            ex_bits: self.num_bits - 1,
+            ex_codes: self
+                .ex_codes
+                .expect("raw-query multi-bit RQ requires ex codes"),
+            ex_add_factors: self
+                .ex_add_factors
+                .expect("raw-query multi-bit RQ requires ex add factors"),
+            ex_scale_factors: self
+                .ex_scale_factors
+                .expect("raw-query multi-bit RQ requires ex scale factors"),
+            query_lower_bound: lower_bound.unwrap_or(f32::MIN),
+            query_upper_bound: upper_bound.unwrap_or(f32::MAX),
+        })
+    }
 
-    // Pack codes batch by batch
-    // Each batch contains codes for 32 vectors
-    let mut col = [0u8; 32];
-    let mut col_0 = [0u8; 32]; // lower 4 bits
-    let mut col_1 = [0u8; 32]; // higher 4 bits
-    for row in (0..num_packed_vectors).step_by(BATCH_SIZE) {
-        // Get quantization codes for each column for each batch
-        // i.e., we get the codes for 8 dims of 32 vectors and reorganize the data layout
-        // based on the shuffle SIMD instruction used during querying
-        for i in 0..code_len {
-            get_column(codes_values, code_len, row, i, &mut col);
+    /// Process one candidate row given its lower bound: the bound checks,
+    /// the exact rerank, and the heap update shared by the sparse scan and
+    /// the dense scan's surviving lanes and tail.
+    #[inline]
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_raw_query_multi_bit_row(
+        &self,
+        ctx: &RawQueryTopkContext<'_>,
+        id: usize,
+        row_id: u64,
+        binary_ip: f32,
+        raw_lower_bound: f32,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        max_dist: &mut Option<OrderedFloat>,
+        counters: &mut RabitPruneCounters,
+    ) {
+        if raw_lower_bound >= ctx.query_upper_bound {
+            counters.pruned_upper_bound += 1;
+            return;
+        }
+        if res.len() >= ctx.k && max_dist.is_some_and(|max_dist| raw_lower_bound >= max_dist.0) {
+            counters.pruned_heap += 1;
+            return;
+        }
 
-            for j in 0..32 {
-                col_0[j] = col[j] & 0xF;
-                col_1[j] = col[j] >> 4;
+        counters.exact += 1;
+        let dist = self.raw_query_multi_bit_exact_distance(
+            id,
+            binary_ip,
+            ctx.ex_bits,
+            ctx.ex_codes,
+            ctx.ex_add_factors,
+            ctx.ex_scale_factors,
+        );
+        if dist < ctx.query_lower_bound || dist >= ctx.query_upper_bound {
+            counters.exact_rejected += 1;
+            return;
+        }
+        let dist = OrderedFloat(dist);
+        if res.len() < ctx.k {
+            res.push(OrderedNode::new(row_id, dist));
+            if res.len() == ctx.k {
+                *max_dist = res.peek().map(|node| node.dist);
             }
+        } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
+            res.pop();
+            res.push(OrderedNode::new(row_id, dist));
+            *max_dist = res.peek().map(|node| node.dist);
+        }
+    }
 
-            let block_offset = (row / BATCH_SIZE) * code_len * BATCH_SIZE + i * BATCH_SIZE;
-            for j in 0..16 {
-                // The lower 4 bits represent vector 0 to 15
-                // The upper 4 bits represent vector 16 to 31
-                let val0 = col_0[PERM0[j]] | (col_0[PERM0[j] + 16] << 4);
-                let val1 = col_1[PERM0[j]] | (col_1[PERM0[j] + 16] << 4);
-                blocks[block_offset + j] = val0;
-                blocks[block_offset + j + 16] = val1;
-            }
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_raw_query_multi_bit_topk_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_ids: impl Iterator<Item = (usize, u64)>,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        let Some(ctx) = self.raw_query_multi_bit_topk_context(
+            k,
+            lower_bound,
+            upper_bound,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        ) else {
+            return;
+        };
+        let mut max_dist = res.peek().map(|node| node.dist);
+        let mut counters = RabitPruneCounters::default();
+
+        for (id, row_id) in row_ids {
+            let Some(binary_ip) = dists.get(id).copied() else {
+                continue;
+            };
+            counters.candidates += 1;
+            let Some(raw_lower_bound) = self.raw_query_lower_bound(id, binary_ip) else {
+                continue;
+            };
+            self.accumulate_raw_query_multi_bit_row(
+                &ctx,
+                id,
+                row_id,
+                binary_ip,
+                raw_lower_bound,
+                res,
+                &mut max_dist,
+                &mut counters,
+            );
         }
+        record_rabit_prune_stats(&counters);
     }
 
-    // for the left codes, transpose them for better cache locality
-    let transposed_codes = transpose(
-        &codes.values().as_primitive::<UInt8Type>().slice(
-            num_packed_vectors * code_len,
-            (codes.len() - num_packed_vectors) * code_len,
-        ),
-        codes.len() - num_packed_vectors,
-        code_len,
-    );
+    /// Top-k scan over all rows `0..n` in order: classify [`PRUNE_LANES`]
+    /// rows at a time with the SIMD lower-bound kernel and run the scalar
+    /// rerank only for the surviving lanes.
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_raw_query_multi_bit_topk_dense_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_id: impl Fn(u32) -> u64,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        let Some(ctx) = self.raw_query_multi_bit_topk_context(
+            k,
+            lower_bound,
+            upper_bound,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        ) else {
+            return;
+        };
+        let dists = dists.as_slice();
+        debug_assert_eq!(dists.len(), ctx.n);
+        let scale_factors = &self.scale_factors[..ctx.n];
+        let add_factors = &self.add_factors[..ctx.n];
+        let error_factors = &self
+            .error_factors
+            .expect("raw-query lower-bound gating requires error factors")[..ctx.n];
+        // Same expression as `raw_query_lower_bound` with `error_factors`
+        // already resolved; the masks below match it bit for bit.
+        let lower_bound_of = |id: usize, binary_ip: f32| {
+            self.raw_query_binary_distance(id, binary_ip) - error_factors[id] * self.query_error
+        };
+        let terms = LowerBoundTerms {
+            half_sum_q: 0.5 * self.sum_q,
+            query_factor: self.query_factor,
+            query_error: self.query_error,
+        };
+        let prune_masks = prune_mask_kernel();
+        let mut max_dist = res.peek().map(|node| node.dist);
+        let mut counters = RabitPruneCounters::default();
+
+        let (dist_groups, dist_tail) = dists.as_chunks::<PRUNE_LANES>();
+        let (scale_groups, _) = scale_factors.as_chunks::<PRUNE_LANES>();
+        let (add_groups, _) = add_factors.as_chunks::<PRUNE_LANES>();
+        let (error_groups, _) = error_factors.as_chunks::<PRUNE_LANES>();
+        for (group, (dist16, scale16, add16, error16)) in
+            izip!(dist_groups, scale_groups, add_groups, error_groups).enumerate()
+        {
+            counters.candidates += PRUNE_LANES;
+            // The heap threshold only ever tightens, so this group-start
+            // snapshot can only over-select survivors (which the per-row
+            // processing below re-checks against live values), never prune a
+            // row the scalar scan would have kept.
+            let heap_threshold = (res.len() >= ctx.k)
+                .then(|| max_dist.map(|max_dist| max_dist.0))
+                .flatten();
+            let (pruned_upper_bound, pruned_heap) = prune_masks(
+                dist16,
+                scale16,
+                add16,
+                error16,
+                terms,
+                ctx.query_upper_bound,
+                heap_threshold,
+            );
+            counters.pruned_upper_bound += pruned_upper_bound.count_ones() as usize;
+            counters.pruned_heap += pruned_heap.count_ones() as usize;
+            let mut survivors = !(pruned_upper_bound | pruned_heap);
+            while survivors != 0 {
+                let lane = survivors.trailing_zeros() as usize;
+                survivors &= survivors - 1;
+                let id = group * PRUNE_LANES + lane;
+                let binary_ip = dists[id];
+                self.accumulate_raw_query_multi_bit_row(
+                    &ctx,
+                    id,
+                    row_id(id as u32),
+                    binary_ip,
+                    lower_bound_of(id, binary_ip),
+                    res,
+                    &mut max_dist,
+                    &mut counters,
+                );
+            }
+        }
 
-    let offset = codes.values().len() - transposed_codes.len();
-    for (i, v) in transposed_codes.values().iter().enumerate() {
-        blocks[offset + i] = *v;
+        let tail_start = ctx.n - dist_tail.len();
+        for (offset, binary_ip) in dist_tail.iter().copied().enumerate() {
+            let id = tail_start + offset;
+            counters.candidates += 1;
+            self.accumulate_raw_query_multi_bit_row(
+                &ctx,
+                id,
+                row_id(id as u32),
+                binary_ip,
+                lower_bound_of(id, binary_ip),
+                res,
+                &mut max_dist,
+                &mut counters,
+            );
+        }
+        record_rabit_prune_stats(&counters);
     }
 
-    assert_eq!(blocks.len(), codes.values().len());
-    FixedSizeListArray::try_new_from_values(UInt8Array::from(blocks), code_len as i32).unwrap()
+    fn raw_query_lower_bound_gating_disabled_reason(&self) -> Option<&'static str> {
+        if self.approx_mode == ApproxMode::Fast {
+            Some("approx_mode_fast")
+        } else if self.query_estimator != RabitQueryEstimator::RawQuery {
+            Some("residual_query_estimator")
+        } else if self.num_bits <= 1 {
+            Some("num_bits_le_one")
+        } else if self.error_factors.is_none() {
+            Some("missing_error_factors")
+        } else {
+            None
+        }
+    }
 }
 
-// Inverse of pack_codes
-pub fn unpack_codes(codes: &FixedSizeListArray) -> FixedSizeListArray {
-    let code_len = codes.value_length() as usize;
-    let num_vectors = codes.len();
+#[inline]
+fn lowbit(x: usize) -> usize {
+    1 << x.trailing_zeros()
+}
 
-    // Calculate number of complete batches
-    let num_blocks = num_vectors / BATCH_SIZE;
-    let num_packed_vectors = num_blocks * BATCH_SIZE;
+#[inline]
+pub fn build_dist_table_direct<T: ArrowFloatType>(qc: &[T::Native]) -> Vec<f32>
+where
+    T::Native: AsPrimitive<f32>,
+{
+    // every 4 bits (SEGMENT_LENGTH) is a segment, and we need to compute the distance between the segment and all the codes
+    // so there are dim/4 segments, and the number of codes is 16 (2^{SEGMENT_LENGTH}),
+    // so we have dim/4 * 16 = dim * 4 elements in the dist_table
+    let mut dist_table = vec![0.0; qc.len() * 4];
+    build_dist_table_direct_into::<T>(qc, &mut dist_table);
+    dist_table
+}
 
-    let mut unpacked = vec![0u8; codes.values().len()];
-
-    let codes_values = codes.values().as_primitive::<UInt8Type>().values();
-
-    // Unpack complete batches
-    for batch_idx in 0..num_blocks {
-        let block_start = batch_idx * code_len * BATCH_SIZE;
-
-        for i in 0..code_len {
-            let block_offset = block_start + i * BATCH_SIZE;
-            let block = &codes_values[block_offset..block_offset + BATCH_SIZE];
+fn build_dist_table_direct_into<T: ArrowFloatType>(qc: &[T::Native], dist_table: &mut [f32])
+where
+    T::Native: AsPrimitive<f32>,
+{
+    debug_assert_eq!(dist_table.len(), qc.len() * 4);
+    qc.chunks_exact(SEGMENT_LENGTH)
+        .zip(dist_table.chunks_exact_mut(SEGMENT_NUM_CODES))
+        .for_each(|(sub_vec, dist_table)| {
+            dist_table[0] = 0.0;
+            build_dist_table_for_subvec::<T>(sub_vec, dist_table);
+        });
+}
 
-            // Reverse the permutation
-            for j in 0..16 {
-                let val0 = block[j];
-                let val1 = block[j + 16];
+#[inline(always)]
+fn build_dist_table_for_subvec<T: ArrowFloatType>(sub_vec: &[T::Native], dist_table: &mut [f32])
+where
+    T::Native: AsPrimitive<f32>,
+{
+    // skip 0 because it's always 0
+    (1..SEGMENT_NUM_CODES).for_each(|j| {
+        // this is a little bit tricky,
+        // j represents a subset of 4 bits, that if the i-th bit of `j` is 1,
+        // then we need to add the distance of the i-th dim of the segment.
+        // but we don't need to check all bits of `j`,
+        // because `j` = `j - lowbit(j)` + `lowbit(j)`,
+        // where `j-lowbit(j)` is less than `j`,
+        // which means dist_table[j-lowbit(j)] is already computed,
+        // and we can use it to compute dist_table[j]
+        // for example, if j = 0b1010, then j - lowbit(j) = 0b1000,
+        // and dist_table[0b1000] is already computed,
+        // so dist_table[0b1010] = dist_table[0b1000] + sub_vec[LOWBIT_IDX[0b1010]];
+        // where lowbit(0b1010) = 0b10, LOWBIT_IDX[0b1010] = LOWBIT_IDX[0b10] = 1.
+        dist_table[j] = dist_table[j - lowbit(j)] + sub_vec[LOWBIT_IDX[j]].as_();
+    })
+}
 
-                let low_0 = val0 & 0xF;
-                let high_0 = val0 >> 4;
-                let low_1 = val1 & 0xF;
-                let high_1 = val1 >> 4;
+/// Build the u8 FastScan LUT for the ex codes directly from the rotated
+/// query (`ex_query`, natural dim order, padding dims zero): the underlying
+/// per-dim table is the pure multiplication `q[d] * code`, so no intermediate
+/// `dim * 2^ex_bits` table is materialized.
+fn quantize_ex_fastscan_dist_table_into(
+    ex_bits: u8,
+    ex_code_len: usize,
+    ex_query: &[f32],
+    quantized_dist_table: &mut Vec<u8>,
+) -> (f32, f32, f32) {
+    debug_assert!(supports_ex_fastscan(ex_bits));
+
+    // One split table per code nibble of the row.
+    let num_split_tables = ex_code_len * 2;
+    let quantization_max = (u16::MAX as usize / num_split_tables)
+        .min(u8::MAX as usize)
+        .max(1) as f32;
+
+    let mut qmin = f32::INFINITY;
+    let mut qmax = f32::NEG_INFINITY;
+    for table_idx in 0..num_split_tables {
+        for code in 0..SEGMENT_NUM_CODES {
+            let value = ex_fastscan_dist_table_value(ex_query, ex_bits, table_idx, code);
+            qmin = qmin.min(value);
+            qmax = qmax.max(value);
+        }
+    }
 
-                let vec_idx_0 = batch_idx * BATCH_SIZE + PERM0[j];
-                let vec_idx_1 = batch_idx * BATCH_SIZE + PERM0[j] + 16;
+    quantized_dist_table.clear();
+    quantized_dist_table.reserve(num_split_tables * SEGMENT_NUM_CODES);
+    if qmin == qmax {
+        quantized_dist_table.resize(num_split_tables * SEGMENT_NUM_CODES, 0);
+        return (qmin, qmax, quantization_max);
+    }
 
-                unpacked[vec_idx_0 * code_len + i] = low_0 | (low_1 << 4);
-                unpacked[vec_idx_1 * code_len + i] = high_0 | (high_1 << 4);
-            }
+    let factor = quantization_max / (qmax - qmin);
+    for table_idx in 0..num_split_tables {
+        for code in 0..SEGMENT_NUM_CODES {
+            let value = ex_fastscan_dist_table_value(ex_query, ex_bits, table_idx, code);
+            quantized_dist_table.push(((value - qmin) * factor).round() as u8);
         }
     }
 
-    // Transpose back the remainder
-    if num_packed_vectors < num_vectors {
-        let remainder = num_vectors - num_packed_vectors;
-        let offset = num_packed_vectors * code_len;
-        let transposed_data = &codes_values[offset..];
+    (qmin, qmax, quantization_max)
+}
 
-        // Transpose from column-major back to row-major
-        for row in 0..remainder {
-            for col in 0..code_len {
-                unpacked[offset + row * code_len + col] = transposed_data[col * remainder + row];
-            }
+#[inline]
+fn supports_ex_fastscan(ex_bits: u8) -> bool {
+    matches!(ex_bits, 2 | 4 | 8)
+}
+
+/// The FastScan LUT value for one nibble of a blocked-layout code byte:
+/// `table_idx / 2` is the byte position within a row and `table_idx % 2`
+/// selects its low/high nibble (see the `ex_dot` module docs for the
+/// byte-to-dim mapping per width). Dims beyond the query length (block
+/// padding) contribute zero.
+#[inline]
+fn ex_fastscan_dist_table_value(
+    ex_query: &[f32],
+    ex_bits: u8,
+    table_idx: usize,
+    code: usize,
+) -> f32 {
+    let query = |dim_idx: usize| ex_query.get(dim_idx).copied().unwrap_or(0.0);
+    let byte_idx = table_idx / 2;
+    let high_nibble = table_idx % 2 == 1;
+    match ex_bits {
+        2 => {
+            // byte 16g+b = dims {64g+b, +16, +32, +48} at bit pairs; the low
+            // nibble covers the first two dims, the high nibble the last two.
+            let dim_idx = 64 * (byte_idx / 16) + byte_idx % 16 + 32 * usize::from(high_nibble);
+            let low = (code & 0b11) as f32;
+            let high = ((code >> 2) & 0b11) as f32;
+            query(dim_idx) * low + query(dim_idx + 16) * high
+        }
+        4 => {
+            // byte 32g+8j+b = dim 64g+16j+b (low nibble) | dim +8 (high).
+            let in_block = byte_idx % 32;
+            let dim_idx = 64 * (byte_idx / 32)
+                + 16 * (in_block / 8)
+                + in_block % 8
+                + 8 * usize::from(high_nibble);
+            query(dim_idx) * code as f32
         }
+        8 => {
+            // byte = dim identity; the high nibble carries code bits 4..8.
+            let code = if high_nibble {
+                code << SEGMENT_LENGTH
+            } else {
+                code
+            };
+            query(byte_idx) * code as f32
+        }
+        _ => unreachable!("unsupported RabitQ ex_bits={ex_bits} for FastScan"),
     }
+}
 
-    FixedSizeListArray::try_new_from_values(UInt8Array::from(unpacked), code_len as i32).unwrap()
+/// Transpose ex codes for the FastScan bulk path. That path is only reachable
+/// when lower-bound gating is disabled, i.e. for legacy indexes without error
+/// factors; gated indexes rerank per candidate with the ex-dot kernels and
+/// never touch this copy, so skip the transpose (and its resident memory).
+fn maybe_pack_ex_codes(
+    ex_codes: Option<&FixedSizeListArray>,
+    ex_bits: u8,
+    error_factors: Option<&Float32Array>,
+) -> Option<FixedSizeListArray> {
+    let ex_codes = ex_codes?;
+    if error_factors.is_some() {
+        return None;
+    }
+    match ex_bits {
+        2 | 4 | 8 => Some(pack_codes(ex_codes)),
+        _ => None,
+    }
 }
 
-#[async_trait]
-impl QuantizerStorage for RabitQuantizationStorage {
-    type Metadata = RabitQuantizationMetadata;
+/// Bring legacy sequential ex codes into the blocked kernel layout: rows are
+/// repacked, except for the widths whose layouts agree byte-for-byte (then
+/// the column is used as stored).
+fn blocked_ex_codes_from_sequential(
+    seq_codes: &FixedSizeListArray,
+    dim: usize,
+    ex_bits: u8,
+) -> Result<FixedSizeListArray> {
+    if sequential_matches_blocked(ex_bits)
+        && seq_codes.value_length() as usize == blocked_ex_code_bytes(dim, ex_bits)
+    {
+        return Ok(seq_codes.clone());
+    }
+    let seq_code_len = seq_codes.value_length() as usize;
+    let seq_values = seq_codes.values().as_primitive::<UInt8Type>().values();
+    let blocked_code_len = blocked_ex_code_bytes(dim, ex_bits);
+    let mut blocked_values = vec![0u8; seq_codes.len() * blocked_code_len];
+    for (seq_row, blocked_row) in seq_values
+        .chunks_exact(seq_code_len)
+        .zip(blocked_values.chunks_exact_mut(blocked_code_len))
+    {
+        repack_sequential_row(seq_row, dim, ex_bits, blocked_row);
+    }
+    Ok(FixedSizeListArray::try_new_from_values(
+        UInt8Array::from(blocked_values),
+        blocked_code_len as i32,
+    )?)
+}
 
-    fn try_from_batch(
-        batch: RecordBatch,
-        metadata: &Self::Metadata,
-        distance_type: DistanceType,
-        _fri: Option<Arc<FragReuseIndex>>,
-    ) -> Result<Self> {
-        validate_supported_rq_num_bits(metadata.num_bits)?;
-        let row_ids = batch[ROW_ID].as_primitive::<UInt64Type>().clone();
-        let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
-        let expected_code_bytes = metadata.binary_code_bytes();
-        if expected_code_bytes > 0 && codes.value_length() as usize != expected_code_bytes {
+/// Load the ex-code column of an index batch into the blocked kernel layout,
+/// accepting both the blocked format and the legacy sequential format. Legacy
+/// batches are normalized in place (the sequential column is replaced by the
+/// blocked one), so rewrites — remap, optimize merges — always emit the
+/// blocked format and legacy indexes upgrade on their next rewrite.
+pub(crate) fn load_blocked_ex_codes(
+    batch: RecordBatch,
+    rotated_dim: usize,
+    num_bits: u8,
+) -> Result<(RecordBatch, FixedSizeListArray)> {
+    let ex_bits = rabit_ex_bits(num_bits)?;
+    if let Some(column) = batch.column_by_name(RABIT_BLOCKED_EX_CODE_COLUMN) {
+        let codes = column.as_fixed_size_list().clone();
+        let expected_bytes = blocked_ex_code_bytes(rotated_dim, ex_bits);
+        if codes.value_length() as usize != expected_bytes {
             return Err(Error::invalid_input(format!(
-                "RabitQ code byte width mismatch: column {} has {} bytes, metadata rotated_dim={} requires {} bytes",
-                RABIT_CODE_COLUMN,
+                "RabitQ ex-code byte width mismatch: column {} has {} bytes, metadata rotated_dim={} ex_bits={} requires {} bytes",
+                RABIT_BLOCKED_EX_CODE_COLUMN,
                 codes.value_length(),
-                metadata.rotated_dim(),
-                expected_code_bytes
+                rotated_dim,
+                ex_bits,
+                expected_bytes
             )));
         }
-        let add_factors = batch[ADD_FACTORS_COLUMN]
-            .as_primitive::<Float32Type>()
-            .clone();
-        let scale_factors = batch[SCALE_FACTORS_COLUMN]
-            .as_primitive::<Float32Type>()
-            .clone();
-
-        let (batch, codes) = if !metadata.packed {
-            let codes = pack_codes(&codes);
-            let batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, Arc::new(codes))?;
-            let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
-            (batch, codes)
-        } else {
-            (batch, codes)
-        };
+        return Ok((batch, codes));
+    }
+    let column = batch.column_by_name(RABIT_EX_CODE_COLUMN).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "RabitQ num_bits={} requires {} column",
+            num_bits, RABIT_BLOCKED_EX_CODE_COLUMN
+        ))
+    })?;
+    let codes = column.as_fixed_size_list().clone();
+    let expected_bytes = rabit_ex_code_bytes(rotated_dim, ex_bits)?;
+    if codes.value_length() as usize != expected_bytes {
+        return Err(Error::invalid_input(format!(
+            "RabitQ ex-code byte width mismatch: column {} has {} bytes, metadata rotated_dim={} ex_bits={} requires {} bytes",
+            RABIT_EX_CODE_COLUMN,
+            codes.value_length(),
+            rotated_dim,
+            ex_bits,
+            expected_bytes
+        )));
+    }
+    let blocked = blocked_ex_codes_from_sequential(&codes, rotated_dim, ex_bits)?;
+    let ex_code_field = rabit_ex_code_field(rotated_dim, num_bits)?
+        .expect("multi-bit RabitQ always has an ex-code field");
+    let batch = batch
+        .drop_column(RABIT_EX_CODE_COLUMN)?
+        .try_with_column(ex_code_field, Arc::new(blocked.clone()))?;
+    Ok((batch, blocked))
+}
 
-        let mut metadata = metadata.clone();
-        metadata.packed = true;
+impl DistCalculator for RabitDistCalculator<'_> {
+    #[inline(always)]
+    fn distance(&self, id: u32) -> f32 {
+        let id = id as usize;
+        let code_len = rabit_binary_code_bytes(self.dim);
+        let num_vectors = self.codes.len() / code_len;
+        let dist =
+            compute_single_rq_distance(self.codes, id, num_vectors, code_len, &self.dist_table);
 
-        Ok(Self {
-            metadata,
-            batch,
-            distance_type,
-            row_ids,
-            codes,
-            add_factors,
-            scale_factors,
-        })
-    }
+        match self.query_estimator {
+            RabitQueryEstimator::ResidualQuery => {
+                // distance between quantized residual vector and residual query vector
+                let dist_vq_qr = (2.0 * dist - self.sum_q) / self.sqrt_d;
+                dist_vq_qr * self.scale_factors[id] + self.add_factors[id] + self.query_factor
+            }
+            RabitQueryEstimator::RawQuery => {
+                let ex_bits = self.num_bits - 1;
+                if ex_bits == 0 || self.approx_mode == ApproxMode::Fast {
+                    return self.raw_query_binary_distance(id, dist);
+                }
 
-    fn metadata(&self) -> &Self::Metadata {
-        &self.metadata
+                let ex_codes = self
+                    .ex_codes
+                    .expect("raw-query multi-bit RQ requires ex codes");
+                let ex_add_factors = self
+                    .ex_add_factors
+                    .expect("raw-query multi-bit RQ requires ex add factors");
+                let ex_scale_factors = self
+                    .ex_scale_factors
+                    .expect("raw-query multi-bit RQ requires ex scale factors");
+                self.raw_query_multi_bit_exact_distance(
+                    id,
+                    dist,
+                    ex_bits,
+                    ex_codes,
+                    ex_add_factors,
+                    ex_scale_factors,
+                )
+            }
+        }
     }
 
-    async fn load_partition(
-        reader: &PreviousFileReader,
-        range: std::ops::Range<usize>,
-        distance_type: DistanceType,
-        metadata: &Self::Metadata,
-        frag_reuse_index: Option<Arc<FragReuseIndex>>,
-    ) -> Result<Self> {
-        let schema = reader.schema();
-        let batch = reader.read_range(range, schema).await?;
-        Self::try_from_batch(batch, metadata, distance_type, frag_reuse_index)
+    #[inline(always)]
+    fn distance_all(&self, _: usize) -> Vec<f32> {
+        let mut dists = Vec::new();
+        let mut quantized_dists = Vec::new();
+        let mut quantized_dists_table = Vec::new();
+        let mut hacc_quantized_dists = Vec::new();
+        self.distance_all_with_scratch(
+            0,
+            &mut dists,
+            &mut quantized_dists,
+            &mut quantized_dists_table,
+            &mut hacc_quantized_dists,
+        );
+        dists
     }
 
-    fn remap(&self, mapping: &HashMap<u64, Option<u64>>) -> Result<Self> {
-        let num_vectors = self.codes.len();
-        let num_code_bytes = self.codes.value_length() as usize;
-        let codes = self.codes.values().as_primitive::<UInt8Type>().values();
-        let mut indices = Vec::with_capacity(num_vectors);
-        let mut new_row_ids = Vec::with_capacity(num_vectors);
-        let mut new_codes = Vec::with_capacity(codes.len());
+    #[inline(always)]
+    #[allow(clippy::uninit_vec)]
+    fn distance_all_with_scratch(
+        &self,
+        _: usize,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        let code_len = rabit_binary_code_bytes(self.dim);
+        let n = self.codes.len() / code_len;
+        if n == 0 {
+            dists.clear();
+            quantized_dists.clear();
+            return;
+        }
 
-        let row_ids = self.row_ids.values();
-        for (i, row_id) in row_ids.iter().enumerate() {
-            match mapping.get(row_id) {
-                Some(Some(new_id)) => {
-                    indices.push(i as u32);
-                    new_row_ids.push(*new_id);
-                    new_codes.extend(get_rq_code(codes, i, num_vectors, num_code_bytes));
-                }
-                Some(None) => {}
-                None => {
-                    indices.push(i as u32);
-                    new_row_ids.push(*row_id);
-                    new_codes.extend(get_rq_code(codes, i, num_vectors, num_code_bytes));
-                }
-            }
+        if self.query_estimator == RabitQueryEstimator::ResidualQuery
+            || self.num_bits == 1
+            || self.approx_mode == ApproxMode::Fast
+        {
+            self.one_bit_distances_with_scratch(
+                n,
+                code_len,
+                dists,
+                quantized_dists,
+                quantized_dists_table,
+                hacc_quantized_dists,
+            );
+            return;
         }
 
-        let new_row_ids = UInt64Array::from(new_row_ids);
-        let new_codes = FixedSizeListArray::try_new_from_values(
-            UInt8Array::from(new_codes),
-            num_code_bytes as i32,
-        )?;
-        let batch = if new_row_ids.is_empty() {
-            RecordBatch::new_empty(self.schema().clone())
-        } else {
-            let codes = Arc::new(pack_codes(&new_codes));
-            self.batch
-                .take(&UInt32Array::from(indices))?
-                .replace_column_by_name(ROW_ID, Arc::new(new_row_ids.clone()))?
-                .replace_column_by_name(RABIT_CODE_COLUMN, codes)?
-        };
-        let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
+        let simd_len = self.binary_distances_with_scratch(
+            n,
+            code_len,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        );
 
-        Ok(Self {
-            metadata: self.metadata.clone(),
-            distance_type: self.distance_type,
-            batch,
-            codes,
-            add_factors: self.add_factors.clone(),
-            scale_factors: self.scale_factors.clone(),
-            row_ids: new_row_ids,
-        })
+        self.apply_raw_query_multi_bit_distances(
+            simd_len,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+        );
     }
-}
 
-/// Compute the raw distance for a single vector without allocating.
-///
-/// Fuses code extraction from the packed layout with distance accumulation
-/// in a single pass, avoiding the intermediate `Vec` allocation that
-/// `get_rq_code` + iterator would require.
-#[inline]
-fn compute_single_rq_distance(
-    codes: &[u8],
-    id: usize,
-    num_vectors: usize,
-    num_code_bytes: usize,
-    dist_table: &[f32],
-) -> f32 {
-    let remainder = num_vectors % BATCH_SIZE;
-    let mut dist_table_iter = dist_table.chunks_exact(SEGMENT_NUM_CODES).tuples();
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_topk_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_id: impl Fn(u32) -> u64,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        if k == 0 {
+            return;
+        }
+        if let Some(reason) = self.raw_query_lower_bound_gating_disabled_reason() {
+            record_rabit_prune_bypass(reason);
+            self.distance_all_with_scratch(
+                k,
+                dists,
+                quantized_dists,
+                quantized_dists_table,
+                hacc_quantized_dists,
+            );
+            accumulate_distances_into_heap(k, lower_bound, upper_bound, row_id, res, dists);
+            return;
+        }
 
-    if id < num_vectors - remainder {
-        let batch_codes = &codes[id / BATCH_SIZE * BATCH_SIZE * num_code_bytes
-            ..(id / BATCH_SIZE + 1) * BATCH_SIZE * num_code_bytes];
+        self.accumulate_raw_query_multi_bit_topk_dense_with_scratch(
+            k,
+            lower_bound,
+            upper_bound,
+            row_id,
+            res,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        );
+    }
 
-        let id_in_batch = id % BATCH_SIZE;
-        let idx = PERM0_INVERSE[id_in_batch % 16];
-        let is_lower = id_in_batch < 16;
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_filtered_topk_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_ids: impl Iterator<Item = (u32, u64)>,
+        accept_row: impl Fn(u64) -> bool,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        dists: &mut Vec<f32>,
+        quantized_dists: &mut Vec<u16>,
+        quantized_dists_table: &mut Vec<u8>,
+        hacc_quantized_dists: &mut Vec<u32>,
+    ) {
+        if k == 0 {
+            return;
+        }
+        if let Some(reason) = self.raw_query_lower_bound_gating_disabled_reason() {
+            record_rabit_prune_bypass(reason);
+            self.distance_all_with_scratch(
+                k,
+                dists,
+                quantized_dists,
+                quantized_dists_table,
+                hacc_quantized_dists,
+            );
+            accumulate_filtered_distances_into_heap(
+                k,
+                lower_bound,
+                upper_bound,
+                row_ids,
+                accept_row,
+                res,
+                dists,
+            );
+            return;
+        }
 
-        let mut dist = 0.0f32;
-        for block in batch_codes.chunks_exact(BATCH_SIZE) {
-            let code_byte = if is_lower {
-                (block[idx] & 0xF) | (block[idx + 16] << 4)
-            } else {
-                (block[idx] >> 4) | (block[idx + 16] & 0xF0)
-            };
-            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
-                let current_code = (code_byte & 0x0F) as usize;
-                let next_code = (code_byte >> 4) as usize;
-                dist += current_dt[current_code] + next_dt[next_code];
+        self.accumulate_raw_query_multi_bit_topk_with_scratch(
+            k,
+            lower_bound,
+            upper_bound,
+            row_ids
+                .filter(|(_, row_id)| accept_row(*row_id))
+                .map(|(id, row_id)| (id as usize, row_id)),
+            res,
+            dists,
+            quantized_dists,
+            quantized_dists_table,
+            hacc_quantized_dists,
+        );
+    }
+}
+
+fn accumulate_distances_into_heap(
+    k: usize,
+    lower_bound: Option<f32>,
+    upper_bound: Option<f32>,
+    row_id: impl Fn(u32) -> u64,
+    res: &mut BinaryHeap<OrderedNode<u64>>,
+    dists: &[f32],
+) {
+    let lower_bound = lower_bound.unwrap_or(f32::MIN).into();
+    let upper_bound = upper_bound.unwrap_or(f32::MAX).into();
+    let mut max_dist = res.peek().map(|node| node.dist);
+    for (id, dist) in dists.iter().copied().enumerate() {
+        let dist = OrderedFloat(dist);
+        if dist < lower_bound || dist >= upper_bound {
+            continue;
+        }
+        if res.len() < k {
+            res.push(OrderedNode::new(row_id(id as u32), dist));
+            if res.len() == k {
+                max_dist = res.peek().map(|node| node.dist);
             }
+        } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
+            res.pop();
+            res.push(OrderedNode::new(row_id(id as u32), dist));
+            max_dist = res.peek().map(|node| node.dist);
         }
-        dist
-    } else {
-        let offset_id = id - (num_vectors - remainder);
-        let remainder_codes = &codes[(num_vectors - remainder) * num_code_bytes..];
+    }
+}
 
-        let mut dist = 0.0f32;
-        for &code_byte in remainder_codes.iter().skip(offset_id).step_by(remainder) {
-            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
-                let current_code = (code_byte & 0x0F) as usize;
-                let next_code = (code_byte >> 4) as usize;
-                dist += current_dt[current_code] + next_dt[next_code];
+fn accumulate_filtered_distances_into_heap(
+    k: usize,
+    lower_bound: Option<f32>,
+    upper_bound: Option<f32>,
+    row_ids: impl Iterator<Item = (u32, u64)>,
+    accept_row: impl Fn(u64) -> bool,
+    res: &mut BinaryHeap<OrderedNode<u64>>,
+    dists: &[f32],
+) {
+    let lower_bound = lower_bound.unwrap_or(f32::MIN).into();
+    let upper_bound = upper_bound.unwrap_or(f32::MAX).into();
+    let mut max_dist = res.peek().map(|node| node.dist);
+    for (id, row_id) in row_ids {
+        if !accept_row(row_id) {
+            continue;
+        }
+        let Some(dist) = dists.get(id as usize).copied() else {
+            continue;
+        };
+        let dist = OrderedFloat(dist);
+        if dist < lower_bound || dist >= upper_bound {
+            continue;
+        }
+        if res.len() < k {
+            res.push(OrderedNode::new(row_id, dist));
+            if res.len() == k {
+                max_dist = res.peek().map(|node| node.dist);
             }
+        } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
+            res.pop();
+            res.push(OrderedNode::new(row_id, dist));
+            max_dist = res.peek().map(|node| node.dist);
         }
-        dist
     }
 }
 
-#[inline]
-fn get_rq_code(
-    codes: &[u8],
-    id: usize,
-    num_vectors: usize,
-    num_code_bytes: usize,
-) -> impl Iterator<Item = u8> + '_ {
-    let remainder = num_vectors % BATCH_SIZE;
+impl VectorStore for RabitQuantizationStorage {
+    type DistanceCalculator<'a> = RabitDistCalculator<'a>;
 
-    if id < num_vectors - remainder {
-        // the codes are packed
-        let codes = &codes[id / BATCH_SIZE * BATCH_SIZE * num_code_bytes
-            ..(id / BATCH_SIZE + 1) * BATCH_SIZE * num_code_bytes];
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn schema(&self) -> &SchemaRef {
+        self.batch.schema_ref()
+    }
+
+    fn to_batches(&self) -> Result<impl Iterator<Item = RecordBatch> + Send> {
+        Ok(std::iter::once(self.batch.clone()))
+    }
+
+    fn append_batch(&self, _batch: RecordBatch, _vector_column: &str) -> Result<Self> {
+        unimplemented!("RabitQ does not support append_batch")
+    }
+
+    fn len(&self) -> usize {
+        self.batch.num_rows()
+    }
+
+    fn row_id(&self, id: u32) -> u64 {
+        self.row_ids.value(id as usize)
+    }
+
+    fn row_ids(&self) -> impl Iterator<Item = &u64> {
+        self.row_ids.values().iter()
+    }
+
+    fn distance_type(&self) -> DistanceType {
+        self.distance_type
+    }
+
+    // qr = (q-c)
+    #[inline(never)]
+    fn dist_calculator(&self, qr: Arc<dyn Array>, dist_q_c: f32) -> Self::DistanceCalculator<'_> {
+        let code_dim = self.code_dim();
+        let rotated_qr = self.rotate_query_vector(code_dim, &qr);
+        let dist_table = build_dist_table_direct::<Float32Type>(&rotated_qr);
+        let query_factor = match self.metadata.query_estimator {
+            RabitQueryEstimator::ResidualQuery => self.residual_query_factor(dist_q_c),
+            RabitQueryEstimator::RawQuery => self.raw_query_factor(dist_q_c, &rotated_qr, None),
+        };
+        let query_error = match self.metadata.query_estimator {
+            RabitQueryEstimator::ResidualQuery => 0.0,
+            RabitQueryEstimator::RawQuery => {
+                self.raw_query_error_for_gating(dist_q_c, &rotated_qr, None)
+            }
+        };
+        let sum_q = rotated_qr.iter().copied().sum();
+        // The kernels read the rotated query directly; only unaligned dims
+        // need a zero-padded copy.
+        let ex_query = if code_dim.is_multiple_of(EX_DOT_BLOCK_DIMS) {
+            rotated_qr
+        } else {
+            let mut padded = vec![0.0; padded_query_len(code_dim)];
+            pad_query_into(&rotated_qr, &mut padded);
+            padded
+        };
+
+        self.distance_calculator_from_parts(RabitDistCalculatorParts {
+            dim: code_dim,
+            dist_table: Cow::Owned(dist_table),
+            ex_query: Cow::Owned(ex_query),
+            sum_q,
+            query_factor,
+            query_error,
+            approx_mode: ApproxMode::Normal,
+        })
+    }
+
+    // qr = (q-c)
+    #[inline(never)]
+    fn dist_calculator_with_scratch<'a>(
+        &'a self,
+        qr: Arc<dyn Array>,
+        dist_q_c: f32,
+        residual: Option<QueryResidual<'a>>,
+        f32_scratch: &'a mut Vec<f32>,
+        options: DistanceCalculatorOptions,
+    ) -> Self::DistanceCalculator<'a> {
+        let code_dim = self.code_dim();
+        if let (
+            RabitQueryEstimator::RawQuery,
+            Some(QueryResidual::RabitRawQuery {
+                rotated_centroid,
+                query: Some(raw_query),
+            }),
+        ) = (self.metadata.query_estimator, residual)
+        {
+            debug_assert_eq!(raw_query.code_dim, code_dim);
+            debug_assert_eq!(raw_query.ex_bits, self.metadata.num_bits - 1);
+            let query_factor =
+                self.raw_query_factor(dist_q_c, &raw_query.rotated_query, rotated_centroid);
+            let query_error = self.raw_query_error_for_gating(
+                dist_q_c,
+                &raw_query.rotated_query,
+                rotated_centroid,
+            );
+            return self.distance_calculator_from_parts(RabitDistCalculatorParts {
+                dim: code_dim,
+                dist_table: Cow::Borrowed(&raw_query.dist_table),
+                ex_query: Cow::Borrowed(kernel_query(
+                    &raw_query.rotated_query,
+                    &raw_query.ex_query,
+                )),
+                sum_q: raw_query.sum_q,
+                query_factor,
+                query_error,
+                approx_mode: options.approx_mode,
+            });
+        }
+
+        let dist_table_len = code_dim * 4;
+        let ex_bits = self.metadata.num_bits - 1;
+        // The kernels read the rotated query in place; a zero-padded copy is
+        // only needed when the rotated dim is not block-aligned.
+        let ex_query_table_len = if ex_bits == 0 || code_dim.is_multiple_of(EX_DOT_BLOCK_DIMS) {
+            0
+        } else {
+            padded_query_len(code_dim)
+        };
+        f32_scratch.resize(code_dim + dist_table_len + ex_query_table_len, 0.0);
+
+        let query_factor;
+        let query_error;
+        let sum_q = {
+            let (rotated_qr, remaining) = f32_scratch.split_at_mut(code_dim);
+            let (dist_table, ex_query) = remaining.split_at_mut(dist_table_len);
+            match residual {
+                Some(QueryResidual::Centroid(residual_centroid)) => {
+                    self.rotate_query_vector_into(
+                        code_dim,
+                        &qr,
+                        Some(residual_centroid),
+                        rotated_qr,
+                    );
+                }
+                Some(QueryResidual::RabitRawQuery { .. }) | None => {
+                    self.rotate_query_vector_into(code_dim, &qr, None, rotated_qr);
+                }
+            }
+            query_factor = match (self.metadata.query_estimator, residual) {
+                (RabitQueryEstimator::ResidualQuery, _) => self.residual_query_factor(dist_q_c),
+                (
+                    RabitQueryEstimator::RawQuery,
+                    Some(QueryResidual::RabitRawQuery {
+                        rotated_centroid, ..
+                    }),
+                ) => self.raw_query_factor(dist_q_c, rotated_qr, rotated_centroid),
+                (RabitQueryEstimator::RawQuery, _) => {
+                    self.raw_query_factor(dist_q_c, rotated_qr, None)
+                }
+            };
+            query_error = match (self.metadata.query_estimator, residual) {
+                (RabitQueryEstimator::ResidualQuery, _) => 0.0,
+                (
+                    RabitQueryEstimator::RawQuery,
+                    Some(QueryResidual::RabitRawQuery {
+                        rotated_centroid, ..
+                    }),
+                ) => self.raw_query_error_for_gating(dist_q_c, rotated_qr, rotated_centroid),
+                (RabitQueryEstimator::RawQuery, _) => {
+                    self.raw_query_error_for_gating(dist_q_c, rotated_qr, None)
+                }
+            };
+            build_dist_table_direct_into::<Float32Type>(rotated_qr, dist_table);
+            if ex_query_table_len > 0 {
+                pad_query_into(rotated_qr, ex_query);
+            }
+            rotated_qr.iter().copied().sum()
+        };
+
+        let ex_query_start = code_dim + dist_table_len;
+        self.distance_calculator_from_parts(RabitDistCalculatorParts {
+            dim: code_dim,
+            dist_table: Cow::Borrowed(&f32_scratch[code_dim..ex_query_start]),
+            ex_query: Cow::Borrowed(kernel_query(
+                &f32_scratch[..code_dim],
+                &f32_scratch[ex_query_start..ex_query_start + ex_query_table_len],
+            )),
+            sum_q,
+            query_factor,
+            query_error,
+            approx_mode: options.approx_mode,
+        })
+    }
+
+    // TODO: implement this
+    // This method is required for HNSW, we can't support HNSW_RABIT before this is implemented
+    fn dist_calculator_from_id(&self, _: u32) -> Self::DistanceCalculator<'_> {
+        unimplemented!("RabitQ does not support dist_calculator_from_id")
+    }
+}
+
+const LOWBIT_IDX: [usize; 16] = {
+    let mut array = [0; 16];
+    let mut i = 1;
+    while i < 16 {
+        array[i] = i.trailing_zeros() as usize;
+        i += 1;
+    }
+    array
+};
+
+fn get_column(
+    quantization_code: &[u8],
+    code_len: usize,
+    row: usize,
+    col_idx: usize,
+    codes: &mut [u8; 32],
+) {
+    for (i, code) in codes.iter_mut().enumerate() {
+        let vec_idx = row + i;
+        *code = quantization_code[vec_idx * code_len + col_idx];
+    }
+}
+
+pub fn pack_codes(codes: &FixedSizeListArray) -> FixedSizeListArray {
+    let code_len = codes.value_length() as usize;
+
+    // round up num of vectors to multiple of batch size (32)
+    let num_blocks = codes.len() / BATCH_SIZE;
+    let num_packed_vectors = num_blocks * BATCH_SIZE;
+
+    // calculate total size for packed blocks
+    // we pack each 32 vectors into a block, each block contains 2 codes (1byte) of each vector
+    // so every 32 vectors would produce code_len blocks
+    // the low 16 bytes of each block is the codes for the low 4 bits of each vector
+    // the high 16 bytes of each block is the codes for the high 4 bits of each vector
+    let mut blocks = vec![0u8; codes.values().len()];
+
+    let codes_values = codes
+        .slice(0, num_packed_vectors)
+        .values()
+        .as_primitive::<UInt8Type>()
+        .clone();
+    let codes_values = codes_values.values();
+
+    // Pack codes batch by batch
+    // Each batch contains codes for 32 vectors
+    let mut col = [0u8; 32];
+    let mut col_0 = [0u8; 32]; // lower 4 bits
+    let mut col_1 = [0u8; 32]; // higher 4 bits
+    for row in (0..num_packed_vectors).step_by(BATCH_SIZE) {
+        // Get quantization codes for each column for each batch
+        // i.e., we get the codes for 8 dims of 32 vectors and reorganize the data layout
+        // based on the shuffle SIMD instruction used during querying
+        for i in 0..code_len {
+            get_column(codes_values, code_len, row, i, &mut col);
+
+            for j in 0..32 {
+                col_0[j] = col[j] & 0xF;
+                col_1[j] = col[j] >> 4;
+            }
+
+            let block_offset = (row / BATCH_SIZE) * code_len * BATCH_SIZE + i * BATCH_SIZE;
+            for j in 0..16 {
+                // The lower 4 bits represent vector 0 to 15
+                // The upper 4 bits represent vector 16 to 31
+                let val0 = col_0[PERM0[j]] | (col_0[PERM0[j] + 16] << 4);
+                let val1 = col_1[PERM0[j]] | (col_1[PERM0[j] + 16] << 4);
+                blocks[block_offset + j] = val0;
+                blocks[block_offset + j + 16] = val1;
+            }
+        }
+    }
+
+    // for the left codes, transpose them for better cache locality
+    let transposed_codes = transpose(
+        &codes.values().as_primitive::<UInt8Type>().slice(
+            num_packed_vectors * code_len,
+            (codes.len() - num_packed_vectors) * code_len,
+        ),
+        codes.len() - num_packed_vectors,
+        code_len,
+    );
+
+    let offset = codes.values().len() - transposed_codes.len();
+    for (i, v) in transposed_codes.values().iter().enumerate() {
+        blocks[offset + i] = *v;
+    }
+
+    assert_eq!(blocks.len(), codes.values().len());
+    FixedSizeListArray::try_new_from_values(UInt8Array::from(blocks), code_len as i32).unwrap()
+}
+
+// Inverse of pack_codes
+pub fn unpack_codes(codes: &FixedSizeListArray) -> FixedSizeListArray {
+    let code_len = codes.value_length() as usize;
+    let num_vectors = codes.len();
+
+    // Calculate number of complete batches
+    let num_blocks = num_vectors / BATCH_SIZE;
+    let num_packed_vectors = num_blocks * BATCH_SIZE;
+
+    let mut unpacked = vec![0u8; codes.values().len()];
+
+    let codes_values = codes.values().as_primitive::<UInt8Type>().values();
+
+    // Unpack complete batches
+    for batch_idx in 0..num_blocks {
+        let block_start = batch_idx * code_len * BATCH_SIZE;
+
+        for i in 0..code_len {
+            let block_offset = block_start + i * BATCH_SIZE;
+            let block = &codes_values[block_offset..block_offset + BATCH_SIZE];
+
+            // Reverse the permutation
+            for j in 0..16 {
+                let val0 = block[j];
+                let val1 = block[j + 16];
+
+                let low_0 = val0 & 0xF;
+                let high_0 = val0 >> 4;
+                let low_1 = val1 & 0xF;
+                let high_1 = val1 >> 4;
+
+                let vec_idx_0 = batch_idx * BATCH_SIZE + PERM0[j];
+                let vec_idx_1 = batch_idx * BATCH_SIZE + PERM0[j] + 16;
+
+                unpacked[vec_idx_0 * code_len + i] = low_0 | (low_1 << 4);
+                unpacked[vec_idx_1 * code_len + i] = high_0 | (high_1 << 4);
+            }
+        }
+    }
+
+    // Transpose back the remainder
+    if num_packed_vectors < num_vectors {
+        let remainder = num_vectors - num_packed_vectors;
+        let offset = num_packed_vectors * code_len;
+        let transposed_data = &codes_values[offset..];
+
+        // Transpose from column-major back to row-major
+        for row in 0..remainder {
+            for col in 0..code_len {
+                unpacked[offset + row * code_len + col] = transposed_data[col * remainder + row];
+            }
+        }
+    }
+
+    FixedSizeListArray::try_new_from_values(UInt8Array::from(unpacked), code_len as i32).unwrap()
+}
+
+/// Build a row-id remapping for the rows present in this partition from a
+/// fragment-reuse index, mirroring the PQ storage frag-reuse path.
+///
+/// Returns `None` when there is nothing to do (no fragment-reuse index, or the
+/// index leaves every present row id unchanged), so callers keep the zero-cost
+/// no-op path. Otherwise, returns a `HashMap` mapping every affected old row id
+/// to `Some(new_id)` for surviving rows or `None` for rows whose covering
+/// fragment was compacted away, suitable for `RabitQuantizationStorage::remap`.
+fn build_frag_reuse_mapping(
+    fri: Option<&FragReuseIndex>,
+    row_ids: &UInt64Array,
+) -> Option<HashMap<u64, Option<u64>>> {
+    let fri = fri?;
+    if fri.row_id_maps.is_empty() {
+        return None;
+    }
+    let mut mapping: HashMap<u64, Option<u64>> = HashMap::new();
+    for row_id in row_ids.values().iter() {
+        match fri.remap_row_id(*row_id) {
+            Some(new_id) if new_id == *row_id => {}
+            mapped => {
+                mapping.insert(*row_id, mapped);
+            }
+        }
+    }
+    if mapping.is_empty() {
+        None
+    } else {
+        Some(mapping)
+    }
+}
+
+#[async_trait]
+impl QuantizerStorage for RabitQuantizationStorage {
+    type Metadata = RabitQuantizationMetadata;
+
+    fn try_from_batch(
+        batch: RecordBatch,
+        metadata: &Self::Metadata,
+        distance_type: DistanceType,
+        fri: Option<Arc<FragReuseIndex>>,
+    ) -> Result<Self> {
+        let distance_type = match (metadata.query_estimator, distance_type) {
+            (RabitQueryEstimator::RawQuery, DistanceType::Cosine) => DistanceType::L2,
+            _ => distance_type,
+        };
+        validate_rq_num_bits(metadata.num_bits)?;
+        let row_ids = batch[ROW_ID].as_primitive::<UInt64Type>().clone();
+        let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
+        let expected_code_bytes = metadata.binary_code_bytes();
+        if expected_code_bytes > 0 && codes.value_length() as usize != expected_code_bytes {
+            return Err(Error::invalid_input(format!(
+                "RabitQ code byte width mismatch: column {} has {} bytes, metadata rotated_dim={} requires {} bytes",
+                RABIT_CODE_COLUMN,
+                codes.value_length(),
+                metadata.rotated_dim(),
+                expected_code_bytes
+            )));
+        }
+        let add_factors = batch[ADD_FACTORS_COLUMN]
+            .as_primitive::<Float32Type>()
+            .clone();
+        let scale_factors = batch[SCALE_FACTORS_COLUMN]
+            .as_primitive::<Float32Type>()
+            .clone();
+        let error_factors = batch
+            .column_by_name(ERROR_FACTORS_COLUMN)
+            .map(|factors| factors.as_primitive::<Float32Type>().clone());
+        let ex_bits = rabit_ex_bits(metadata.num_bits)?;
+        let mut batch = batch;
+        let mut ex_codes = None;
+        let mut ex_add_factors = None;
+        let mut ex_scale_factors = None;
+        if ex_bits != 0 {
+            let (normalized_batch, codes) =
+                load_blocked_ex_codes(batch, metadata.rotated_dim(), metadata.num_bits)?;
+            batch = normalized_batch;
+            ex_codes = Some(codes);
+            ex_add_factors = Some(
+                batch
+                    .column_by_name(EX_ADD_FACTORS_COLUMN)
+                    .ok_or_else(|| {
+                        Error::invalid_input(format!(
+                            "RabitQ num_bits={} requires {} column",
+                            metadata.num_bits, EX_ADD_FACTORS_COLUMN
+                        ))
+                    })?
+                    .as_primitive::<Float32Type>()
+                    .clone(),
+            );
+            ex_scale_factors = Some(
+                batch
+                    .column_by_name(EX_SCALE_FACTORS_COLUMN)
+                    .ok_or_else(|| {
+                        Error::invalid_input(format!(
+                            "RabitQ num_bits={} requires {} column",
+                            metadata.num_bits, EX_SCALE_FACTORS_COLUMN
+                        ))
+                    })?
+                    .as_primitive::<Float32Type>()
+                    .clone(),
+            );
+        } else if metadata.query_estimator == RabitQueryEstimator::RawQuery {
+            if batch.column_by_name(EX_ADD_FACTORS_COLUMN).is_some()
+                || batch.column_by_name(EX_SCALE_FACTORS_COLUMN).is_some()
+                || batch.column_by_name(RABIT_EX_CODE_COLUMN).is_some()
+                || batch.column_by_name(RABIT_BLOCKED_EX_CODE_COLUMN).is_some()
+            {
+                return Err(Error::invalid_input(
+                    "RabitQ num_bits=1 raw-query indexes must not contain ex-code columns"
+                        .to_string(),
+                ));
+            }
+        } else if batch.column_by_name(RABIT_EX_CODE_COLUMN).is_some()
+            || batch.column_by_name(RABIT_BLOCKED_EX_CODE_COLUMN).is_some()
+        {
+            return Err(Error::invalid_input(format!(
+                "RabitQ num_bits={} does not support ex-code columns",
+                metadata.num_bits
+            )));
+        }
+
+        let (batch, codes) = if !metadata.packed {
+            let codes = pack_codes(&codes);
+            let batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, Arc::new(codes))?;
+            let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
+            (batch, codes)
+        } else {
+            (batch, codes)
+        };
+
+        let mut metadata = metadata.clone();
+        metadata.packed = true;
+        let packed_ex_codes =
+            maybe_pack_ex_codes(ex_codes.as_ref(), ex_bits, error_factors.as_ref());
+
+        let storage = Self {
+            metadata,
+            batch,
+            distance_type,
+            row_ids,
+            codes,
+            add_factors,
+            scale_factors,
+            error_factors,
+            ex_codes,
+            packed_ex_codes,
+            ex_add_factors,
+            ex_scale_factors,
+        };
+
+        match build_frag_reuse_mapping(fri.as_deref(), &storage.row_ids) {
+            Some(mapping) => storage.remap(&RowAddrRemap::direct(mapping)),
+            None => Ok(storage),
+        }
+    }
+
+    fn metadata(&self) -> &Self::Metadata {
+        &self.metadata
+    }
+
+    async fn load_partition(
+        reader: &PreviousFileReader,
+        range: std::ops::Range<usize>,
+        distance_type: DistanceType,
+        metadata: &Self::Metadata,
+        frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    ) -> Result<Self> {
+        let schema = reader.schema();
+        let batch = reader.read_range(range, schema).await?;
+        Self::try_from_batch(batch, metadata, distance_type, frag_reuse_index)
+    }
+
+    fn remap(&self, mapping: &RowAddrRemap) -> Result<Self> {
+        let num_vectors = self.codes.len();
+        let num_code_bytes = self.codes.value_length() as usize;
+        let codes = self.codes.values().as_primitive::<UInt8Type>().values();
+        let mut indices = Vec::with_capacity(num_vectors);
+        let mut new_row_ids = Vec::with_capacity(num_vectors);
+        let mut new_codes = Vec::with_capacity(codes.len());
+
+        let row_ids = self.row_ids.values();
+        for (i, row_id) in row_ids.iter().enumerate() {
+            match mapping.get(*row_id) {
+                Some(Some(new_id)) => {
+                    indices.push(i as u32);
+                    new_row_ids.push(new_id);
+                    new_codes.extend(get_rq_code(codes, i, num_vectors, num_code_bytes));
+                }
+                Some(None) => {}
+                None => {
+                    indices.push(i as u32);
+                    new_row_ids.push(*row_id);
+                    new_codes.extend(get_rq_code(codes, i, num_vectors, num_code_bytes));
+                }
+            }
+        }
+
+        let new_row_ids = UInt64Array::from(new_row_ids);
+        let new_codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from(new_codes),
+            num_code_bytes as i32,
+        )?;
+        let batch = if new_row_ids.is_empty() {
+            RecordBatch::new_empty(self.schema().clone())
+        } else {
+            let codes = Arc::new(pack_codes(&new_codes));
+            self.batch
+                .take(&UInt32Array::from(indices))?
+                .replace_column_by_name(ROW_ID, Arc::new(new_row_ids.clone()))?
+                .replace_column_by_name(RABIT_CODE_COLUMN, codes)?
+        };
+        let codes = batch[RABIT_CODE_COLUMN].as_fixed_size_list().clone();
+        let add_factors = batch[ADD_FACTORS_COLUMN]
+            .as_primitive::<Float32Type>()
+            .clone();
+        let scale_factors = batch[SCALE_FACTORS_COLUMN]
+            .as_primitive::<Float32Type>()
+            .clone();
+        let error_factors = batch
+            .column_by_name(ERROR_FACTORS_COLUMN)
+            .map(|factors| factors.as_primitive::<Float32Type>().clone());
+        let ex_bits = rabit_ex_bits(self.metadata.num_bits)?;
+        let (batch, ex_codes) = if ex_bits == 0 {
+            (batch, None)
+        } else {
+            // `self.batch` is already normalized at load, so this is a
+            // zero-copy column lookup.
+            let (batch, codes) =
+                load_blocked_ex_codes(batch, self.metadata.rotated_dim(), self.metadata.num_bits)?;
+            (batch, Some(codes))
+        };
+        let packed_ex_codes =
+            maybe_pack_ex_codes(ex_codes.as_ref(), ex_bits, error_factors.as_ref());
+        let ex_add_factors = batch
+            .column_by_name(EX_ADD_FACTORS_COLUMN)
+            .map(|factors| factors.as_primitive::<Float32Type>().clone());
+        let ex_scale_factors = batch
+            .column_by_name(EX_SCALE_FACTORS_COLUMN)
+            .map(|factors| factors.as_primitive::<Float32Type>().clone());
+
+        Ok(Self {
+            metadata: self.metadata.clone(),
+            distance_type: self.distance_type,
+            batch,
+            codes,
+            add_factors,
+            scale_factors,
+            error_factors,
+            ex_codes,
+            packed_ex_codes,
+            ex_add_factors,
+            ex_scale_factors,
+            row_ids: new_row_ids,
+        })
+    }
+}
+
+/// Compute the raw distance for a single vector without allocating.
+///
+/// Fuses code extraction from the packed layout with distance accumulation
+/// in a single pass, avoiding the intermediate `Vec` allocation that
+/// `get_rq_code` + iterator would require.
+#[inline]
+fn compute_single_rq_distance(
+    codes: &[u8],
+    id: usize,
+    num_vectors: usize,
+    num_code_bytes: usize,
+    dist_table: &[f32],
+) -> f32 {
+    let remainder = num_vectors % BATCH_SIZE;
+    let mut dist_table_iter = dist_table.chunks_exact(SEGMENT_NUM_CODES).tuples();
+
+    if id < num_vectors - remainder {
+        let batch_codes = &codes[id / BATCH_SIZE * BATCH_SIZE * num_code_bytes
+            ..(id / BATCH_SIZE + 1) * BATCH_SIZE * num_code_bytes];
+
+        let id_in_batch = id % BATCH_SIZE;
+        let idx = PERM0_INVERSE[id_in_batch % 16];
+        let is_lower = id_in_batch < 16;
+
+        let mut dist = 0.0f32;
+        for block in batch_codes.chunks_exact(BATCH_SIZE) {
+            let code_byte = if is_lower {
+                (block[idx] & 0xF) | (block[idx + 16] << 4)
+            } else {
+                (block[idx] >> 4) | (block[idx + 16] & 0xF0)
+            };
+            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
+                let current_code = (code_byte & 0x0F) as usize;
+                let next_code = (code_byte >> 4) as usize;
+                dist += current_dt[current_code] + next_dt[next_code];
+            }
+        }
+        dist
+    } else {
+        let offset_id = id - (num_vectors - remainder);
+        let remainder_codes = &codes[(num_vectors - remainder) * num_code_bytes..];
+
+        let mut dist = 0.0f32;
+        for &code_byte in remainder_codes.iter().skip(offset_id).step_by(remainder) {
+            if let Some((current_dt, next_dt)) = dist_table_iter.next() {
+                let current_code = (code_byte & 0x0F) as usize;
+                let next_code = (code_byte >> 4) as usize;
+                dist += current_dt[current_code] + next_dt[next_code];
+            }
+        }
+        dist
+    }
+}
+
+#[inline]
+fn get_rq_code(
+    codes: &[u8],
+    id: usize,
+    num_vectors: usize,
+    num_code_bytes: usize,
+) -> impl Iterator<Item = u8> + '_ {
+    let remainder = num_vectors % BATCH_SIZE;
+
+    if id < num_vectors - remainder {
+        // the codes are packed
+        let codes = &codes[id / BATCH_SIZE * BATCH_SIZE * num_code_bytes
+            ..(id / BATCH_SIZE + 1) * BATCH_SIZE * num_code_bytes];
+
+        let id_in_batch = id % BATCH_SIZE;
+        if id_in_batch < 16 {
+            let idx = PERM0_INVERSE[id_in_batch];
+            codes
+                .chunks_exact(BATCH_SIZE)
+                .map(|block| (block[idx] & 0xF) | (block[idx + 16] << 4))
+                .exact_size(num_code_bytes)
+                .collect_vec()
+                .into_iter()
+        } else {
+            let idx = PERM0_INVERSE[id_in_batch - 16];
+            codes
+                .chunks_exact(BATCH_SIZE)
+                .map(|block| (block[idx] >> 4) | (block[idx + 16] & 0xF0))
+                .exact_size(num_code_bytes)
+                .collect_vec()
+                .into_iter()
+        }
+    } else {
+        let id = id - (num_vectors - remainder);
+        let codes = &codes[(num_vectors - remainder) * num_code_bytes..];
+        codes
+            .iter()
+            .skip(id)
+            .step_by(remainder)
+            .copied()
+            .exact_size(num_code_bytes)
+            .collect_vec()
+            .into_iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+    use std::collections::{BinaryHeap, HashMap};
+
+    use arrow_array::{ArrayRef, Float32Array, Float64Array, UInt64Array};
+    use lance_core::ROW_ID;
+    use lance_linalg::distance::DistanceType;
+    use rand::rngs::SmallRng;
+    use rand::{Rng, SeedableRng};
+
+    use crate::vector::bq::{RQRotationType, builder::RabitQuantizer};
+    use crate::vector::quantizer::{Quantization, QuantizerStorage};
+
+    fn build_dist_table_not_optimized<T: ArrowFloatType>(
+        sub_vec: &[T::Native],
+        dist_table: &mut [f32],
+    ) where
+        T::Native: AsPrimitive<f32>,
+    {
+        for (j, dist) in dist_table.iter_mut().enumerate().take(SEGMENT_NUM_CODES) {
+            for (k, v) in sub_vec.iter().enumerate().take(SEGMENT_LENGTH) {
+                if j & (1 << k) != 0 {
+                    *dist += v.as_();
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn test_build_dist_table_not_optimized() {
+        let sub_vec = vec![1.0, 2.0, 3.0, 4.0];
+        let mut expected = vec![0.0; SEGMENT_NUM_CODES];
+        build_dist_table_not_optimized::<Float32Type>(&sub_vec, &mut expected);
+        let mut dist_table = vec![0.0; SEGMENT_NUM_CODES];
+        build_dist_table_for_subvec::<Float32Type>(&sub_vec, &mut dist_table);
+        assert_eq!(dist_table, expected);
+    }
+
+    #[test]
+    fn test_dist_calculator_with_scratch_matches_owned_and_reuses_buffer() {
+        let code_dim = 64;
+        let original_codes = make_test_codes(50, code_dim);
+        let metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(original_codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+        let query = Arc::new(Float32Array::from_iter_values(
+            (0..code_dim).map(|idx| idx as f32 / code_dim as f32),
+        )) as ArrayRef;
+
+        let expected = storage.dist_calculator(query.clone(), 0.25).distance_all(0);
+        let expected_scratch_len = code_dim as usize + code_dim as usize * 4;
+        let mut scratch = Vec::with_capacity(expected_scratch_len);
+        let initial_ptr = scratch.as_ptr();
+        {
+            let calc = storage.dist_calculator_with_scratch(
+                query.clone(),
+                0.25,
+                None,
+                &mut scratch,
+                DistanceCalculatorOptions::default(),
+            );
+            assert_eq!(calc.distance_all(0), expected);
+        }
+        assert_eq!(scratch.len(), expected_scratch_len);
+        assert_eq!(scratch.as_ptr(), initial_ptr);
+
+        scratch.fill(f32::NAN);
+        {
+            let calc = storage.dist_calculator_with_scratch(
+                query,
+                0.25,
+                None,
+                &mut scratch,
+                DistanceCalculatorOptions::default(),
+            );
+            assert_eq!(calc.distance_all(0), expected);
+        }
+        assert_eq!(scratch.as_ptr(), initial_ptr);
+    }
+
+    #[test]
+    fn test_dist_calculator_with_scratch_applies_residual_centroid_without_residual_array() {
+        let code_dim = 64usize;
+        let original_codes = make_test_codes(50, code_dim as i32);
+        let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+        metadata.query_estimator = RabitQueryEstimator::ResidualQuery;
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(original_codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+        let query_values = (0..code_dim)
+            .map(|idx| idx as f32 / code_dim as f32)
+            .collect::<Vec<_>>();
+        let centroid_values = (0..code_dim)
+            .map(|idx| (idx % 7) as f32 / code_dim as f32)
+            .collect::<Vec<_>>();
+        let residual_values = query_values
+            .iter()
+            .zip(centroid_values.iter())
+            .map(|(query, centroid)| query - centroid)
+            .collect::<Vec<_>>();
+        let query = Arc::new(Float32Array::from(query_values)) as ArrayRef;
+        let centroid = Arc::new(Float32Array::from(centroid_values)) as ArrayRef;
+        let residual = Arc::new(Float32Array::from(residual_values)) as ArrayRef;
+
+        let expected = storage.dist_calculator(residual, 0.25).distance_all(0);
+        let mut scratch = Vec::new();
+        let calc = storage.dist_calculator_with_scratch(
+            query.clone(),
+            0.25,
+            Some(QueryResidual::Centroid(centroid.as_ref())),
+            &mut scratch,
+            DistanceCalculatorOptions::default(),
+        );
+
+        assert_eq!(calc.distance_all(0), expected);
+    }
+
+    #[test]
+    fn test_dist_calculator_with_scratch_applies_float64_residual_before_f32_cast() {
+        let code_dim = 64usize;
+        let original_codes = make_test_codes(50, code_dim as i32);
+        let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+        metadata.query_estimator = RabitQueryEstimator::ResidualQuery;
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(original_codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+        let query_values = (0..code_dim)
+            .map(|idx| 1.0 + idx as f64 * 1.0e-9)
+            .collect::<Vec<_>>();
+        let centroid_values = vec![1.0; code_dim];
+        let residual_values = query_values
+            .iter()
+            .zip(centroid_values.iter())
+            .map(|(query, centroid)| query - centroid)
+            .collect::<Vec<_>>();
+        let query = Arc::new(Float64Array::from(query_values)) as ArrayRef;
+        let centroid = Arc::new(Float64Array::from(centroid_values)) as ArrayRef;
+        let residual = Arc::new(Float64Array::from(residual_values)) as ArrayRef;
+
+        let expected = storage.dist_calculator(residual, 0.25).distance_all(0);
+        let mut scratch = Vec::new();
+        let calc = storage.dist_calculator_with_scratch(
+            query,
+            0.25,
+            Some(QueryResidual::Centroid(centroid.as_ref())),
+            &mut scratch,
+            DistanceCalculatorOptions::default(),
+        );
+
+        assert_eq!(calc.distance_all(0), expected);
+    }
+
+    #[test]
+    fn test_pack_unpack_codes() {
+        // Test with multiple batch sizes to cover both packed and transposed sections
+        for num_vectors in [10, 32, 50, 64, 100] {
+            let code_len = 8;
+
+            // Create test data with known pattern
+            let mut codes_data = Vec::new();
+            for i in 0..num_vectors {
+                for j in 0..code_len {
+                    codes_data.push((i * code_len + j) as u8);
+                }
+            }
+
+            let original_codes = FixedSizeListArray::try_new_from_values(
+                UInt8Array::from(codes_data.clone()),
+                code_len,
+            )
+            .unwrap();
+
+            // Pack and then unpack
+            let packed = pack_codes(&original_codes);
+            let unpacked = unpack_codes(&packed);
+
+            // Verify they match
+            assert_eq!(original_codes.len(), unpacked.len());
+            assert_eq!(original_codes.value_length(), unpacked.value_length());
+
+            let original_values = original_codes.values().as_primitive::<UInt8Type>().values();
+            let unpacked_values = unpacked.values().as_primitive::<UInt8Type>().values();
+
+            assert_eq!(
+                original_values, unpacked_values,
+                "Mismatch for num_vectors={}",
+                num_vectors
+            );
+        }
+    }
+
+    #[test]
+    fn test_rabit_split_code_fields() {
+        let bin_field = rabit_binary_code_field(128);
+        let DataType::FixedSizeList(_, bin_code_bytes) = bin_field.data_type() else {
+            panic!("binary code field should be FixedSizeList");
+        };
+        assert_eq!(*bin_code_bytes, 16);
+
+        assert!(rabit_ex_code_field(128, 1).unwrap().is_none());
+        let ex_field = rabit_ex_code_field(128, 9).unwrap().unwrap();
+        assert_eq!(ex_field.name(), RABIT_BLOCKED_EX_CODE_COLUMN);
+        let DataType::FixedSizeList(_, ex_code_bytes) = ex_field.data_type() else {
+            panic!("ex-code field should be FixedSizeList");
+        };
+        assert_eq!(*ex_code_bytes, 128);
+    }
+
+    fn make_test_codes(num_vectors: usize, code_dim: i32) -> FixedSizeListArray {
+        let quantizer =
+            RabitQuantizer::new_with_rotation::<Float32Type>(1, code_dim, RQRotationType::Fast);
+        let values = Float32Array::from_iter_values(
+            (0..num_vectors * code_dim as usize).map(|idx| idx as f32 / code_dim as f32),
+        );
+        let vectors = FixedSizeListArray::try_new_from_values(values, code_dim).unwrap();
+        quantizer
+            .quantize(&vectors)
+            .unwrap()
+            .as_fixed_size_list()
+            .clone()
+    }
+
+    fn make_test_metadata(code_dim: usize) -> RabitQuantizationMetadata {
+        RabitQuantizer::new_with_rotation::<Float32Type>(1, code_dim as i32, RQRotationType::Fast)
+            .metadata(None)
+    }
+
+    #[test]
+    fn test_rabit_metadata_defaults_old_indexes_to_residual_query() {
+        let metadata: RabitQuantizationMetadata = serde_json::from_str(
+            r#"{"rotate_mat_position":0,"rotation_type":"matrix","code_dim":64,"num_bits":1,"packed":true}"#,
+        )
+        .unwrap();
+        assert_eq!(metadata.query_estimator, RabitQueryEstimator::ResidualQuery);
+    }
+
+    #[test]
+    fn test_new_rabit_metadata_uses_raw_query_estimator() {
+        let metadata = make_test_metadata(64);
+        assert_eq!(metadata.query_estimator, RabitQueryEstimator::RawQuery);
+    }
+
+    fn make_test_batch(codes: FixedSizeListArray) -> RecordBatch {
+        let num_rows = codes.len();
+        RecordBatch::try_from_iter(vec![
+            (
+                ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)) as ArrayRef,
+            ),
+            (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32),
+                )) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 0.5),
+                )) as ArrayRef,
+            ),
+            (
+                ERROR_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 0.25),
+                )) as ArrayRef,
+            ),
+        ])
+        .unwrap()
+    }
+
+    fn make_test_ex_codes(num_vectors: usize, code_dim: usize, num_bits: u8) -> FixedSizeListArray {
+        let ex_bits = rabit_ex_bits(num_bits).unwrap();
+        let ex_code_bytes = rabit_ex_code_bytes(code_dim, ex_bits).unwrap();
+        let values = (0..num_vectors * ex_code_bytes)
+            .map(|idx| (idx % 251) as u8)
+            .collect::<Vec<_>>();
+        FixedSizeListArray::try_new_from_values(UInt8Array::from(values), ex_code_bytes as i32)
+            .unwrap()
+    }
+
+    fn make_test_batch_with_ex(
+        codes: FixedSizeListArray,
+        ex_codes: FixedSizeListArray,
+    ) -> RecordBatch {
+        let num_rows = codes.len();
+        RecordBatch::try_from_iter(vec![
+            (
+                ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)) as ArrayRef,
+            ),
+            (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32),
+                )) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 0.5),
+                )) as ArrayRef,
+            ),
+            (
+                ERROR_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 0.25),
+                )) as ArrayRef,
+            ),
+            (RABIT_EX_CODE_COLUMN, Arc::new(ex_codes) as ArrayRef),
+            (
+                EX_ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 10.5),
+                )) as ArrayRef,
+            ),
+            (
+                EX_SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from_iter_values(
+                    (0..num_rows).map(|v| v as f32 + 1.5),
+                )) as ArrayRef,
+            ),
+        ])
+        .unwrap()
+    }
 
-        let id_in_batch = id % BATCH_SIZE;
-        if id_in_batch < 16 {
-            let idx = PERM0_INVERSE[id_in_batch];
-            codes
-                .chunks_exact(BATCH_SIZE)
-                .map(|block| (block[idx] & 0xF) | (block[idx + 16] << 4))
-                .exact_size(num_code_bytes)
-                .collect_vec()
-                .into_iter()
-        } else {
-            let idx = PERM0_INVERSE[id_in_batch - 16];
-            codes
-                .chunks_exact(BATCH_SIZE)
-                .map(|block| (block[idx] >> 4) | (block[idx + 16] & 0xF0))
-                .exact_size(num_code_bytes)
-                .collect_vec()
-                .into_iter()
-        }
-    } else {
-        let id = id - (num_vectors - remainder);
-        let codes = &codes[(num_vectors - remainder) * num_code_bytes..];
-        codes
-            .iter()
-            .skip(id)
-            .step_by(remainder)
-            .copied()
-            .exact_size(num_code_bytes)
-            .collect_vec()
-            .into_iter()
+    fn assert_codes_eq(actual: &FixedSizeListArray, expected: &FixedSizeListArray) {
+        assert_eq!(actual.len(), expected.len());
+        assert_eq!(actual.value_length(), expected.value_length());
+        assert_eq!(
+            actual.values().as_primitive::<UInt8Type>().values(),
+            expected.values().as_primitive::<UInt8Type>().values()
+        );
     }
-}
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::collections::HashMap;
+    #[test]
+    fn test_raw_query_multi_bit_distance_uses_ex_factors() {
+        let code_dim = 8usize;
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits: 2,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0xff, 0xff]), 1).unwrap();
+        let ex_codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0x00, 0xff]), 1).unwrap();
+        let batch = RecordBatch::try_from_iter(vec![
+            (ROW_ID, Arc::new(UInt64Array::from(vec![0, 1])) as ArrayRef),
+            (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0, 0.0])) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0, 0.0])) as ArrayRef,
+            ),
+            (RABIT_EX_CODE_COLUMN, Arc::new(ex_codes) as ArrayRef),
+            (
+                EX_ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![100.0, 10.0])) as ArrayRef,
+            ),
+            (
+                EX_SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![1.0, 1.0])) as ArrayRef,
+            ),
+        ])
+        .unwrap();
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let calc = storage.dist_calculator(query, 0.0);
+
+        assert_eq!(calc.distance(0), 104.0);
+        assert_eq!(calc.distance(1), 22.0);
+        let mut distances = Vec::new();
+        let mut u16_scratch = Vec::new();
+        let mut u8_scratch = Vec::new();
+        let mut u32_scratch = Vec::new();
+        calc.distance_all_with_scratch(
+            0,
+            &mut distances,
+            &mut u16_scratch,
+            &mut u8_scratch,
+            &mut u32_scratch,
+        );
+        assert_eq!(distances, vec![104.0, 22.0]);
+    }
 
-    use arrow_array::{ArrayRef, Float32Array, Float64Array, UInt64Array};
-    use lance_core::ROW_ID;
-    use lance_linalg::distance::DistanceType;
+    /// Exercise the ex-dot kernel through the storage API for every ex width,
+    /// including the widths without FastScan support ({1, 3, 5, 6, 7}), and a
+    /// dim that is not a multiple of the 64-dim kernel group.
+    ///
+    /// The dim must be a multiple of 8: the binary distance stage consumes
+    /// two 4-dim segments per code byte and ignores trailing dims otherwise.
+    #[test]
+    fn test_raw_query_multi_bit_distance_matches_reference_for_all_ex_widths() {
+        use rand::rngs::SmallRng;
+        use rand::{Rng, SeedableRng};
+
+        // 72 exercises the kernels' padded-tail path; 1536 is a production
+        // embedding dim exercising the full-group path. Both the blocked
+        // format and the legacy sequential format must produce the same
+        // distances.
+        for (code_dim, num_rows) in [(72usize, 33usize), (1536, 33)] {
+            for num_bits in 2..=9u8 {
+                for legacy_format in [false, true] {
+                    let ex_bits = num_bits - 1;
+                    let mut rng = SmallRng::seed_from_u64(num_bits as u64);
+
+                    let sign_bits = (0..num_rows * code_dim)
+                        .map(|_| rng.random_bool(0.5))
+                        .collect::<Vec<_>>();
+                    let max_code = ((1u16 << ex_bits) - 1) as u8;
+                    let ex_values = (0..num_rows * code_dim)
+                        .map(|_| rng.random_range(0..=max_code))
+                        .collect::<Vec<_>>();
+
+                    let code_len = rabit_binary_code_bytes(code_dim);
+                    let mut code_bytes = vec![0u8; num_rows * code_len];
+                    for (row, bits) in sign_bits.chunks_exact(code_dim).enumerate() {
+                        for (dim, &bit) in bits.iter().enumerate() {
+                            code_bytes[row * code_len + dim / 8] |= (bit as u8) << (dim % 8);
+                        }
+                    }
+                    let (ex_code_column, ex_code_len, ex_code_bytes) = if legacy_format {
+                        let ex_code_len = rabit_ex_code_bytes(code_dim, ex_bits).unwrap();
+                        let mut ex_code_bytes = vec![0u8; num_rows * ex_code_len];
+                        for (row, values) in ex_values.chunks_exact(code_dim).enumerate() {
+                            for (dim, &value) in values.iter().enumerate() {
+                                let bit_offset = dim * ex_bits as usize;
+                                let bits = (value as u16) << (bit_offset % 8);
+                                ex_code_bytes[row * ex_code_len + bit_offset / 8] |= bits as u8;
+                                if bits >> 8 != 0 {
+                                    ex_code_bytes[row * ex_code_len + bit_offset / 8 + 1] |=
+                                        (bits >> 8) as u8;
+                                }
+                            }
+                        }
+                        (RABIT_EX_CODE_COLUMN, ex_code_len, ex_code_bytes)
+                    } else {
+                        let ex_code_len = blocked_ex_code_bytes(code_dim, ex_bits);
+                        let mut ex_code_bytes = vec![0u8; num_rows * ex_code_len];
+                        for (row, values) in ex_code_bytes
+                            .chunks_exact_mut(ex_code_len)
+                            .zip(ex_values.chunks_exact(code_dim))
+                        {
+                            crate::vector::bq::ex_dot::pack_blocked_row(values, ex_bits, row);
+                        }
+                        (RABIT_BLOCKED_EX_CODE_COLUMN, ex_code_len, ex_code_bytes)
+                    };
+
+                    let identity = Float32Array::from_iter_values((0..code_dim).flat_map(|row| {
+                        (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })
+                    }));
+                    let rotate_mat =
+                        FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+                    let metadata = RabitQuantizationMetadata {
+                        rotate_mat: Some(rotate_mat),
+                        rotate_mat_position: None,
+                        fast_rotation_signs: None,
+                        rotation_type: RQRotationType::Matrix,
+                        code_dim: code_dim as u32,
+                        num_bits,
+                        packed: false,
+                        query_estimator: RabitQueryEstimator::RawQuery,
+                    };
+                    let codes = FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(code_bytes),
+                        code_len as i32,
+                    )
+                    .unwrap();
+                    let ex_codes = FixedSizeListArray::try_new_from_values(
+                        UInt8Array::from(ex_code_bytes),
+                        ex_code_len as i32,
+                    )
+                    .unwrap();
+                    let ex_add_factors = (0..num_rows)
+                        .map(|_| rng.random_range(-1.0f32..1.0))
+                        .collect::<Vec<_>>();
+                    let ex_scale_factors = (0..num_rows)
+                        .map(|_| rng.random_range(0.1f32..1.0))
+                        .collect::<Vec<_>>();
+                    let batch = RecordBatch::try_from_iter(vec![
+                        (
+                            ROW_ID,
+                            Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)) as ArrayRef,
+                        ),
+                        (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
+                        (
+                            ADD_FACTORS_COLUMN,
+                            Arc::new(Float32Array::from(vec![0.0; num_rows])) as ArrayRef,
+                        ),
+                        (
+                            SCALE_FACTORS_COLUMN,
+                            Arc::new(Float32Array::from(vec![0.0; num_rows])) as ArrayRef,
+                        ),
+                        (ex_code_column, Arc::new(ex_codes) as ArrayRef),
+                        (
+                            EX_ADD_FACTORS_COLUMN,
+                            Arc::new(Float32Array::from(ex_add_factors.clone())) as ArrayRef,
+                        ),
+                        (
+                            EX_SCALE_FACTORS_COLUMN,
+                            Arc::new(Float32Array::from(ex_scale_factors.clone())) as ArrayRef,
+                        ),
+                    ])
+                    .unwrap();
+                    let storage = RabitQuantizationStorage::try_from_batch(
+                        batch,
+                        &metadata,
+                        DistanceType::L2,
+                        None,
+                    )
+                    .unwrap();
+
+                    let query = (0..code_dim)
+                        .map(|_| rng.random_range(-1.0f32..1.0))
+                        .collect::<Vec<_>>();
+                    let sum_q = query.iter().sum::<f32>();
+                    let calc = storage.dist_calculator(
+                        Arc::new(Float32Array::from(query.clone())) as ArrayRef,
+                        0.0,
+                    );
 
-    use crate::vector::bq::{RQRotationType, builder::RabitQuantizer};
-    use crate::vector::quantizer::{Quantization, QuantizerStorage};
+                    let code_scale = (1u32 << ex_bits) as f32;
+                    let code_bias = -(code_scale - 0.5);
+                    let expected = (0..num_rows)
+                        .map(|row| {
+                            let binary_ip = (0..code_dim)
+                                .map(|dim| {
+                                    query[dim] * sign_bits[row * code_dim + dim] as u8 as f32
+                                })
+                                .sum::<f32>();
+                            let ex_dist = (0..code_dim)
+                                .map(|dim| query[dim] * ex_values[row * code_dim + dim] as f32)
+                                .sum::<f32>();
+                            let full_dot = code_scale * binary_ip + ex_dist + code_bias * sum_q;
+                            full_dot * ex_scale_factors[row] + ex_add_factors[row]
+                        })
+                        .collect::<Vec<_>>();
+
+                    for (row, &want) in expected.iter().enumerate() {
+                        let got = calc.distance(row as u32);
+                        assert!(
+                            (got - want).abs() <= 1e-3 * want.abs().max(1.0),
+                            "num_bits={num_bits} row={row}: {got} != {want}"
+                        );
+                    }
 
-    fn build_dist_table_not_optimized<T: ArrowFloatType>(
-        sub_vec: &[T::Native],
-        dist_table: &mut [f32],
-    ) where
-        T::Native: AsPrimitive<f32>,
-    {
-        for (j, dist) in dist_table.iter_mut().enumerate().take(SEGMENT_NUM_CODES) {
-            for (k, v) in sub_vec.iter().enumerate().take(SEGMENT_LENGTH) {
-                if j & (1 << k) != 0 {
-                    *dist += v.as_();
+                    let mut distances = Vec::new();
+                    let mut u16_scratch = Vec::new();
+                    let mut u8_scratch = Vec::new();
+                    let mut u32_scratch = Vec::new();
+                    calc.distance_all_with_scratch(
+                        0,
+                        &mut distances,
+                        &mut u16_scratch,
+                        &mut u8_scratch,
+                        &mut u32_scratch,
+                    );
+                    assert_eq!(distances.len(), num_rows);
+                    // The bulk path quantizes the binary LUT to u8, and that error is
+                    // amplified by 2^ex_bits in the multi-bit estimate, so the value
+                    // assertions need a quantization-aware bound. The FastScan ex
+                    // widths additionally quantize the ex LUT and are covered by
+                    // `test_raw_query_multi_bit_distance_all_uses_fastscan_for_split_ex_codes`.
+                    if !matches!(ex_bits, 2 | 4 | 8) {
+                        // Worst-case |error| of one u8-quantized binary LUT lookup is
+                        // (table range) / 255 / 2, accumulated over one lookup per
+                        // 8-dim pair of segments.
+                        let num_tables = code_dim.div_ceil(4);
+                        let mut table_min = f32::INFINITY;
+                        let mut table_max = f32::NEG_INFINITY;
+                        for segment in query.chunks(4) {
+                            for subset in 0..16usize {
+                                let value = segment
+                                    .iter()
+                                    .enumerate()
+                                    .filter(|(idx, _)| subset & (1 << idx) != 0)
+                                    .map(|(_, q)| *q)
+                                    .sum::<f32>();
+                                table_min = table_min.min(value);
+                                table_max = table_max.max(value);
+                            }
+                        }
+                        let binary_bound =
+                            code_scale * num_tables as f32 * (table_max - table_min) / 255.0 / 2.0
+                                * ex_scale_factors.iter().fold(0.0f32, |max, &s| max.max(s));
+                        for (row, (&got, &want)) in
+                            distances.iter().zip(expected.iter()).enumerate()
+                        {
+                            assert!(
+                                (got - want).abs() <= binary_bound + 1e-3,
+                                "num_bits={num_bits} row={row} (distance_all): {got} != {want} (bound {binary_bound})"
+                            );
+                        }
+                        // Rows past the SIMD batch use the exact binary path, so the
+                        // final remainder row must match the per-candidate distance.
+                        let remainder_row = num_rows - 1;
+                        let got = distances[remainder_row];
+                        let want = calc.distance(remainder_row as u32);
+                        assert!(
+                            (got - want).abs() <= 1e-3 * want.abs().max(1.0),
+                            "num_bits={num_bits} remainder row (distance_all): {got} != {want}"
+                        );
+                    }
                 }
             }
         }
     }
 
     #[test]
-    fn test_build_dist_table_not_optimized() {
-        let sub_vec = vec![1.0, 2.0, 3.0, 4.0];
-        let mut expected = vec![0.0; SEGMENT_NUM_CODES];
-        build_dist_table_not_optimized::<Float32Type>(&sub_vec, &mut expected);
-        let mut dist_table = vec![0.0; SEGMENT_NUM_CODES];
-        build_dist_table_for_subvec::<Float32Type>(&sub_vec, &mut dist_table);
-        assert_eq!(dist_table, expected);
+    fn test_fast_approx_mode_uses_one_bit_scores_for_multi_bit_raw_query() {
+        let code_dim = 8usize;
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits: 2,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0xff, 0xff]), 1).unwrap();
+        let ex_codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0x00, 0xff]), 1).unwrap();
+        let batch = make_test_batch_with_ex(codes, ex_codes)
+            .replace_column_by_name(
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0, 0.0])),
+            )
+            .unwrap();
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let normal = storage.dist_calculator(query.clone(), 0.0).distance_all(0);
+
+        let mut f32_scratch = Vec::new();
+        let calc = storage.dist_calculator_with_scratch(
+            query,
+            0.0,
+            None,
+            &mut f32_scratch,
+            DistanceCalculatorOptions {
+                approx_mode: ApproxMode::Fast,
+            },
+        );
+        let mut distances = Vec::new();
+        let mut u16_scratch = Vec::new();
+        let mut u8_scratch = Vec::new();
+        let mut u32_scratch = Vec::new();
+        calc.distance_all_with_scratch(
+            0,
+            &mut distances,
+            &mut u16_scratch,
+            &mut u8_scratch,
+            &mut u32_scratch,
+        );
+
+        let expected_fast = (0..2)
+            .map(|id| calc.distance(id as u32))
+            .collect::<Vec<_>>();
+        assert_ne!(normal, distances);
+        assert_eq!(distances, expected_fast);
+        assert_eq!(
+            calc.raw_query_lower_bound_gating_disabled_reason(),
+            Some("approx_mode_fast")
+        );
     }
 
     #[test]
-    fn test_dist_calculator_with_scratch_matches_owned_and_reuses_buffer() {
-        let code_dim = 64;
-        let original_codes = make_test_codes(50, code_dim);
-        let metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+    fn test_accurate_approx_mode_reduces_binary_lut_quantization_error() {
+        let code_dim = 64usize;
+        let num_rows = BATCH_SIZE;
+        let original_codes = make_test_codes(num_rows, code_dim as i32);
+        let metadata = make_test_metadata(code_dim);
         let storage = RabitQuantizationStorage::try_from_batch(
             make_test_batch(original_codes),
             &metadata,
@@ -1221,211 +3465,799 @@ mod tests {
         )
         .unwrap();
         let query = Arc::new(Float32Array::from_iter_values(
-            (0..code_dim).map(|idx| idx as f32 / code_dim as f32),
+            (0..code_dim).map(|idx| (idx as f32 * 0.137).sin() + idx as f32 * 0.003),
         )) as ArrayRef;
+        let exact_calc = storage.dist_calculator(query.clone(), 0.0);
+        let exact = (0..num_rows)
+            .map(|id| exact_calc.distance(id as u32))
+            .collect::<Vec<_>>();
 
-        let expected = storage.dist_calculator(query.clone(), 0.25).distance_all(0);
-        let expected_scratch_len = code_dim as usize + code_dim as usize * 4;
-        let mut scratch = Vec::with_capacity(expected_scratch_len);
-        let initial_ptr = scratch.as_ptr();
-        {
-            let calc =
-                storage.dist_calculator_with_scratch(query.clone(), 0.25, None, &mut scratch);
-            assert_eq!(calc.distance_all(0), expected);
-        }
-        assert_eq!(scratch.len(), expected_scratch_len);
-        assert_eq!(scratch.as_ptr(), initial_ptr);
+        let normal = {
+            let mut f32_scratch = Vec::new();
+            let calc = storage.dist_calculator_with_scratch(
+                query.clone(),
+                0.0,
+                None,
+                &mut f32_scratch,
+                DistanceCalculatorOptions::default(),
+            );
+            let mut distances = Vec::new();
+            let mut u16_scratch = Vec::new();
+            let mut u8_scratch = Vec::new();
+            let mut u32_scratch = Vec::new();
+            calc.distance_all_with_scratch(
+                0,
+                &mut distances,
+                &mut u16_scratch,
+                &mut u8_scratch,
+                &mut u32_scratch,
+            );
+            distances
+        };
 
-        scratch.fill(f32::NAN);
-        {
-            let calc = storage.dist_calculator_with_scratch(query, 0.25, None, &mut scratch);
-            assert_eq!(calc.distance_all(0), expected);
-        }
-        assert_eq!(scratch.as_ptr(), initial_ptr);
+        let (accurate, hacc_table_len, hacc_packed_table_len, hacc_accum_len) = {
+            let mut f32_scratch = Vec::new();
+            let calc = storage.dist_calculator_with_scratch(
+                query,
+                0.0,
+                None,
+                &mut f32_scratch,
+                DistanceCalculatorOptions {
+                    approx_mode: ApproxMode::Accurate,
+                },
+            );
+            let mut distances = Vec::new();
+            let mut u16_scratch = Vec::new();
+            let mut u8_scratch = Vec::new();
+            let mut u32_scratch = Vec::new();
+            calc.distance_all_with_scratch(
+                0,
+                &mut distances,
+                &mut u16_scratch,
+                &mut u8_scratch,
+                &mut u32_scratch,
+            );
+            (
+                distances,
+                u16_scratch.len(),
+                u8_scratch.len(),
+                u32_scratch.len(),
+            )
+        };
+
+        let normal_error = normal
+            .iter()
+            .zip(exact.iter())
+            .map(|(actual, expected)| (actual - expected).abs())
+            .sum::<f32>();
+        let accurate_error = accurate
+            .iter()
+            .zip(exact.iter())
+            .map(|(actual, expected)| (actual - expected).abs())
+            .sum::<f32>();
+
+        assert!(normal_error > 0.0);
+        assert!(
+            accurate_error < normal_error,
+            "accurate_error={accurate_error}, normal_error={normal_error}"
+        );
+        assert_eq!(hacc_table_len, code_dim * 4);
+        assert_eq!(hacc_packed_table_len, code_dim * 8);
+        assert_eq!(hacc_accum_len, num_rows);
     }
 
-    #[test]
-    fn test_dist_calculator_with_scratch_applies_residual_centroid_without_residual_array() {
-        let code_dim = 64usize;
-        let original_codes = make_test_codes(50, code_dim as i32);
-        let metadata = make_test_metadata(original_codes.value_length() as usize * 8);
-        let storage = RabitQuantizationStorage::try_from_batch(
-            make_test_batch(original_codes),
-            &metadata,
-            DistanceType::L2,
-            None,
+    fn assert_raw_query_multi_bit_distance_all_uses_fastscan(
+        num_bits: u8,
+        legacy_format: bool,
+        with_error_factors: bool,
+    ) {
+        // Not a multiple of 64, so the padded-tail LUT entries are exercised;
+        // a multiple of 8 as the binary stage requires.
+        let code_dim = 72usize;
+        let num_rows = BATCH_SIZE + 1;
+        let ex_bits = rabit_ex_bits(num_bits).unwrap();
+        let max_code = ((1u16 << ex_bits) - 1) as u8;
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let code_len = rabit_binary_code_bytes(code_dim);
+        let codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from_iter_values((0..num_rows * code_len).map(|idx| (idx * 13) as u8)),
+            code_len as i32,
         )
         .unwrap();
-        let query_values = (0..code_dim)
-            .map(|idx| idx as f32 / code_dim as f32)
-            .collect::<Vec<_>>();
-        let centroid_values = (0..code_dim)
-            .map(|idx| (idx % 7) as f32 / code_dim as f32)
+        let ex_values = (0..num_rows * code_dim)
+            .map(|idx| ((idx * 37) % (max_code as usize + 1)) as u8)
             .collect::<Vec<_>>();
-        let residual_values = query_values
-            .iter()
-            .zip(centroid_values.iter())
-            .map(|(query, centroid)| query - centroid)
+        let (ex_code_column, ex_code_len, ex_code_bytes) = if legacy_format {
+            let ex_code_len = rabit_ex_code_bytes(code_dim, ex_bits).unwrap();
+            let mut ex_code_bytes = vec![0u8; num_rows * ex_code_len];
+            for (row, values) in ex_values.chunks_exact(code_dim).enumerate() {
+                for (dim, &value) in values.iter().enumerate() {
+                    let bit_offset = dim * ex_bits as usize;
+                    let bits = (value as u16) << (bit_offset % 8);
+                    ex_code_bytes[row * ex_code_len + bit_offset / 8] |= bits as u8;
+                    if bits >> 8 != 0 {
+                        ex_code_bytes[row * ex_code_len + bit_offset / 8 + 1] |= (bits >> 8) as u8;
+                    }
+                }
+            }
+            (RABIT_EX_CODE_COLUMN, ex_code_len, ex_code_bytes)
+        } else {
+            let ex_code_len = blocked_ex_code_bytes(code_dim, ex_bits);
+            let mut ex_code_bytes = vec![0u8; num_rows * ex_code_len];
+            for (row, values) in ex_code_bytes
+                .chunks_exact_mut(ex_code_len)
+                .zip(ex_values.chunks_exact(code_dim))
+            {
+                crate::vector::bq::ex_dot::pack_blocked_row(values, ex_bits, row);
+            }
+            (RABIT_BLOCKED_EX_CODE_COLUMN, ex_code_len, ex_code_bytes)
+        };
+        let ex_codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from(ex_code_bytes),
+            ex_code_len as i32,
+        )
+        .unwrap();
+        let batch = RecordBatch::try_from_iter(vec![
+            (
+                ROW_ID,
+                Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)) as ArrayRef,
+            ),
+            (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
+            (
+                ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0; num_rows])) as ArrayRef,
+            ),
+            (
+                SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![1.0; num_rows])) as ArrayRef,
+            ),
+            (ex_code_column, Arc::new(ex_codes) as ArrayRef),
+            (
+                EX_ADD_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![0.0; num_rows])) as ArrayRef,
+            ),
+            (
+                EX_SCALE_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![1.0; num_rows])) as ArrayRef,
+            ),
+        ])
+        .unwrap();
+        let batch = if with_error_factors {
+            batch
+                .try_with_column(
+                    crate::vector::bq::transform::ERROR_FACTORS_FIELD.clone(),
+                    Arc::new(Float32Array::from(vec![1000.0; num_rows])) as ArrayRef,
+                )
+                .unwrap()
+        } else {
+            batch
+        };
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        // The FastScan transpose only exists for indexes that can reach the
+        // bulk bypass path (no error factors); gated indexes fall through to
+        // the exact per-row kernels in `distance_all`.
+        assert_eq!(storage.packed_ex_codes.is_some(), !with_error_factors);
+
+        // A per-dim varying query so that any dim-mapping error in the
+        // FastScan LUT shows up as a value mismatch.
+        let query_values = (0..code_dim)
+            .map(|dim| (dim % 11) as f32 * 0.3 - 1.5)
             .collect::<Vec<_>>();
-        let query = Arc::new(Float32Array::from(query_values)) as ArrayRef;
-        let centroid = Arc::new(Float32Array::from(centroid_values)) as ArrayRef;
-        let residual = Arc::new(Float32Array::from(residual_values)) as ArrayRef;
-
-        let expected = storage.dist_calculator(residual, 0.25).distance_all(0);
-        let mut scratch = Vec::new();
-        let calc = storage.dist_calculator_with_scratch(
-            query.clone(),
-            0.25,
-            Some(QueryResidual::Centroid(centroid.as_ref())),
-            &mut scratch,
+        let query = Arc::new(Float32Array::from(query_values.clone())) as ArrayRef;
+        let calc = storage.dist_calculator(query, 0.0);
+        let mut distances = Vec::new();
+        let mut u16_scratch = Vec::new();
+        let mut u8_scratch = Vec::new();
+        let mut u32_scratch = Vec::new();
+        calc.distance_all_with_scratch(
+            0,
+            &mut distances,
+            &mut u16_scratch,
+            &mut u8_scratch,
+            &mut u32_scratch,
         );
 
-        assert_eq!(calc.distance_all(0), expected);
+        assert_eq!(distances.len(), num_rows);
+        assert_eq!(u16_scratch.len(), BATCH_SIZE);
+        let loaded_ex_code_len = storage.ex_codes.as_ref().unwrap().value_length() as usize;
+        if with_error_factors {
+            // The gated path never builds the ex LUT; the scratch holds the
+            // binary LUT only.
+            assert_eq!(u8_scratch.len(), code_dim * 4);
+        } else {
+            assert_eq!(u8_scratch.len(), loaded_ex_code_len * 2 * SEGMENT_NUM_CODES);
+        }
+
+        // The fastscan estimate differs from the exact path only by the u8
+        // quantization of the binary LUT (amplified by 2^ex_bits) and of the
+        // ex LUT, so bound the comparison by those quantization errors.
+        let mut table_min = f32::INFINITY;
+        let mut table_max = f32::NEG_INFINITY;
+        for segment in query_values.chunks(4) {
+            for subset in 0..SEGMENT_NUM_CODES {
+                let value = segment
+                    .iter()
+                    .enumerate()
+                    .filter(|(idx, _)| subset & (1 << idx) != 0)
+                    .map(|(_, q)| *q)
+                    .sum::<f32>();
+                table_min = table_min.min(value);
+                table_max = table_max.max(value);
+            }
+        }
+        let code_scale = (1u32 << ex_bits) as f32;
+        let binary_bound =
+            code_scale * code_dim.div_ceil(4) as f32 * (table_max - table_min) / 510.0;
+        let mut padded_query = vec![0.0f32; crate::vector::bq::ex_dot::padded_query_len(code_dim)];
+        crate::vector::bq::ex_dot::pad_query_into(&query_values, &mut padded_query);
+        let mut quantized_table = Vec::new();
+        let (ex_qmin, ex_qmax, ex_qcap) = quantize_ex_fastscan_dist_table_into(
+            ex_bits,
+            loaded_ex_code_len,
+            &padded_query,
+            &mut quantized_table,
+        );
+        // Without the FastScan transpose the ex stage is exact, so only the
+        // binary LUT quantization remains.
+        let ex_bound = if with_error_factors {
+            0.0
+        } else {
+            (loaded_ex_code_len * 2) as f32 * (ex_qmax - ex_qmin) / ex_qcap / 2.0
+        };
+        let bound = (binary_bound + ex_bound) * 1.5 + 1e-3;
+        for (id, distance) in distances.iter().take(BATCH_SIZE).enumerate() {
+            let exact = calc.distance(id as u32);
+            assert!(
+                (*distance - exact).abs() <= bound,
+                "distance_all fastscan mismatch for id {id} (num_bits={num_bits} legacy={legacy_format}): actual={distance}, exact={exact}, bound={bound}"
+            );
+        }
+        assert_eq!(distances[BATCH_SIZE], calc.distance(BATCH_SIZE as u32));
     }
 
     #[test]
-    fn test_dist_calculator_with_scratch_applies_float64_residual_before_f32_cast() {
-        let code_dim = 64usize;
-        let original_codes = make_test_codes(50, code_dim as i32);
-        let metadata = make_test_metadata(original_codes.value_length() as usize * 8);
-        let storage = RabitQuantizationStorage::try_from_batch(
-            make_test_batch(original_codes),
-            &metadata,
-            DistanceType::L2,
-            None,
+    fn test_raw_query_multi_bit_distance_all_uses_fastscan_for_split_ex_codes() {
+        for num_bits in [3, 5, 9] {
+            for legacy_format in [false, true] {
+                assert_raw_query_multi_bit_distance_all_uses_fastscan(
+                    num_bits,
+                    legacy_format,
+                    false,
+                );
+            }
+            // Gated indexes (with error factors) skip the FastScan artifacts
+            // and score the bulk path with the exact kernels.
+            assert_raw_query_multi_bit_distance_all_uses_fastscan(num_bits, false, true);
+        }
+    }
+
+    /// A dist table whose `num_tables`-scaled reconstruction overflows `f32`
+    /// must fall back to exact distances rather than the affine dequant's
+    /// `0 * inf = NaN`. Covers both the u8 (Normal) and u16 (Accurate) LUT
+    /// paths end-to-end through `distance_all`, asserting the result is
+    /// NaN-free and bit-identical to the always-exact per-row computation.
+    #[rstest]
+    fn test_degenerate_dist_table_falls_back_to_exact_distances(
+        #[values(ApproxMode::Normal, ApproxMode::Accurate)] approx_mode: ApproxMode,
+    ) {
+        let code_dim = 8usize;
+        let num_rows = BATCH_SIZE + 5;
+        let num_bits = 3;
+        let ex_bits = rabit_ex_bits(num_bits).unwrap();
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from_iter_values((0..num_rows).map(|idx| (idx * 19) as u8)),
+            rabit_binary_code_bytes(code_dim) as i32,
         )
         .unwrap();
-        let query_values = (0..code_dim)
-            .map(|idx| 1.0 + idx as f64 * 1.0e-9)
+        let ex_codes = make_test_ex_codes(num_rows, code_dim, num_bits);
+        let batch = make_test_batch_with_ex(codes, ex_codes);
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+
+        let mut calc = storage.dist_calculator(query, 4.0);
+        calc.approx_mode = approx_mode;
+        // num_tables = (code_dim * 4) / SEGMENT_NUM_CODES = 2; the extrema sum
+        // (qmax - qmin = 4e38) overflows when scaled by num_tables, so the
+        // quantizer returns `Exact`. Per-row sums stay finite (each row reads
+        // one entry per segment), so the exact path is well-defined.
+        let mut degenerate = vec![0.0f32; code_dim * 4];
+        degenerate[0] = -2e38;
+        degenerate[1] = 2e38;
+        calc.dist_table = Cow::Owned(degenerate);
+
+        let code_len = rabit_binary_code_bytes(code_dim);
+        let ex_codes = calc.ex_codes.unwrap();
+        let ex_add_factors = calc.ex_add_factors.unwrap();
+        let ex_scale_factors = calc.ex_scale_factors.unwrap();
+        let expected = (0..num_rows)
+            .map(|id| {
+                let binary_ip = compute_single_rq_distance(
+                    calc.codes,
+                    id,
+                    num_rows,
+                    code_len,
+                    &calc.dist_table,
+                );
+                calc.raw_query_multi_bit_exact_distance(
+                    id,
+                    binary_ip,
+                    ex_bits,
+                    ex_codes,
+                    ex_add_factors,
+                    ex_scale_factors,
+                )
+            })
             .collect::<Vec<_>>();
-        let centroid_values = vec![1.0; code_dim];
-        let residual_values = query_values
+
+        let actual = calc.distance_all(0);
+        assert_eq!(actual.len(), num_rows);
+        for id in 0..num_rows {
+            assert!(
+                !actual[id].is_nan(),
+                "approx_mode={approx_mode:?} id={id}: degenerate table produced NaN"
+            );
+            assert_eq!(
+                actual[id].to_bits(),
+                expected[id].to_bits(),
+                "approx_mode={approx_mode:?} id={id}: distance_all must match the exact path"
+            );
+        }
+    }
+
+    #[test]
+    fn test_raw_query_multi_bit_accumulate_topk_uses_lower_bound_gating() {
+        let code_dim = 8usize;
+        let num_rows = BATCH_SIZE + 9;
+        let num_bits = 3;
+        let ex_bits = rabit_ex_bits(num_bits).unwrap();
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from_iter_values((0..num_rows).map(|idx| (idx * 19) as u8)),
+            1,
+        )
+        .unwrap();
+        let ex_code_len = rabit_ex_code_bytes(code_dim, ex_bits).unwrap();
+        let ex_codes = FixedSizeListArray::try_new_from_values(
+            UInt8Array::from_iter_values(
+                (0..num_rows * ex_code_len).map(|idx| (idx * 29 % 251) as u8),
+            ),
+            ex_code_len as i32,
+        )
+        .unwrap();
+        let batch = make_test_batch_with_ex(codes, ex_codes)
+            .replace_column_by_name(
+                ERROR_FACTORS_COLUMN,
+                Arc::new(Float32Array::from(vec![1000.0; num_rows])),
+            )
+            .unwrap();
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let calc = storage.dist_calculator(query, 4.0);
+        assert!(
+            calc.raw_query_lower_bound_gating_disabled_reason()
+                .is_none()
+        );
+
+        let k = 5;
+        let mut binary_ips = Vec::new();
+        let mut binary_u16_scratch = Vec::new();
+        let mut binary_u8_scratch = Vec::new();
+        let mut binary_u32_scratch = Vec::new();
+        calc.binary_distances_with_scratch(
+            num_rows,
+            rabit_binary_code_bytes(code_dim),
+            &mut binary_ips,
+            &mut binary_u16_scratch,
+            &mut binary_u8_scratch,
+            &mut binary_u32_scratch,
+        );
+        let ex_codes = calc.ex_codes.unwrap();
+        let ex_add_factors = calc.ex_add_factors.unwrap();
+        let ex_scale_factors = calc.ex_scale_factors.unwrap();
+        let mut expected = binary_ips
             .iter()
-            .zip(centroid_values.iter())
-            .map(|(query, centroid)| query - centroid)
+            .copied()
+            .enumerate()
+            .map(|(id, binary_ip)| {
+                (
+                    id,
+                    calc.raw_query_multi_bit_exact_distance(
+                        id,
+                        binary_ip,
+                        ex_bits,
+                        ex_codes,
+                        ex_add_factors,
+                        ex_scale_factors,
+                    ),
+                )
+            })
             .collect::<Vec<_>>();
-        let query = Arc::new(Float64Array::from(query_values)) as ArrayRef;
-        let centroid = Arc::new(Float64Array::from(centroid_values)) as ArrayRef;
-        let residual = Arc::new(Float64Array::from(residual_values)) as ArrayRef;
-
-        let expected = storage.dist_calculator(residual, 0.25).distance_all(0);
-        let mut scratch = Vec::new();
-        let calc = storage.dist_calculator_with_scratch(
-            query,
-            0.25,
-            Some(QueryResidual::Centroid(centroid.as_ref())),
-            &mut scratch,
+        expected.sort_by(|left, right| left.1.total_cmp(&right.1));
+        expected.truncate(k);
+        let mut expected = expected
+            .into_iter()
+            .map(|(id, dist)| (id as u64, dist))
+            .collect::<Vec<_>>();
+        expected.sort_by_key(|left| left.0);
+
+        let mut heap = BinaryHeap::with_capacity(k);
+        let mut distances = Vec::new();
+        let mut u16_scratch = Vec::new();
+        let mut u8_scratch = Vec::new();
+        let mut u32_scratch = Vec::new();
+        calc.accumulate_topk_with_scratch(
+            k,
+            None,
+            None,
+            |id| id as u64,
+            &mut heap,
+            &mut distances,
+            &mut u16_scratch,
+            &mut u8_scratch,
+            &mut u32_scratch,
         );
+        let mut actual = heap
+            .into_iter()
+            .map(|node| (node.id, node.dist.0))
+            .collect::<Vec<_>>();
+        actual.sort_by_key(|left| left.0);
 
-        assert_eq!(calc.distance_all(0), expected);
+        assert_eq!(actual.len(), expected.len());
+        for ((actual_id, actual_dist), (expected_id, expected_dist)) in
+            actual.into_iter().zip(expected)
+        {
+            assert_eq!(actual_id, expected_id);
+            assert!(
+                (actual_dist - expected_dist).abs() < 1e-5,
+                "actual={actual_dist}, expected={expected_dist}"
+            );
+        }
     }
 
-    #[test]
-    fn test_pack_unpack_codes() {
-        // Test with multiple batch sizes to cover both packed and transposed sections
-        for num_vectors in [10, 32, 50, 64, 100] {
-            let code_len = 8;
+    /// Inputs crafted so the top-k scan outcomes are fully determined by the
+    /// factor columns: with zero scale factors, a zero query factor, and a
+    /// query error of one, the lower bound is
+    /// `add_factors[id] - error_factors[id]`, and with zero ex scale factors
+    /// the exact distance is `ex_add_factors[id]`, regardless of the random
+    /// codes and query.
+    struct CraftedTopkData {
+        codes: Vec<u8>,
+        ex_codes: Vec<u8>,
+        dist_table: Vec<f32>,
+        ex_query: Vec<f32>,
+        scale_factors: Vec<f32>,
+        add_factors: Vec<f32>,
+        error_factors: Vec<f32>,
+        ex_scale_factors: Vec<f32>,
+        ex_add_factors: Vec<f32>,
+    }
 
-            // Create test data with known pattern
-            let mut codes_data = Vec::new();
-            for i in 0..num_vectors {
-                for j in 0..code_len {
-                    codes_data.push((i * code_len + j) as u8);
-                }
+    const CRAFTED_TOPK_DIM: usize = 64;
+    const CRAFTED_TOPK_NUM_BITS: u8 = 5;
+
+    impl CraftedTopkData {
+        fn new(
+            exact_dists: &[f32],
+            lower_bound_margins: &[f32],
+            error_factors: Vec<f32>,
+            rng: &mut SmallRng,
+        ) -> Self {
+            let n = exact_dists.len();
+            let code_len = rabit_binary_code_bytes(CRAFTED_TOPK_DIM);
+            let ex_code_len = blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1);
+            let add_factors = izip!(exact_dists, lower_bound_margins, &error_factors)
+                .map(|(dist, margin, error)| dist - margin + error)
+                .collect();
+            Self {
+                codes: (0..n * code_len).map(|_| rng.random()).collect(),
+                ex_codes: (0..n * ex_code_len).map(|_| rng.random()).collect(),
+                dist_table: (0..CRAFTED_TOPK_DIM * 4)
+                    .map(|_| rng.random_range(-1.0f32..1.0))
+                    .collect(),
+                ex_query: (0..CRAFTED_TOPK_DIM)
+                    .map(|_| rng.random_range(-1.0f32..1.0))
+                    .collect(),
+                scale_factors: vec![0.0; n],
+                add_factors,
+                error_factors,
+                ex_scale_factors: vec![0.0; n],
+                ex_add_factors: exact_dists.to_vec(),
             }
+        }
 
-            let original_codes = FixedSizeListArray::try_new_from_values(
-                UInt8Array::from(codes_data.clone()),
-                code_len,
+        fn calculator(&self, approx_mode: ApproxMode) -> RabitDistCalculator<'_> {
+            RabitDistCalculator::new(
+                CRAFTED_TOPK_DIM,
+                CRAFTED_TOPK_NUM_BITS,
+                RabitQueryEstimator::RawQuery,
+                Cow::Borrowed(self.dist_table.as_slice()),
+                Cow::Borrowed(self.ex_query.as_slice()),
+                0.7,
+                &self.codes,
+                Some(&self.ex_codes),
+                blocked_ex_code_bytes(CRAFTED_TOPK_DIM, CRAFTED_TOPK_NUM_BITS - 1),
+                &self.add_factors,
+                &self.scale_factors,
+                Some(&self.error_factors),
+                Some(&self.ex_add_factors),
+                Some(&self.ex_scale_factors),
+                None,
+                0.0,
+                1.0,
+                approx_mode,
             )
-            .unwrap();
+        }
+    }
 
-            // Pack and then unpack
-            let packed = pack_codes(&original_codes);
-            let unpacked = unpack_codes(&packed);
+    fn canonical_heap_rows(heap: BinaryHeap<OrderedNode<u64>>) -> Vec<(u32, u64)> {
+        let mut rows = heap
+            .into_iter()
+            .map(|node| (node.dist.0.to_bits(), node.id))
+            .collect::<Vec<_>>();
+        rows.sort_unstable();
+        rows
+    }
 
-            // Verify they match
-            assert_eq!(original_codes.len(), unpacked.len());
-            assert_eq!(original_codes.value_length(), unpacked.value_length());
+    /// The dense (SIMD-pruned) scan must reproduce the sparse scalar scan
+    /// exactly: identical heap contents including row ids, and the k smallest
+    /// in-bounds exact distances overall.
+    #[rstest]
+    fn test_raw_query_multi_bit_topk_dense_matches_sparse(
+        #[values(ApproxMode::Normal, ApproxMode::Accurate)] approx_mode: ApproxMode,
+        #[values("descending", "ascending", "random", "duplicates", "duplicate_ties")]
+        ordering: &str,
+    ) {
+        for n in [1usize, 15, 16, 17, 100, 4109] {
+            let mut rng = SmallRng::seed_from_u64(n as u64 * 31 + ordering.len() as u64);
+            let exact_dists: Vec<f32> = match ordering {
+                // Improving rows force constant heap updates.
+                "descending" => (0..n).map(|id| (n - id) as f32).collect(),
+                // Worsening rows force mass pruning, the common regime.
+                "ascending" => (0..n).map(|id| id as f32).collect(),
+                "random" => (0..n).map(|_| rng.random_range(0.0..n as f32)).collect(),
+                "duplicates" => (0..n).map(|id| (id % 7) as f32).collect(),
+                // Lower bound equals the distance, so heap-threshold and
+                // upper-bound comparisons hit exact `>=` ties.
+                "duplicate_ties" => (0..n).map(|id| (id % 5) as f32).collect(),
+                _ => unreachable!(),
+            };
+            let (margins, error_factors) = if ordering == "duplicate_ties" {
+                (vec![0.0; n], vec![0.0; n])
+            } else if ordering == "random" {
+                (
+                    (0..n).map(|_| rng.random_range(0.0f32..2.0)).collect(),
+                    (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(),
+                )
+            } else {
+                (
+                    vec![1.0; n],
+                    (0..n).map(|_| rng.random_range(0.0f32..1.0)).collect(),
+                )
+            };
+            let data = CraftedTopkData::new(&exact_dists, &margins, error_factors, &mut rng);
+            let calc = data.calculator(approx_mode);
+            assert!(
+                calc.raw_query_lower_bound_gating_disabled_reason()
+                    .is_none()
+            );
 
-            let original_values = original_codes.values().as_primitive::<UInt8Type>().values();
-            let unpacked_values = unpacked.values().as_primitive::<UInt8Type>().values();
+            let max_dist = exact_dists.iter().fold(0.0f32, |acc, dist| acc.max(*dist));
+            for k in [1usize, 10, n + 7] {
+                for bounds in [(None, None), (Some(max_dist * 0.25), Some(max_dist * 0.7))] {
+                    let (lower_bound, upper_bound) = bounds;
+                    let mut dense_heap = BinaryHeap::new();
+                    let mut sparse_heap = BinaryHeap::new();
+                    let mut dists = Vec::new();
+                    let mut u16_scratch = Vec::new();
+                    let mut u8_scratch = Vec::new();
+                    let mut u32_scratch = Vec::new();
+                    // Two passes sharing the heap, as IVF partition probing
+                    // does: the second pass starts with a full, tight heap.
+                    for pass in 0..2u64 {
+                        let offset = pass * n as u64;
+                        calc.accumulate_topk_with_scratch(
+                            k,
+                            lower_bound,
+                            upper_bound,
+                            |id| id as u64 + offset,
+                            &mut dense_heap,
+                            &mut dists,
+                            &mut u16_scratch,
+                            &mut u8_scratch,
+                            &mut u32_scratch,
+                        );
+                        calc.accumulate_filtered_topk_with_scratch(
+                            k,
+                            lower_bound,
+                            upper_bound,
+                            (0..n as u32).map(|id| (id, id as u64 + offset)),
+                            |_| true,
+                            &mut sparse_heap,
+                            &mut dists,
+                            &mut u16_scratch,
+                            &mut u8_scratch,
+                            &mut u32_scratch,
+                        );
+                    }
+                    let dense = canonical_heap_rows(dense_heap);
+                    let sparse = canonical_heap_rows(sparse_heap);
+                    assert_eq!(
+                        dense, sparse,
+                        "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}"
+                    );
 
-            assert_eq!(
-                original_values, unpacked_values,
-                "Mismatch for num_vectors={}",
-                num_vectors
-            );
+                    // The distance multiset must be the k smallest in-bounds
+                    // distances over both passes. Row ids are not compared:
+                    // evictions among tied maxima depend on heap layout.
+                    let query_lower_bound = lower_bound.unwrap_or(f32::MIN);
+                    let query_upper_bound = upper_bound.unwrap_or(f32::MAX);
+                    let mut expected = (0..2 * n)
+                        .map(|row| exact_dists[row % n])
+                        .filter(|dist| *dist >= query_lower_bound && *dist < query_upper_bound)
+                        .map(|dist| dist.to_bits())
+                        .collect::<Vec<_>>();
+                    expected.sort_unstable();
+                    expected.truncate(k);
+                    let actual = dense.iter().map(|(dist, _)| *dist).collect::<Vec<_>>();
+                    assert_eq!(
+                        actual, expected,
+                        "ordering={ordering} n={n} k={k} bounds={bounds:?} mode={approx_mode:?}"
+                    );
+                }
+            }
         }
     }
 
     #[test]
-    fn test_rabit_split_code_fields() {
-        let bin_field = rabit_binary_code_field(128);
-        let DataType::FixedSizeList(_, bin_code_bytes) = bin_field.data_type() else {
-            panic!("binary code field should be FixedSizeList");
+    fn test_raw_query_one_bit_distance_uses_binary_factors_without_ex_columns() {
+        let code_dim = 8usize;
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
+        );
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits: 1,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
         };
-        assert_eq!(*bin_code_bytes, 16);
+        let codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0xff, 0x00]), 1).unwrap();
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let calc = storage.dist_calculator(query, 3.0);
 
-        assert!(rabit_ex_code_field(128, 1).unwrap().is_none());
-        let ex_field = rabit_ex_code_field(128, 4).unwrap().unwrap();
-        assert_eq!(ex_field.name(), RABIT_EX_CODE_COLUMN);
-        let DataType::FixedSizeList(_, ex_code_bytes) = ex_field.data_type() else {
-            panic!("ex-code field should be FixedSizeList");
-        };
-        assert_eq!(*ex_code_bytes, 48);
+        assert_eq!(calc.distance_all(0), vec![5.0, -2.0]);
     }
 
-    fn make_test_codes(num_vectors: usize, code_dim: i32) -> FixedSizeListArray {
-        let quantizer =
-            RabitQuantizer::new_with_rotation::<Float32Type>(1, code_dim, RQRotationType::Fast);
-        let values = Float32Array::from_iter_values(
-            (0..num_vectors * code_dim as usize).map(|idx| idx as f32 / code_dim as f32),
+    #[test]
+    fn test_raw_query_context_matches_fallback_and_only_updates_partition_factor() {
+        let code_dim = 8usize;
+        let identity = Float32Array::from_iter_values(
+            (0..code_dim)
+                .flat_map(|row| (0..code_dim).map(move |col| if row == col { 1.0 } else { 0.0 })),
         );
-        let vectors = FixedSizeListArray::try_new_from_values(values, code_dim).unwrap();
-        quantizer
-            .quantize(&vectors)
-            .unwrap()
-            .as_fixed_size_list()
-            .clone()
-    }
-
-    fn make_test_metadata(code_dim: usize) -> RabitQuantizationMetadata {
-        RabitQuantizer::new_with_rotation::<Float32Type>(1, code_dim as i32, RQRotationType::Fast)
-            .metadata(None)
-    }
-
-    fn make_test_batch(codes: FixedSizeListArray) -> RecordBatch {
-        let num_rows = codes.len();
-        RecordBatch::try_from_iter(vec![
-            (
-                ROW_ID,
-                Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)) as ArrayRef,
-            ),
-            (RABIT_CODE_COLUMN, Arc::new(codes) as ArrayRef),
-            (
-                ADD_FACTORS_COLUMN,
-                Arc::new(Float32Array::from_iter_values(
-                    (0..num_rows).map(|v| v as f32),
-                )) as ArrayRef,
-            ),
-            (
-                SCALE_FACTORS_COLUMN,
-                Arc::new(Float32Array::from_iter_values(
-                    (0..num_rows).map(|v| v as f32 + 0.5),
-                )) as ArrayRef,
-            ),
-        ])
-        .unwrap()
-    }
+        let rotate_mat =
+            FixedSizeListArray::try_new_from_values(identity, code_dim as i32).unwrap();
+        let metadata = RabitQuantizationMetadata {
+            rotate_mat: Some(rotate_mat),
+            rotate_mat_position: None,
+            fast_rotation_signs: None,
+            rotation_type: RQRotationType::Matrix,
+            code_dim: code_dim as u32,
+            num_bits: 2,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+        let codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0xff, 0xff]), 1).unwrap();
+        let ex_codes =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0x00, 0xff]), 1).unwrap();
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch_with_ex(codes, ex_codes),
+            &metadata,
+            DistanceType::Dot,
+            None,
+        )
+        .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let rotated_centroid = vec![0.25; code_dim];
+        let raw_query = metadata.prepare_raw_query_context(query.as_ref()).unwrap();
+
+        let mut fallback_scratch = Vec::new();
+        let expected = storage
+            .dist_calculator_with_scratch(
+                query.clone(),
+                123.0,
+                Some(QueryResidual::RabitRawQuery {
+                    rotated_centroid: Some(&rotated_centroid),
+                    query: None,
+                }),
+                &mut fallback_scratch,
+                DistanceCalculatorOptions::default(),
+            )
+            .distance_all(0);
+
+        let mut prepared_scratch = Vec::new();
+        let actual = storage
+            .dist_calculator_with_scratch(
+                query,
+                456.0,
+                Some(QueryResidual::RabitRawQuery {
+                    rotated_centroid: Some(&rotated_centroid),
+                    query: Some(&raw_query),
+                }),
+                &mut prepared_scratch,
+                DistanceCalculatorOptions::default(),
+            )
+            .distance_all(0);
 
-    fn assert_codes_eq(actual: &FixedSizeListArray, expected: &FixedSizeListArray) {
-        assert_eq!(actual.len(), expected.len());
-        assert_eq!(actual.value_length(), expected.value_length());
-        assert_eq!(
-            actual.values().as_primitive::<UInt8Type>().values(),
-            expected.values().as_primitive::<UInt8Type>().values()
-        );
+        assert_eq!(actual, expected);
+        assert!(prepared_scratch.is_empty());
     }
 
     #[test]
@@ -1450,7 +4282,40 @@ mod tests {
     }
 
     #[test]
-    fn test_try_from_batch_rejects_unsupported_rq_num_bits() {
+    fn test_try_from_batch_uses_l2_for_cosine() {
+        let original_codes = make_test_codes(50, 64);
+        let metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(original_codes),
+            &metadata,
+            DistanceType::Cosine,
+            None,
+        )
+        .unwrap();
+
+        assert_eq!(storage.distance_type(), DistanceType::L2);
+    }
+
+    #[test]
+    fn test_try_from_batch_keeps_cosine_for_legacy_residual_query() {
+        let original_codes = make_test_codes(50, 64);
+        let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8);
+        metadata.query_estimator = RabitQueryEstimator::ResidualQuery;
+
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch(original_codes),
+            &metadata,
+            DistanceType::Cosine,
+            None,
+        )
+        .unwrap();
+
+        assert_eq!(storage.distance_type(), DistanceType::Cosine);
+    }
+
+    #[test]
+    fn test_try_from_batch_requires_ex_columns_for_multi_bit_rq() {
         let original_codes = make_test_codes(50, 64);
         let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8);
         metadata.num_bits = 2;
@@ -1463,12 +4328,87 @@ mod tests {
         )
         .unwrap_err();
         assert!(
-            err.to_string().contains("only num_bits=1 is supported"),
+            err.to_string()
+                .contains("requires __blocked_ex_codes column"),
+            "{}",
+            err
+        );
+    }
+
+    #[test]
+    fn test_try_from_batch_requires_ex_add_factors_for_multi_bit_rq() {
+        let original_codes = make_test_codes(50, 64);
+        let code_dim = original_codes.value_length() as usize * 8;
+        let ex_codes = make_test_ex_codes(original_codes.len(), code_dim, 9);
+        let mut metadata = make_test_metadata(code_dim);
+        metadata.num_bits = 9;
+        let batch = make_test_batch_with_ex(original_codes, ex_codes)
+            .drop_column(EX_ADD_FACTORS_COLUMN)
+            .unwrap();
+
+        let err =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap_err();
+        assert!(
+            err.to_string().contains("requires __add_factors_ex column"),
             "{}",
             err
         );
     }
 
+    #[test]
+    fn test_try_from_batch_accepts_multi_bit_rq_split_codes() {
+        let original_codes = make_test_codes(50, 64);
+        let code_dim = original_codes.value_length() as usize * 8;
+        let ex_codes = make_test_ex_codes(original_codes.len(), code_dim, 9);
+        let mut metadata = make_test_metadata(code_dim);
+        metadata.num_bits = 9;
+
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch_with_ex(original_codes, ex_codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+
+        assert!(storage.metadata().packed);
+        // Legacy batches are normalized to the blocked column at load.
+        let stored_batch = storage.to_batches().unwrap().next().unwrap();
+        assert!(stored_batch.column_by_name(RABIT_EX_CODE_COLUMN).is_none());
+        assert_eq!(
+            stored_batch[RABIT_BLOCKED_EX_CODE_COLUMN]
+                .as_fixed_size_list()
+                .value_length(),
+            64
+        );
+        assert!(stored_batch.column_by_name(ERROR_FACTORS_COLUMN).is_some());
+    }
+
+    #[test]
+    fn test_try_from_batch_accepts_missing_error_factors_for_compatibility() {
+        let original_codes = make_test_codes(50, 64);
+        let code_dim = original_codes.value_length() as usize * 8;
+        let ex_codes = make_test_ex_codes(original_codes.len(), code_dim, 9);
+        let mut metadata = make_test_metadata(code_dim);
+        metadata.num_bits = 9;
+        let batch = make_test_batch_with_ex(original_codes, ex_codes)
+            .drop_column(ERROR_FACTORS_COLUMN)
+            .unwrap();
+
+        let storage =
+            RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None)
+                .unwrap();
+        let query = Arc::new(Float32Array::from(vec![1.0; code_dim])) as ArrayRef;
+        let calc = storage.dist_calculator(query, 4.0);
+
+        assert!(storage.error_factors.is_none());
+        assert_eq!(
+            calc.raw_query_lower_bound_gating_disabled_reason(),
+            Some("missing_error_factors")
+        );
+    }
+
     #[test]
     fn test_remap_preserves_packed_rq_storage_layout() {
         let original_codes = make_test_codes(50, 64);
@@ -1486,7 +4426,7 @@ mod tests {
         mapping.insert(3, None);
         mapping.insert(4, Some(104));
 
-        let remapped = storage.remap(&mapping).unwrap();
+        let remapped = storage.remap(&RowAddrRemap::direct(mapping)).unwrap();
         assert!(remapped.metadata().packed);
 
         let remapped_batch = remapped.to_batches().unwrap().next().unwrap();
@@ -1502,4 +4442,86 @@ mod tests {
         let repacked = pack_codes(&unpack_codes(remapped_codes));
         assert_codes_eq(remapped_codes, &repacked);
     }
+
+    #[test]
+    fn test_remap_preserves_multi_bit_rq_split_columns() {
+        // num_bits=9 keeps sequential ex codes; num_bits 4/6/8 (ex_bits
+        // 3/5/7) also exercise the bit-plane repack rebuild in `remap`.
+        for num_bits in [4, 6, 8, 9u8] {
+            test_remap_preserves_multi_bit_rq_split_columns_impl(num_bits);
+        }
+    }
+
+    fn test_remap_preserves_multi_bit_rq_split_columns_impl(num_bits: u8) {
+        let original_codes = make_test_codes(50, 64);
+        let code_dim = original_codes.value_length() as usize * 8;
+        let ex_codes = make_test_ex_codes(original_codes.len(), code_dim, num_bits);
+        let mut metadata = make_test_metadata(code_dim);
+        metadata.num_bits = num_bits;
+        let storage = RabitQuantizationStorage::try_from_batch(
+            make_test_batch_with_ex(original_codes.clone(), ex_codes),
+            &metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+
+        let mut mapping = HashMap::new();
+        mapping.insert(1, Some(101));
+        mapping.insert(3, None);
+        mapping.insert(4, Some(104));
+
+        let remapped = storage.remap(&RowAddrRemap::direct(mapping)).unwrap();
+        let remapped_batch = remapped.to_batches().unwrap().next().unwrap();
+        let remapped_row_ids = remapped_batch[ROW_ID].as_primitive::<UInt64Type>().values();
+        let expected_row_ids = UInt64Array::from_iter_values(
+            [0, 101, 2, 104]
+                .into_iter()
+                .chain(5..original_codes.len() as u64),
+        );
+        assert_eq!(remapped_row_ids, expected_row_ids.values());
+
+        // Legacy batches are normalized to the blocked format at load, so the
+        // remapped batch carries the blocked column.
+        let ex_code_len = blocked_ex_code_bytes(code_dim, rabit_ex_bits(num_bits).unwrap());
+        assert_eq!(
+            remapped_batch[RABIT_BLOCKED_EX_CODE_COLUMN]
+                .as_fixed_size_list()
+                .value_length(),
+            ex_code_len as i32
+        );
+        assert_eq!(
+            &remapped_batch[EX_ADD_FACTORS_COLUMN]
+                .as_primitive::<Float32Type>()
+                .values()[..5],
+            &[10.5, 11.5, 12.5, 14.5, 15.5]
+        );
+        assert_eq!(
+            &remapped_batch[EX_SCALE_FACTORS_COLUMN]
+                .as_primitive::<Float32Type>()
+                .values()[..5],
+            &[1.5, 2.5, 3.5, 5.5, 6.5]
+        );
+        assert_eq!(
+            &remapped_batch[ERROR_FACTORS_COLUMN]
+                .as_primitive::<Float32Type>()
+                .values()[..5],
+            &[0.25, 1.25, 2.25, 4.25, 5.25]
+        );
+
+        // The remapped storage must hold the same kernel-layout ex codes as a
+        // storage freshly loaded from the remapped batch.
+        let reloaded = RabitQuantizationStorage::try_from_batch(
+            remapped_batch,
+            &remapped.metadata,
+            DistanceType::L2,
+            None,
+        )
+        .unwrap();
+        assert_eq!(remapped.ex_codes, reloaded.ex_codes);
+        assert_eq!(
+            remapped.ex_codes.as_ref().unwrap().value_length() as usize,
+            blocked_ex_code_bytes(code_dim, rabit_ex_bits(num_bits).unwrap())
+        );
+    }
 }
diff --git a/rust/lance-index/src/vector/bq/transform.rs b/rust/lance-index/src/vector/bq/transform.rs
index af3e563c173..c87695e14cd 100644
--- a/rust/lance-index/src/vector/bq/transform.rs
+++ b/rust/lance-index/src/vector/bq/transform.rs
@@ -6,7 +6,9 @@ use std::sync::{Arc, LazyLock};
 
 use arrow::array::AsArray;
 use arrow::datatypes::{Float16Type, Float32Type, Float64Type, UInt32Type};
-use arrow_array::{Array, ArrowNativeTypeOp, FixedSizeListArray, Float32Array, RecordBatch};
+use arrow_array::{
+    Array, ArrowNativeTypeOp, FixedSizeListArray, Float32Array, RecordBatch, UInt32Array,
+};
 use arrow_schema::DataType;
 use lance_arrow::RecordBatchExt;
 use lance_core::{Error, Result};
@@ -14,7 +16,10 @@ use lance_linalg::distance::{DistanceType, norm_squared_fsl};
 use tracing::instrument;
 
 use crate::vector::bq::builder::RabitQuantizer;
-use crate::vector::bq::storage::RABIT_CODE_COLUMN;
+use crate::vector::bq::rabit_ex_bits;
+use crate::vector::bq::storage::{
+    RABIT_BLOCKED_EX_CODE_COLUMN, RABIT_CODE_COLUMN, RabitQueryEstimator,
+};
 use crate::vector::quantizer::Quantization;
 use crate::vector::transform::Transformer;
 use crate::vector::{CENTROID_DIST_COLUMN, PART_ID_COLUMN};
@@ -25,6 +30,9 @@ pub const ADD_FACTORS_COLUMN: &str = "__add_factors";
 pub const SCALE_FACTORS_COLUMN: &str = "__scale_factors";
 pub const EX_ADD_FACTORS_COLUMN: &str = "__add_factors_ex";
 pub const EX_SCALE_FACTORS_COLUMN: &str = "__scale_factors_ex";
+pub const ERROR_FACTORS_COLUMN: &str = "__error_factors";
+
+const RABIT_ERROR_EPSILON: f32 = 1.9;
 
 pub static ADD_FACTORS_FIELD: LazyLock<arrow_schema::Field> = LazyLock::new(|| {
     arrow_schema::Field::new(ADD_FACTORS_COLUMN, arrow_schema::DataType::Float32, true)
@@ -42,11 +50,15 @@ pub static EX_SCALE_FACTORS_FIELD: LazyLock<arrow_schema::Field> = LazyLock::new
         true,
     )
 });
+pub static ERROR_FACTORS_FIELD: LazyLock<arrow_schema::Field> = LazyLock::new(|| {
+    arrow_schema::Field::new(ERROR_FACTORS_COLUMN, arrow_schema::DataType::Float32, true)
+});
 
 pub struct RQTransformer {
     rq: RabitQuantizer,
     distance_type: DistanceType,
     centroids_norm_square: Option<Float32Array>,
+    rotated_centroids: Option<Vec<f32>>,
     vector_column: String,
 }
 
@@ -56,18 +68,209 @@ impl RQTransformer {
         distance_type: DistanceType,
         centroids: FixedSizeListArray,
         vector_column: impl Into<String>,
-    ) -> Self {
+    ) -> Result<Self> {
         // for dot product, the add factor is `1 - v*c + |c|^2`, so we need to compute |c|^2
         let centroids_norm_square = (distance_type == DistanceType::Dot)
             .then(|| Float32Array::from(norm_squared_fsl(&centroids)));
+        let rotated_centroids = (rq.metadata_ref().query_estimator
+            == RabitQueryEstimator::RawQuery)
+            .then(|| rq.rotate_fsl_to_f32(&centroids))
+            .transpose()?;
 
-        Self {
+        Ok(Self {
             rq,
             distance_type,
             centroids_norm_square,
+            rotated_centroids,
             vector_column: vector_column.into(),
+        })
+    }
+}
+
+struct RabitRawQueryFactors {
+    add_factors: Float32Array,
+    scale_factors: Float32Array,
+    error_factors: Float32Array,
+    ex_add_factors: Option<Float32Array>,
+    ex_scale_factors: Option<Float32Array>,
+}
+
+#[inline]
+fn factor_ratio(numerator: f32, denominator: f32) -> f32 {
+    if denominator == 0.0 {
+        0.0
+    } else {
+        numerator / denominator
+    }
+}
+
+#[inline]
+fn binary_factor_value(rotated_residual: f32) -> f32 {
+    if rotated_residual.is_sign_positive() {
+        0.5
+    } else {
+        -0.5
+    }
+}
+
+#[inline]
+fn error_factor_value(
+    distance_type: DistanceType,
+    norm_square: f32,
+    binary_res_dot: f32,
+    code_dim: usize,
+) -> f32 {
+    if code_dim <= 1 || norm_square <= 0.0 || binary_res_dot == 0.0 {
+        return 0.0;
+    }
+
+    let code_norm_square = code_dim as f32 * 0.25;
+    let alignment = norm_square * code_norm_square / binary_res_dot.powi(2);
+    let angular_error = ((alignment - 1.0).max(0.0) / (code_dim as f32 - 1.0)).sqrt();
+    let error = norm_square.sqrt() * RABIT_ERROR_EPSILON * angular_error;
+    match distance_type {
+        DistanceType::L2 => 2.0 * error,
+        DistanceType::Dot => error,
+        _ => unreachable!(),
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn compute_raw_query_factors(
+    distance_type: DistanceType,
+    res_norm_square: &Float32Array,
+    rotated_residuals: &[f32],
+    rotated_centroids: &[f32],
+    part_ids: &UInt32Array,
+    ex_code_values: Option<&[u8]>,
+    ex_res_dot_dists: Option<&[f32]>,
+    ex_bits: u8,
+    code_dim: usize,
+) -> Result<RabitRawQueryFactors> {
+    if !matches!(distance_type, DistanceType::L2 | DistanceType::Dot) {
+        return Err(Error::index(format!(
+            "RQ Transform: distance type {} not supported",
+            distance_type
+        )));
+    }
+
+    let num_rows = res_norm_square.len();
+    debug_assert_eq!(rotated_residuals.len(), num_rows * code_dim);
+    if let Some(ex_code_values) = ex_code_values {
+        debug_assert_eq!(ex_code_values.len(), num_rows * code_dim);
+    }
+    if let Some(ex_res_dot_dists) = ex_res_dot_dists {
+        debug_assert_eq!(ex_res_dot_dists.len(), num_rows);
+    }
+
+    let has_ex_codes = ex_bits != 0;
+    let ex_code_bias = -((1u32 << ex_bits) as f32 - 0.5);
+    let mut add_factors = Vec::with_capacity(num_rows);
+    let mut scale_factors = Vec::with_capacity(num_rows);
+    let mut error_factors = Vec::with_capacity(num_rows);
+    let mut ex_add_factors = has_ex_codes.then(|| Vec::with_capacity(num_rows));
+    let mut ex_scale_factors = has_ex_codes.then(|| Vec::with_capacity(num_rows));
+
+    for (row_idx, &norm_square) in res_norm_square.values().iter().enumerate() {
+        let part_id = part_ids.value(row_idx) as usize;
+        let centroid_start = part_id.checked_mul(code_dim).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "RQ Transform: partition id {} overflows code_dim {}",
+                part_id, code_dim
+            ))
+        })?;
+        let centroid_end = centroid_start.checked_add(code_dim).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "RQ Transform: partition id {} plus code_dim {} overflows",
+                part_id, code_dim
+            ))
+        })?;
+        if centroid_end > rotated_centroids.len() {
+            return Err(Error::invalid_input(format!(
+                "RQ Transform: partition id {} out of range for {} rotated centroids",
+                part_id,
+                rotated_centroids.len() / code_dim
+            )));
+        }
+
+        let row_start = row_idx * code_dim;
+        let row_end = row_start + code_dim;
+        let residual = &rotated_residuals[row_start..row_end];
+        let centroid = &rotated_centroids[centroid_start..centroid_end];
+        let ex_values = ex_code_values.map(|values| &values[row_start..row_end]);
+
+        let mut binary_res_dot = 0.0f32;
+        let mut binary_cent_dot = 0.0f32;
+        let mut ex_cent_dot = 0.0f32;
+        let mut residual_centroid_dot = 0.0f32;
+        for (dim_idx, (&residual_value, &centroid_value)) in
+            residual.iter().zip(centroid.iter()).enumerate()
+        {
+            let residual_value: f32 = residual_value;
+            let centroid_value: f32 = centroid_value;
+            let binary_code = if residual_value.is_sign_positive() {
+                1u32
+            } else {
+                0u32
+            };
+            let binary_factor = binary_factor_value(residual_value);
+
+            binary_res_dot += residual_value * binary_factor;
+            binary_cent_dot += centroid_value * binary_factor;
+            if let Some(ex_values) = ex_values {
+                let ex_code_value = ex_values[dim_idx];
+                let ex_factor =
+                    ((binary_code << ex_bits) + ex_code_value as u32) as f32 + ex_code_bias;
+                ex_cent_dot += centroid_value * ex_factor;
+            }
+            residual_centroid_dot += residual_value * centroid_value;
+        }
+
+        let binary_correction = factor_ratio(norm_square * binary_cent_dot, binary_res_dot);
+        let ex_res_dot = ex_res_dot_dists
+            .map(|values| values[row_idx])
+            .unwrap_or_default();
+        let ex_correction = factor_ratio(norm_square * ex_cent_dot, ex_res_dot);
+        error_factors.push(error_factor_value(
+            distance_type,
+            norm_square,
+            binary_res_dot,
+            code_dim,
+        ));
+
+        match distance_type {
+            DistanceType::L2 => {
+                add_factors.push(norm_square + 2.0 * binary_correction);
+                scale_factors.push(factor_ratio(-2.0 * norm_square, binary_res_dot));
+                if let Some(ex_add_factors) = ex_add_factors.as_mut() {
+                    ex_add_factors.push(norm_square + 2.0 * ex_correction);
+                }
+                if let Some(ex_scale_factors) = ex_scale_factors.as_mut() {
+                    ex_scale_factors.push(factor_ratio(-2.0 * norm_square, ex_res_dot));
+                }
+            }
+            DistanceType::Dot => {
+                let dot_base = 1.0 - residual_centroid_dot;
+                add_factors.push(dot_base + binary_correction);
+                scale_factors.push(factor_ratio(-norm_square, binary_res_dot));
+                if let Some(ex_add_factors) = ex_add_factors.as_mut() {
+                    ex_add_factors.push(dot_base + ex_correction);
+                }
+                if let Some(ex_scale_factors) = ex_scale_factors.as_mut() {
+                    ex_scale_factors.push(factor_ratio(-norm_square, ex_res_dot));
+                }
+            }
+            _ => unreachable!(),
         }
     }
+
+    Ok(RabitRawQueryFactors {
+        add_factors: Float32Array::from(add_factors),
+        scale_factors: Float32Array::from(scale_factors),
+        error_factors: Float32Array::from(error_factors),
+        ex_add_factors: ex_add_factors.map(Float32Array::from),
+        ex_scale_factors: ex_scale_factors.map(Float32Array::from),
+    })
 }
 
 impl Debug for RQTransformer {
@@ -79,7 +282,11 @@ impl Debug for RQTransformer {
 impl Transformer for RQTransformer {
     #[instrument(name = "RQTransformer::transform", level = "debug", skip_all)]
     fn transform(&self, batch: &RecordBatch) -> Result<RecordBatch> {
-        if batch.column_by_name(RABIT_CODE_COLUMN).is_some() {
+        let has_split_codes = self.rq.num_bits() == 1
+            || (batch.column_by_name(RABIT_BLOCKED_EX_CODE_COLUMN).is_some()
+                && batch.column_by_name(EX_ADD_FACTORS_COLUMN).is_some()
+                && batch.column_by_name(EX_SCALE_FACTORS_COLUMN).is_some());
+        if batch.column_by_name(RABIT_CODE_COLUMN).is_some() && has_split_codes {
             return Ok(batch.clone());
         }
 
@@ -117,89 +324,158 @@ impl Transformer for RQTransformer {
             }
         };
 
-        let rq_codes = self.rq.quantize(&residual_vectors)?;
-        let codes_fsl = rq_codes.as_fixed_size_list();
-
-        let ip_rq_res = match residual_vectors.value_type() {
-            DataType::Float16 => Float32Array::from(
-                self.rq
-                    .codes_res_dot_dists::<Float16Type>(residual_vectors)?,
-            ),
-            DataType::Float32 => Float32Array::from(
-                self.rq
-                    .codes_res_dot_dists::<Float32Type>(residual_vectors)?,
-            ),
-            DataType::Float64 => Float32Array::from(
-                self.rq
-                    .codes_res_dot_dists::<Float64Type>(residual_vectors)?,
-            ),
-            _ => {
-                return Err(Error::index(format!(
-                    "RQ Transform: unsupported residual vector data type: {}",
-                    residual_vectors.data_type()
-                )));
-            }
-        };
+        let rq_codes = self.rq.quantize_split(residual_vectors)?;
+        let codes_fsl = rq_codes.binary_codes.as_fixed_size_list();
         debug_assert_eq!(codes_fsl.len(), batch.num_rows());
 
-        let add_factors = match self.distance_type {
-            DistanceType::L2 => res_norm_square.clone(),
-            DistanceType::Dot => {
-                // for dot, the add factor is `1 - v*c + |c|^2 = dist_v_c + |c|^2`
-                let part_ids = &batch[PART_ID_COLUMN];
-                let part_ids = part_ids.as_primitive::<UInt32Type>();
-                let centroids_norm_square = self.centroids_norm_square.as_ref().ok_or(
-                    Error::index("RQ Transform: centroids norm square not found".to_string()),
-                )?;
-                let centroids_norm_square =
-                    arrow::compute::take(centroids_norm_square, part_ids, None)?;
-                let centroids_norm_square = centroids_norm_square.as_primitive::<Float32Type>();
-                Float32Array::from_iter_values(
-                    dist_v_c
-                        .values()
-                        .iter()
-                        .zip(centroids_norm_square.values().iter())
-                        .map(|(dist_v_c, centroids_norm_square)| dist_v_c + centroids_norm_square),
-                )
-            }
-            _ => {
-                return Err(Error::index(format!(
-                    "RQ Transform: distance type {} not supported",
-                    self.distance_type
-                )));
-            }
-        };
+        let mut batch = batch.try_with_column(self.rq.field(), rq_codes.binary_codes)?;
+        if self.rq.metadata_ref().query_estimator == RabitQueryEstimator::ResidualQuery {
+            // Preserve the released residual-query estimator and factor layout.
+            let ip_rq_res = match residual_vectors.value_type() {
+                DataType::Float16 => Float32Array::from(
+                    self.rq
+                        .codes_res_dot_dists::<Float16Type>(residual_vectors)?,
+                ),
+                DataType::Float32 => Float32Array::from(
+                    self.rq
+                        .codes_res_dot_dists::<Float32Type>(residual_vectors)?,
+                ),
+                DataType::Float64 => Float32Array::from(
+                    self.rq
+                        .codes_res_dot_dists::<Float64Type>(residual_vectors)?,
+                ),
+                _ => {
+                    return Err(Error::index(format!(
+                        "RQ Transform: unsupported residual vector data type: {}",
+                        residual_vectors.data_type()
+                    )));
+                }
+            };
+
+            let add_factors = match self.distance_type {
+                DistanceType::L2 => res_norm_square.clone(),
+                DistanceType::Dot => {
+                    // for dot, the add factor is `1 - v*c + |c|^2 = dist_v_c + |c|^2`
+                    let part_ids = &batch[PART_ID_COLUMN];
+                    let part_ids = part_ids.as_primitive::<UInt32Type>();
+                    let centroids_norm_square = self.centroids_norm_square.as_ref().ok_or(
+                        Error::index("RQ Transform: centroids norm square not found".to_string()),
+                    )?;
+                    let centroids_norm_square =
+                        arrow::compute::take(centroids_norm_square, part_ids, None)?;
+                    let centroids_norm_square = centroids_norm_square.as_primitive::<Float32Type>();
+                    Float32Array::from_iter_values(
+                        dist_v_c
+                            .values()
+                            .iter()
+                            .zip(centroids_norm_square.values().iter())
+                            .map(|(dist_v_c, centroids_norm_square)| {
+                                dist_v_c + centroids_norm_square
+                            }),
+                    )
+                }
+                _ => {
+                    return Err(Error::index(format!(
+                        "RQ Transform: distance type {} not supported",
+                        self.distance_type
+                    )));
+                }
+            };
 
-        let scale_factors = match self.distance_type {
-            DistanceType::L2 => Float32Array::from_iter_values(
-                res_norm_square.values().iter().zip(ip_rq_res.values()).map(
-                    |(res_norm_square, ip_rq_res)| {
-                        (-2.0 * res_norm_square)
-                            .div_checked(*ip_rq_res)
-                            .unwrap_or_default()
-                    },
+            let scale_factors = match self.distance_type {
+                DistanceType::L2 => Float32Array::from_iter_values(
+                    res_norm_square.values().iter().zip(ip_rq_res.values()).map(
+                        |(res_norm_square, ip_rq_res)| {
+                            (-2.0 * res_norm_square)
+                                .div_checked(*ip_rq_res)
+                                .unwrap_or_default()
+                        },
+                    ),
                 ),
-            ),
-            DistanceType::Dot => Float32Array::from_iter_values(
-                res_norm_square.values().iter().zip(ip_rq_res.values()).map(
-                    |(res_norm_square, ip_rq_res)| {
-                        -res_norm_square.div_checked(*ip_rq_res).unwrap_or_default()
-                    },
+                DistanceType::Dot => Float32Array::from_iter_values(
+                    res_norm_square.values().iter().zip(ip_rq_res.values()).map(
+                        |(res_norm_square, ip_rq_res)| {
+                            -res_norm_square.div_checked(*ip_rq_res).unwrap_or_default()
+                        },
+                    ),
                 ),
-            ),
-            _ => {
-                return Err(Error::index(format!(
-                    "RQ Transform: distance type {} not supported",
-                    self.distance_type
-                )));
+                _ => {
+                    return Err(Error::index(format!(
+                        "RQ Transform: distance type {} not supported",
+                        self.distance_type
+                    )));
+                }
+            };
+
+            batch = batch
+                .try_with_column(ADD_FACTORS_FIELD.clone(), Arc::new(add_factors))?
+                .try_with_column(SCALE_FACTORS_FIELD.clone(), Arc::new(scale_factors))?;
+        } else {
+            // New RQ indexes use the RaBitQ-Library raw-query estimator.
+            let ex_bits = rabit_ex_bits(self.rq.num_bits())?;
+            let ex_codes = rq_codes.ex_codes;
+            let ex_res_dot_dists = rq_codes.ex_res_dot_dists;
+            let rotated_residuals = rq_codes.rotated_residuals.ok_or_else(|| {
+                Error::internal("RabitQ quantization did not return rotated residuals".to_string())
+            })?;
+            let ex_code_values = rq_codes.ex_code_values;
+            if ex_bits != 0
+                && (ex_codes.is_none() || ex_res_dot_dists.is_none() || ex_code_values.is_none())
+            {
+                return Err(Error::internal(
+                    "RabitQ multi-bit quantization did not return split-code values".to_string(),
+                ));
             }
-        };
 
-        let batch = batch.try_with_column(self.rq.field(), Arc::new(rq_codes))?;
-        let batch = batch
-            .try_with_column(ADD_FACTORS_FIELD.clone(), Arc::new(add_factors))?
-            .drop_column(CENTROID_DIST_COLUMN)?;
-        let batch = batch.try_with_column(SCALE_FACTORS_FIELD.clone(), Arc::new(scale_factors))?;
+            let part_ids = batch[PART_ID_COLUMN].as_primitive::<UInt32Type>();
+            let rotated_centroids = self.rotated_centroids.as_ref().ok_or_else(|| {
+                Error::internal("RabitQ raw-query transformer is missing rotated centroids")
+            })?;
+            let raw_query_factors = compute_raw_query_factors(
+                self.distance_type,
+                &res_norm_square,
+                &rotated_residuals,
+                rotated_centroids,
+                part_ids,
+                ex_code_values.as_deref(),
+                ex_res_dot_dists.as_deref(),
+                ex_bits,
+                self.rq.dim(),
+            )?;
+
+            batch = batch
+                .try_with_column(
+                    ADD_FACTORS_FIELD.clone(),
+                    Arc::new(raw_query_factors.add_factors),
+                )?
+                .try_with_column(
+                    SCALE_FACTORS_FIELD.clone(),
+                    Arc::new(raw_query_factors.scale_factors),
+                )?
+                .try_with_column(
+                    ERROR_FACTORS_FIELD.clone(),
+                    Arc::new(raw_query_factors.error_factors),
+                )?;
+
+            if let Some(ex_codes) = ex_codes {
+                batch = batch.try_with_column(
+                    crate::vector::bq::storage::rabit_ex_code_field(
+                        self.rq.dim(),
+                        self.rq.num_bits(),
+                    )?
+                    .expect("ex-code field should exist for num_bits > 1"),
+                    ex_codes,
+                )?;
+            }
+            if let Some(ex_add_factors) = raw_query_factors.ex_add_factors {
+                batch = batch
+                    .try_with_column(EX_ADD_FACTORS_FIELD.clone(), Arc::new(ex_add_factors))?;
+            }
+            if let Some(ex_scale_factors) = raw_query_factors.ex_scale_factors {
+                batch = batch
+                    .try_with_column(EX_SCALE_FACTORS_FIELD.clone(), Arc::new(ex_scale_factors))?;
+            }
+        }
 
         let batch = batch
             .drop_column(&self.vector_column)?
@@ -207,3 +483,207 @@ impl Transformer for RQTransformer {
         Ok(batch)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::array::AsArray;
+    use arrow::datatypes::{Float32Type, UInt8Type};
+    use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt32Array};
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_linalg::distance::DistanceType;
+
+    use crate::vector::bq::RQRotationType;
+    use crate::vector::bq::builder::RabitQuantizer;
+    use crate::vector::bq::ex_dot::blocked_ex_code_bytes;
+    use crate::vector::bq::storage::RABIT_BLOCKED_EX_CODE_COLUMN;
+    use crate::vector::transform::Transformer;
+    use crate::vector::{CENTROID_DIST_COLUMN, PART_ID_COLUMN};
+
+    use super::{
+        ADD_FACTORS_COLUMN, ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN,
+        RQTransformer, compute_raw_query_factors, error_factor_value,
+    };
+
+    #[test]
+    fn test_rq_transformer_writes_multi_bit_ex_factors() {
+        let rq = RabitQuantizer::new_with_rotation::<Float32Type>(4, 8, RQRotationType::Fast);
+        let centroids =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0f32; 8]), 8)
+                .unwrap();
+        let transformer =
+            RQTransformer::new(rq.clone(), DistanceType::L2, centroids, "vector").unwrap();
+
+        let residual_vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from(vec![
+                1.0, -2.0, 3.0, -4.0, 1.5, -2.5, 3.5, -4.5, 0.5, -1.0, 1.5, -2.0, 2.5, -3.0, 3.5,
+                -4.0,
+            ]),
+            8,
+        )
+        .unwrap();
+        let res_norm_square = Float32Array::from(vec![73.0f32, 47.0]);
+        let batch = RecordBatch::try_from_iter(vec![
+            ("vector", Arc::new(residual_vectors.clone()) as ArrayRef),
+            (
+                PART_ID_COLUMN,
+                Arc::new(UInt32Array::from(vec![0, 0])) as ArrayRef,
+            ),
+            (
+                CENTROID_DIST_COLUMN,
+                Arc::new(res_norm_square.clone()) as ArrayRef,
+            ),
+        ])
+        .unwrap();
+
+        let transformed = transformer.transform(&batch).unwrap();
+        assert!(
+            transformed
+                .column_by_name(RABIT_BLOCKED_EX_CODE_COLUMN)
+                .is_some()
+        );
+        assert_eq!(
+            transformed[RABIT_BLOCKED_EX_CODE_COLUMN]
+                .as_fixed_size_list()
+                .value_length(),
+            blocked_ex_code_bytes(8, 3) as i32
+        );
+        assert!(
+            transformed[RABIT_BLOCKED_EX_CODE_COLUMN]
+                .as_fixed_size_list()
+                .values()
+                .as_primitive::<UInt8Type>()
+                .values()
+                .iter()
+                .any(|value| *value != 0)
+        );
+        let expected_ex_dots = rq
+            .quantize_split(&residual_vectors)
+            .unwrap()
+            .ex_res_dot_dists
+            .unwrap();
+        let ex_add_factors = transformed[EX_ADD_FACTORS_COLUMN].as_primitive::<Float32Type>();
+        assert_eq!(ex_add_factors.values(), res_norm_square.values());
+        let ex_scale_factors = transformed[EX_SCALE_FACTORS_COLUMN].as_primitive::<Float32Type>();
+        for ((actual, norm), ex_dot) in ex_scale_factors
+            .values()
+            .iter()
+            .zip(res_norm_square.values())
+            .zip(expected_ex_dots)
+        {
+            let expected = if ex_dot == 0.0 {
+                0.0
+            } else {
+                -2.0 * norm / ex_dot
+            };
+            assert!((actual - expected).abs() < 1e-6);
+        }
+        assert!(transformed.column_by_name("vector").is_none());
+        assert!(transformed.column_by_name(CENTROID_DIST_COLUMN).is_none());
+        assert!(transformed.column_by_name(ADD_FACTORS_COLUMN).is_some());
+        assert!(transformed.column_by_name(ERROR_FACTORS_COLUMN).is_some());
+    }
+
+    #[test]
+    fn test_rq_transformer_caches_rotated_centroids_for_raw_query() {
+        let centroids =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0f32; 8]), 8)
+                .unwrap();
+        let raw_query_rq =
+            RabitQuantizer::new_with_rotation::<Float32Type>(1, 8, RQRotationType::Fast);
+        let raw_query_transformer =
+            RQTransformer::new(raw_query_rq, DistanceType::L2, centroids.clone(), "vector")
+                .unwrap();
+        assert_eq!(
+            raw_query_transformer
+                .rotated_centroids
+                .as_ref()
+                .unwrap()
+                .len(),
+            8
+        );
+
+        let multi_bit_rq =
+            RabitQuantizer::new_with_rotation::<Float32Type>(4, 8, RQRotationType::Fast);
+        let multi_bit_transformer =
+            RQTransformer::new(multi_bit_rq, DistanceType::L2, centroids, "vector").unwrap();
+        assert_eq!(
+            multi_bit_transformer
+                .rotated_centroids
+                .as_ref()
+                .unwrap()
+                .len(),
+            8
+        );
+    }
+
+    #[test]
+    fn test_raw_query_factors_match_reference_l2_formula() {
+        let res_norm_square = Float32Array::from(vec![5.0f32, 7.0]);
+        let rotated_residuals = vec![2.0, -1.0, 0.0, 0.0];
+        let rotated_centroids = vec![3.0, 4.0];
+        let part_ids = UInt32Array::from(vec![0, 0]);
+        let ex_code_values = vec![1, 0, 0, 0];
+        let ex_res_dot_dists = vec![4.5, 0.0];
+
+        let factors = compute_raw_query_factors(
+            DistanceType::L2,
+            &res_norm_square,
+            &rotated_residuals,
+            &rotated_centroids,
+            &part_ids,
+            Some(&ex_code_values),
+            Some(&ex_res_dot_dists),
+            1,
+            2,
+        )
+        .unwrap();
+
+        assert!((factors.add_factors.value(0) - 1.6666667).abs() < 1e-5);
+        assert!((factors.scale_factors.value(0) + 6.6666665).abs() < 1e-5);
+        let expected_error = error_factor_value(DistanceType::L2, 5.0, 1.5, 2);
+        assert!((factors.error_factors.value(0) - expected_error).abs() < 1e-5);
+        let ex_add_factors = factors.ex_add_factors.unwrap();
+        let ex_scale_factors = factors.ex_scale_factors.unwrap();
+        assert!((ex_add_factors.value(0) - 1.6666667).abs() < 1e-5);
+        assert!((ex_scale_factors.value(0) + 2.2222223).abs() < 1e-5);
+        assert_eq!(factors.add_factors.value(1), 7.0);
+        assert_eq!(factors.scale_factors.value(1), 0.0);
+        assert_eq!(factors.error_factors.value(1), 0.0);
+        assert_eq!(ex_add_factors.value(1), 7.0);
+        assert_eq!(ex_scale_factors.value(1), 0.0);
+    }
+
+    #[test]
+    fn test_raw_query_factors_match_reference_dot_formula() {
+        let res_norm_square = Float32Array::from(vec![5.0f32]);
+        let rotated_residuals = vec![2.0, -1.0];
+        let rotated_centroids = vec![3.0, 4.0];
+        let part_ids = UInt32Array::from(vec![0]);
+        let ex_code_values = vec![1, 0];
+        let ex_res_dot_dists = vec![4.5];
+
+        let factors = compute_raw_query_factors(
+            DistanceType::Dot,
+            &res_norm_square,
+            &rotated_residuals,
+            &rotated_centroids,
+            &part_ids,
+            Some(&ex_code_values),
+            Some(&ex_res_dot_dists),
+            1,
+            2,
+        )
+        .unwrap();
+
+        assert!((factors.add_factors.value(0) + 2.6666667).abs() < 1e-5);
+        assert!((factors.scale_factors.value(0) + 3.3333333).abs() < 1e-5);
+        let expected_error = error_factor_value(DistanceType::Dot, 5.0, 1.5, 2);
+        assert!((factors.error_factors.value(0) - expected_error).abs() < 1e-5);
+        let ex_add_factors = factors.ex_add_factors.unwrap();
+        let ex_scale_factors = factors.ex_scale_factors.unwrap();
+        assert!((ex_add_factors.value(0) + 2.6666667).abs() < 1e-5);
+        assert!((ex_scale_factors.value(0) + 1.1111112).abs() < 1e-5);
+    }
+}
diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs
index 9753c8a88c8..19aa3fa177d 100755
--- a/rust/lance-index/src/vector/distributed/index_merger.rs
+++ b/rust/lance-index/src/vector/distributed/index_merger.rs
@@ -20,11 +20,14 @@ use std::sync::Arc;
 use crate::IndexMetadata as IndexMetaSchema;
 use crate::pb;
 use crate::vector::bq::storage::{
-    RABIT_CODE_COLUMN, RABIT_METADATA_KEY, RabitQuantizationMetadata, pack_codes,
-    rabit_binary_code_field,
+    RABIT_CODE_COLUMN, RABIT_METADATA_KEY, RabitQuantizationMetadata, RabitQueryEstimator,
+    pack_codes, rabit_binary_code_field, rabit_ex_code_field,
 };
-use crate::vector::bq::transform::{ADD_FACTORS_FIELD, SCALE_FACTORS_FIELD};
-use crate::vector::bq::validate_supported_rq_num_bits;
+use crate::vector::bq::transform::{
+    ADD_FACTORS_FIELD, ERROR_FACTORS_FIELD, EX_ADD_FACTORS_FIELD, EX_SCALE_FACTORS_FIELD,
+    SCALE_FACTORS_FIELD,
+};
+use crate::vector::bq::validate_rq_num_bits;
 use crate::vector::flat::index::FlatMetadata;
 use crate::vector::ivf::storage::{IVF_METADATA_KEY, IvfModel as IvfStorageModel};
 use crate::vector::pq::storage::{PQ_METADATA_KEY, ProductQuantizationMetadata, transpose};
@@ -109,6 +112,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
                 return false;
             }
             for i in 0..av.len() {
+                if av[i].is_nan() || bv[i].is_nan() {
+                    return false;
+                }
                 if (av[i] - bv[i]).abs() > tol {
                     return false;
                 }
@@ -124,6 +130,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
                 return false;
             }
             for i in 0..av.len() {
+                if av[i].is_nan() || bv[i].is_nan() {
+                    return false;
+                }
                 if (av[i] - bv[i]).abs() > tol as f64 {
                     return false;
                 }
@@ -141,6 +150,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
             for i in 0..av.len() {
                 let da = av[i].to_f32();
                 let db = bv[i].to_f32();
+                if da.is_nan() || db.is_nan() {
+                    return false;
+                }
                 if (da - db).abs() > tol {
                     return false;
                 }
@@ -151,6 +163,63 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray,
     }
 }
 
+fn ensure_fixed_size_list_compatible(
+    what: &str,
+    reference: &FixedSizeListArray,
+    candidate: &FixedSizeListArray,
+) -> Result<()> {
+    if !fixed_size_list_equal(reference, candidate) {
+        const TOL: f32 = 1e-5;
+        if !fixed_size_list_almost_equal(reference, candidate, TOL) {
+            return Err(Error::index(format!("{what} mismatch across shards")));
+        }
+        log::warn!("{what} differs within tolerance; proceeding with first shard value");
+    }
+    Ok(())
+}
+
+async fn try_read_ivf_proto(reader: &V2Reader) -> Result<Option<pb::Ivf>> {
+    let Some(ivf_idx) = reader.metadata().file_schema.metadata.get(IVF_METADATA_KEY) else {
+        return Ok(None);
+    };
+    let ivf_idx = ivf_idx
+        .parse()
+        .map_err(|_| Error::index("IVF index parse error".to_string()))?;
+    let bytes = reader.read_global_buffer(ivf_idx).await?;
+    Ok(Some(pb::Ivf::decode(bytes)?))
+}
+
+fn ivf_centroids_from_proto(ivf: &pb::Ivf) -> Result<Option<FixedSizeListArray>> {
+    ivf.centroids_tensor
+        .as_ref()
+        .map(FixedSizeListArray::try_from)
+        .transpose()
+}
+
+async fn open_sibling_index_reader(
+    object_store: &lance_io::object_store::ObjectStore,
+    sched: &Arc<ScanScheduler>,
+    idx_path: &object_store::path::Path,
+) -> Result<Option<V2Reader>> {
+    if !object_store.exists(idx_path).await? {
+        return Ok(None);
+    }
+
+    let fh = sched
+        .open_file(idx_path, &CachedFileSize::unknown())
+        .await?;
+    Ok(Some(
+        V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &lance_core::cache::LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await?,
+    ))
+}
+
 /// Initialize schema-level metadata on a writer for a given storage.
 ///
 /// It writes the distance type and the storage metadata (as a vector payload),
@@ -299,12 +368,21 @@ pub async fn init_writer_for_rq(
     rq_meta: &RabitQuantizationMetadata,
     format_version: LanceFileVersion,
 ) -> Result<FileWriter> {
-    let arrow_schema = ArrowSchema::new(vec![
+    let mut fields = vec![
         (*ROW_ID_FIELD).clone(),
         rabit_binary_code_field(rq_meta.rotated_dim()),
         ADD_FACTORS_FIELD.clone(),
         SCALE_FACTORS_FIELD.clone(),
-    ]);
+    ];
+    if rq_meta.query_estimator == RabitQueryEstimator::RawQuery {
+        fields.push(ERROR_FACTORS_FIELD.clone());
+    }
+    if let Some(ex_code_field) = rabit_ex_code_field(rq_meta.rotated_dim(), rq_meta.num_bits)? {
+        fields.push(ex_code_field);
+        fields.push(EX_ADD_FACTORS_FIELD.clone());
+        fields.push(EX_SCALE_FACTORS_FIELD.clone());
+    }
+    let arrow_schema = ArrowSchema::new(fields);
     let writer = object_store.create(aux_out).await?;
     let mut w = FileWriter::try_new(
         writer,
@@ -701,7 +779,7 @@ pub async fn merge_partial_vector_auxiliary_files(
     aux_paths: &[object_store::path::Path],
     target_dir: &object_store::path::Path,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<lance_table::format::IndexFile> {
     if aux_paths.is_empty() {
         return Err(Error::index(
             "No partial auxiliary files were selected for merge".to_string(),
@@ -759,6 +837,12 @@ pub async fn merge_partial_vector_auxiliary_files(
         )
         .await?;
         let meta = reader.metadata();
+        let idx_path = aux
+            .parent()
+            .unwrap_or_default()
+            .join(crate::INDEX_FILE_NAME);
+        let mut idx_reader: Option<V2Reader> = None;
+        let mut idx_reader_checked = false;
 
         // Inherit format version from the first shard file
         if format_version.is_none() {
@@ -783,55 +867,33 @@ pub async fn merge_partial_vector_auxiliary_files(
         // Detect index type (first iteration only)
         if detected_index_type.is_none() {
             // Try to derive precise type from sibling partial index.idx metadata if available
-            // Try resolve sibling index.idx path by trimming the last component of aux path
-            let parent_str = {
-                let s = aux.as_ref();
-                if let Some((p, _)) = s.trim_end_matches('/').rsplit_once('/') {
-                    p.to_string()
-                } else {
-                    s.to_string()
-                }
-            };
-            let idx_path = object_store::path::Path::from(format!(
-                "{}/{}",
-                parent_str,
-                crate::INDEX_FILE_NAME
-            ));
-            if object_store.exists(&idx_path).await.unwrap_or(false) {
-                let fh2 = sched
-                    .open_file(&idx_path, &CachedFileSize::unknown())
-                    .await?;
-                let idx_reader = V2Reader::try_open(
-                    fh2,
-                    None,
-                    Arc::default(),
-                    &lance_core::cache::LanceCache::no_cache(),
-                    V2ReaderOptions::default(),
-                )
-                .await?;
-                if let Some(idx_meta_json) = idx_reader
+            if !idx_reader_checked {
+                idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?;
+                idx_reader_checked = true;
+            }
+            if let Some(idx_reader) = idx_reader.as_ref()
+                && let Some(idx_meta_json) = idx_reader
                     .metadata()
                     .file_schema
                     .metadata
                     .get(INDEX_METADATA_SCHEMA_KEY)
-                {
-                    let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?;
-                    detected_index_type = Some(match idx_meta.index_type.as_str() {
-                        "IVF_FLAT" => SupportedIvfIndexType::IvfFlat,
-                        "IVF_PQ" => SupportedIvfIndexType::IvfPq,
-                        "IVF_SQ" => SupportedIvfIndexType::IvfSq,
-                        "IVF_RQ" => SupportedIvfIndexType::IvfRq,
-                        "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat,
-                        "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq,
-                        "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq,
-                        other => {
-                            return Err(Error::index(format!(
-                                "Unsupported index type in shard index.idx: {}",
-                                other
-                            )));
-                        }
-                    });
-                }
+            {
+                let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?;
+                detected_index_type = Some(match idx_meta.index_type.as_str() {
+                    "IVF_FLAT" => SupportedIvfIndexType::IvfFlat,
+                    "IVF_PQ" => SupportedIvfIndexType::IvfPq,
+                    "IVF_SQ" => SupportedIvfIndexType::IvfSq,
+                    "IVF_RQ" => SupportedIvfIndexType::IvfRq,
+                    "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat,
+                    "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq,
+                    "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq,
+                    other => {
+                        return Err(Error::index(format!(
+                            "Unsupported index type in shard index.idx: {}",
+                            other
+                        )));
+                    }
+                });
             }
             // Fallback: infer from auxiliary schema
             if detected_index_type.is_none() {
@@ -841,35 +903,52 @@ pub async fn merge_partial_vector_auxiliary_files(
         }
 
         // Read IVF lengths from global buffer
-        let ivf_idx: u32 = reader
-            .metadata()
-            .file_schema
-            .metadata
-            .get(IVF_METADATA_KEY)
-            .ok_or_else(|| Error::index("IVF meta missing".to_string()))?
-            .parse()
-            .map_err(|_| Error::index("IVF index parse error".to_string()))?;
-        let bytes = reader.read_global_buffer(ivf_idx).await?;
-        let pb_ivf: pb::Ivf = prost::Message::decode(bytes)?;
+        let pb_ivf = try_read_ivf_proto(&reader)
+            .await?
+            .ok_or_else(|| Error::index("IVF meta missing".to_string()))?;
         let lengths = pb_ivf.lengths.clone();
         let nlist = lengths.len();
 
+        let mut current_centroids = ivf_centroids_from_proto(&pb_ivf)?;
+        if current_centroids.is_none() {
+            if !idx_reader_checked {
+                idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?;
+            }
+            if let Some(idx_reader) = idx_reader.as_ref()
+                && let Some(index_ivf) = try_read_ivf_proto(idx_reader).await?
+            {
+                current_centroids = ivf_centroids_from_proto(&index_ivf)?;
+            }
+        }
         if nlist_opt.is_none() {
             nlist_opt = Some(nlist);
             accumulated_lengths = vec![0; nlist];
-            // Try load centroids tensor if present
-            if let Some(tensor) = pb_ivf.centroids_tensor.as_ref() {
-                let arr = FixedSizeListArray::try_from(tensor)?;
-                first_centroids = Some(arr.clone());
+            if let Some(arr) = current_centroids {
                 let d0 = arr.value_length() as usize;
                 if dim.is_none() {
                     dim = Some(d0);
                 }
+                first_centroids = Some(arr);
             }
         } else if nlist_opt.as_ref().map(|v| *v != nlist).unwrap_or(false) {
             return Err(Error::index(
                 "IVF partition count mismatch across shards".to_string(),
             ));
+        } else {
+            match (&first_centroids, &current_centroids) {
+                (Some(reference), Some(candidate)) => {
+                    ensure_fixed_size_list_compatible("IVF centroids", reference, candidate)?;
+                }
+                (Some(_), None) => {
+                    return Err(Error::index("IVF centroids missing from shard".to_string()));
+                }
+                (None, Some(_)) => {
+                    return Err(Error::index(
+                        "IVF centroids missing from first shard".to_string(),
+                    ));
+                }
+                (None, None) => {}
+            }
         }
 
         // Handle logic based on detected index type
@@ -982,7 +1061,12 @@ pub async fn merge_partial_vector_auxiliary_files(
                     let rotate_mat_bytes = reader.read_global_buffer(buf_idx).await?;
                     rq_meta_parsed.parse_buffer(rotate_mat_bytes)?;
                 }
-                validate_supported_rq_num_bits(rq_meta_parsed.num_bits)?;
+                validate_rq_num_bits(rq_meta_parsed.num_bits)?;
+                if rq_meta_parsed.packed {
+                    return Err(Error::index(format!(
+                        "Distributed RQ merge: source shard {idx} stores packed RQ codes; expected row-major distributed shard"
+                    )));
+                }
 
                 let d0 = rq_meta_parsed.rotated_dim();
                 if d0 == 0 {
@@ -999,7 +1083,9 @@ pub async fn merge_partial_vector_auxiliary_files(
                 if let Some(existing_rq) = rq_meta.as_ref()
                     && (existing_rq.code_dim != rq_meta_parsed.code_dim
                         || existing_rq.num_bits != rq_meta_parsed.num_bits
-                        || existing_rq.rotation_type != rq_meta_parsed.rotation_type)
+                        || existing_rq.rotation_type != rq_meta_parsed.rotation_type
+                        || existing_rq.query_estimator != rq_meta_parsed.query_estimator
+                        || existing_rq.fast_rotation_signs != rq_meta_parsed.fast_rotation_signs)
                 {
                     return Err(Error::index(format!(
                         "Distributed RQ merge: structural mismatch across shards; first(code_dim={}, num_bits={}, rotation_type={:?}), current(code_dim={}, num_bits={}, rotation_type={:?})",
@@ -1011,6 +1097,24 @@ pub async fn merge_partial_vector_auxiliary_files(
                         rq_meta_parsed.rotation_type
                     )));
                 }
+                if let Some(existing_rq) = rq_meta.as_ref() {
+                    match (&existing_rq.rotate_mat, &rq_meta_parsed.rotate_mat) {
+                        (Some(reference), Some(candidate)) => {
+                            ensure_fixed_size_list_compatible(
+                                "RQ rotation matrix",
+                                reference,
+                                candidate,
+                            )?;
+                        }
+                        (Some(_), None) | (None, Some(_)) => {
+                            return Err(Error::index(
+                                "Distributed RQ merge: rotation matrix mismatch across shards"
+                                    .to_string(),
+                            ));
+                        }
+                        (None, None) => {}
+                    }
+                }
                 if rq_meta.is_none() {
                     rq_meta = Some(rq_meta_parsed.clone());
                 }
@@ -1059,6 +1163,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                 };
                 let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json)
                     .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?;
+                if pm.transposed {
+                    return Err(Error::index(format!(
+                        "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard"
+                    )));
+                }
                 // Load codebook from global buffer if not present
                 if pm.codebook.is_none() {
                     let tensor_bytes = reader
@@ -1098,18 +1207,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                         .codebook
                         .as_ref()
                         .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?;
-                    if !fixed_size_list_equal(existing_cb, current_cb) {
-                        const TOL: f32 = 1e-5;
-                        if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) {
-                            return Err(Error::index(
-                                "PQ codebook content mismatch across shards".to_string(),
-                            ));
-                        } else {
-                            log::warn!(
-                                "PQ codebook differs within tolerance; proceeding with first shard codebook"
-                            );
-                        }
-                    }
+                    ensure_fixed_size_list_compatible(
+                        "PQ codebook content",
+                        existing_cb,
+                        current_cb,
+                    )?;
                 }
                 if pq_meta.is_none() {
                     pq_meta = Some(pm.clone());
@@ -1220,6 +1322,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                 };
                 let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json)
                     .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?;
+                if pm.transposed {
+                    return Err(Error::index(format!(
+                        "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard"
+                    )));
+                }
                 if pm.codebook.is_none() {
                     let tensor_bytes = reader
                         .read_global_buffer(pm.codebook_position as u32)
@@ -1258,18 +1365,11 @@ pub async fn merge_partial_vector_auxiliary_files(
                         .codebook
                         .as_ref()
                         .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?;
-                    if !fixed_size_list_equal(existing_cb, current_cb) {
-                        const TOL: f32 = 1e-5;
-                        if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) {
-                            return Err(Error::index(
-                                "PQ codebook content mismatch across shards".to_string(),
-                            ));
-                        } else {
-                            log::warn!(
-                                "PQ codebook differs within tolerance; proceeding with first shard codebook"
-                            );
-                        }
-                    }
+                    ensure_fixed_size_list_compatible(
+                        "PQ codebook content",
+                        existing_cb,
+                        current_cb,
+                    )?;
                 }
                 if pq_meta.is_none() {
                     pq_meta = Some(pm.clone());
@@ -1438,6 +1538,25 @@ pub async fn merge_partial_vector_auxiliary_files(
                     )));
                 }
 
+                // Shards written by older lance versions carry sequential ex
+                // codes; normalize every batch to the blocked layout before
+                // concatenation so mixed-version shards merge correctly
+                // (concat_batches combines columns by position and would
+                // otherwise mix the two layouts silently).
+                let batches = match rq_meta.as_ref() {
+                    Some(meta) if meta.num_bits > 1 => batches
+                        .into_iter()
+                        .map(|batch| {
+                            crate::vector::bq::storage::load_blocked_ex_codes(
+                                batch,
+                                meta.rotated_dim(),
+                                meta.num_bits,
+                            )
+                            .map(|(batch, _)| batch)
+                        })
+                        .collect::<Result<Vec<_>>>()?,
+                    _ => batches,
+                };
                 let schema = batches[0].schema();
                 let partition_batch = concat_batches(&schema, batches.iter())?;
                 if let Some(w) = v2w_opt.as_mut() {
@@ -1490,16 +1609,18 @@ pub async fn merge_partial_vector_auxiliary_files(
         }
         let dt2 = distance_type.ok_or_else(|| Error::index("Distance type missing".to_string()))?;
         write_unified_ivf_and_index_metadata(w, &ivf_model, dt2, idx_type_final).await?;
-        w.finish().await?;
+        let summary = w.finish().await?;
         progress.stage_progress("write_auxiliary_index", 1).await?;
         progress.stage_complete("write_auxiliary_index").await?;
+        Ok(lance_table::format::IndexFile {
+            path: INDEX_AUXILIARY_FILE_NAME.to_string(),
+            size_bytes: summary.size_bytes,
+        })
     } else {
-        return Err(Error::index(
+        Err(Error::index(
             "Failed to initialize unified writer".to_string(),
-        ));
+        ))
     }
-
-    Ok(())
 }
 
 #[cfg(test)]
@@ -1523,6 +1644,8 @@ mod tests {
     use prost::Message;
 
     use crate::vector::bq::RQRotationType;
+    use crate::vector::bq::storage::{RABIT_BLOCKED_EX_CODE_COLUMN, RabitQueryEstimator};
+    use crate::vector::bq::transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN};
     lance_testing::define_stage_event_progress!(
         RecordingProgress,
         IndexBuildProgress,
@@ -1951,6 +2074,7 @@ mod tests {
         base_row_id: u64,
         distance_type: DistanceType,
         codebook: &FixedSizeListArray,
+        transposed: bool,
     ) -> Result<usize> {
         let num_bytes = if nbits == 4 {
             // Two 4-bit codes per byte.
@@ -1989,7 +2113,7 @@ mod tests {
             dimension,
             codebook: Some(codebook.clone()),
             codebook_tensor: Vec::new(),
-            transposed: true,
+            transposed,
         };
 
         let codebook_tensor: pb::Tensor = pb::Tensor::try_from(codebook)?;
@@ -2053,7 +2177,14 @@ mod tests {
         distance_type: DistanceType,
     ) -> Result<usize> {
         let num_bytes = (metadata.code_dim as usize).div_ceil(u8::BITS as usize);
-        let arrow_schema = ArrowSchema::new(vec![
+        let ex_code_field = rabit_ex_code_field(metadata.code_dim as usize, metadata.num_bits)?;
+        let ex_code_bytes = ex_code_field.as_ref().map(|field| {
+            let DataType::FixedSizeList(_, num_bytes) = field.data_type() else {
+                panic!("RQ ex-code field should be FixedSizeList");
+            };
+            *num_bytes as usize
+        });
+        let mut fields = vec![
             (*ROW_ID_FIELD).clone(),
             Field::new(
                 RABIT_CODE_COLUMN,
@@ -2065,7 +2196,16 @@ mod tests {
             ),
             ADD_FACTORS_FIELD.clone(),
             SCALE_FACTORS_FIELD.clone(),
-        ]);
+        ];
+        if metadata.query_estimator == RabitQueryEstimator::RawQuery {
+            fields.push(ERROR_FACTORS_FIELD.clone());
+        }
+        if let Some(field) = ex_code_field {
+            fields.push(field);
+            fields.push(EX_ADD_FACTORS_FIELD.clone());
+            fields.push(EX_SCALE_FACTORS_FIELD.clone());
+        }
+        let arrow_schema = ArrowSchema::new(fields);
 
         let writer = store.create(aux_path).await?;
         let mut v2w = V2Writer::try_new(
@@ -2094,6 +2234,11 @@ mod tests {
         let mut codes = Vec::with_capacity(total_rows * num_bytes);
         let mut add_factors = Vec::with_capacity(total_rows);
         let mut scale_factors = Vec::with_capacity(total_rows);
+        let mut error_factors = Vec::with_capacity(total_rows);
+        let mut ex_codes =
+            ex_code_bytes.map(|num_bytes| Vec::with_capacity(total_rows * num_bytes));
+        let mut ex_add_factors = Vec::with_capacity(total_rows);
+        let mut ex_scale_factors = Vec::with_capacity(total_rows);
 
         let mut current_row_id = base_row_id;
         for (pid, len) in lengths.iter().enumerate() {
@@ -2105,21 +2250,38 @@ mod tests {
                 }
                 add_factors.push(pid as f32 + row_offset as f32 * 0.1);
                 scale_factors.push(pid as f32 + row_offset as f32 * 0.2);
+                error_factors.push(pid as f32 + row_offset as f32 * 0.3);
+                if let (Some(ex_codes), Some(ex_code_bytes)) = (ex_codes.as_mut(), ex_code_bytes) {
+                    for b in 0..ex_code_bytes {
+                        ex_codes.push((17 + pid + row_offset + b) as u8);
+                    }
+                    ex_add_factors.push(pid as f32 + 10.0 + row_offset as f32 * 0.2);
+                    ex_scale_factors.push(pid as f32 + 1.0 + row_offset as f32 * 0.2);
+                }
             }
         }
 
-        let batch = RecordBatch::try_new(
-            Arc::new(arrow_schema),
-            vec![
-                Arc::new(UInt64Array::from(row_ids)),
-                Arc::new(FixedSizeListArray::try_new_from_values(
-                    UInt8Array::from(codes),
-                    num_bytes as i32,
-                )?),
-                Arc::new(Float32Array::from(add_factors)),
-                Arc::new(Float32Array::from(scale_factors)),
-            ],
-        )?;
+        let mut columns: Vec<Arc<dyn Array>> = vec![
+            Arc::new(UInt64Array::from(row_ids)),
+            Arc::new(FixedSizeListArray::try_new_from_values(
+                UInt8Array::from(codes),
+                num_bytes as i32,
+            )?),
+            Arc::new(Float32Array::from(add_factors)),
+            Arc::new(Float32Array::from(scale_factors)),
+        ];
+        if metadata.query_estimator == RabitQueryEstimator::RawQuery {
+            columns.push(Arc::new(Float32Array::from(error_factors)));
+        }
+        if let (Some(ex_codes), Some(ex_code_bytes)) = (ex_codes, ex_code_bytes) {
+            columns.push(Arc::new(FixedSizeListArray::try_new_from_values(
+                UInt8Array::from(ex_codes),
+                ex_code_bytes as i32,
+            )?));
+            columns.push(Arc::new(Float32Array::from(ex_add_factors)));
+            columns.push(Arc::new(Float32Array::from(ex_scale_factors)));
+        }
+        let batch = RecordBatch::try_new(Arc::new(arrow_schema), columns)?;
 
         v2w.write_batch(&batch).await?;
         v2w.finish().await?;
@@ -2162,6 +2324,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2176,6 +2339,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2256,6 +2420,66 @@ mod tests {
         assert!(fixed_size_list_equal(&codebook, &merged_codebook));
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_pq_rejects_transposed_source_shard() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_pq_transposed");
+
+        let partial0 = index_dir.clone().join("partial_0");
+        let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let lengths = vec![2_u32, 1_u32];
+
+        let nbits = 4_u32;
+        let num_sub_vectors = 2_usize;
+        let dimension = 8_usize;
+        let num_centroids = 1_usize << nbits;
+        let num_codebook_vectors = num_centroids * num_sub_vectors;
+        let total_values = num_codebook_vectors * dimension;
+        let values = Float32Array::from_iter((0..total_values).map(|v| v as f32));
+        let codebook = FixedSizeListArray::try_new_from_values(values, dimension as i32).unwrap();
+
+        write_pq_partial_aux(
+            &object_store,
+            &aux0,
+            nbits,
+            num_sub_vectors,
+            dimension,
+            &lengths,
+            0,
+            DistanceType::L2,
+            &codebook,
+            true,
+        )
+        .await
+        .unwrap();
+
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            std::slice::from_ref(&aux0),
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
+        match res {
+            Err(Error::Index { message, .. }) => {
+                assert!(
+                    message.contains("source shard 0"),
+                    "unexpected message: {}",
+                    message
+                );
+                assert!(
+                    message.contains("transposed PQ codes"),
+                    "unexpected message: {}",
+                    message
+                );
+            }
+            other => panic!(
+                "expected Error::Index for transposed PQ source shard, got {:?}",
+                other
+            ),
+        }
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_rq_success() {
         let object_store = ObjectStore::memory();
@@ -2277,6 +2501,7 @@ mod tests {
             code_dim: 16,
             num_bits: 1,
             packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
         };
 
         write_rq_partial_aux(
@@ -2391,6 +2616,181 @@ mod tests {
         assert_eq!(total_rows, expected_total);
     }
 
+    #[tokio::test]
+    async fn test_merge_ivf_rq_rejects_packed_source_shard() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_rq_packed");
+
+        let partial0 = index_dir.clone().join("partial_0");
+        let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let lengths = vec![2_u32, 1_u32];
+
+        let rq_meta = RabitQuantizationMetadata {
+            rotate_mat: None,
+            rotate_mat_position: None,
+            fast_rotation_signs: Some(vec![0xAA; 2]),
+            rotation_type: RQRotationType::Fast,
+            code_dim: 16,
+            num_bits: 1,
+            packed: true,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+
+        write_rq_partial_aux(
+            &object_store,
+            &aux0,
+            &rq_meta,
+            &lengths,
+            0,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+
+        let res = merge_partial_vector_auxiliary_files(
+            &object_store,
+            std::slice::from_ref(&aux0),
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await;
+        match res {
+            Err(Error::Index { message, .. }) => {
+                assert!(
+                    message.contains("source shard 0"),
+                    "unexpected message: {}",
+                    message
+                );
+                assert!(
+                    message.contains("packed RQ codes"),
+                    "unexpected message: {}",
+                    message
+                );
+            }
+            other => panic!(
+                "expected Error::Index for packed RQ source shard, got {:?}",
+                other
+            ),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_merge_ivf_rq_multi_bit_preserves_split_columns() {
+        let object_store = ObjectStore::memory();
+        let index_dir = Path::from("index/uuid_rq_multi_bit");
+
+        let partial0 = index_dir.clone().join("partial_0");
+        let partial1 = index_dir.clone().join("partial_1");
+        let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let aux1 = partial1.clone().join(INDEX_AUXILIARY_FILE_NAME);
+
+        let lengths0 = vec![2_u32, 1_u32];
+        let lengths1 = vec![1_u32, 2_u32];
+
+        let rq_meta = RabitQuantizationMetadata {
+            rotate_mat: None,
+            rotate_mat_position: None,
+            fast_rotation_signs: Some(vec![0xAA; 2]),
+            rotation_type: RQRotationType::Fast,
+            code_dim: 16,
+            num_bits: 4,
+            packed: false,
+            query_estimator: RabitQueryEstimator::RawQuery,
+        };
+
+        write_rq_partial_aux(
+            &object_store,
+            &aux0,
+            &rq_meta,
+            &lengths0,
+            0,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+        write_rq_partial_aux(
+            &object_store,
+            &aux1,
+            &rq_meta,
+            &lengths1,
+            1_000,
+            DistanceType::L2,
+        )
+        .await
+        .unwrap();
+
+        merge_partial_vector_auxiliary_files(
+            &object_store,
+            &[aux0.clone(), aux1.clone()],
+            &index_dir,
+            crate::progress::noop_progress(),
+        )
+        .await
+        .unwrap();
+
+        let aux_out = index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME);
+        let sched = ScanScheduler::new(
+            Arc::new(object_store.clone()),
+            SchedulerConfig::max_bandwidth(&object_store),
+        );
+        let fh = sched
+            .open_file(&aux_out, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        let reader = V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &lance_core::cache::LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await
+        .unwrap();
+        let meta = reader.metadata();
+        let rq_meta_json = meta.file_schema.metadata.get(RABIT_METADATA_KEY).unwrap();
+        let merged_rq_meta: RabitQuantizationMetadata = serde_json::from_str(rq_meta_json).unwrap();
+        assert_eq!(merged_rq_meta.num_bits, 4);
+        assert!(merged_rq_meta.packed);
+
+        let mut total_rows = 0usize;
+        let mut checked_split_columns = false;
+        let mut stream = reader
+            .read_stream(
+                lance_io::ReadBatchParams::RangeFull,
+                u32::MAX,
+                4,
+                lance_encoding::decoder::FilterExpression::no_filter(),
+            )
+            .await
+            .unwrap();
+        while let Some(batch) = stream.next().await {
+            let batch = batch.unwrap();
+            if !checked_split_columns {
+                let schema = batch.schema();
+                let ex_code_field = schema
+                    .field_with_name(RABIT_BLOCKED_EX_CODE_COLUMN)
+                    .unwrap();
+                let DataType::FixedSizeList(_, ex_code_bytes) = ex_code_field.data_type() else {
+                    panic!("RQ ex-code field should be FixedSizeList");
+                };
+                // code_dim=16 padded to one 64-dim block at ex_bits=3.
+                assert_eq!(*ex_code_bytes, 24);
+                assert!(schema.field_with_name(ERROR_FACTORS_FIELD.name()).is_ok());
+                assert!(schema.field_with_name(EX_ADD_FACTORS_COLUMN).is_ok());
+                assert!(schema.field_with_name(EX_SCALE_FACTORS_COLUMN).is_ok());
+                checked_split_columns = true;
+            }
+            total_rows += batch.num_rows();
+        }
+        assert!(checked_split_columns);
+        let expected_total: usize = lengths0
+            .iter()
+            .zip(lengths1.iter())
+            .map(|(a, b)| (*a + *b) as usize)
+            .sum();
+        assert_eq!(total_rows, expected_total);
+    }
+
     #[tokio::test]
     async fn test_merge_ivf_pq_codebook_mismatch() {
         let object_store = ObjectStore::memory();
@@ -2431,6 +2831,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook0,
+            false,
         )
         .await
         .unwrap();
@@ -2445,6 +2846,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook1,
+            false,
         )
         .await
         .unwrap();
@@ -2510,6 +2912,7 @@ mod tests {
             0,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
@@ -2525,6 +2928,7 @@ mod tests {
             1_000,
             DistanceType::L2,
             &codebook,
+            false,
         )
         .await
         .unwrap();
diff --git a/rust/lance-index/src/vector/flat/index.rs b/rust/lance-index/src/vector/flat/index.rs
index 6ebab23688f..89c28a6ffd7 100644
--- a/rust/lance-index/src/vector/flat/index.rs
+++ b/rust/lance-index/src/vector/flat/index.rs
@@ -4,13 +4,14 @@
 //! Flat Vector Index.
 //!
 
-use std::collections::{BinaryHeap, HashMap};
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::collections::BinaryHeap;
 use std::sync::Arc;
 
 use arrow::array::AsArray;
 use arrow_array::{Array, ArrayRef, Float32Array, RecordBatch, UInt64Array};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID_FIELD, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_linalg::distance::DistanceType;
@@ -20,10 +21,12 @@ use crate::{
     metrics::MetricsCollector,
     prefilter::PreFilter,
     vector::{
-        DIST_COL, Query,
+        ApproxMode, DIST_COL, Query,
         graph::{OrderedFloat, OrderedNode},
         quantizer::{Quantization, QuantizationType, Quantizer, QuantizerMetadata},
-        storage::{DistCalculator, QueryResidual, QueryScratch, VectorStore},
+        storage::{
+            DistCalculator, DistanceCalculatorOptions, QueryResidual, QueryScratch, VectorStore,
+        },
         v3::subindex::IvfSubIndex,
     },
 };
@@ -48,29 +51,6 @@ fn push_candidate_local(
     }
 }
 
-#[inline(always)]
-fn push_candidate_global(
-    res: &mut BinaryHeap<OrderedNode<u64>>,
-    k: usize,
-    row_id: u64,
-    dist: OrderedFloat,
-    max_dist: &mut Option<OrderedFloat>,
-) {
-    if k == 0 {
-        return;
-    }
-    if res.len() < k {
-        res.push(OrderedNode::new(row_id, dist));
-        if res.len() == k {
-            *max_dist = res.peek().map(|node| node.dist);
-        }
-    } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
-        res.pop();
-        res.push(OrderedNode::new(row_id, dist));
-        *max_dist = res.peek().map(|node| node.dist);
-    }
-}
-
 /// A Flat index is any index that stores no metadata, and
 /// during query, it simply scans over the storage and returns the top k results
 #[derive(Debug, Clone, Default, DeepSizeOf)]
@@ -91,6 +71,7 @@ pub struct FlatQueryParams {
     lower_bound: Option<f32>,
     upper_bound: Option<f32>,
     dist_q_c: f32,
+    approx_mode: ApproxMode,
 }
 
 impl From<&Query> for FlatQueryParams {
@@ -99,6 +80,7 @@ impl From<&Query> for FlatQueryParams {
             lower_bound: q.lower_bound,
             upper_bound: q.upper_bound,
             dist_q_c: q.dist_q_c,
+            approx_mode: q.approx_mode,
         }
     }
 }
@@ -159,6 +141,9 @@ impl IvfSubIndex for FlatIndex {
             params.dist_q_c,
             residual,
             &mut scratch.query_f32,
+            DistanceCalculatorOptions {
+                approx_mode: params.approx_mode,
+            },
         );
         let mut res = BinaryHeap::with_capacity(k);
         metrics.record_comparisons(storage.len());
@@ -170,6 +155,7 @@ impl IvfSubIndex for FlatIndex {
                     &mut scratch.distances,
                     &mut scratch.u16,
                     &mut scratch.u8,
+                    &mut scratch.u32,
                 );
                 let dists = scratch.distances.iter().copied();
 
@@ -271,70 +257,46 @@ impl IvfSubIndex for FlatIndex {
         scratch: &mut QueryScratch,
         metrics: &dyn MetricsCollector,
     ) -> Result<()> {
-        let is_range_query = params.lower_bound.is_some() || params.upper_bound.is_some();
         let row_ids = storage.row_ids();
         let dist_calc = storage.dist_calculator_with_scratch(
             query,
             params.dist_q_c,
             residual,
             &mut scratch.query_f32,
+            DistanceCalculatorOptions {
+                approx_mode: params.approx_mode,
+            },
         );
-        let mut max_dist = res.peek().map(|node| node.dist);
         metrics.record_comparisons(storage.len());
 
         match prefilter.is_empty() {
             true => {
-                dist_calc.distance_all_with_scratch(
+                dist_calc.accumulate_topk_with_scratch(
                     k,
+                    params.lower_bound,
+                    params.upper_bound,
+                    |id| storage.row_id(id),
+                    res,
                     &mut scratch.distances,
                     &mut scratch.u16,
                     &mut scratch.u8,
+                    &mut scratch.u32,
                 );
-                let dists = scratch.distances.iter().copied();
-
-                if is_range_query {
-                    let lower_bound = params.lower_bound.unwrap_or(f32::MIN).into();
-                    let upper_bound = params.upper_bound.unwrap_or(f32::MAX).into();
-
-                    for (&row_id, dist) in row_ids.zip(dists) {
-                        let dist = dist.into();
-                        if dist < lower_bound || dist >= upper_bound {
-                            continue;
-                        }
-                        push_candidate_global(res, k, row_id, dist, &mut max_dist);
-                    }
-                } else {
-                    for (&row_id, dist) in row_ids.zip(dists) {
-                        let dist = dist.into();
-                        push_candidate_global(res, k, row_id, dist, &mut max_dist);
-                    }
-                }
             }
             false => {
                 let row_addr_mask = prefilter.mask();
-                if is_range_query {
-                    let lower_bound = params.lower_bound.unwrap_or(f32::MIN).into();
-                    let upper_bound = params.upper_bound.unwrap_or(f32::MAX).into();
-                    for (id, &row_addr) in row_ids.enumerate() {
-                        if !row_addr_mask.selected(row_addr) {
-                            continue;
-                        }
-                        let dist = dist_calc.distance(id as u32).into();
-                        if dist < lower_bound || dist >= upper_bound {
-                            continue;
-                        }
-
-                        push_candidate_global(res, k, row_addr, dist, &mut max_dist);
-                    }
-                } else {
-                    for (id, &row_addr) in row_ids.enumerate() {
-                        if !row_addr_mask.selected(row_addr) {
-                            continue;
-                        }
-                        let dist = dist_calc.distance(id as u32).into();
-                        push_candidate_global(res, k, row_addr, dist, &mut max_dist);
-                    }
-                }
+                dist_calc.accumulate_filtered_topk_with_scratch(
+                    k,
+                    params.lower_bound,
+                    params.upper_bound,
+                    row_ids.enumerate().map(|(id, &row_id)| (id as u32, row_id)),
+                    |row_id| row_addr_mask.selected(row_id),
+                    res,
+                    &mut scratch.distances,
+                    &mut scratch.u16,
+                    &mut scratch.u8,
+                    &mut scratch.u32,
+                );
             }
         };
         Ok(())
@@ -351,7 +313,7 @@ impl IvfSubIndex for FlatIndex {
         Ok(Self {})
     }
 
-    fn remap(&self, _: &HashMap<u64, Option<u64>>, _: &impl VectorStore) -> Result<Self> {
+    fn remap(&self, _: &RowAddrRemap, _: &impl VectorStore) -> Result<Self> {
         Ok(self.clone())
     }
 
@@ -608,7 +570,7 @@ mod tests {
             .zip(dists.values().iter())
             .map(|(row_id, dist)| (*row_id, *dist))
             .collect::<Vec<_>>();
-        results.sort_by(|left, right| left.0.cmp(&right.0));
+        results.sort_by_key(|left| left.0);
         results
     }
 
@@ -617,7 +579,7 @@ mod tests {
             .into_iter()
             .map(|node| (node.id, node.dist.0))
             .collect::<Vec<_>>();
-        results.sort_by(|left, right| left.0.cmp(&right.0));
+        results.sort_by_key(|left| left.0);
         results
     }
 
diff --git a/rust/lance-index/src/vector/flat/storage.rs b/rust/lance-index/src/vector/flat/storage.rs
index 4d1bf840111..ff8756cbe41 100644
--- a/rust/lance-index/src/vector/flat/storage.rs
+++ b/rust/lance-index/src/vector/flat/storage.rs
@@ -17,7 +17,7 @@ use arrow_array::{
     types::{Float32Type, UInt64Type},
 };
 use arrow_schema::{DataType, SchemaRef};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_linalg::distance::hamming::hamming;
@@ -38,7 +38,7 @@ pub struct FlatFloatStorage {
 }
 
 impl DeepSizeOf for FlatFloatStorage {
-    fn deep_size_of_children(&self, _: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize {
         self.batch.get_array_memory_size()
     }
 }
@@ -134,7 +134,7 @@ impl VectorStore for FlatFloatStorage {
 
     fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result<Self> {
         // TODO: use chunked storage
-        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
+        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?;
         let mut storage = self.clone();
         storage.row_ids = Arc::new(
             new_batch
@@ -200,7 +200,7 @@ pub struct FlatBinStorage {
 }
 
 impl DeepSizeOf for FlatBinStorage {
-    fn deep_size_of_children(&self, _: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize {
         self.batch.get_array_memory_size()
     }
 }
@@ -296,7 +296,7 @@ impl VectorStore for FlatBinStorage {
 
     fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result<Self> {
         // TODO: use chunked storage
-        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?;
+        let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?;
         let mut storage = self.clone();
         storage.row_ids = Arc::new(
             new_batch
diff --git a/rust/lance-index/src/vector/flat/transform.rs b/rust/lance-index/src/vector/flat/transform.rs
index 75a465ce262..f9fdca0819c 100644
--- a/rust/lance-index/src/vector/flat/transform.rs
+++ b/rust/lance-index/src/vector/flat/transform.rs
@@ -26,7 +26,7 @@ impl FlatTransformer {
 
 impl Transformer for FlatTransformer {
     #[instrument(name = "FlatTransformer::transform", level = "debug", skip_all)]
-    fn transform(&self, batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    fn transform(&self, batch: &RecordBatch) -> lance_core::Result<RecordBatch> {
         let input_arr = batch
             .column_by_name(&self.input_column)
             .ok_or(Error::index(format!(
diff --git a/rust/lance-index/src/vector/graph.rs b/rust/lance-index/src/vector/graph.rs
index b79ce4e9947..097aa064d67 100644
--- a/rust/lance-index/src/vector/graph.rs
+++ b/rust/lance-index/src/vector/graph.rs
@@ -9,7 +9,7 @@ use std::collections::BinaryHeap;
 use std::sync::Arc;
 
 use arrow_schema::{DataType, Field};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 use crate::vector::hnsw::builder::HnswQueryParams;
 
diff --git a/rust/lance-index/src/vector/graph/builder.rs b/rust/lance-index/src/vector/graph/builder.rs
index 8183ae3acfe..36e42b7cf9a 100644
--- a/rust/lance-index/src/vector/graph/builder.rs
+++ b/rust/lance-index/src/vector/graph/builder.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 use super::OrderedFloat;
 use super::OrderedNode;
diff --git a/rust/lance-index/src/vector/hnsw.rs b/rust/lance-index/src/vector/hnsw.rs
index 02e9d1682e2..a618f34753f 100644
--- a/rust/lance-index/src/vector/hnsw.rs
+++ b/rust/lance-index/src/vector/hnsw.rs
@@ -7,8 +7,8 @@
 //!
 
 use arrow_schema::{DataType, Field};
-use deepsize::DeepSizeOf;
 use itertools::Itertools;
+use lance_core::deepsize::DeepSizeOf;
 use serde::{Deserialize, Serialize};
 
 use self::builder::HnswBuildParams;
diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs
index cf5c8864281..a020d011fba 100644
--- a/rust/lance-index/src/vector/hnsw/builder.rs
+++ b/rust/lance-index/src/vector/hnsw/builder.rs
@@ -5,11 +5,12 @@
 
 use arrow::array::{AsArray, ListBuilder, UInt32Builder};
 use arrow::compute::concat_batches;
-use arrow::datatypes::{DataType, Float32Type, UInt32Type};
-use arrow_array::{ArrayRef, Float32Array, RecordBatch, UInt64Array};
+use arrow::datatypes::{DataType, UInt32Type};
+use arrow_array::{ArrayRef, Float32Array, ListArray, RecordBatch, UInt64Array};
 use crossbeam_queue::ArrayQueue;
-use deepsize::DeepSizeOf;
 use itertools::Itertools;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_linalg::distance::DistanceType;
@@ -42,7 +43,7 @@ use crate::vector::graph::{
 use crate::vector::graph::{Visited, beam_search_borrowed, greedy_search, greedy_search_borrowed};
 use crate::vector::storage::{DistCalculator, VectorStore};
 use crate::vector::v3::subindex::IvfSubIndex;
-use crate::vector::{DIST_COL, Query, VECTOR_RESULT_SCHEMA};
+use crate::vector::{Query, VECTOR_RESULT_SCHEMA};
 
 pub const HNSW_METADATA_KEY: &str = "lance:hnsw";
 
@@ -158,16 +159,16 @@ pub struct HNSW {
 
 struct HnswCore {
     params: HnswBuildParams,
-    nodes: Arc<Vec<GraphBuilderNode>>,
+    graph: HnswGraph,
     level_count: Vec<usize>,
     entry_point: u32,
     visited_generator_queue: Arc<ArrayQueue<VisitedGenerator>>,
 }
 
 impl DeepSizeOf for HnswCore {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.params.deep_size_of_children(context)
-            + self.nodes.deep_size_of_children(context)
+            + self.graph.deep_size_of_children(context)
             + self.level_count.deep_size_of_children(context)
         // Skipping the visited_generator_queue
     }
@@ -181,10 +182,6 @@ impl HnswCore {
     fn num_nodes(&self, level: usize) -> usize {
         self.level_count[level]
     }
-
-    fn nodes(&self) -> Arc<Vec<GraphBuilderNode>> {
-        self.nodes.clone()
-    }
 }
 
 impl Debug for HNSW {
@@ -210,7 +207,7 @@ impl HNSW {
         Self {
             inner: Arc::new(HnswCore {
                 params,
-                nodes: Arc::new(nodes),
+                graph: HnswGraph::Built(Arc::new(nodes)),
                 level_count,
                 entry_point,
                 visited_generator_queue,
@@ -222,7 +219,7 @@ impl HNSW {
         Self {
             inner: Arc::new(HnswCore {
                 params: HnswBuildParams::default(),
-                nodes: Arc::new(Vec::new()),
+                graph: HnswGraph::Built(Arc::new(Vec::new())),
                 level_count: Vec::new(),
                 entry_point: 0,
                 visited_generator_queue: Arc::new(ArrayQueue::new(1)),
@@ -231,7 +228,11 @@ impl HNSW {
     }
 
     pub fn len(&self) -> usize {
-        self.inner.nodes.len()
+        match &self.inner.graph {
+            HnswGraph::Built(nodes) => nodes.len(),
+            // `level_count[0]` is the bottom-level (== total) node count.
+            HnswGraph::Loaded(graph) => graph.level_count[0],
+        }
     }
 
     pub fn is_empty(&self) -> bool {
@@ -246,8 +247,15 @@ impl HNSW {
         self.inner.num_nodes(level)
     }
 
-    pub fn nodes(&self) -> Arc<Vec<GraphBuilderNode>> {
-        self.inner.nodes()
+    /// Returns the in-memory builder nodes, if this graph was freshly built.
+    ///
+    /// A disk-loaded graph is Arrow-backed and has no `GraphBuilderNode`s,
+    /// so this returns `None` for it.
+    pub fn nodes(&self) -> Option<Arc<Vec<GraphBuilderNode>>> {
+        match &self.inner.graph {
+            HnswGraph::Built(nodes) => Some(nodes.clone()),
+            HnswGraph::Loaded(_) => None,
+        }
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -263,32 +271,96 @@ impl HNSW {
     ) -> Result<Vec<OrderedNode>> {
         let dist_calc = storage.dist_calculator(query, params.dist_q_c);
         let entry = self.inner.entry_point;
-        let mut ep = OrderedNode::new(entry, dist_calc.distance(entry).into());
-        let nodes = self.inner.nodes.as_ref();
+        let ep = OrderedNode::new(entry, dist_calc.distance(entry).into());
+
+        // The level descent + bottom beam search are identical across
+        // graph backends; only the view types differ. `run_search` is
+        // generic over those view types so the loop is single-sourced:
+        // each backend supplies a per-level view closure and a
+        // bottom-level view.
+        let result = match &self.inner.graph {
+            HnswGraph::Built(nodes) => {
+                let nodes = nodes.as_slice();
+                self.run_search(
+                    ep,
+                    k,
+                    params,
+                    bitset.as_ref(),
+                    visited_generator,
+                    storage.len(),
+                    prefetch_distance,
+                    &dist_calc,
+                    |level| ImmutableHnswLevelView::new(level, nodes),
+                    ImmutableHnswBottomView::new(nodes),
+                )
+            }
+            HnswGraph::Loaded(graph) => {
+                let graph = graph.as_ref();
+                self.run_search(
+                    ep,
+                    k,
+                    params,
+                    bitset.as_ref(),
+                    visited_generator,
+                    storage.len(),
+                    prefetch_distance,
+                    &dist_calc,
+                    |level| LoadedHnswLevelView::new(level, graph),
+                    LoadedHnswBottomView::new(graph),
+                )
+            }
+        };
+        Ok(result)
+    }
+
+    /// Drives the shared HNSW query path over backend-specific graph
+    /// views: a per-level view produced by `make_level` and a
+    /// bottom-level view `bottom`. The views borrow their backing store
+    /// and are created, used, and dropped entirely within this call;
+    /// only the owned result escapes. Monomorphizing over `L`/`B` is the
+    /// single seam that lets the in-memory and disk-loaded backends
+    /// share one search loop.
+    #[allow(clippy::too_many_arguments)]
+    fn run_search<L, B>(
+        &self,
+        ep: OrderedNode,
+        k: usize,
+        params: &HnswQueryParams,
+        bitset: Option<&Visited>,
+        visited_generator: &mut VisitedGenerator,
+        storage_len: usize,
+        prefetch_distance: Option<usize>,
+        dist_calc: &impl DistCalculator,
+        make_level: impl Fn(u16) -> L,
+        bottom: B,
+    ) -> Vec<OrderedNode>
+    where
+        L: BorrowingGraph,
+        B: BorrowingGraph,
+    {
+        let mut ep = ep;
         for level in (0..self.max_level()).rev() {
-            let cur_level = ImmutableHnswLevelView::new(level, nodes);
+            let cur_level = make_level(level);
             ep = greedy_search_borrowed(
                 &cur_level,
                 ep,
-                &dist_calc,
+                dist_calc,
                 self.inner.params.prefetch_distance,
             );
         }
-
-        let bottom_level = ImmutableHnswBottomView::new(nodes);
-        let mut visited = visited_generator.generate(storage.len());
-        Ok(beam_search_borrowed(
-            &bottom_level,
+        let mut visited = visited_generator.generate(storage_len);
+        beam_search_borrowed(
+            &bottom,
             &ep,
             params,
-            &dist_calc,
-            bitset.as_ref(),
+            dist_calc,
+            bitset,
             prefetch_distance,
             &mut visited,
         )
         .into_iter()
         .take(k)
-        .collect())
+        .collect::<Vec<OrderedNode>>()
     }
 
     #[instrument(level = "debug", skip(self, query, bitset, storage))]
@@ -426,7 +498,7 @@ struct HnswBuilder {
 }
 
 impl DeepSizeOf for HnswBuilder {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.params.deep_size_of_children(context)
             + self.nodes.deep_size_of_children(context)
             + self.level_count.deep_size_of_children(context)
@@ -456,7 +528,7 @@ impl HnswBuilder {
         HNSW {
             inner: Arc::new(HnswCore {
                 params: self.params,
-                nodes: Arc::new(nodes),
+                graph: HnswGraph::Built(Arc::new(nodes)),
                 level_count,
                 entry_point: self.entry_point,
                 visited_generator_queue: self.visited_generator_queue,
@@ -716,6 +788,177 @@ impl BorrowingGraph for ImmutableHnswBottomView<'_> {
     }
 }
 
+/// Per-level node-id -> row-index lookup for a disk-loaded HNSW graph.
+enum LevelLookup {
+    /// `row == node id`. Used only for level 0, where [`HNSW::to_batch`]
+    /// writes every node once in ascending `__vector_id` (== node id) order,
+    /// so the level-0 slice is exactly `[0, N)` with `row == id`.
+    Dense,
+    /// Upper level: an explicit `node_id -> row` map built from the level's
+    /// `__vector_id` column.
+    ///
+    /// We do *not* assume the column is sorted or that the slice is aligned
+    /// to a true level boundary: `level_offsets`/`level_count` omit the
+    /// entry-point node (it is written at every level by `to_batch` but only
+    /// counted at level 0), so upper-level slices can be off-by-one and
+    /// non-monotonic. Keying by the `__vector_id` value -- exactly what the
+    /// old per-node `load` did -- preserves behavior bit-for-bit. Upper
+    /// levels shrink geometrically, so this map stays tiny.
+    Sparse(HashMap<u32, u32>),
+}
+
+/// A search-only HNSW graph backed directly by the Arrow buffers of the
+/// on-disk `RecordBatch`.
+///
+/// Loading performs no per-node reconstruction: neighbor adjacency is served
+/// as `&[u32]` slices straight out of the `__neighbors` `ListArray` value
+/// buffer (zero copy). The full `batch` is retained so [`HNSW::to_batch`] is a
+/// near-free passthrough -- required, because the IVF partition cache
+/// re-serializes loaded indices through `to_batch()`
+/// (`lance/src/index/vector/ivf/partition_serde.rs`) -- and so a future
+/// zero-copy `CacheCodec` (#6745) can write/read it through
+/// `lance_arrow::ipc` without rebuilding the graph.
+struct LoadedHnswGraph {
+    /// The full loaded batch (all levels concatenated, level 0 first),
+    /// retained verbatim for `to_batch()` and #6745.
+    batch: RecordBatch,
+    /// Per-level `__neighbors` `List<UInt32>`, zero-copy slices of `batch`.
+    level_neighbors: Vec<ListArray>,
+    /// Per-level node-id -> row lookup (see [`LevelLookup`]).
+    level_lookup: Vec<LevelLookup>,
+    /// Number of nodes present at each level (`level_count[0]` == total).
+    level_count: Vec<usize>,
+}
+
+impl DeepSizeOf for LoadedHnswGraph {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
+        // `level_neighbors` are zero-copy views into `batch`, so counting
+        // `batch` alone avoids double counting (mirrors
+        // `vector/flat/storage.rs`). The upper-level `level_lookup` maps are
+        // sized to the geometrically-shrinking node counts above level 0 --
+        // negligible next to the batch and not separately accounted here.
+        self.batch.get_array_memory_size()
+    }
+}
+
+impl LoadedHnswGraph {
+    /// Borrow the neighbor ids of `key` at `level` directly from the Arrow
+    /// `ListArray` value buffer -- no allocation, no copy.
+    #[inline]
+    fn neighbors_at(&self, level: usize, key: u32) -> &[u32] {
+        let row = match &self.level_lookup[level] {
+            LevelLookup::Dense => key as usize,
+            LevelLookup::Sparse(id_to_row) => match id_to_row.get(&key) {
+                Some(&row) => row as usize,
+                // The node is absent at this level -- e.g. an empty upper
+                // level the search descends through, or a node that only
+                // exists at lower levels. Mirror the old representation
+                // (`level_neighbors[level]` defaulted to empty): no
+                // neighbors here, so greedy search stays put and descends.
+                None => return &[],
+            },
+        };
+        let list = &self.level_neighbors[level];
+        let offsets = list.value_offsets();
+        let start = offsets[row] as usize;
+        let end = offsets[row + 1] as usize;
+        // The `__neighbors` list child is `UInt32` per `HNSW::schema()`.
+        // Validity bitmap is ignored on purpose: `to_batch` never writes null
+        // neighbor lists, matching the previous `.unwrap()`-based load.
+        let values = list.values().as_primitive::<UInt32Type>();
+        &values.values()[start..end]
+    }
+}
+
+/// Per-level search view over a disk-loaded [`LoadedHnswGraph`].
+pub(crate) struct LoadedHnswLevelView<'a> {
+    level: usize,
+    graph: &'a LoadedHnswGraph,
+}
+
+impl<'a> LoadedHnswLevelView<'a> {
+    fn new(level: u16, graph: &'a LoadedHnswGraph) -> Self {
+        Self {
+            level: level as usize,
+            graph,
+        }
+    }
+}
+
+impl Graph for LoadedHnswLevelView<'_> {
+    fn len(&self) -> usize {
+        // Mirrors `ImmutableHnswLevelView::len` (total node count).
+        self.graph.level_count[0]
+    }
+
+    fn neighbors(&self, key: u32) -> Arc<Vec<u32>> {
+        // Non-hot fallback: HNSW search goes through `BorrowingGraph`. Kept
+        // only so the `Graph` trait / legacy `greedy_search` need no
+        // special-casing for loaded graphs.
+        Arc::new(self.graph.neighbors_at(self.level, key).to_vec())
+    }
+}
+
+impl BorrowingGraph for LoadedHnswLevelView<'_> {
+    fn len(&self) -> usize {
+        self.graph.level_count[0]
+    }
+
+    fn neighbors(&self, key: u32) -> &[u32] {
+        self.graph.neighbors_at(self.level, key)
+    }
+}
+
+/// Bottom-level (level 0) search view over a disk-loaded [`LoadedHnswGraph`].
+pub(crate) struct LoadedHnswBottomView<'a> {
+    graph: &'a LoadedHnswGraph,
+}
+
+impl<'a> LoadedHnswBottomView<'a> {
+    fn new(graph: &'a LoadedHnswGraph) -> Self {
+        Self { graph }
+    }
+}
+
+impl Graph for LoadedHnswBottomView<'_> {
+    fn len(&self) -> usize {
+        self.graph.level_count[0]
+    }
+
+    fn neighbors(&self, key: u32) -> Arc<Vec<u32>> {
+        Arc::new(self.graph.neighbors_at(0, key).to_vec())
+    }
+}
+
+impl BorrowingGraph for LoadedHnswBottomView<'_> {
+    fn len(&self) -> usize {
+        self.graph.level_count[0]
+    }
+
+    fn neighbors(&self, key: u32) -> &[u32] {
+        self.graph.neighbors_at(0, key)
+    }
+}
+
+/// The graph backing an [`HNSW`]: either built in memory or disk-loaded.
+enum HnswGraph {
+    /// Built in memory by the (online) builder / `index_vectors` /
+    /// `from_parts`. Mutable-shaped `GraphBuilderNode`s; `to_batch()`
+    /// re-encodes from these (it needs the per-node ranked distances).
+    Built(Arc<Vec<GraphBuilderNode>>),
+    /// Loaded from disk, Arrow-backed, search-only.
+    Loaded(Arc<LoadedHnswGraph>),
+}
+
+impl DeepSizeOf for HnswGraph {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        match self {
+            Self::Built(nodes) => nodes.deep_size_of_children(context),
+            Self::Loaded(graph) => graph.deep_size_of_children(context),
+        }
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct HnswQueryParams {
     pub ef: usize,
@@ -760,39 +1003,81 @@ impl IvfSubIndex for HNSW {
             ))
         })?;
 
-        let levels: Vec<_> = hnsw_metadata
+        // Slice the concatenated batch into one (zero-copy) view per level.
+        let level_batches: Vec<RecordBatch> = hnsw_metadata
             .level_offsets
             .iter()
             .tuple_windows()
             .map(|(start, end)| data.slice(*start, end - start))
             .collect();
 
-        let level_count = levels.iter().map(|b| b.num_rows()).collect::<Vec<_>>();
+        let level_count = level_batches
+            .iter()
+            .map(|b| b.num_rows())
+            .collect::<Vec<_>>();
 
-        let bottom_level_len = levels[0].num_rows();
-        let mut nodes = Vec::with_capacity(bottom_level_len);
-        for i in 0..bottom_level_len {
-            nodes.push(GraphBuilderNode::new(i as u32, levels.len()));
-        }
-        for (level, batch) in levels.into_iter().enumerate() {
+        // No per-node reconstruction: keep the Arrow adjacency buffers as-is
+        // and only build the tiny per-upper-level id->row lookups. The
+        // `__distance` column is never materialized here -- search doesn't
+        // need it, and `to_batch()` returns the retained `data` verbatim.
+        let mut level_neighbors = Vec::with_capacity(level_batches.len());
+        let mut level_lookup = Vec::with_capacity(level_batches.len());
+        for (level, batch) in level_batches.iter().enumerate() {
+            // `.clone()` on an Arrow array bumps a refcount; buffers stay
+            // shared with `data` (zero copy).
+            let neighbors = batch[NEIGHBORS_COL].as_list::<i32>().clone();
             let ids = batch[VECTOR_ID_COL].as_primitive::<UInt32Type>();
-            let neighbors = batch[NEIGHBORS_COL].as_list::<i32>();
-            let distances = batch[DIST_COL].as_list::<i32>();
-
-            for ((node, neighbors), distances) in
-                ids.iter().zip(neighbors.iter()).zip(distances.iter())
-            {
-                let node = node.unwrap();
-                let neighbors = neighbors.as_ref().unwrap().as_primitive::<UInt32Type>();
-                let distances = distances.as_ref().unwrap().as_primitive::<Float32Type>();
-
-                nodes[node as usize].level_neighbors_ranked[level] = neighbors
+            if level == 0 {
+                // `to_batch` writes every node at level 0 exactly once in
+                // ascending `__vector_id` (== node id) order, so the level-0
+                // slice is exactly `[0, N)` and the row index *is* the node
+                // id. The `Dense` lookup below depends on this: in a release
+                // build a violated invariant would silently make search read
+                // the wrong neighbor list, so enforce it at load time (not via
+                // `debug_assert!`) and reject a malformed or version-
+                // incompatible batch.
+                if let Some((row, id)) = ids
+                    .values()
+                    .iter()
+                    .enumerate()
+                    .find(|&(row, id)| *id != row as u32)
+                {
+                    return Err(Error::index(format!(
+                        "HNSW level-0 __vector_id must equal the row index, but \
+                         row {row} has __vector_id {id}; the on-disk batch is \
+                         malformed or was written by an incompatible version"
+                    )));
+                }
+                level_lookup.push(LevelLookup::Dense);
+            } else {
+                // Upper levels: explicit id -> row map. No ordering/alignment
+                // assumption (see `LevelLookup::Sparse`). On the rare
+                // duplicate id (a misaligned slice can repeat one across a
+                // level boundary) the last wins, matching the old load's
+                // `nodes[id].level_neighbors[level] = ...` last-write.
+                let id_to_row: HashMap<u32, u32> = ids
+                    .values()
                     .iter()
-                    .zip(distances.iter())
-                    .map(|(n, dist)| OrderedNode::new(n.unwrap(), OrderedFloat(dist.unwrap())))
+                    .enumerate()
+                    .map(|(row, id)| (*id, row as u32))
                     .collect();
-                nodes[node as usize].update_from_ranked_neighbors(level as u16);
+                level_lookup.push(LevelLookup::Sparse(id_to_row));
             }
+            level_neighbors.push(neighbors);
+        }
+
+        // `entry_point` is read from untrusted metadata and indexes the `Dense`
+        // level-0 lookup directly; an out-of-range value would read past the
+        // level-0 neighbor buffer during search. Validate it under the same
+        // persisted-format invariant as the level-0 ids above.
+        let num_nodes = level_count[0];
+        if hnsw_metadata.entry_point as usize >= num_nodes {
+            return Err(Error::index(format!(
+                "HNSW entry_point {} is out of range for a graph with {num_nodes} \
+                 nodes; the on-disk batch is malformed or was written by an \
+                 incompatible version",
+                hnsw_metadata.entry_point
+            )));
         }
 
         let visited_generator_queue =
@@ -802,9 +1087,16 @@ impl IvfSubIndex for HNSW {
                 .push(VisitedGenerator::new(0))
                 .unwrap();
         }
+
+        let graph = LoadedHnswGraph {
+            batch: data,
+            level_neighbors,
+            level_lookup,
+            level_count: level_count.clone(),
+        };
         let inner = HnswCore {
             params: hnsw_metadata.params,
-            nodes: Arc::new(nodes),
+            graph: HnswGraph::Loaded(Arc::new(graph)),
             level_count,
             entry_point: hnsw_metadata.entry_point,
             visited_generator_queue,
@@ -931,7 +1223,7 @@ impl IvfSubIndex for HNSW {
 
     fn remap(
         &self,
-        _mapping: &HashMap<u64, Option<u64>>, // we don't need the mapping here because we rebuild the graph from remapped storage
+        _mapping: &RowAddrRemap, // we don't need the mapping here because we rebuild the graph from remapped storage
         store: &impl VectorStore,
     ) -> Result<Self> {
         // We can't simply remap the row ids in the graph because the vectors are changed,
@@ -941,6 +1233,34 @@ impl IvfSubIndex for HNSW {
 
     /// Encode the sub index into a record batch
     fn to_batch(&self) -> Result<RecordBatch> {
+        let nodes = match &self.inner.graph {
+            HnswGraph::Built(nodes) => nodes,
+            HnswGraph::Loaded(graph) => {
+                // A loaded graph is already Arrow-backed: return the retained
+                // batch verbatim, re-stamped with up-to-date HNSW metadata.
+                // The IVF partition cache re-serializes loaded indices through
+                // here (`ivf/partition_serde.rs`), so this must round-trip.
+                //
+                // Merge into (not replace) the existing schema metadata: a
+                // disk-loaded batch inherits other keys from the index file
+                // schema (e.g. `INDEX_METADATA_SCHEMA_KEY`, `IVF_METADATA_KEY`),
+                // and `RecordBatch::with_schema` requires the new metadata to be
+                // a superset of the current one. Dropping those keys here makes
+                // the new schema a non-superset and fails the round-trip with
+                // "target schema is not superset of current schema".
+                let metadata = serde_json::to_string(&self.metadata())?;
+                let mut schema_metadata = graph.batch.schema_ref().metadata().clone();
+                schema_metadata.insert(HNSW_METADATA_KEY.to_string(), metadata);
+                let schema = graph
+                    .batch
+                    .schema()
+                    .as_ref()
+                    .clone()
+                    .with_metadata(schema_metadata);
+                return Ok(graph.batch.clone().with_schema(Arc::new(schema))?);
+            }
+        };
+
         let mut vector_id_builder = UInt32Builder::with_capacity(self.len());
         let mut neighbors_builder = ListBuilder::with_capacity(UInt32Builder::new(), self.len());
         let mut distances_builder =
@@ -948,7 +1268,7 @@ impl IvfSubIndex for HNSW {
         let mut batches = Vec::with_capacity(self.max_level() as usize);
         for level in 0..self.max_level() {
             let level = level as usize;
-            for (id, node) in self.inner.nodes.iter().enumerate() {
+            for (id, node) in nodes.iter().enumerate() {
                 if level >= node.level_neighbors.len() {
                     continue;
                 }
@@ -991,9 +1311,10 @@ impl IvfSubIndex for HNSW {
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{FixedSizeListArray, UInt8Array};
+    use arrow_array::{ArrayRef, FixedSizeListArray, RecordBatch, UInt8Array, UInt32Array};
     use arrow_schema::Schema;
     use lance_arrow::FixedSizeListArrayExt;
+    use lance_core::deepsize::DeepSizeOf;
     use lance_file::previous::{
         reader::FileReader as PreviousFileReader,
         writer::{
@@ -1006,8 +1327,10 @@ mod tests {
     use lance_table::io::manifest::ManifestDescribing;
     use lance_testing::datagen::generate_random_array;
     use object_store::path::Path;
+    use rstest::rstest;
 
-    use crate::scalar::IndexWriter;
+    use super::HnswGraph;
+    use crate::vector::storage::{DistCalculator, VectorStore};
     use crate::vector::v3::subindex::IvfSubIndex;
     use crate::vector::{
         flat::storage::{FlatBinStorage, FlatFloatStorage},
@@ -1051,7 +1374,7 @@ mod tests {
         .unwrap();
         let batch = builder.to_batch().unwrap();
         let metadata = batch.schema_ref().metadata().clone();
-        writer.write_record_batch(batch).await.unwrap();
+        writer.write(&[batch]).await.unwrap();
         writer.finish_with_metadata(&metadata).await.unwrap();
 
         let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None)
@@ -1112,7 +1435,7 @@ mod tests {
         .unwrap();
         let batch = builder.to_batch().unwrap();
         let metadata = batch.schema_ref().metadata().clone();
-        writer.write_record_batch(batch).await.unwrap();
+        writer.write(&[batch]).await.unwrap();
         writer.finish_with_metadata(&metadata).await.unwrap();
 
         let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None)
@@ -1140,4 +1463,393 @@ mod tests {
             .unwrap();
         assert_eq!(builder_results, loaded_results);
     }
+
+    /// Brute-force top-`k` node ids by distance -- recall ground truth.
+    fn brute_force_topk(store: &FlatFloatStorage, query: ArrayRef, k: usize) -> Vec<u32> {
+        let dist_calc = store.dist_calculator(query, 0.0);
+        let mut all: Vec<(f32, u32)> = (0..store.len() as u32)
+            .map(|id| (dist_calc.distance(id), id))
+            .collect();
+        all.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+        all.into_iter().take(k).map(|(_, id)| id).collect()
+    }
+
+    /// The Arrow-backed loaded graph must search bit-identically to the
+    /// in-memory build, across distance types and graph sizes (single node,
+    /// pair, and a multi-level graph exercising the sparse upper-level
+    /// id->row lookup).
+    #[rstest]
+    #[case::l2_single(DistanceType::L2, 1)]
+    #[case::l2_pair(DistanceType::L2, 2)]
+    #[case::l2_multi_level(DistanceType::L2, 2048)]
+    #[case::dot_multi_level(DistanceType::Dot, 2048)]
+    #[tokio::test]
+    async fn test_loaded_search_parity_and_recall(
+        #[case] distance_type: DistanceType,
+        #[case] total: usize,
+    ) {
+        const DIM: usize = 32;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(total * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl.clone(), distance_type));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+        assert!(!matches!(builder.inner.graph, HnswGraph::Loaded(_)));
+
+        let loaded = HNSW::load(builder.to_batch().unwrap()).unwrap();
+        assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+        assert_eq!(loaded.len(), total);
+
+        let k = total.min(10);
+        let params = HnswQueryParams {
+            ef: 50,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+        };
+        let query = fsl.value(0);
+
+        let builder_results = builder
+            .search_basic(query.clone(), k, &params, None, store.as_ref())
+            .unwrap();
+        let loaded_results = loaded
+            .search_basic(query.clone(), k, &params, None, store.as_ref())
+            .unwrap();
+        assert_eq!(builder_results, loaded_results);
+
+        // Recall vs brute-force ground truth (project rule: >= 0.5).
+        let truth: std::collections::HashSet<u32> = brute_force_topk(store.as_ref(), query, k)
+            .into_iter()
+            .collect();
+        let hits = loaded_results
+            .iter()
+            .filter(|n| truth.contains(&n.id))
+            .count();
+        let recall = hits as f32 / k as f32;
+        assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})");
+    }
+
+    /// Regression guard for the `level_offsets` misalignment (issue #6746).
+    /// `to_batch` writes the entry-point node at *every* level, but
+    /// `level_count` only counts it at level 0, so the serialized batch has
+    /// strictly more rows than `sum(level_count)` and the upper-level
+    /// `level_offsets` slices are off-by-one / non-monotonic. The Arrow-backed
+    /// loaded graph must still search bit-identically to the in-memory build:
+    /// it keys upper levels by `__vector_id` value via the `Sparse` map
+    /// (last-write-wins), never `row == id`. A naive `row == id`
+    /// reimplementation would pass the small cases but break here.
+    #[tokio::test]
+    async fn test_loaded_level_offsets_misalignment_invariant() {
+        use arrow::array::AsArray;
+        use arrow::datatypes::UInt32Type;
+
+        const DIM: usize = 32;
+        const TOTAL: usize = 2048;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+
+        // The scenario only exists on a multi-level graph.
+        assert!(
+            builder.max_level() >= 2,
+            "expected a multi-level graph (got max_level {})",
+            builder.max_level()
+        );
+
+        let batch = builder.to_batch().unwrap();
+        let md = builder.metadata();
+        let total_counted = *md.level_offsets.last().unwrap();
+
+        // The exact misalignment: more serialized rows than `level_count` sums
+        // to, because the entry-point node is written at every level yet
+        // counted only at level 0.
+        assert!(
+            batch.num_rows() > total_counted,
+            "expected serialized rows ({}) to exceed sum(level_count) ({}) -- \
+             entry point should be written at every level",
+            batch.num_rows(),
+            total_counted,
+        );
+
+        // Level-0 slice must still be exactly `[0, N)` with
+        // `__vector_id == row` -- the precondition for `LevelLookup::Dense`.
+        let n = md.level_offsets[1];
+        assert_eq!(n, TOTAL);
+        let level0 = batch.slice(0, n);
+        let ids = level0.column(0).as_primitive::<UInt32Type>();
+        assert!(
+            ids.values()
+                .iter()
+                .enumerate()
+                .all(|(row, id)| *id == row as u32),
+            "level-0 __vector_id must equal the row index",
+        );
+
+        // Despite the surplus rows and off-by-one upper slices, the loaded
+        // graph searches bit-identically to the in-memory build (old `load`
+        // semantics preserved via the `Sparse` last-write-wins map).
+        let loaded = HNSW::load(batch).unwrap();
+        assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+        let params = HnswQueryParams {
+            ef: 50,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+        };
+        let query = fsl.value(0);
+        let builder_results = builder
+            .search_basic(query.clone(), 10, &params, None, store.as_ref())
+            .unwrap();
+        let loaded_results = loaded
+            .search_basic(query, 10, &params, None, store.as_ref())
+            .unwrap();
+        assert_eq!(builder_results, loaded_results);
+    }
+
+    /// `load()` must reject a batch whose level-0 `__vector_id` no longer
+    /// matches the row index. The `LevelLookup::Dense` fast path relies on
+    /// `row == id`, and the old `debug_assert!` was compiled out of release
+    /// builds -- so a corrupt batch must fail at the `load()` boundary instead
+    /// of silently searching the wrong neighbor lists.
+    #[tokio::test]
+    async fn test_load_rejects_misaligned_level0_id() {
+        use arrow::array::AsArray;
+        use arrow::datatypes::UInt32Type;
+
+        const DIM: usize = 16;
+        const TOTAL: usize = 256;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+
+        let batch = builder.to_batch().unwrap();
+        // Row 0 is always a level-0 node; break its `__vector_id == row`
+        // invariant while preserving the (metadata-bearing) schema.
+        let mut ids = batch
+            .column(0)
+            .as_primitive::<UInt32Type>()
+            .values()
+            .to_vec();
+        ids[0] = ids.len() as u32;
+        let mut columns = batch.columns().to_vec();
+        columns[0] = Arc::new(UInt32Array::from(ids));
+        let corrupted = RecordBatch::try_new(batch.schema(), columns).unwrap();
+
+        assert!(
+            HNSW::load(corrupted).is_err(),
+            "load() must reject a misaligned level-0 __vector_id"
+        );
+    }
+
+    /// `load()` must reject metadata whose `entry_point` is out of range for
+    /// the node count: it indexes the `Dense` level-0 lookup directly, so an
+    /// out-of-range value would read past the level-0 neighbor buffer at search
+    /// time.
+    #[tokio::test]
+    async fn test_load_rejects_out_of_range_entry_point() {
+        use super::{HNSW_METADATA_KEY, HnswMetadata};
+
+        const DIM: usize = 16;
+        const TOTAL: usize = 256;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+
+        let batch = builder.to_batch().unwrap();
+        let mut metadata = batch.schema_ref().metadata().clone();
+        let mut md: HnswMetadata =
+            serde_json::from_str(metadata.get(HNSW_METADATA_KEY).unwrap()).unwrap();
+        // Valid entry points are `[0, N)`; `level_offsets[1]` == N is one past.
+        let n = md.level_offsets[1];
+        md.entry_point = n as u32;
+        metadata.insert(
+            HNSW_METADATA_KEY.to_string(),
+            serde_json::to_string(&md).unwrap(),
+        );
+        // Rebuild the batch under the rewritten metadata. `with_schema` would
+        // reject this: it requires the new metadata to be a superset, but we
+        // are changing an existing key's value, not adding one.
+        let schema = batch.schema().as_ref().clone().with_metadata(metadata);
+        let corrupted = RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec()).unwrap();
+
+        assert!(
+            HNSW::load(corrupted).is_err(),
+            "load() must reject an out-of-range entry_point"
+        );
+    }
+
+    /// An empty index round-trips: 0-row `to_batch` -> `load` -> empty graph.
+    #[tokio::test]
+    async fn test_loaded_empty_index() {
+        const DIM: usize = 16;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(0), DIM as i32).unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2));
+        let builder = HNSW::index_vectors(store.as_ref(), HnswBuildParams::default()).unwrap();
+        assert!(builder.is_empty());
+
+        let batch = builder.to_batch().unwrap();
+        assert_eq!(batch.num_rows(), 0);
+
+        let loaded = HNSW::load(batch).unwrap();
+        assert!(loaded.is_empty());
+        assert_eq!(loaded.len(), 0);
+        // A 0-row load short-circuits to the empty (Built) graph.
+        assert!(!matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+        assert_eq!(loaded.to_batch().unwrap().num_rows(), 0);
+    }
+
+    /// build -> `to_batch` (b1) -> `load` -> `to_batch` (b2) must satisfy
+    /// `b1 == b2`, and the round-tripped batch must reload and search
+    /// identically. This is exactly the IVF partition-cache path:
+    /// `ivf/partition_serde.rs` calls `to_batch()` on a *loaded* index.
+    #[tokio::test]
+    async fn test_to_batch_roundtrip_loaded() {
+        const DIM: usize = 24;
+        const TOTAL: usize = 1500;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(16).ef_construction(50),
+        )
+        .unwrap();
+
+        let b1 = builder.to_batch().unwrap();
+        let loaded = HNSW::load(b1.clone()).unwrap();
+        assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+        let b2 = loaded.to_batch().unwrap();
+        assert_eq!(b1, b2);
+
+        let reloaded = HNSW::load(b2).unwrap();
+        let params = HnswQueryParams {
+            ef: 50,
+            lower_bound: None,
+            upper_bound: None,
+            dist_q_c: 0.0,
+        };
+        let query = fsl.value(7);
+        let a = builder
+            .search_basic(query.clone(), 10, &params, None, store.as_ref())
+            .unwrap();
+        let b = reloaded
+            .search_basic(query, 10, &params, None, store.as_ref())
+            .unwrap();
+        assert_eq!(a, b);
+    }
+
+    /// Regression for the IVF partition-cache round-trip: a disk-loaded batch
+    /// inherits extra schema metadata keys from the index file (e.g.
+    /// `INDEX_METADATA_SCHEMA_KEY`, `IVF_METADATA_KEY`). `to_batch()` on the
+    /// loaded graph must *merge* the HNSW key into that metadata rather than
+    /// replacing it -- otherwise the new schema is not a superset of the
+    /// current one and `RecordBatch::with_schema` fails with "target schema is
+    /// not superset of current schema".
+    #[tokio::test]
+    async fn test_to_batch_loaded_preserves_extra_schema_metadata() {
+        use super::HNSW_METADATA_KEY;
+
+        const DIM: usize = 24;
+        const TOTAL: usize = 512;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(16).ef_construction(50),
+        )
+        .unwrap();
+
+        // Simulate the disk-load path (`ivf/v2.rs::load_partition_entry`):
+        // the batch reaching `HNSW::load` carries the index file's schema
+        // metadata in addition to the HNSW key.
+        let built_batch = builder.to_batch().unwrap();
+        let mut metadata = built_batch.schema_ref().metadata().clone();
+        metadata.insert(
+            "lance:index_metadata".to_string(),
+            "{\"distance_type\":\"l2\"}".to_string(),
+        );
+        metadata.insert("lance:ivf".to_string(), "42".to_string());
+        let schema = built_batch
+            .schema()
+            .as_ref()
+            .clone()
+            .with_metadata(metadata);
+        let batch_with_extra =
+            RecordBatch::try_new(Arc::new(schema), built_batch.columns().to_vec()).unwrap();
+
+        let loaded = HNSW::load(batch_with_extra).unwrap();
+        assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+
+        // Before the fix this fails: the re-stamped schema dropped the extra
+        // keys, so `with_schema`'s superset check rejected the round-trip.
+        let out = loaded.to_batch().unwrap();
+        let out_metadata = out.schema_ref().metadata();
+        assert!(out_metadata.contains_key(HNSW_METADATA_KEY));
+        assert_eq!(
+            out_metadata.get("lance:index_metadata").map(String::as_str),
+            Some("{\"distance_type\":\"l2\"}"),
+        );
+        assert_eq!(
+            out_metadata.get("lance:ivf").map(String::as_str),
+            Some("42")
+        );
+
+        // The HNSW key must still decode to valid metadata after the merge.
+        let reloaded = HNSW::load(out).unwrap();
+        assert_eq!(reloaded.len(), loaded.len());
+    }
+
+    /// The loaded graph shares the Arrow batch and reconstructs no per-node
+    /// `Vec<GraphBuilderNode>` / `Vec<OrderedNode>`, so it is strictly
+    /// lighter than the in-memory build representation.
+    #[tokio::test]
+    async fn test_loaded_graph_is_arrow_backed() {
+        const DIM: usize = 32;
+        const TOTAL: usize = 2048;
+        let fsl =
+            FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32)
+                .unwrap();
+        let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2));
+        let builder = HNSW::index_vectors(
+            store.as_ref(),
+            HnswBuildParams::default().num_edges(20).ef_construction(50),
+        )
+        .unwrap();
+        assert!(!matches!(builder.inner.graph, HnswGraph::Loaded(_)));
+
+        let loaded = HNSW::load(builder.to_batch().unwrap()).unwrap();
+        assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_)));
+        assert!(
+            loaded.deep_size_of() < builder.deep_size_of(),
+            "loaded graph ({}) should be lighter than built ({})",
+            loaded.deep_size_of(),
+            builder.deep_size_of(),
+        );
+    }
 }
diff --git a/rust/lance-index/src/vector/hnsw/index.rs b/rust/lance-index/src/vector/hnsw/index.rs
index 8d19c2e634a..dca9ea78955 100644
--- a/rust/lance-index/src/vector/hnsw/index.rs
+++ b/rust/lance-index/src/vector/hnsw/index.rs
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
     any::Any,
-    collections::HashMap,
     fmt::{Debug, Formatter},
     sync::Arc,
 };
@@ -12,9 +12,9 @@ use arrow_array::{Float32Array, RecordBatch, UInt32Array};
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use deepsize::DeepSizeOf;
 use lance_arrow::RecordBatchExt;
 use lance_core::ROW_ID;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, datatypes::Schema};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_io::traits::Reader;
@@ -119,11 +119,6 @@ impl<Q: Quantization + Send + Sync + 'static> Index for HNSWIndex<Q> {
         self
     }
 
-    /// Cast to [VectorIndex]
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Ok(self)
-    }
-
     /// Retrieve index statistics as a JSON Value
     fn statistics(&self) -> Result<serde_json::Value> {
         Ok(json!({
@@ -312,7 +307,7 @@ impl<Q: Quantization + Send + Sync + 'static> VectorIndex for HNSWIndex<Q> {
         Box::new(self.storage.as_ref().unwrap().row_ids())
     }
 
-    async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
         Err(Error::index(
             "Remapping HNSW in this way not supported".to_string(),
         ))
diff --git a/rust/lance-index/src/vector/ivf.rs b/rust/lance-index/src/vector/ivf.rs
index 700c8f193d8..93436ee4dfd 100644
--- a/rust/lance-index/src/vector/ivf.rs
+++ b/rust/lance-index/src/vector/ivf.rs
@@ -79,13 +79,9 @@ pub fn new_ivf_transformer_with_quantizer(
             sq,
             range,
         )),
-        Quantizer::Rabit(rq) => Ok(IvfTransformer::with_rq(
-            centroids,
-            metric_type,
-            vector_column,
-            rq,
-            range,
-        )),
+        Quantizer::Rabit(rq) => {
+            IvfTransformer::with_rq(centroids, metric_type, vector_column, rq, range)
+        }
     }
 }
 
@@ -284,7 +280,7 @@ impl IvfTransformer {
         vector_column: &str,
         rq: RabitQuantizer,
         range: Option<Range<u32>>,
-    ) -> Self {
+    ) -> Result<Self> {
         let mut transforms: Vec<Arc<dyn Transformer>> =
             vec![Arc::new(super::transform::Flatten::new(vector_column))];
 
@@ -322,9 +318,9 @@ impl IvfTransformer {
             distance_type,
             centroids.clone(),
             vector_column,
-        )));
+        )?));
 
-        Self::new(centroids, distance_type, transforms)
+        Ok(Self::new(centroids, distance_type, transforms))
     }
 
     #[inline]
diff --git a/rust/lance-index/src/vector/ivf/storage.rs b/rust/lance-index/src/vector/ivf/storage.rs
index ed5e70f5514..5d58401bb12 100644
--- a/rust/lance-index/src/vector/ivf/storage.rs
+++ b/rust/lance-index/src/vector/ivf/storage.rs
@@ -4,9 +4,9 @@
 use std::ops::Range;
 
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array, UInt32Array};
-use deepsize::DeepSizeOf;
 use itertools::Itertools;
 use lance_arrow::FixedSizeListArrayExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use lance_file::previous::{
     reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter,
@@ -41,10 +41,10 @@ pub struct IvfModel {
 }
 
 impl DeepSizeOf for IvfModel {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.centroids
             .as_ref()
-            .map(|centroids| centroids.get_array_memory_size())
+            .map(|centroids| (centroids as &dyn arrow_array::Array).deep_size_of_children(context))
             .unwrap_or_default()
             + self.lengths.deep_size_of_children(context)
             + self.offsets.deep_size_of_children(context)
diff --git a/rust/lance-index/src/vector/kmeans.rs b/rust/lance-index/src/vector/kmeans.rs
index d0e86028b89..07dc067b263 100644
--- a/rust/lance-index/src/vector/kmeans.rs
+++ b/rust/lance-index/src/vector/kmeans.rs
@@ -45,7 +45,7 @@ use {
 };
 
 use crate::vector::utils::SimpleIndex;
-use crate::{Error, Result};
+use lance_core::{Error, Result};
 
 /// KMean initialization method.
 #[derive(Debug, PartialEq)]
@@ -800,14 +800,14 @@ impl KMeans {
         let mut adjusted_balance_factor = f32::MAX;
 
         // TODO: use seed for Rng.
-        let rng = SmallRng::from_os_rng();
+        let mut rng = SmallRng::from_os_rng();
         for redo in 1..=params.redos {
             let mut kmeans: Self = match &params.init {
                 KMeanInit::Random => Self::init_random::<T>(
                     data.values(),
                     dimension,
                     k,
-                    rng.clone(),
+                    &mut rng,
                     params.distance_type,
                 ),
                 KMeanInit::Incremental(centroids) => Self::with_centroids(
@@ -1592,6 +1592,21 @@ mod tests {
         assert_eq!(expected, actual);
     }
 
+    #[test]
+    fn test_random_init_advances_rng() {
+        let values = Float32Array::from_iter_values((0..64).map(|value| value as f32));
+        let mut rng = SmallRng::seed_from_u64(42);
+        let first =
+            KMeans::init_random::<Float32Type>(values.values(), 1, 8, &mut rng, DistanceType::L2);
+        let second =
+            KMeans::init_random::<Float32Type>(values.values(), 1, 8, &mut rng, DistanceType::L2);
+
+        assert_ne!(
+            first.centroids.as_primitive::<Float32Type>().values(),
+            second.centroids.as_primitive::<Float32Type>().values(),
+        );
+    }
+
     #[tokio::test]
     async fn test_compute_membership_and_loss() {
         const DIM: usize = 256;
diff --git a/rust/lance-index/src/vector/pq.rs b/rust/lance-index/src/vector/pq.rs
index 3d11fc4a99e..5749e56ed31 100644
--- a/rust/lance-index/src/vector/pq.rs
+++ b/rust/lance-index/src/vector/pq.rs
@@ -10,9 +10,9 @@ use arrow::datatypes::{self, ArrowPrimitiveType};
 use arrow_array::{Array, FixedSizeListArray, UInt8Array, cast::AsArray};
 use arrow_array::{ArrayRef, Float32Array, PrimitiveArray};
 use arrow_schema::{DataType, Field};
-use deepsize::DeepSizeOf;
 use distance::build_distance_table_dot;
 use lance_arrow::*;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, assume_eq};
 use lance_linalg::distance::{DistanceType, Dot, L2, l2::L2Prepared};
 use lance_table::utils::LanceIteratorExtension;
@@ -49,20 +49,20 @@ pub struct ProductQuantizer {
     /// Pre-transposed L2 targets per sub-vector for fast f32 L2 batch computation.
     /// Only populated when codebook is f32 and distance_type is L2
     /// (Cosine is converted to L2 before construction, so it benefits too).
-    l2_targets: Option<Vec<L2Prepared>>,
+    /// Wrapped in Arc so all clones (one per cached IVF partition) share one copy.
+    l2_targets: Option<Arc<Vec<L2Prepared>>>,
 }
 
 impl DeepSizeOf for ProductQuantizer {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         self.codebook.get_array_memory_size()
             + self.num_sub_vectors.deep_size_of_children(_context)
             + self.num_bits.deep_size_of_children(_context)
             + self.dimension.deep_size_of_children(_context)
             + self.distance_type.deep_size_of_children(_context)
-            + self
-                .l2_targets
-                .as_ref()
-                .map_or(0, |v| v.iter().map(|t| t.size_bytes()).sum())
+            // deep_size_of_children on the Arc de-duplicates shared allocations
+            // via the context, so partitions sharing one l2_targets are counted once.
+            + self.l2_targets.deep_size_of_children(_context)
     }
 }
 
@@ -74,7 +74,7 @@ impl ProductQuantizer {
         num_sub_vectors: usize,
         num_bits: u32,
         dimension: usize,
-    ) -> Option<Vec<L2Prepared>> {
+    ) -> Option<Arc<Vec<L2Prepared>>> {
         if codebook.value_type() != DataType::Float32 || distance_type != DistanceType::L2 {
             return None;
         }
@@ -86,14 +86,14 @@ impl ProductQuantizer {
         let num_centroids = 2_usize.pow(num_bits);
         let block_size = sub_dim * num_centroids;
 
-        let targets = (0..num_sub_vectors)
+        let targets: Vec<L2Prepared> = (0..num_sub_vectors)
             .map(|sub_idx| {
                 let block_start = sub_idx * block_size;
                 let block = &values[block_start..block_start + block_size];
                 L2Prepared::new(block, sub_dim)
             })
             .collect();
-        Some(targets)
+        Some(Arc::new(targets))
     }
 
     pub fn new(
@@ -361,7 +361,7 @@ impl ProductQuantizer {
             DataType::Float32 => {
                 if let Some(targets) = &self.l2_targets {
                     let query = key.as_primitive::<datatypes::Float32Type>().values();
-                    Ok(build_distance_table_l2_prepared(targets, query))
+                    Ok(build_distance_table_l2_prepared(targets.as_slice(), query))
                 } else {
                     Ok(self
                         .build_l2_distance_table_impl::<datatypes::Float32Type>(key.as_primitive()))
diff --git a/rust/lance-index/src/vector/pq/storage.rs b/rust/lance-index/src/vector/pq/storage.rs
index 17443d15ea6..4e98df9906d 100644
--- a/rust/lance-index/src/vector/pq/storage.rs
+++ b/rust/lance-index/src/vector/pq/storage.rs
@@ -5,14 +5,14 @@
 //!
 //! Used as storage backend for Graph based algorithms.
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::{
     cmp::min,
-    collections::HashMap,
     sync::{Arc, OnceLock},
 };
 
 use arrow::datatypes::{self, UInt8Type};
-use arrow_array::{Array, ArrayRef, ArrowPrimitiveType, PrimitiveArray};
+use arrow_array::{ArrayRef, ArrowPrimitiveType, PrimitiveArray};
 use arrow_array::{
     FixedSizeListArray, RecordBatch, UInt8Array, UInt64Array,
     cast::AsArray,
@@ -21,8 +21,8 @@ use arrow_array::{
 use arrow_schema::{DataType, SchemaRef};
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
-use deepsize::DeepSizeOf;
 use lance_arrow::{FixedSizeListArrayExt, RecordBatchExt};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::{
     reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter,
@@ -70,10 +70,10 @@ pub struct ProductQuantizationMetadata {
 }
 
 impl DeepSizeOf for ProductQuantizationMetadata {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.codebook
             .as_ref()
-            .map(|codebook| codebook.get_array_memory_size())
+            .map(|codebook| (codebook as &dyn arrow_array::Array).deep_size_of_children(context))
             .unwrap_or(0)
     }
 }
@@ -166,18 +166,20 @@ pub struct ProductQuantizationStorage {
 }
 
 impl DeepSizeOf for ProductQuantizationStorage {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
-        self.batch.get_array_memory_size()
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.batch.deep_size_of_children(context)
             + self
                 .metadata
                 .codebook
                 .as_ref()
-                .map(|codebook| codebook.get_array_memory_size())
+                .map(|codebook| {
+                    (codebook as &dyn arrow_array::Array).deep_size_of_children(context)
+                })
                 .unwrap_or(0)
             + self
                 .pairwise_distance_table
                 .get()
-                .map(|table| table.deep_size_of_children(_context))
+                .map(|table| table.deep_size_of_children(context))
                 .unwrap_or(0)
     }
 }
@@ -219,7 +221,7 @@ impl ProductQuantizationStorage {
                 "Row ID column not found from PQ storage".to_string(),
             ));
         };
-        let row_ids: Arc<UInt64Array> = row_ids
+        let mut row_ids: Arc<UInt64Array> = row_ids
             .as_primitive_opt::<UInt64Type>()
             .ok_or(Error::index(
                 "Row ID column is not of type UInt64".to_string(),
@@ -291,6 +293,11 @@ impl ProductQuantizationStorage {
                 .as_primitive::<UInt8Type>()
                 .clone()
                 .into();
+            // Refresh the stored row ids from the remapped batch. Without this
+            // the storage keeps the pre-remap (compacted-away) addresses while
+            // its codes are remapped, so search returns stale row ids and the
+            // take fails with "fragment ... does not exist".
+            row_ids = batch[ROW_ID].as_primitive::<UInt64Type>().clone().into();
         }
 
         let distance_type = match distance_type {
@@ -543,16 +550,16 @@ impl QuantizerStorage for ProductQuantizationStorage {
 
     // we can't use the default implementation of remap,
     // because PQ Storage transposed the PQ codes
-    fn remap(&self, mapping: &HashMap<u64, Option<u64>>) -> Result<Self> {
+    fn remap(&self, mapping: &RowAddrRemap) -> Result<Self> {
         let transposed_codes = self.pq_code.values();
         let mut new_row_ids = Vec::with_capacity(self.len());
         let mut new_codes = Vec::with_capacity(self.len() * self.metadata.num_sub_vectors);
 
         let row_ids = self.row_ids.values();
         for (i, row_id) in row_ids.iter().enumerate() {
-            match mapping.get(row_id) {
+            match mapping.get(*row_id) {
                 Some(Some(new_id)) => {
-                    new_row_ids.push(*new_id);
+                    new_row_ids.push(new_id);
                     new_codes.extend(get_pq_code(
                         transposed_codes,
                         self.metadata.nbits,
@@ -1147,6 +1154,7 @@ mod tests {
     use lance_arrow::FixedSizeListArrayExt;
     use lance_core::ROW_ID_FIELD;
     use rand::Rng;
+    use rstest::rstest;
 
     const DIM: usize = 32;
     const TOTAL: usize = 512;
@@ -1293,21 +1301,40 @@ mod tests {
         assert!((storage.dist_between(u, v) - expected).abs() < 1e-4);
     }
 
+    // The first half of the rows is rewritten in order into frag 1; the second
+    // half is deleted. remap must behave the same in either RowAddrRemap mode.
+    fn pq_remap_compact() -> RowAddrRemap {
+        use lance_core::utils::row_addr_remap::GroupInput;
+        use roaring::RoaringTreemap;
+        RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: RoaringTreemap::from_iter((0..TOTAL / 2).map(|i| i as u64)),
+            old_frag_ids: vec![0],
+            new_frags: vec![(1, (TOTAL / 2) as u32)],
+        }])
+        .unwrap()
+    }
+
+    fn pq_remap_explicit() -> RowAddrRemap {
+        RowAddrRemap::direct(
+            (0..TOTAL / 2)
+                .map(|i| (i as u64, Some((1u64 << 32) | i as u64)))
+                .chain((TOTAL / 2..TOTAL).map(|i| (i as u64, None)))
+                .collect(),
+        )
+    }
+
+    #[rstest]
+    #[case(pq_remap_compact())]
+    #[case(pq_remap_explicit())]
     #[tokio::test]
-    async fn test_remap_with_extra_column() {
+    async fn test_remap_with_extra_column(#[case] remap: RowAddrRemap) {
         let storage = create_pq_storage_with_extra_column().await;
-        let mut mapping = HashMap::new();
-        for i in 0..TOTAL / 2 {
-            mapping.insert(i as u64, Some((TOTAL + i) as u64));
-        }
-        for i in TOTAL / 2..TOTAL {
-            mapping.insert(i as u64, None);
-        }
-        let new_storage = storage.remap(&mapping).unwrap();
+        let new_storage = storage.remap(&remap).unwrap();
         assert_eq!(new_storage.len(), TOTAL / 2);
         assert_eq!(new_storage.row_ids.len(), TOTAL / 2);
         for (i, row_id) in new_storage.row_ids().enumerate() {
-            assert_eq!(*row_id, (TOTAL + i) as u64);
+            // Rewritten row i lands at offset i of frag 1.
+            assert_eq!(*row_id, (1u64 << 32) | i as u64);
         }
         assert_eq!(new_storage.batch.num_columns(), 2);
         assert!(new_storage.batch.column_by_name(ROW_ID).is_some());
diff --git a/rust/lance-index/src/vector/quantizer.rs b/rust/lance-index/src/vector/quantizer.rs
index 8e2dd333606..6f4b191098b 100644
--- a/rust/lance-index/src/vector/quantizer.rs
+++ b/rust/lance-index/src/vector/quantizer.rs
@@ -2,17 +2,18 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use core::fmt;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::fmt::Debug;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::{collections::HashMap, fmt::Debug};
 
 use arrow::{array::AsArray, compute::concat_batches, datatypes::UInt64Type};
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, UInt32Array, UInt64Array};
 use arrow_schema::Field;
 use async_trait::async_trait;
 use bytes::Bytes;
-use deepsize::DeepSizeOf;
 use lance_arrow::RecordBatchExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_io::traits::Reader;
@@ -240,7 +241,7 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore {
 
     fn metadata(&self) -> &Self::Metadata;
 
-    fn remap(&self, mapping: &HashMap<u64, Option<u64>>) -> Result<Self> {
+    fn remap(&self, mapping: &RowAddrRemap) -> Result<Self> {
         let batches = self
             .to_batches()?
             .map(|b| {
@@ -249,10 +250,10 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore {
 
                 let row_ids = b.column(0).as_primitive::<UInt64Type>().values();
                 for (i, row_id) in row_ids.iter().enumerate() {
-                    match mapping.get(row_id) {
+                    match mapping.get(*row_id) {
                         Some(Some(new_id)) => {
                             indices.push(i as u32);
-                            new_row_ids.push(*new_id);
+                            new_row_ids.push(new_id);
                         }
                         Some(None) => {}
                         None => {
@@ -296,7 +297,7 @@ pub struct IvfQuantizationStorage<Q: Quantization> {
 }
 
 impl<Q: Quantization> DeepSizeOf for IvfQuantizationStorage<Q> {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.reader.deep_size_of_children(context)
             + self.quantizer.deep_size_of_children(context)
             + self.metadata.deep_size_of_children(context)
diff --git a/rust/lance-index/src/vector/sq.rs b/rust/lance-index/src/vector/sq.rs
index 9ddfed8e9af..bdc3204021e 100644
--- a/rust/lance-index/src/vector/sq.rs
+++ b/rust/lance-index/src/vector/sq.rs
@@ -9,9 +9,9 @@ use arrow_array::{Array, ArrayRef, FixedSizeListArray, UInt8Array};
 
 use arrow_schema::{DataType, Field};
 use builder::SQBuildParams;
-use deepsize::DeepSizeOf;
 use itertools::Itertools;
 use lance_arrow::*;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use lance_linalg::distance::DistanceType;
 use num_traits::*;
@@ -34,7 +34,7 @@ pub struct ScalarQuantizer {
 }
 
 impl DeepSizeOf for ScalarQuantizer {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         0
     }
 }
diff --git a/rust/lance-index/src/vector/sq/storage.rs b/rust/lance-index/src/vector/sq/storage.rs
index 58165e31caf..785aadc289d 100644
--- a/rust/lance-index/src/vector/sq/storage.rs
+++ b/rust/lance-index/src/vector/sq/storage.rs
@@ -12,12 +12,14 @@ use arrow_array::{
 };
 use arrow_schema::{DataType, SchemaRef};
 use async_trait::async_trait;
-use deepsize::DeepSizeOf;
+use lance_arrow::ArrowFloatType;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_io::object_store::ObjectStore;
-use lance_linalg::distance::{DistanceType, dot_distance, l2_u8::l2_u8};
+use lance_linalg::distance::{DistanceType, dot_u8::dot_u8, l2_u8::l2_u8};
 use lance_table::format::SelfDescribingFileReader;
+use num_traits::AsPrimitive;
 use object_store::path::Path;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
@@ -29,7 +31,7 @@ use crate::{
     vector::{
         SQ_CODE_COLUMN,
         quantizer::{QuantizerMetadata, QuantizerStorage},
-        storage::{DistCalculator, VectorStore},
+        storage::{DistCalculator, DistanceCalculatorOptions, QueryResidual, VectorStore},
         transform::Transformer,
     },
 };
@@ -44,7 +46,7 @@ pub struct ScalarQuantizationMetadata {
 }
 
 impl DeepSizeOf for ScalarQuantizationMetadata {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         0
     }
 }
@@ -138,8 +140,8 @@ impl SQStorageChunk {
 }
 
 impl DeepSizeOf for SQStorageChunk {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
-        self.batch.get_array_memory_size()
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.batch.deep_size_of_children(context)
     }
 }
 
@@ -155,7 +157,7 @@ pub struct ScalarQuantizationStorage {
 }
 
 impl DeepSizeOf for ScalarQuantizationStorage {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.chunks
             .iter()
             .map(|c| c.deep_size_of_children(context))
@@ -370,30 +372,118 @@ impl VectorStore for ScalarQuantizationStorage {
         SQDistCalculator::new(query, self, self.quantizer.bounds())
     }
 
+    fn dist_calculator_with_scratch<'a>(
+        &'a self,
+        query: ArrayRef,
+        _dist_q_c: f32,
+        _residual: Option<QueryResidual<'a>>,
+        f32_scratch: &'a mut Vec<f32>,
+        _options: DistanceCalculatorOptions,
+    ) -> Self::DistanceCalculator<'a> {
+        SQDistCalculator::new_with_scratch(query, self, self.quantizer.bounds(), f32_scratch)
+    }
+
     fn dist_calculator_from_id(&self, id: u32) -> Self::DistanceCalculator<'_> {
         let (offset, chunk) = self.chunk(id);
         let query_sq_code = chunk.sq_code_slice(id - offset);
         let bounds = self.quantizer.bounds();
+        let lower_bound = bounds.start as f32;
+        let value_scale = sq_value_scale(&bounds);
+        let query_dot = match self.distance_type {
+            DistanceType::Dot => Some(SQDotQuery::from_sq_code(query_sq_code)),
+            _ => None,
+        };
         SQDistCalculator {
             query_sq_code: SQQueryCode::Borrowed(query_sq_code),
+            query_dot,
             scale: sq_distance_scale(&bounds),
+            lower_bound,
+            value_scale,
             storage: self,
         }
     }
 }
 
+#[inline]
+fn sq_value_scale(bounds: &Range<f64>) -> f32 {
+    (bounds.end - bounds.start) as f32 / 255.0_f32
+}
+
 #[inline]
 fn sq_distance_scale(bounds: &Range<f64>) -> f32 {
-    let range = (bounds.end - bounds.start) as f32;
-    (range * range) / (255.0_f32 * 255.0_f32)
+    let scale = sq_value_scale(bounds);
+    scale * scale
 }
 
 pub struct SQDistCalculator<'a> {
     query_sq_code: SQQueryCode<'a>,
+    query_dot: Option<SQDotQuery<'a>>,
     scale: f32,
+    lower_bound: f32,
+    value_scale: f32,
     storage: &'a ScalarQuantizationStorage,
 }
 
+enum SQDotQuery<'a> {
+    Values { values: SQFloatQuery<'a>, sum: f32 },
+    SqCode { code: &'a [u8], sum: f32 },
+}
+
+enum SQFloatQuery<'a> {
+    Borrowed(&'a [f32]),
+    Owned(Vec<f32>),
+}
+
+impl SQFloatQuery<'_> {
+    fn as_slice(&self) -> &[f32] {
+        match self {
+            Self::Borrowed(values) => values,
+            Self::Owned(values) => values,
+        }
+    }
+}
+
+impl<'a> SQDotQuery<'a> {
+    fn from_values<T: ArrowFloatType>(values: &[T::Native]) -> Self
+    where
+        T::Native: AsPrimitive<f32>,
+    {
+        let values: Vec<_> = values.iter().map(|v| v.as_()).collect();
+        let sum = values.iter().sum();
+        Self::Values {
+            values: SQFloatQuery::Owned(values),
+            sum,
+        }
+    }
+
+    fn from_values_with_scratch<T: ArrowFloatType>(
+        values: &[T::Native],
+        scratch: &'a mut Vec<f32>,
+    ) -> Self
+    where
+        T::Native: AsPrimitive<f32>,
+    {
+        scratch.clear();
+        scratch.extend(values.iter().map(|v| v.as_()));
+        let sum = scratch.iter().sum();
+        Self::Values {
+            values: SQFloatQuery::Borrowed(scratch.as_slice()),
+            sum,
+        }
+    }
+
+    fn from_sq_code(sq_code: &'a [u8]) -> Self {
+        Self::SqCode {
+            code: sq_code,
+            sum: sq_code_sum(sq_code),
+        }
+    }
+}
+
+fn sq_code_sum(sq_code: &[u8]) -> f32 {
+    sq_code.iter().map(|code| *code as u32).sum::<u32>() as f32
+}
+
 enum SQQueryCode<'a> {
     Borrowed(&'a [u8]),
     Owned(Vec<u8>),
@@ -415,26 +505,127 @@ impl<'a> SQDistCalculator<'a> {
         // since we search 10s-100s of partitions, we can afford the overhead
         // this could be annoying at indexing time for HNSW, which requires constructing the
         // dist calculator frequently. However, HNSW isn't first-class citizen in Lance yet. so be it.
-        let query_sq_code = match query.data_type() {
-            DataType::Float16 => {
-                scale_to_u8::<Float16Type>(query.as_primitive::<Float16Type>().values(), &bounds)
-            }
-            DataType::Float32 => {
-                scale_to_u8::<Float32Type>(query.as_primitive::<Float32Type>().values(), &bounds)
+        let (query_sq_code, query_dot) = match storage.distance_type {
+            DistanceType::Dot => {
+                let query_dot = match query.data_type() {
+                    DataType::Float16 => SQDotQuery::from_values::<Float16Type>(
+                        query.as_primitive::<Float16Type>().values(),
+                    ),
+                    DataType::Float32 => SQDotQuery::from_values::<Float32Type>(
+                        query.as_primitive::<Float32Type>().values(),
+                    ),
+                    DataType::Float64 => SQDotQuery::from_values::<Float64Type>(
+                        query.as_primitive::<Float64Type>().values(),
+                    ),
+                    _ => {
+                        panic!("Unsupported data type for ScalarQuantizationStorage");
+                    }
+                };
+                (SQQueryCode::Owned(Vec::new()), Some(query_dot))
             }
-            DataType::Float64 => {
-                scale_to_u8::<Float64Type>(query.as_primitive::<Float64Type>().values(), &bounds)
+            DistanceType::L2 | DistanceType::Cosine => {
+                let query_sq_code = match query.data_type() {
+                    DataType::Float16 => scale_to_u8::<Float16Type>(
+                        query.as_primitive::<Float16Type>().values(),
+                        &bounds,
+                    ),
+                    DataType::Float32 => scale_to_u8::<Float32Type>(
+                        query.as_primitive::<Float32Type>().values(),
+                        &bounds,
+                    ),
+                    DataType::Float64 => scale_to_u8::<Float64Type>(
+                        query.as_primitive::<Float64Type>().values(),
+                        &bounds,
+                    ),
+                    _ => {
+                        panic!("Unsupported data type for ScalarQuantizationStorage");
+                    }
+                };
+                (SQQueryCode::Owned(query_sq_code), None)
             }
+            _ => panic!("We should not reach here: sq distance can only be L2 or Dot"),
+        };
+        let lower_bound = bounds.start as f32;
+        let value_scale = sq_value_scale(&bounds);
+        Self {
+            query_sq_code,
+            query_dot,
+            scale: sq_distance_scale(&bounds),
+            lower_bound,
+            value_scale,
+            storage,
+        }
+    }
+
+    fn new_with_scratch(
+        query: ArrayRef,
+        storage: &'a ScalarQuantizationStorage,
+        bounds: Range<f64>,
+        f32_scratch: &'a mut Vec<f32>,
+    ) -> Self {
+        if storage.distance_type != DistanceType::Dot {
+            return Self::new(query, storage, bounds);
+        }
+
+        let query_dot = match query.data_type() {
+            DataType::Float16 => SQDotQuery::from_values_with_scratch::<Float16Type>(
+                query.as_primitive::<Float16Type>().values(),
+                f32_scratch,
+            ),
+            DataType::Float32 => SQDotQuery::from_values_with_scratch::<Float32Type>(
+                query.as_primitive::<Float32Type>().values(),
+                f32_scratch,
+            ),
+            DataType::Float64 => SQDotQuery::from_values_with_scratch::<Float64Type>(
+                query.as_primitive::<Float64Type>().values(),
+                f32_scratch,
+            ),
             _ => {
                 panic!("Unsupported data type for ScalarQuantizationStorage");
             }
         };
+        let lower_bound = bounds.start as f32;
+        let value_scale = sq_value_scale(&bounds);
         Self {
-            query_sq_code: SQQueryCode::Owned(query_sq_code),
+            query_sq_code: SQQueryCode::Owned(Vec::new()),
+            query_dot: Some(query_dot),
             scale: sq_distance_scale(&bounds),
+            lower_bound,
+            value_scale,
             storage,
         }
     }
+
+    fn dot_distance(&self, sq_code: &[u8]) -> f32 {
+        let query = self
+            .query_dot
+            .as_ref()
+            .expect("SQ dot distance requires a dot query");
+        let dot = match query {
+            SQDotQuery::Values { values, sum } => {
+                let values = values.as_slice();
+                self.lower_bound * *sum
+                    + self.value_scale
+                        * sq_code
+                            .iter()
+                            .zip(values.iter())
+                            .map(|(code, query_value)| *code as f32 * *query_value)
+                            .sum::<f32>()
+            }
+            SQDotQuery::SqCode {
+                code: query_sq_code,
+                sum: query_code_sum,
+            } => {
+                let dim = sq_code.len() as f32;
+                let code_dot = dot_u8(sq_code, query_sq_code) as f32;
+                let code_sum = sq_code_sum(sq_code);
+                dim * self.lower_bound * self.lower_bound
+                    + self.lower_bound * self.value_scale * (code_sum + *query_code_sum)
+                    + self.scale * code_dot
+            }
+        };
+        1.0 - dot
+    }
 }
 
 impl DistCalculator for SQDistCalculator<'_> {
@@ -442,12 +633,13 @@ impl DistCalculator for SQDistCalculator<'_> {
         let (offset, chunk) = self.storage.chunk(id);
         let sq_code = chunk.sq_code_slice(id - offset);
         let query_sq_code = self.query_sq_code.as_slice();
-        let dist = match self.storage.distance_type {
-            DistanceType::L2 | DistanceType::Cosine => l2_u8(sq_code, query_sq_code) as f32,
-            DistanceType::Dot => dot_distance(sq_code, query_sq_code),
+        match self.storage.distance_type {
+            DistanceType::L2 | DistanceType::Cosine => {
+                l2_u8(sq_code, query_sq_code) as f32 * self.scale
+            }
+            DistanceType::Dot => self.dot_distance(sq_code),
             _ => panic!("We should not reach here: sq distance can only be L2 or Dot"),
-        };
-        dist * self.scale
+        }
     }
 
     fn distance_all(&self, _k_hint: usize) -> Vec<f32> {
@@ -473,9 +665,8 @@ impl DistCalculator for SQDistCalculator<'_> {
                     c.sq_codes
                         .values()
                         .chunks_exact(c.dim())
-                        .map(|sq_codes| dot_distance(sq_codes, query_sq_code))
+                        .map(|sq_codes| self.dot_distance(sq_codes))
                 })
-                .map(|dist| dist * self.scale)
                 .collect(),
             _ => panic!("We should not reach here: sq distance can only be L2 or Dot"),
         }
@@ -511,7 +702,7 @@ mod tests {
     use std::iter::repeat_with;
     use std::sync::Arc;
 
-    use arrow_array::FixedSizeListArray;
+    use arrow_array::{FixedSizeListArray, Float32Array};
     use arrow_schema::{DataType, Field, Schema};
     use lance_arrow::FixedSizeListArrayExt;
     use lance_testing::datagen::generate_random_array;
@@ -541,6 +732,31 @@ mod tests {
         RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(code_arr)]).unwrap()
     }
 
+    fn create_record_batch_with_sq_codes(
+        row_ids: Vec<u64>,
+        sq_codes: Vec<u8>,
+        dim: usize,
+    ) -> RecordBatch {
+        assert_eq!(sq_codes.len(), row_ids.len() * dim);
+
+        let row_ids = UInt64Array::from_iter_values(row_ids);
+        let sq_code = UInt8Array::from_iter_values(sq_codes);
+        let code_arr = FixedSizeListArray::try_new_from_values(sq_code, dim as i32).unwrap();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new(ROW_ID, DataType::UInt64, false),
+            Field::new(
+                SQ_CODE_COLUMN,
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::UInt8, true)),
+                    dim as i32,
+                ),
+                false,
+            ),
+        ]));
+        RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(code_arr)]).unwrap()
+    }
+
     #[test]
     fn test_get_chunks() {
         const DIM: usize = 64;
@@ -592,4 +808,53 @@ mod tests {
         assert_eq!(offset, 400);
         assert_eq!(chunk.row_id(5), 105);
     }
+
+    #[test]
+    fn test_dot_distance_accounts_for_sq_offset() {
+        const DIM: usize = 2;
+
+        let storage = ScalarQuantizationStorage::try_new(
+            8,
+            DistanceType::Dot,
+            -1.0..1.0,
+            [create_record_batch_with_sq_codes(
+                vec![0, 1],
+                vec![
+                    255, 255, // [1.0, 1.0]
+                    0, 191, // [-1.0, 0.498]
+                ],
+                DIM,
+            )],
+            None,
+        )
+        .unwrap();
+
+        let query = Arc::new(Float32Array::from(vec![-1.0, 1.0])) as ArrayRef;
+        let calculator = storage.dist_calculator(query.clone(), 0.0);
+        let distances = calculator.distance_all(2);
+
+        assert!(
+            distances[1] < distances[0],
+            "expected [-1.0, 0.498] to rank before [1.0, 1.0], got {distances:?}"
+        );
+        assert!((calculator.distance(0) - 1.0).abs() < 1e-6);
+        assert!((calculator.distance(1) - -0.49803925).abs() < 1e-6);
+
+        let mut scratch = Vec::new();
+        let scratch_distances = {
+            let scratch_calculator = storage.dist_calculator_with_scratch(
+                query,
+                0.0,
+                None,
+                &mut scratch,
+                DistanceCalculatorOptions::default(),
+            );
+            scratch_calculator.distance_all(2)
+        };
+        assert_eq!(scratch_distances, distances);
+        assert_eq!(scratch.len(), DIM);
+
+        let stored_query_calculator = storage.dist_calculator_from_id(0);
+        assert!((stored_query_calculator.distance(1) - 1.5019608).abs() < 1e-6);
+    }
 }
diff --git a/rust/lance-index/src/vector/storage.rs b/rust/lance-index/src/vector/storage.rs
index 1443a1f355d..a14308197ed 100644
--- a/rust/lance-index/src/vector/storage.rs
+++ b/rust/lance-index/src/vector/storage.rs
@@ -7,17 +7,20 @@ use crate::vector::quantizer::QuantizerStorage;
 use arrow::compute::concat_batches;
 use arrow_array::{ArrayRef, RecordBatch};
 use arrow_schema::SchemaRef;
-use deepsize::DeepSizeOf;
 use futures::prelude::stream::TryStreamExt;
 use lance_arrow::RecordBatchExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, ROW_ID, Result};
 use lance_encoding::decoder::FilterExpression;
 use lance_file::reader::FileReader;
 use lance_io::ReadBatchParams;
+use lance_io::scheduler::IoStats;
 use lance_linalg::distance::DistanceType;
 use prost::Message;
 use std::{
     any::Any,
+    borrow::Cow,
+    collections::BinaryHeap,
     mem::size_of,
     ops::{Deref, DerefMut},
     sync::Arc,
@@ -34,10 +37,10 @@ use crate::{
     },
 };
 
-use super::DISTANCE_TYPE_KEY;
 use super::graph::OrderedFloat;
 use super::graph::OrderedNode;
 use super::quantizer::{Quantizer, QuantizerMetadata};
+use super::{ApproxMode, DISTANCE_TYPE_KEY};
 
 /// <section class="warning">
 ///  Internal API
@@ -58,11 +61,95 @@ pub trait DistCalculator {
         dists: &mut Vec<f32>,
         _u16_scratch: &mut Vec<u16>,
         _u8_scratch: &mut Vec<u8>,
+        _u32_scratch: &mut Vec<u32>,
     ) {
         *dists = self.distance_all(k_hint);
     }
 
     fn prefetch(&self, _id: u32) {}
+
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_topk_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_id: impl Fn(u32) -> u64,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        dists: &mut Vec<f32>,
+        u16_scratch: &mut Vec<u16>,
+        u8_scratch: &mut Vec<u8>,
+        u32_scratch: &mut Vec<u32>,
+    ) {
+        if k == 0 {
+            return;
+        }
+
+        self.distance_all_with_scratch(k, dists, u16_scratch, u8_scratch, u32_scratch);
+        let lower_bound = lower_bound.unwrap_or(f32::MIN).into();
+        let upper_bound = upper_bound.unwrap_or(f32::MAX).into();
+        let mut max_dist = res.peek().map(|node| node.dist);
+
+        for (id, dist) in dists.iter().copied().enumerate() {
+            let dist = OrderedFloat(dist);
+            if dist < lower_bound || dist >= upper_bound {
+                continue;
+            }
+            if res.len() < k {
+                res.push(OrderedNode::new(row_id(id as u32), dist));
+                if res.len() == k {
+                    max_dist = res.peek().map(|node| node.dist);
+                }
+            } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
+                res.pop();
+                res.push(OrderedNode::new(row_id(id as u32), dist));
+                max_dist = res.peek().map(|node| node.dist);
+            }
+        }
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn accumulate_filtered_topk_with_scratch(
+        &self,
+        k: usize,
+        lower_bound: Option<f32>,
+        upper_bound: Option<f32>,
+        row_ids: impl Iterator<Item = (u32, u64)>,
+        accept_row: impl Fn(u64) -> bool,
+        res: &mut BinaryHeap<OrderedNode<u64>>,
+        _dists: &mut Vec<f32>,
+        _u16_scratch: &mut Vec<u16>,
+        _u8_scratch: &mut Vec<u8>,
+        _u32_scratch: &mut Vec<u32>,
+    ) {
+        if k == 0 {
+            return;
+        }
+
+        let lower_bound = lower_bound.unwrap_or(f32::MIN).into();
+        let upper_bound = upper_bound.unwrap_or(f32::MAX).into();
+        let mut max_dist = res.peek().map(|node| node.dist);
+
+        for (id, row_id) in row_ids {
+            if !accept_row(row_id) {
+                continue;
+            }
+            let dist = OrderedFloat(self.distance(id));
+            if dist < lower_bound || dist >= upper_bound {
+                continue;
+            }
+            if res.len() < k {
+                res.push(OrderedNode::new(row_id, dist));
+                if res.len() == k {
+                    max_dist = res.peek().map(|node| node.dist);
+                }
+            } else if max_dist.is_some_and(|max_dist| max_dist > dist) {
+                res.pop();
+                res.push(OrderedNode::new(row_id, dist));
+                max_dist = res.peek().map(|node| node.dist);
+            }
+        }
+    }
 }
 
 pub const STORAGE_METADATA_KEY: &str = "storage_metadata";
@@ -73,6 +160,7 @@ pub struct QueryScratch {
     pub query_f32: Vec<f32>,
     pub u16: Vec<u16>,
     pub u8: Vec<u8>,
+    pub u32: Vec<u32>,
 }
 
 impl QueryScratch {
@@ -82,6 +170,7 @@ impl QueryScratch {
             query_f32: Vec::new(),
             u16: Vec::new(),
             u8: Vec::new(),
+            u32: Vec::new(),
         }
     }
 
@@ -91,6 +180,7 @@ impl QueryScratch {
             query_f32: vec![0.0; capacity.query_f32],
             u16: vec![0; capacity.u16],
             u8: vec![0; capacity.u8],
+            u32: vec![0; capacity.u32],
         }
     }
 }
@@ -102,11 +192,12 @@ impl Default for QueryScratch {
 }
 
 impl DeepSizeOf for QueryScratch {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         self.distances.capacity() * size_of::<f32>()
             + self.query_f32.capacity() * size_of::<f32>()
             + self.u16.capacity() * size_of::<u16>()
             + self.u8.capacity() * size_of::<u8>()
+            + self.u32.capacity() * size_of::<u32>()
     }
 }
 
@@ -116,15 +207,27 @@ pub struct QueryScratchCapacity {
     pub query_f32: usize,
     pub u16: usize,
     pub u8: usize,
+    pub u32: usize,
 }
 
 impl QueryScratchCapacity {
     pub const fn new(distances: usize, query_f32: usize, u16: usize, u8: usize) -> Self {
+        Self::new_with_u32(distances, query_f32, u16, u8, 0)
+    }
+
+    pub const fn new_with_u32(
+        distances: usize,
+        query_f32: usize,
+        u16: usize,
+        u8: usize,
+        u32: usize,
+    ) -> Self {
         Self {
             distances,
             query_f32,
             u16,
             u8,
+            u32,
         }
     }
 
@@ -133,12 +236,35 @@ impl QueryScratchCapacity {
             + self.query_f32 * size_of::<f32>()
             + self.u16 * size_of::<u16>()
             + self.u8 * size_of::<u8>()
+            + self.u32 * size_of::<u32>()
     }
 }
 
+#[derive(Clone, Copy, Debug, Default)]
+pub struct DistanceCalculatorOptions {
+    pub approx_mode: ApproxMode,
+}
+
+#[derive(Debug)]
+pub struct RabitRawQueryContext {
+    pub code_dim: usize,
+    pub ex_bits: u8,
+    pub rotated_query: Vec<f32>,
+    pub dist_table: Vec<f32>,
+    /// The rotated query zero-padded to a 64-dim multiple for the ex-dot
+    /// kernels; empty when `code_dim` is already aligned (the kernels then
+    /// read `rotated_query` directly).
+    pub ex_query: Vec<f32>,
+    pub sum_q: f32,
+}
+
 #[derive(Clone, Copy)]
 pub enum QueryResidual<'a> {
     Centroid(&'a dyn arrow_array::Array),
+    RabitRawQuery {
+        rotated_centroid: Option<&'a [f32]>,
+        query: Option<&'a RabitRawQueryContext>,
+    },
 }
 
 #[derive(Debug)]
@@ -224,7 +350,7 @@ impl Drop for QueryScratchGuard<'_> {
 }
 
 impl DeepSizeOf for QueryScratchPool {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let mut total = self.scratches.capacity() * size_of::<QueryScratch>();
         let mut scratches = Vec::new();
         while let Some(scratch) = self.scratches.pop() {
@@ -295,8 +421,9 @@ pub trait VectorStore: Send + Sync + Sized + Clone {
         &'a self,
         query: ArrayRef,
         dist_q_c: f32,
-        _residual: Option<QueryResidual<'_>>,
+        _residual: Option<QueryResidual<'a>>,
         _f32_scratch: &'a mut Vec<f32>,
+        _options: DistanceCalculatorOptions,
     ) -> Self::DistanceCalculator<'a> {
         self.dist_calculator(query, dist_q_c)
     }
@@ -381,7 +508,7 @@ pub struct IvfQuantizationStorage<Q: Quantization> {
 }
 
 impl<Q: Quantization> DeepSizeOf for IvfQuantizationStorage<Q> {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.metadata.deep_size_of_children(context) + self.ivf.deep_size_of_children(context)
     }
 }
@@ -498,15 +625,29 @@ impl<Q: Quantization> IvfQuantizationStorage<Q> {
         self.ivf.num_partitions()
     }
 
-    pub async fn load_partition(&self, part_id: usize) -> Result<Q::Storage> {
+    /// Load a partition's quantization storage, optionally measuring the exact
+    /// I/O it performs into `io_stats`.
+    ///
+    /// When `io_stats` is `Some`, the partition is read through a reader whose
+    /// scheduler also records into the sink (a cheap clone that shares all
+    /// cached metadata, so no file is re-opened).  When `None`, the normal
+    /// uninstrumented reader is used.
+    pub async fn load_partition(
+        &self,
+        part_id: usize,
+        io_stats: Option<IoStats>,
+    ) -> Result<Q::Storage> {
         let range = self.ivf.row_range(part_id);
         let batch = if range.is_empty() {
             let schema = self.reader.schema();
             let arrow_schema = arrow_schema::Schema::from(schema.as_ref());
             RecordBatch::new_empty(Arc::new(arrow_schema))
         } else {
-            let batches = self
-                .reader
+            let reader = match &io_stats {
+                Some(io_stats) => Cow::Owned(self.reader.with_io_stats(io_stats.recorder())),
+                None => Cow::Borrowed(&self.reader),
+            };
+            let batches = reader
                 .read_stream(
                     ReadBatchParams::Range(range),
                     u32::MAX,
@@ -531,7 +672,7 @@ impl<Q: Quantization> IvfQuantizationStorage<Q> {
 #[cfg(test)]
 mod tests {
     use super::{QueryScratchCapacity, QueryScratchPool};
-    use deepsize::DeepSizeOf;
+    use lance_core::deepsize::DeepSizeOf;
 
     #[test]
     fn test_query_scratch_pool_reuses_buffers() {
@@ -545,11 +686,14 @@ mod tests {
             scratch.u16.resize(4, 3);
             scratch.u8.clear();
             scratch.u8.resize(2, 4);
+            scratch.u32.clear();
+            scratch.u32.resize(3, 5);
             (
                 scratch.query_f32.as_ptr(),
                 scratch.distances.as_ptr(),
                 scratch.u16.as_ptr(),
                 scratch.u8.as_ptr(),
+                scratch.u32.as_ptr(),
             )
         });
 
@@ -562,11 +706,14 @@ mod tests {
             assert!(scratch.u16.iter().all(|value| *value == 3));
             assert_eq!(scratch.u8.len(), 2);
             assert!(scratch.u8.iter().all(|value| *value == 4));
+            assert_eq!(scratch.u32.len(), 3);
+            assert!(scratch.u32.iter().all(|value| *value == 5));
             (
                 scratch.query_f32.as_ptr(),
                 scratch.distances.as_ptr(),
                 scratch.u16.as_ptr(),
                 scratch.u8.as_ptr(),
+                scratch.u32.as_ptr(),
             )
         });
 
@@ -592,7 +739,8 @@ mod tests {
 
     #[test]
     fn test_query_scratch_pool_uses_temporary_scratch_when_empty() {
-        let pool = QueryScratchPool::with_capacity(1, QueryScratchCapacity::new(8, 16, 4, 2));
+        let pool =
+            QueryScratchPool::with_capacity(1, QueryScratchCapacity::new_with_u32(8, 16, 4, 2, 3));
         let pooled = pool.scratch();
         assert!(pooled.pooled);
 
@@ -602,12 +750,14 @@ mod tests {
         assert_eq!(temporary.query_f32.len(), 16);
         assert_eq!(temporary.u16.len(), 4);
         assert_eq!(temporary.u8.len(), 2);
+        assert_eq!(temporary.u32.len(), 3);
     }
 
     #[test]
     fn test_query_scratch_pool_deep_size_includes_buffer_capacity() {
         let empty_size = QueryScratchPool::new(1).deep_size_of();
-        let pool = QueryScratchPool::with_capacity(1, QueryScratchCapacity::new(8, 16, 4, 2));
+        let pool =
+            QueryScratchPool::with_capacity(1, QueryScratchCapacity::new_with_u32(8, 16, 4, 2, 3));
 
         assert!(pool.deep_size_of() > empty_size);
 
@@ -619,7 +769,8 @@ mod tests {
 
     #[test]
     fn test_query_scratch_pool_initializes_buffer_capacity() {
-        let pool = QueryScratchPool::with_capacity(1, QueryScratchCapacity::new(8, 16, 4, 2));
+        let pool =
+            QueryScratchPool::with_capacity(1, QueryScratchCapacity::new_with_u32(8, 16, 4, 2, 3));
 
         pool.with_scratch(|scratch| {
             assert_eq!(scratch.distances.len(), 8);
@@ -630,6 +781,8 @@ mod tests {
             assert_eq!(scratch.u16.capacity(), 4);
             assert_eq!(scratch.u8.len(), 2);
             assert_eq!(scratch.u8.capacity(), 2);
+            assert_eq!(scratch.u32.len(), 3);
+            assert_eq!(scratch.u32.capacity(), 3);
         });
     }
 }
diff --git a/rust/lance-index/src/vector/utils.rs b/rust/lance-index/src/vector/utils.rs
index 1e56370613e..587e7dc3f15 100644
--- a/rust/lance-index/src/vector/utils.rs
+++ b/rust/lance-index/src/vector/utils.rs
@@ -277,6 +277,7 @@ mod tests {
     use half::f16;
     use lance_arrow::FixedSizeListArrayExt;
     use num_traits::identities::Zero;
+    use rayon::ThreadPoolBuilder;
 
     use arrow::compute::cast;
     use rstest::rstest;
@@ -302,15 +303,17 @@ mod tests {
     #[rstest]
     #[case::f16(Arc::new(Float16Array::from(
         (0..100).flat_map(|i| std::iter::repeat_n(f16::from_f32(i as f32), 16)).collect::<Vec<_>>(),
-    )) as ArrayRef)]
+    )) as ArrayRef, 42.0f32)]
     #[case::f32(Arc::new(Float32Array::from(
         (0..100).flat_map(|i| std::iter::repeat_n(i as f32, 16)).collect::<Vec<_>>(),
-    )) as ArrayRef)]
-    fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef) {
-        let index = build_index(centroids, 16);
-        let query: ArrayRef = Arc::new(Float32Array::from(vec![42.1f32; 16]));
-        let (id, _) = index.search(query).unwrap();
+    )) as ArrayRef, 42.0f32)]
+    fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef, #[case] query_val: f32) {
+        let thread_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap();
+        let index = thread_pool.install(|| build_index(centroids, 16));
+        let query: ArrayRef = Arc::new(Float32Array::from(vec![query_val; 16]));
+        let (id, dist) = index.search(query).unwrap();
         assert_eq!(id, 42);
+        assert_eq!(dist, 0.0);
     }
 
     #[test]
diff --git a/rust/lance-index/src/vector/v3/shuffler.rs b/rust/lance-index/src/vector/v3/shuffler.rs
index 0b76517e1c2..4203d099d0b 100644
--- a/rust/lance-index/src/vector/v3/shuffler.rs
+++ b/rust/lance-index/src/vector/v3/shuffler.rs
@@ -5,8 +5,7 @@
 //! the corresponding IVF partitions.
 
 use std::ops::Range;
-use std::sync::atomic::AtomicU64;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 
 use arrow::compute::concat_batches;
 use arrow::datatypes::UInt64Type;
@@ -14,8 +13,7 @@ use arrow::{array::AsArray, compute::sort_to_indices};
 use arrow_array::{RecordBatch, UInt32Array, UInt64Array};
 use arrow_schema::{DataType, Field, Schema};
 use futures::{future::try_join_all, prelude::*};
-use lance_arrow::stream::rechunk_stream_by_size;
-use lance_arrow::{RecordBatchExt, SchemaExt};
+use lance_arrow::{RecordBatchExt, SchemaExt, interleave_batches};
 use lance_core::{
     Error, Result,
     cache::LanceCache,
@@ -341,6 +339,11 @@ pub fn create_ivf_shuffler(
 
 const DEFAULT_SHUFFLE_BATCH_BYTES: usize = 128 * 1024 * 1024;
 
+/// Number of rows per output batch when streaming sorted data via interleave.
+/// Small enough to keep the output chunk's memory footprint modest relative to
+/// the accumulated source data.
+const SHUFFLE_WRITE_CHUNK_ROWS: usize = 8 * 1024;
+
 /// Limit of how much transformed data we accumulate before spilling to disk.
 ///
 /// A larger value will use more RAM but require less random access during the
@@ -407,6 +410,51 @@ impl TwoFileShuffler {
     }
 }
 
+/// `(batch_idx, row_idx)` pairs produced by [`sort_to_interleave_indices`], paired with
+/// per-partition row counts.
+type InterleaveResult = (Vec<(usize, usize)>, Vec<u64>);
+
+/// Sorts rows from multiple batches by partition ID and returns interleave indices.
+///
+/// Builds a sort key of `(part_id, batch_idx, row_idx)` for every row across all
+/// batches, sorts by `part_id`, then emits `(batch_idx, row_idx)` pairs in that
+/// order. This avoids concatenating the full data: only the `UInt32` partition-ID
+/// columns are touched here.
+///
+/// Also returns per-partition row counts (derived from the same sorted keys at no
+/// extra cost).
+///
+/// Returns an error if any partition ID is out of range `[0, num_partitions)`.
+fn sort_to_interleave_indices(
+    part_id_columns: &[&UInt32Array],
+    num_partitions: usize,
+) -> Result<InterleaveResult> {
+    let total_rows: usize = part_id_columns.iter().map(|a| a.len()).sum();
+    let mut keys: Vec<(u32, u32, u32)> = Vec::with_capacity(total_rows);
+    for (batch_idx, col) in part_id_columns.iter().enumerate() {
+        let batch_idx = batch_idx as u32;
+        for (row_idx, &part_id) in col.values().iter().enumerate() {
+            keys.push((part_id, batch_idx, row_idx as u32));
+        }
+    }
+    keys.sort_unstable_by_key(|k| k.0);
+
+    let mut partition_counts = vec![0u64; num_partitions];
+    let mut interleave_indices = Vec::with_capacity(total_rows);
+    for (part_id, batch_idx, row_idx) in &keys {
+        let pid = *part_id as usize;
+        if pid >= num_partitions {
+            return Err(Error::invalid_input(format!(
+                "partition ID {} is out of range [0, {})",
+                pid, num_partitions
+            )));
+        }
+        partition_counts[pid] += 1;
+        interleave_indices.push((*batch_idx as usize, *row_idx as usize));
+    }
+    Ok((interleave_indices, partition_counts))
+}
+
 #[async_trait::async_trait]
 impl Shuffler for TwoFileShuffler {
     async fn shuffle(
@@ -414,8 +462,7 @@ impl Shuffler for TwoFileShuffler {
         data: Box<dyn RecordBatchStream + Unpin + 'static>,
     ) -> Result<Box<dyn ShuffleReader>> {
         let num_partitions = self.num_partitions;
-        let full_schema = Arc::new(data.schema().as_ref().clone());
-        // No need to write partition ids since we can infer this
+        // No need to write partition ids since we can infer this from offsets
         let schema = data.schema().without_column(PART_ID_COLUMN);
         let offsets_schema = Arc::new(Schema::new(vec![Field::new(
             "offset",
@@ -424,28 +471,6 @@ impl Shuffler for TwoFileShuffler {
         )]));
         let batch_size_bytes = self.batch_size_bytes;
 
-        // Extract loss from batch metadata before rechunking (concat_batches drops metadata)
-        let total_loss = Arc::new(Mutex::new(0.0f64));
-        let loss_ref = total_loss.clone();
-        let loss_stream = data.map(move |result| {
-            result.inspect(|batch| {
-                let loss = batch
-                    .metadata()
-                    .get(LOSS_METADATA_KEY)
-                    .and_then(|s| s.parse::<f64>().ok())
-                    .unwrap_or(0.0);
-                *loss_ref.lock().unwrap() += loss;
-            })
-        });
-
-        // Rechunk to target batch size
-        let rechunked = rechunk_stream_by_size(
-            loss_stream,
-            full_schema,
-            batch_size_bytes,
-            batch_size_bytes * 2,
-        );
-
         // Create data file writer
         let data_path = self.output_dir.clone().join("shuffle_data.lance");
         let spill_path = self.output_dir.clone().join("shuffle_data.spill");
@@ -468,72 +493,63 @@ impl Shuffler for TwoFileShuffler {
         )?
         .with_page_metadata_spill(self.object_store.clone(), spill_path);
 
-        let num_batches = Arc::new(AtomicU64::new(0));
-        let num_batches_ref = num_batches.clone();
+        let mut num_batches: u64 = 0;
         let mut partition_counts: Vec<u64> = vec![0; num_partitions];
         let mut global_row_count: u64 = 0;
         let mut rows_processed: u64 = 0;
+        let mut total_loss = 0.0f64;
+        let mut accumulated: Vec<RecordBatch> = Vec::new();
+        let mut acc_bytes: usize = 0;
 
-        let mut rechunked = std::pin::pin!(rechunked);
-        while let Some(batch) = rechunked.next().await {
-            num_batches_ref.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let mut data = std::pin::pin!(data);
+        while let Some(batch) = data.next().await {
             let batch = batch?;
-            let np = num_partitions;
-            let num_rows = batch.num_rows() as u64;
-
-            // Sort by partition ID and compute offsets on CPU
-            let (sorted_batch, batch_offsets) = spawn_cpu(move || {
-                let part_ids: &UInt32Array = batch[PART_ID_COLUMN].as_primitive();
-                let indices = sort_to_indices(part_ids, None, None)?;
-                let batch = batch.take(&indices)?;
-
-                let part_ids: &UInt32Array = batch[PART_ID_COLUMN].as_primitive();
-                let batch = batch.drop_column(PART_ID_COLUMN)?;
-
-                // Count rows per partition by scanning sorted part IDs
-                let mut partition_counts = vec![0u64; np];
-                for i in 0..part_ids.len() {
-                    let pid = part_ids.value(i) as usize;
-                    if pid < np {
-                        partition_counts[pid] += 1;
-                    } else {
-                        log::warn!("Partition ID {} is out of range [0, {})", pid, np);
-                    }
-                }
-
-                // Build cumulative offsets (end positions) for this batch
-                let mut batch_offsets = Vec::with_capacity(np);
-                let mut running = 0u64;
-                for count in &partition_counts {
-                    running += count;
-                    batch_offsets.push(running);
+            total_loss += batch
+                .metadata()
+                .get(LOSS_METADATA_KEY)
+                .and_then(|s| s.parse::<f64>().ok())
+                .unwrap_or(0.0);
+            acc_bytes += batch.get_array_memory_size();
+            accumulated.push(batch);
+
+            if acc_bytes >= batch_size_bytes {
+                let (total_rows, counts) = flush_shuffle_batch(
+                    std::mem::take(&mut accumulated),
+                    &mut file_writer,
+                    &mut offsets_writer,
+                    offsets_schema.clone(),
+                    num_partitions,
+                    global_row_count,
+                )
+                .await?;
+                acc_bytes = 0;
+                for (p, c) in counts.iter().enumerate() {
+                    partition_counts[p] += c;
                 }
+                global_row_count += total_rows;
+                rows_processed += total_rows;
+                num_batches += 1;
+                self.progress
+                    .stage_progress("shuffle", rows_processed)
+                    .await?;
+            }
+        }
 
-                Ok::<(RecordBatch, Vec<u64>), Error>((batch, batch_offsets))
-            })
+        if !accumulated.is_empty() {
+            let (total_rows, counts) = flush_shuffle_batch(
+                accumulated,
+                &mut file_writer,
+                &mut offsets_writer,
+                offsets_schema,
+                num_partitions,
+                global_row_count,
+            )
             .await?;
-
-            // Write sorted batch to data file
-            file_writer.write_batch(&sorted_batch).await?;
-
-            // Record offsets adjusted by global row count
-            let mut adjusted_offsets = Vec::with_capacity(batch_offsets.len());
-            let mut last_offset = 0;
-            for (idx, offset) in batch_offsets.iter().enumerate() {
-                adjusted_offsets.push(global_row_count + offset);
-                partition_counts[idx] += offset - last_offset;
-                last_offset = *offset;
+            for (p, c) in counts.iter().enumerate() {
+                partition_counts[p] += c;
             }
-            global_row_count += sorted_batch.num_rows() as u64;
-
-            // Write offsets to offsets file
-            let offsets_batch = RecordBatch::try_new(
-                offsets_schema.clone(),
-                vec![Arc::new(UInt64Array::from(adjusted_offsets))],
-            )?;
-            offsets_writer.write_batch(&offsets_batch).await?;
-
-            rows_processed += num_rows;
+            rows_processed += total_rows;
+            num_batches += 1;
             self.progress
                 .stage_progress("shuffle", rows_processed)
                 .await?;
@@ -543,22 +559,76 @@ impl Shuffler for TwoFileShuffler {
         file_writer.finish().await?;
         offsets_writer.finish().await?;
 
-        let num_batches = num_batches.load(std::sync::atomic::Ordering::Relaxed);
-
-        let total_loss_val = *total_loss.lock().unwrap();
-
         TwoFileShuffleReader::try_new(
             self.object_store.clone(),
             self.output_dir.clone(),
             num_partitions,
             num_batches,
             partition_counts,
-            total_loss_val,
+            total_loss,
         )
         .await
     }
 }
 
+/// Sorts `accumulated` batches by partition ID and writes the result to the data
+/// and offsets files.
+///
+/// Returns `(total_rows_written, per_partition_row_counts)`.
+async fn flush_shuffle_batch(
+    accumulated: Vec<RecordBatch>,
+    file_writer: &mut FileWriter,
+    offsets_writer: &mut FileWriter,
+    offsets_schema: Arc<Schema>,
+    num_partitions: usize,
+    global_row_count: u64,
+) -> Result<(u64, Vec<u64>)> {
+    let total_rows: u64 = accumulated.iter().map(|b| b.num_rows() as u64).sum();
+
+    // Clone part-id columns into the CPU task (cheap: Arc ref bump, not data copy).
+    let part_id_cols: Vec<UInt32Array> = accumulated
+        .iter()
+        .map(|b| {
+            let col: &UInt32Array = b[PART_ID_COLUMN].as_primitive();
+            col.clone()
+        })
+        .collect();
+
+    let np = num_partitions;
+    let (interleave_indices, batch_partition_counts) =
+        spawn_cpu(move || sort_to_interleave_indices(&part_id_cols.iter().collect::<Vec<_>>(), np))
+            .await?;
+
+    // Drop part-id column from source batches before interleaving.
+    let source_batches: Vec<RecordBatch> = accumulated
+        .into_iter()
+        .map(|b| b.drop_column(PART_ID_COLUMN).map_err(Error::from))
+        .collect::<Result<_>>()?;
+
+    // Stream sorted output to the data file in fixed-size chunks so the peak
+    // memory for the interleave output stays small relative to the source data.
+    for chunk in interleave_indices.chunks(SHUFFLE_WRITE_CHUNK_ROWS) {
+        let out = interleave_batches(&source_batches, chunk)?;
+        file_writer.write_batch(&out).await?;
+    }
+
+    // Compute cumulative end-row offsets (adjusted by global position) and write
+    // one offsets batch for this flush group.
+    let mut adjusted_offsets = Vec::with_capacity(num_partitions);
+    let mut running = 0u64;
+    for count in &batch_partition_counts {
+        running += count;
+        adjusted_offsets.push(global_row_count + running);
+    }
+    let offsets_batch = RecordBatch::try_new(
+        offsets_schema,
+        vec![Arc::new(UInt64Array::from(adjusted_offsets))],
+    )?;
+    offsets_writer.write_batch(&offsets_batch).await?;
+
+    Ok((total_rows, batch_partition_counts))
+}
+
 pub struct TwoFileShuffleReader {
     _scheduler: Arc<ScanScheduler>,
     file_reader: FileReader,
@@ -934,4 +1004,65 @@ mod tests {
 
         assert!((reader.total_loss().unwrap() - 6.0).abs() < 1e-10);
     }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_multi_batch_single_flush() {
+        // All three batches fit within the default batch_size_bytes, so they
+        // accumulate and are interleaved in a single flush group. This exercises
+        // the cross-batch interleave path.
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+        let num_partitions = 3;
+
+        let batch1 = make_batch(&[0, 1, 2], &[10, 20, 30], None);
+        let batch2 = make_batch(&[2, 0, 1], &[40, 50, 60], None);
+        let batch3 = make_batch(&[1, 2, 0], &[70, 80, 90], None);
+
+        // Large batch_size_bytes so all three batches flush together.
+        let shuffler =
+            TwoFileShuffler::new(output_dir, num_partitions).with_batch_size_bytes(1024 * 1024);
+        let stream = batches_to_stream(vec![batch1, batch2, batch3]);
+        let reader = shuffler.shuffle(stream).await.unwrap();
+
+        assert_eq!(reader.partition_size(0).unwrap(), 3);
+        assert_eq!(reader.partition_size(1).unwrap(), 3);
+        assert_eq!(reader.partition_size(2).unwrap(), 3);
+
+        let p0 = collect_partition(reader.as_ref(), 0).await.unwrap();
+        let vals: &Int32Array = p0.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![10, 50, 90]);
+
+        let p1 = collect_partition(reader.as_ref(), 1).await.unwrap();
+        let vals: &Int32Array = p1.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![20, 60, 70]);
+
+        let p2 = collect_partition(reader.as_ref(), 2).await.unwrap();
+        let vals: &Int32Array = p2.column_by_name("val").unwrap().as_primitive();
+        let mut v: Vec<i32> = vals.iter().map(|x| x.unwrap()).collect();
+        v.sort();
+        assert_eq!(v, vec![30, 40, 80]);
+    }
+
+    #[tokio::test]
+    async fn test_two_file_shuffler_out_of_range_partition_id() {
+        let dir = TempStrDir::default();
+        let output_dir = Path::from(dir.as_ref());
+
+        // Row with partition ID 5 is out of range for num_partitions=3.
+        let batch = make_batch(&[0, 5, 1], &[10, 20, 30], None);
+
+        let shuffler = TwoFileShuffler::new(output_dir, 3);
+        let stream = batches_to_stream(vec![batch]);
+        let Err(err) = shuffler.shuffle(stream).await else {
+            panic!("expected an error for out-of-range partition ID");
+        };
+        assert!(
+            err.to_string().contains("partition ID 5 is out of range"),
+            "unexpected error: {err}"
+        );
+    }
 }
diff --git a/rust/lance-index/src/vector/v3/subindex.rs b/rust/lance-index/src/vector/v3/subindex.rs
index 7c9667859f4..9e82921347f 100644
--- a/rust/lance-index/src/vector/v3/subindex.rs
+++ b/rust/lance-index/src/vector/v3/subindex.rs
@@ -1,12 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::{BinaryHeap, HashMap};
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::collections::BinaryHeap;
 use std::fmt::Debug;
 use std::sync::Arc;
 
 use arrow_array::{ArrayRef, RecordBatch};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 
 use crate::metrics::MetricsCollector;
@@ -106,7 +107,7 @@ pub trait IvfSubIndex: Send + Sync + Debug + DeepSizeOf {
     where
         Self: Sized;
 
-    fn remap(&self, mapping: &HashMap<u64, Option<u64>>, store: &impl VectorStore) -> Result<Self>
+    fn remap(&self, mapping: &RowAddrRemap, store: &impl VectorStore) -> Result<Self>
     where
         Self: Sized;
 
diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml
index d0c8a3c3c33..643fef8c14a 100644
--- a/rust/lance-io/Cargo.toml
+++ b/rust/lance-io/Cargo.toml
@@ -34,10 +34,11 @@ aws-credential-types = { workspace = true, optional = true }
 byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
-deepsize.workspace = true
 futures.workspace = true
+goosefs-sdk = { workspace = true, optional = true }
 http.workspace = true
 log.workspace = true
+metrics = { workspace = true, optional = true }
 moka.workspace = true
 pin-project.workspace = true
 prost.workspace = true
@@ -59,7 +60,9 @@ test-log.workspace = true
 mockall.workspace = true
 rstest.workspace = true
 mock_instant.workspace = true
+tokio = { workspace = true, features = ["test-util"] }
 tracing-mock = { workspace = true }
+metrics-util = { workspace = true }
 
 [[bench]]
 name = "scheduler"
@@ -67,13 +70,19 @@ harness = false
 
 [features]
 default = ["aws", "azure", "gcp"]
+metrics = ["dep:metrics"]
 gcs-test = []
+goosefs-test = []
 gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"]
 aws = ["object_store/aws", "dep:aws-config", "dep:aws-credential-types", "dep:opendal", "opendal/services-s3", "dep:object_store_opendal"]
 azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "opendal/services-azdls", "dep:object_store_opendal"]
 oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"]
+# Pin goosefs-sdk until a release fixes the missing CString import in 0.1.6's Linux backend.
+goosefs = ["dep:goosefs-sdk", "dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal"]
 tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
 huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
+tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"]
+tos-test = ["tos"]
 test-util = []
 
 [lints]
diff --git a/rust/lance-io/src/lib.rs b/rust/lance-io/src/lib.rs
index c327a91c1ba..2ef686fd551 100644
--- a/rust/lance-io/src/lib.rs
+++ b/rust/lance-io/src/lib.rs
@@ -18,6 +18,7 @@ pub mod object_reader;
 pub mod object_store;
 pub mod object_writer;
 pub mod scheduler;
+pub mod spill;
 pub mod stream;
 #[cfg(test)]
 pub mod testing;
diff --git a/rust/lance-io/src/local.rs b/rust/lance-io/src/local.rs
index 12f846bcc52..2b8a339331a 100644
--- a/rust/lance-io/src/local.rs
+++ b/rust/lance-io/src/local.rs
@@ -16,8 +16,8 @@ use std::os::windows::fs::FileExt;
 
 use async_trait::async_trait;
 use bytes::{Bytes, BytesMut};
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use object_store::path::Path;
 use tokio::io::AsyncSeekExt;
@@ -89,7 +89,7 @@ pub struct LocalObjectReader {
 }
 
 impl DeepSizeOf for LocalObjectReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // Skipping `file` as it should just be a file handle
         self.path.as_ref().deep_size_of_children(context)
     }
diff --git a/rust/lance-io/src/object_reader.rs b/rust/lance-io/src/object_reader.rs
index d6d5de98f0b..1c27800c90f 100644
--- a/rust/lance-io/src/object_reader.rs
+++ b/rust/lance-io/src/object_reader.rs
@@ -11,12 +11,12 @@ use crate::local::read_exact_at;
 use std::os::unix::fs::FileExt;
 
 use bytes::Bytes;
-use deepsize::DeepSizeOf;
 use futures::{
     FutureExt,
     future::{BoxFuture, Shared},
     stream::{self, StreamExt},
 };
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, error::CloneableError};
 use object_store::ObjectStoreExt;
 use object_store::{GetOptions, GetResult, ObjectStore, Result as OSResult, path::Path};
@@ -74,7 +74,7 @@ pub struct CloudObjectReader {
 }
 
 impl DeepSizeOf for CloudObjectReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // Skipping object_store because there is no easy way to do that and it shouldn't be too big
         self.path.as_ref().deep_size_of_children(context)
     }
@@ -449,7 +449,7 @@ pub(crate) fn stream_local_range(
 }
 
 impl DeepSizeOf for SmallReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let mut size = self.inner.path.as_ref().deep_size_of_children(context);
 
         if let Ok(guard) = self.inner.state.try_lock()
diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs
index 698378c5b9e..15905ff4e69 100644
--- a/rust/lance-io/src/object_store.rs
+++ b/rust/lance-io/src/object_store.rs
@@ -3,6 +3,7 @@
 
 //! Extend [object_store::ObjectStore] functionalities
 
+use std::borrow::Cow;
 use std::collections::HashMap;
 use std::ops::Range;
 use std::pin::Pin;
@@ -13,9 +14,9 @@ use std::time::Duration;
 use async_trait::async_trait;
 use bytes::Bytes;
 use chrono::{DateTime, Utc};
-use deepsize::DeepSizeOf;
 use futures::{FutureExt, Stream};
 use futures::{StreamExt, TryStreamExt, future, stream::BoxStream};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::error::LanceOptionExt;
 use lance_core::utils::parse::str_is_truthy;
 use list_retry::ListRetryStream;
@@ -25,7 +26,7 @@ use object_store::ObjectStoreExt as OSObjectStoreExt;
 use object_store::aws::AwsCredentialProvider;
 #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
 use object_store::{ClientOptions, HeaderMap, HeaderValue};
-use object_store::{ObjectMeta, ObjectStore as OSObjectStore, path::Path};
+use object_store::{ListResult, ObjectMeta, ObjectStore as OSObjectStore, path::Path};
 use providers::local::FileStoreProvider;
 use providers::memory::MemoryStoreProvider;
 use tokio::io::AsyncWriteExt;
@@ -36,9 +37,11 @@ use super::local::LocalObjectReader;
 use crate::uring::{UringCurrentThreadReader, UringReader};
 #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
 pub(crate) mod dynamic_credentials;
-#[cfg(any(feature = "oss", feature = "huggingface"))]
+#[cfg(any(feature = "oss", feature = "huggingface", feature = "tos"))]
 pub(crate) mod dynamic_opendal;
 mod list_retry;
+#[cfg(feature = "metrics")]
+pub mod metrics;
 pub mod providers;
 pub mod storage_options;
 #[cfg(test)]
@@ -61,7 +64,16 @@ pub const DEFAULT_LOCAL_IO_PARALLELISM: usize = 8;
 pub const DEFAULT_CLOUD_IO_PARALLELISM: usize = 64;
 
 const DEFAULT_LOCAL_BLOCK_SIZE: usize = 4 * 1024; // 4KB block size
-#[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))]
+#[cfg(any(
+    feature = "aws",
+    feature = "gcp",
+    feature = "azure",
+    feature = "oss",
+    feature = "tencent",
+    feature = "huggingface",
+    feature = "tos",
+    feature = "goosefs",
+))]
 const DEFAULT_CLOUD_BLOCK_SIZE: usize = 64 * 1024; // 64KB block size
 
 pub static DEFAULT_MAX_IOP_SIZE: std::sync::LazyLock<u64> = std::sync::LazyLock::new(|| {
@@ -74,8 +86,10 @@ pub const DEFAULT_DOWNLOAD_RETRY_COUNT: usize = 3;
 
 pub use providers::{ObjectStoreProvider, ObjectStoreRegistry};
 pub use storage_options::{
-    EXPIRES_AT_MILLIS_KEY, LanceNamespaceStorageOptionsProvider, REFRESH_OFFSET_MILLIS_KEY,
-    StorageOptionsAccessor, StorageOptionsProvider,
+    BASE_SCOPED_OPTION_PREFIX, BaseScopedStorageOptionsProvider, EXPIRES_AT_MILLIS_KEY,
+    LanceNamespaceStorageOptionsProvider, REFRESH_OFFSET_MILLIS_KEY, StorageOptionsAccessor,
+    StorageOptionsProvider, has_base_scoped_options, parse_base_scoped_key,
+    resolve_base_scoped_options,
 };
 
 #[async_trait]
@@ -145,7 +159,7 @@ pub struct ObjectStore {
 }
 
 impl DeepSizeOf for ObjectStore {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // We aren't counting `inner` here which is problematic but an ObjectStore
         // shouldn't be too big.  The only exception might be the write cache but, if
         // the writer cache has data, it means we're using it somewhere else that isn't
@@ -249,6 +263,27 @@ impl ObjectStoreParams {
             .as_ref()
             .and_then(|a| a.initial_storage_options())
     }
+
+    /// Resolve these params for a single base path scope.
+    ///
+    /// Storage options may carry base-scoped entries (`base_<id>.<key>`) that
+    /// apply only to one registered base path; see
+    /// [`StorageOptionsAccessor::scoped_to_base`]. Returns the params unchanged
+    /// when the storage options contain no base-scoped entries.
+    pub fn scoped_to_base(&self, base_id: Option<u32>) -> Cow<'_, Self> {
+        let Some(accessor) = &self.storage_options_accessor else {
+            return Cow::Borrowed(self);
+        };
+        let scoped = accessor.scoped_to_base(base_id);
+        if Arc::ptr_eq(&scoped, accessor) {
+            Cow::Borrowed(self)
+        } else {
+            Cow::Owned(Self {
+                storage_options_accessor: Some(scoped),
+                ..self.clone()
+            })
+        }
+    }
 }
 
 // We implement hash for caching
@@ -570,10 +605,17 @@ impl ObjectStore {
         self.max_iop_size
     }
 
+    /// The amount of parallelism to use for I/O operations.
+    ///
+    /// Honors the `LANCE_IO_THREADS` override when set, otherwise the store's configured value.
+    /// Always at least 1: callers feed this straight into `buffered` / `buffer_unordered`, and a
+    /// window of 0 makes those streams never poll their input — e.g. a metadata-only `count_rows`
+    /// would hang rather than return.
     pub fn io_parallelism(&self) -> usize {
         std::env::var("LANCE_IO_THREADS")
             .map(|val| val.parse::<usize>().unwrap())
             .unwrap_or(self.io_parallelism)
+            .max(1)
     }
 
     /// Get the IO tracker for this object store
@@ -830,10 +872,20 @@ impl ObjectStore {
             .common_prefixes
             .iter()
             .chain(output.objects.iter().map(|o| &o.location))
-            .map(|s| s.filename().unwrap().to_string())
+            .filter_map(|s| s.filename().map(|f| f.to_string()))
             .collect())
     }
 
+    /// Non-recursive, path-segment delimited list of a single directory level.
+    ///
+    /// Unlike [`Self::list`], which recurses into the entire subtree, this returns
+    /// only the immediate children of `prefix`: the child "directories" as
+    /// [`ListResult::common_prefixes`] and the direct child files as
+    /// [`ListResult::objects`].
+    pub async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
+        Ok(self.inner.list_with_delimiter(prefix).await?)
+    }
+
     pub fn list(
         &self,
         path: Option<Path>,
@@ -1150,6 +1202,35 @@ mod tests {
         Ok(contents)
     }
 
+    #[test]
+    fn test_io_parallelism_clamped_to_nonzero() {
+        // `io_parallelism()` feeds `buffered`/`buffer_unordered` windows; a value of 0 makes those
+        // streams never poll, hanging callers (e.g. a metadata-only `count_rows`). It must clamp.
+        let store = ObjectStore::local();
+
+        // SAFETY: process-global env var, set and restored within this test. `io_parallelism()`
+        // only reads it, and a concurrent reader observes a valid clamped value, never 0.
+        unsafe { std::env::set_var("LANCE_IO_THREADS", "0") };
+        assert_eq!(
+            store.io_parallelism(),
+            1,
+            "LANCE_IO_THREADS=0 must clamp to 1"
+        );
+
+        unsafe { std::env::set_var("LANCE_IO_THREADS", "8") };
+        assert_eq!(
+            store.io_parallelism(),
+            8,
+            "a positive override must pass through unchanged"
+        );
+
+        unsafe { std::env::remove_var("LANCE_IO_THREADS") };
+        assert!(
+            store.io_parallelism() >= 1,
+            "the configured default parallelism must be at least 1"
+        );
+    }
+
     #[tokio::test]
     async fn test_absolute_paths() {
         let tmp_path = TempStrDir::default();
diff --git a/rust/lance-io/src/object_store/metrics.rs b/rust/lance-io/src/object_store/metrics.rs
new file mode 100644
index 00000000000..d11ae76bd22
--- /dev/null
+++ b/rust/lance-io/src/object_store/metrics.rs
@@ -0,0 +1,1602 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Publishes object store metrics via the [`metrics`] crate.
+//!
+//! Two layers cooperate:
+//!
+//! * [`MeteredObjectStore`] wraps any [`object_store::ObjectStore`] and records
+//!   per-operation request counts, transferred bytes, latency, errors, and the
+//!   number of requests currently in flight. It works for every store
+//!   regardless of backend.
+//! * [`MeteringHttpConnector`] wraps the HTTP client used by the native cloud
+//!   stores (S3 / GCS / Azure) and records throttle / retryable responses per
+//!   attempt. Because `object_store`'s retry loop re-issues each request
+//!   through the [`HttpService`](object_store::client::HttpService), this sees
+//!   every retried response, which a store-level wrapper cannot observe.
+//!
+//! The two layers have different coverage: every store gets the request-level
+//! metrics from [`MeteredObjectStore`], but only the native cloud stores get
+//! the HTTP-level throttle metrics. Opendal-backed stores (tos, oss, etc.)
+//! bypass `object_store`'s HTTP client, so there is no place to install the
+//! connector for them.
+//!
+//! Metrics carry a `base` label identifying the store. Its cardinality is
+//! controlled by the `LANCE_OBJECT_STORE_METRICS_LABEL` environment variable
+//! ([`BASE_LABEL_ENV_VAR`]):
+//!
+//! * `scheme` (default) — scheme only, e.g. `s3`; low, bounded cardinality.
+//! * `full` — the full store prefix, e.g. `s3$bucket` or `az$container@account`,
+//!   so multiple buckets on the same cloud can be told apart.
+//! * `off` — omit the `base` label entirely.
+//!
+//! The metric name constants ([`METRIC_REQUESTS`] etc.) and the recording
+//! helpers ([`record_request`], [`record_count`], [`record_error`],
+//! [`InFlightGuard`]) are public so custom object stores can emit the same
+//! metrics.
+
+use std::ops::Range;
+use std::pin::Pin;
+use std::sync::{Arc, OnceLock};
+use std::task::{Context, Poll};
+use std::time::Instant;
+
+use bytes::Bytes;
+use futures::stream::BoxStream;
+use futures::{FutureExt, Stream, StreamExt};
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
+    UploadPart,
+};
+
+/// Total number of object store requests, labelled by `operation` and `base`.
+pub const METRIC_REQUESTS: &str = "lance_object_store_requests_total";
+/// Total bytes transferred by object store requests, labelled by `operation` and `base`.
+pub const METRIC_BYTES: &str = "lance_object_store_request_bytes_total";
+/// Object store request latency in seconds, labelled by `operation` and `base`.
+pub const METRIC_DURATION: &str = "lance_object_store_request_duration_seconds";
+/// Total number of failed object store requests, labelled by `operation` and `base`.
+pub const METRIC_ERRORS: &str = "lance_object_store_errors_total";
+/// Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer,
+/// labelled by `status` and `base`. Counts every attempt, including retries.
+pub const METRIC_THROTTLE: &str = "lance_object_store_throttle_total";
+/// Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP
+/// layer, labelled by `status` and `base`. Counts every attempt, including
+/// retries. This is a superset of [`METRIC_THROTTLE`]; 409 (conflict) is
+/// deliberately excluded so commit conflicts are not counted as retries.
+pub const METRIC_RETRYABLE: &str = "lance_object_store_retryable_responses_total";
+/// Number of object store requests currently in flight, labelled by `operation`
+/// and `base`.
+pub const METRIC_IN_FLIGHT: &str = "lance_object_store_in_flight_requests";
+
+/// Environment variable controlling the cardinality of the `base` label.
+pub const BASE_LABEL_ENV_VAR: &str = "LANCE_OBJECT_STORE_METRICS_LABEL";
+
+/// Controls how much of a store's identity the `base` label carries, traded off
+/// against metric cardinality. Selected via [`BASE_LABEL_ENV_VAR`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum BaseLabelMode {
+    /// Full store prefix, e.g. `s3$bucket` or `az$container@account`. Highest
+    /// cardinality: one series family per bucket/container.
+    Full,
+    /// Scheme only, e.g. `s3`. The default: low, bounded cardinality.
+    Scheme,
+    /// Omit the `base` label entirely.
+    Off,
+}
+
+fn parse_base_label_mode(value: Option<&str>) -> BaseLabelMode {
+    match value {
+        Some("full") => BaseLabelMode::Full,
+        Some("off") | Some("none") => BaseLabelMode::Off,
+        Some("scheme") | None => BaseLabelMode::Scheme,
+        Some(other) => {
+            tracing::warn!(
+                "Unrecognized {BASE_LABEL_ENV_VAR}={other:?}; \
+                 expected one of full, scheme, off. Defaulting to scheme."
+            );
+            BaseLabelMode::Scheme
+        }
+    }
+}
+
+/// The label mode is read once from the environment and cached for the process.
+fn base_label_mode() -> BaseLabelMode {
+    static MODE: OnceLock<BaseLabelMode> = OnceLock::new();
+    *MODE.get_or_init(|| parse_base_label_mode(std::env::var(BASE_LABEL_ENV_VAR).ok().as_deref()))
+}
+
+/// Reduce a full store prefix (`scheme$authority`, or just `scheme` for stores
+/// without buckets) to the configured `base` label value, or `None` when the
+/// label should be omitted.
+fn scoped_base(mode: BaseLabelMode, base: &str) -> Option<String> {
+    match mode {
+        BaseLabelMode::Full => Some(base.to_owned()),
+        BaseLabelMode::Scheme => Some(base.split('$').next().unwrap_or(base).to_owned()),
+        BaseLabelMode::Off => None,
+    }
+}
+
+/// Build the `operation` (+ optional `base`) label set shared by all
+/// store-level metrics, honoring the configured label mode.
+fn operation_labels(base: &str, operation: &'static str) -> Vec<metrics::Label> {
+    let mut labels = vec![metrics::Label::new("operation", operation)];
+    if let Some(base) = scoped_base(base_label_mode(), base) {
+        labels.push(metrics::Label::new("base", base));
+    }
+    labels
+}
+
+/// Recommended histogram bucket boundaries for [`METRIC_DURATION`], in seconds.
+///
+/// Object store requests can take anywhere from a few milliseconds to the
+/// client timeout (commonly ~120s), so the boundaries are dense below 10s and
+/// keep useful resolution through the timeout band out to 5 minutes. Exporters
+/// that aggregate into fixed buckets (e.g. the OpenTelemetry bridge in the
+/// Python bindings) use these.
+pub const REQUEST_DURATION_BOUNDS: &[f64] = &[
+    0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, // sub-10s
+    10.0, 20.0, 30.0, 45.0, 60.0, 90.0, 120.0, 150.0, 180.0, 240.0, 300.0, // 10s–5min
+];
+
+/// Register descriptions (units and help text) for the object store metrics.
+///
+/// This routes through whatever [`metrics::Recorder`] is currently installed,
+/// so it must be called *after* the recorder is set. Exporters that build a
+/// catalog of available metrics (such as the OpenTelemetry bridge) rely on
+/// these descriptions to discover metric names, kinds, and units up front.
+pub fn describe_metrics() {
+    metrics::describe_counter!(
+        METRIC_REQUESTS,
+        metrics::Unit::Count,
+        "Total number of object store requests, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_BYTES,
+        metrics::Unit::Bytes,
+        "Total bytes transferred by object store requests, by operation and scheme."
+    );
+    metrics::describe_histogram!(
+        METRIC_DURATION,
+        metrics::Unit::Seconds,
+        "Object store request latency in seconds, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_ERRORS,
+        metrics::Unit::Count,
+        "Total number of failed object store requests, by operation and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_THROTTLE,
+        metrics::Unit::Count,
+        "Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, by status and scheme."
+    );
+    metrics::describe_counter!(
+        METRIC_RETRYABLE,
+        metrics::Unit::Count,
+        "Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, by status and scheme."
+    );
+    metrics::describe_gauge!(
+        METRIC_IN_FLIGHT,
+        metrics::Unit::Count,
+        "Number of object store requests currently in flight, by operation and scheme."
+    );
+}
+
+/// Recommended fixed bucket boundaries for the histogram metrics defined here,
+/// as `(metric_name, boundaries)` pairs. Exporters that aggregate histograms
+/// into fixed buckets read this to configure each histogram.
+pub fn histogram_bounds() -> &'static [(&'static str, &'static [f64])] {
+    &[(METRIC_DURATION, REQUEST_DURATION_BOUNDS)]
+}
+
+/// Record the outcome of a unary request: count, latency, bytes (on success), and errors.
+pub fn record_request<T>(
+    base: &str,
+    operation: &'static str,
+    start: Instant,
+    bytes: u64,
+    result: &OSResult<T>,
+) {
+    record_outcome(base, operation, start, bytes, result.is_err());
+}
+
+/// Record count, latency, and either transferred bytes or an error for a
+/// completed request. Used both for unary requests and for streamed GETs whose
+/// bytes are only known once the body finishes.
+pub fn record_outcome(
+    base: &str,
+    operation: &'static str,
+    start: Instant,
+    bytes: u64,
+    is_error: bool,
+) {
+    let elapsed = start.elapsed().as_secs_f64();
+    let labels = operation_labels(base, operation);
+    metrics::counter!(METRIC_REQUESTS, labels.clone()).increment(1);
+    metrics::histogram!(METRIC_DURATION, labels.clone()).record(elapsed);
+    if is_error {
+        metrics::counter!(METRIC_ERRORS, labels).increment(1);
+    } else if bytes > 0 {
+        metrics::counter!(METRIC_BYTES, labels).increment(bytes);
+    }
+}
+
+/// Record a single request count without latency, used for streaming operations
+/// (list / delete) whose work happens lazily as the stream is polled.
+pub fn record_count(base: &str, operation: &'static str) {
+    metrics::counter!(METRIC_REQUESTS, operation_labels(base, operation)).increment(1);
+}
+
+/// Record a single error for an operation.
+pub fn record_error(base: &str, operation: &'static str) {
+    metrics::counter!(METRIC_ERRORS, operation_labels(base, operation)).increment(1);
+}
+
+/// Raises the in-flight gauge for an operation on creation and lowers it on
+/// drop, so the count stays balanced even if the request future or stream is
+/// cancelled or dropped before completing.
+pub struct InFlightGuard {
+    labels: Vec<metrics::Label>,
+}
+
+impl InFlightGuard {
+    pub fn new(base: &str, operation: &'static str) -> Self {
+        let labels = operation_labels(base, operation);
+        metrics::gauge!(METRIC_IN_FLIGHT, labels.clone()).increment(1.0);
+        Self { labels }
+    }
+}
+
+impl Drop for InFlightGuard {
+    fn drop(&mut self) {
+        metrics::gauge!(METRIC_IN_FLIGHT, self.labels.clone()).decrement(1.0);
+    }
+}
+
+#[derive(Debug)]
+pub struct MeteredObjectStore {
+    target: Arc<dyn object_store::ObjectStore>,
+    base: String,
+}
+
+impl std::fmt::Display for MeteredObjectStore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "MeteredObjectStore({})", self.target)
+    }
+}
+
+#[async_trait::async_trait]
+#[deny(clippy::missing_trait_methods)]
+impl object_store::ObjectStore for MeteredObjectStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        let size = bytes.content_length() as u64;
+        let _in_flight = InFlightGuard::new(&self.base, "put");
+        let start = Instant::now();
+        let result = self.target.put_opts(location, bytes, opts).await;
+        record_request(&self.base, "put", start, size, &result);
+        result
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        let upload = self.target.put_multipart_opts(location, opts).await?;
+        Ok(Box::new(MeteredMultipartUpload {
+            target: upload,
+            base: self.base.clone(),
+        }))
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        // `head()` is implemented as a `get_opts` call with `head = true`, so we
+        // distinguish it here to keep HEAD and GET as separate operations.
+        let is_head = options.head;
+        let operation = if is_head { "head" } else { "get" };
+        let in_flight = InFlightGuard::new(&self.base, operation);
+        let start = Instant::now();
+        let result = self.target.get_opts(location, options).await;
+
+        // A HEAD transfers only metadata, and errors carry no payload, so both
+        // are recorded immediately. `get_opts` only resolves once the response
+        // headers arrive; the body is streamed afterwards, so for a successful
+        // GET we defer recording until the body has been drained (see below).
+        if is_head || result.is_err() {
+            record_request(&self.base, operation, start, 0, &result);
+            return result;
+        }
+
+        let result = result.expect("checked to be Ok above");
+        Ok(meter_get_result(
+            result,
+            self.base.clone(),
+            start,
+            in_flight,
+        ))
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        let _in_flight = InFlightGuard::new(&self.base, "get");
+        let start = Instant::now();
+        let result = self.target.get_ranges(location, ranges).await;
+        let bytes = match &result {
+            Ok(parts) => parts.iter().map(|b| b.len() as u64).sum(),
+            Err(_) => 0,
+        };
+        record_request(&self.base, "get", start, bytes, &result);
+        result
+    }
+
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OSResult<Path>>,
+    ) -> BoxStream<'static, OSResult<Path>> {
+        let base = self.base.clone();
+        // Count one logical delete request per call, matching `list`: a single
+        // `delete_stream` maps to one batched request on stores that support it
+        // (e.g. S3's `DeleteObjects`), so counting per yielded path would
+        // over-count. Errors are still recorded per failing path.
+        record_count(&self.base, "delete");
+        let in_flight = InFlightGuard::new(&self.base, "delete");
+        self.target
+            .delete_stream(locations)
+            .map(move |result| {
+                // Reference `in_flight` so this `move` closure captures (owns)
+                // the guard, keeping the gauge raised until the stream is
+                // dropped (a move closure only captures the variables it uses).
+                let _in_flight = &in_flight;
+                if result.is_err() {
+                    record_error(&base, "delete");
+                }
+                result
+            })
+            .boxed()
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        record_count(&self.base, "list");
+        meter_list_stream(
+            self.target.list(prefix),
+            self.base.clone(),
+            InFlightGuard::new(&self.base, "list"),
+        )
+    }
+
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        record_count(&self.base, "list");
+        meter_list_stream(
+            self.target.list_with_offset(prefix, offset),
+            self.base.clone(),
+            InFlightGuard::new(&self.base, "list"),
+        )
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        let _in_flight = InFlightGuard::new(&self.base, "list");
+        let start = Instant::now();
+        let result = self.target.list_with_delimiter(prefix).await;
+        record_request(&self.base, "list", start, 0, &result);
+        result
+    }
+
+    async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "copy");
+        let start = Instant::now();
+        let result = self.target.copy_opts(from, to, opts).await;
+        record_request(&self.base, "copy", start, 0, &result);
+        result
+    }
+
+    async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "rename");
+        let start = Instant::now();
+        let result = self.target.rename_opts(from, to, opts).await;
+        record_request(&self.base, "rename", start, 0, &result);
+        result
+    }
+}
+
+/// Count errors yielded while draining a list stream. The request itself is
+/// counted once when the stream is created (a single LIST may return many items).
+fn meter_list_stream(
+    stream: BoxStream<'static, OSResult<ObjectMeta>>,
+    base: String,
+    in_flight: InFlightGuard,
+) -> BoxStream<'static, OSResult<ObjectMeta>> {
+    stream
+        .map(move |result| {
+            // Reference `in_flight` so this `move` closure captures (owns) the
+            // guard: a move closure only captures the variables it uses, and
+            // holding it here keeps the gauge raised until the stream is dropped.
+            let _in_flight = &in_flight;
+            if result.is_err() {
+                record_error(&base, "list");
+            }
+            result
+        })
+        .boxed()
+}
+
+/// Wrap a successful GET so the request is recorded once its body has been
+/// fully read, capturing the true transfer duration and byte count rather than
+/// the time-to-first-byte and declared range. For payloads without a body
+/// stream (e.g. a local file handle) the request is recorded immediately.
+fn meter_get_result(
+    mut result: GetResult,
+    base: String,
+    start: Instant,
+    in_flight: InFlightGuard,
+) -> GetResult {
+    match result.payload {
+        GetResultPayload::Stream(stream) => {
+            result.payload = GetResultPayload::Stream(
+                MeteredGetStream {
+                    inner: stream,
+                    base,
+                    start,
+                    bytes: 0,
+                    errored: false,
+                    recorded: false,
+                    _in_flight: in_flight,
+                }
+                .boxed(),
+            );
+            result
+        }
+        // No body stream to observe (e.g. a local file), so record now.
+        other => {
+            let bytes = result.range.end - result.range.start;
+            record_outcome(&base, "get", start, bytes, false);
+            result.payload = other;
+            result
+        }
+    }
+}
+
+/// Stream wrapper over a GET body that records the request (count, duration,
+/// bytes, errors) once the body is fully drained or the stream is dropped.
+struct MeteredGetStream {
+    inner: BoxStream<'static, OSResult<Bytes>>,
+    base: String,
+    start: Instant,
+    bytes: u64,
+    errored: bool,
+    recorded: bool,
+    _in_flight: InFlightGuard,
+}
+
+impl MeteredGetStream {
+    fn record(&mut self) {
+        if self.recorded {
+            return;
+        }
+        self.recorded = true;
+        record_outcome(&self.base, "get", self.start, self.bytes, self.errored);
+    }
+}
+
+impl Stream for MeteredGetStream {
+    type Item = OSResult<Bytes>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        match self.inner.poll_next_unpin(cx) {
+            Poll::Ready(Some(Ok(chunk))) => {
+                self.bytes += chunk.len() as u64;
+                Poll::Ready(Some(Ok(chunk)))
+            }
+            Poll::Ready(Some(Err(e))) => {
+                self.errored = true;
+                Poll::Ready(Some(Err(e)))
+            }
+            Poll::Ready(None) => {
+                self.record();
+                Poll::Ready(None)
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl Drop for MeteredGetStream {
+    fn drop(&mut self) {
+        // Records the partial transfer if the body was dropped before it drained.
+        self.record();
+    }
+}
+
+#[derive(Debug)]
+struct MeteredMultipartUpload {
+    target: Box<dyn MultipartUpload>,
+    base: String,
+}
+
+#[async_trait::async_trait]
+impl MultipartUpload for MeteredMultipartUpload {
+    fn put_part(&mut self, data: PutPayload) -> UploadPart {
+        // Each part upload is a distinct request, recorded under the `put_part`
+        // operation with the same count / bytes / latency / error set as a
+        // unary put.
+        let base = self.base.clone();
+        let size = data.content_length() as u64;
+        let inner = self.target.put_part(data);
+        async move {
+            let _in_flight = InFlightGuard::new(&base, "put_part");
+            let start = Instant::now();
+            let result = inner.await;
+            record_request(&base, "put_part", start, size, &result);
+            result
+        }
+        .boxed()
+    }
+
+    async fn complete(&mut self) -> OSResult<PutResult> {
+        // Completing a multipart upload issues its own request that can throttle
+        // or fail, so it is metered like any other operation.
+        let _in_flight = InFlightGuard::new(&self.base, "complete_multipart");
+        let start = Instant::now();
+        let result = self.target.complete().await;
+        record_request(&self.base, "complete_multipart", start, 0, &result);
+        result
+    }
+
+    async fn abort(&mut self) -> OSResult<()> {
+        let _in_flight = InFlightGuard::new(&self.base, "abort_multipart");
+        let start = Instant::now();
+        let result = self.target.abort().await;
+        record_request(&self.base, "abort_multipart", start, 0, &result);
+        result
+    }
+}
+
+pub trait ObjectStoreMetricsExt {
+    /// Wrap this store so its operations publish metrics under the given `base` label.
+    fn metered(self, base: String) -> Arc<dyn object_store::ObjectStore>;
+}
+
+impl ObjectStoreMetricsExt for Arc<dyn object_store::ObjectStore> {
+    fn metered(self, base: String) -> Arc<dyn object_store::ObjectStore> {
+        Arc::new(MeteredObjectStore { target: self, base })
+    }
+}
+
+// --- Layer 2: HTTP-level throttle metrics for native cloud stores ---
+
+#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
+mod http {
+    use super::*;
+    use object_store::client::{
+        ClientOptions, HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse,
+        HttpService, ReqwestConnector,
+    };
+
+    /// An [`HttpConnector`] that records throttle and retryable responses
+    /// observed by the underlying HTTP client. Install it on the S3 / GCS /
+    /// Azure builders via `with_http_connector`.
+    #[derive(Debug)]
+    pub struct MeteringHttpConnector {
+        base: String,
+        inner: ReqwestConnector,
+    }
+
+    impl MeteringHttpConnector {
+        pub fn new(base: String) -> Self {
+            Self {
+                base,
+                inner: ReqwestConnector::default(),
+            }
+        }
+    }
+
+    impl HttpConnector for MeteringHttpConnector {
+        fn connect(&self, options: &ClientOptions) -> object_store::Result<HttpClient> {
+            let client = self.inner.connect(options)?;
+            Ok(HttpClient::new(MeteringHttpService {
+                base: self.base.clone(),
+                inner: client,
+            }))
+        }
+    }
+
+    #[derive(Debug)]
+    struct MeteringHttpService {
+        base: String,
+        inner: HttpClient,
+    }
+
+    #[async_trait::async_trait]
+    impl HttpService for MeteringHttpService {
+        async fn call(&self, req: HttpRequest) -> Result<HttpResponse, HttpError> {
+            let response = self.inner.execute(req).await?;
+            let status = response.status().as_u16();
+            // Each attempt that object_store may retry is recorded with its
+            // numeric status. Throttles (429 / 503) are a distinct, narrower
+            // signal than the broader set of retryable responses, so they get
+            // their own counter. 409 (conflict) is intentionally excluded from
+            // the retryable set so commit conflicts are not counted as retries.
+            let is_throttle = status == 429 || status == 503;
+            let is_retryable = status == 429 || status == 408 || (500..600).contains(&status);
+            if is_throttle {
+                metrics::counter!(METRIC_THROTTLE, status_labels(&self.base, status)).increment(1);
+            }
+            if is_retryable {
+                metrics::counter!(METRIC_RETRYABLE, status_labels(&self.base, status)).increment(1);
+            }
+            Ok(response)
+        }
+    }
+
+    /// Build the `status` (+ optional `base`) label set for HTTP-layer metrics,
+    /// honoring the configured label mode.
+    fn status_labels(base: &str, status: u16) -> Vec<metrics::Label> {
+        let mut labels = vec![metrics::Label::new("status", status.to_string())];
+        if let Some(base) = scoped_base(base_label_mode(), base) {
+            labels.push(metrics::Label::new("base", base));
+        }
+        labels
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+        use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+        use object_store::client::{HttpRequestBody, HttpResponseBody};
+
+        /// A mock [`HttpService`] that always responds with a fixed status code.
+        #[derive(Debug)]
+        struct StaticStatusService {
+            status: u16,
+        }
+
+        #[async_trait::async_trait]
+        impl HttpService for StaticStatusService {
+            async fn call(&self, _req: HttpRequest) -> Result<HttpResponse, HttpError> {
+                Ok(::http::Response::builder()
+                    .status(self.status)
+                    .body(HttpResponseBody::from(Bytes::new()))
+                    .unwrap())
+            }
+        }
+
+        fn request() -> HttpRequest {
+            ::http::Request::builder()
+                .method("GET")
+                .uri("http://example.com/obj")
+                .body(HttpRequestBody::empty())
+                .unwrap()
+        }
+
+        fn metric_count(
+            metrics: &[(metrics::Key, DebugValue)],
+            name: &str,
+            base: &str,
+            status: &str,
+        ) -> u64 {
+            for (key, value) in metrics {
+                if key.name() != name {
+                    continue;
+                }
+                let labels: std::collections::HashMap<&str, &str> =
+                    key.labels().map(|l| (l.key(), l.value())).collect();
+                if labels.get("base") == Some(&base)
+                    && labels.get("status") == Some(&status)
+                    && let DebugValue::Counter(v) = value
+                {
+                    return *v;
+                }
+            }
+            0
+        }
+
+        #[test]
+        fn test_throttle_and_retryable_responses_counted_by_status() {
+            let recorder = DebuggingRecorder::new();
+            let snapshotter = recorder.snapshotter();
+            metrics::with_local_recorder(&recorder, || {
+                let rt = tokio::runtime::Builder::new_current_thread()
+                    .build()
+                    .unwrap();
+                rt.block_on(async {
+                    // Each attempt that object_store retries flows through `call`
+                    // again; here we simulate that by issuing several responses.
+                    // The base is baked into the connector, so it labels the
+                    // metric. Bases here have no `$`, so they are unaffected by
+                    // the label mode and this test isolates status handling.
+                    for (base, status) in [
+                        ("s3", 429u16),
+                        ("s3", 503),
+                        ("s3", 503),
+                        ("s3", 500),
+                        ("s3", 408),
+                        ("s3", 409),
+                        ("s3", 200),
+                        ("s3", 404),
+                        ("gs", 429),
+                    ] {
+                        let service = MeteringHttpService {
+                            base: base.into(),
+                            inner: HttpClient::new(StaticStatusService { status }),
+                        };
+                        service.call(request()).await.unwrap();
+                    }
+                });
+            });
+
+            let recorded: Vec<_> = snapshotter
+                .snapshot()
+                .into_vec()
+                .into_iter()
+                .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value))
+                .collect();
+
+            let throttle = |base, status| metric_count(&recorded, METRIC_THROTTLE, base, status);
+            let retryable = |base, status| metric_count(&recorded, METRIC_RETRYABLE, base, status);
+
+            // Throttles are only 429 and 503.
+            assert_eq!(throttle("s3", "429"), 1);
+            assert_eq!(throttle("s3", "503"), 2);
+            assert_eq!(throttle("s3", "500"), 0);
+            assert_eq!(throttle("s3", "408"), 0);
+
+            // Retryable is the broader set: 5xx, 429, 408 (but not 409).
+            assert_eq!(retryable("s3", "429"), 1);
+            assert_eq!(retryable("s3", "503"), 2);
+            assert_eq!(retryable("s3", "500"), 1);
+            assert_eq!(retryable("s3", "408"), 1);
+            // 409 conflict is excluded so commit conflicts are not counted as retries.
+            assert_eq!(retryable("s3", "409"), 0);
+
+            // Success and non-retryable client errors count as neither.
+            assert_eq!(throttle("s3", "200"), 0);
+            assert_eq!(retryable("s3", "404"), 0);
+
+            // The base label is taken from the connector, not shared across stores.
+            assert_eq!(throttle("gs", "429"), 1);
+            assert_eq!(retryable("gs", "429"), 1);
+            assert_eq!(throttle("gs", "503"), 0);
+        }
+    }
+}
+
+#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
+pub use http::MeteringHttpConnector;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
+    use object_store::memory::InMemory;
+    use object_store::{ObjectStoreExt, PutPayload};
+
+    fn payload(data: &[u8]) -> PutPayload {
+        PutPayload::from_bytes(Bytes::copy_from_slice(data))
+    }
+
+    fn metered_store() -> Arc<dyn object_store::ObjectStore> {
+        (Arc::new(InMemory::new()) as Arc<dyn object_store::ObjectStore>).metered("memory".into())
+    }
+
+    /// A single materialized snapshot of recorded metrics. It must be taken
+    /// only once: the snapshotter *drains* histogram samples on every
+    /// `snapshot()` call, so a second snapshot would see empty histograms.
+    type Metrics = Vec<(metrics::Key, DebugValue)>;
+
+    /// Materialize the current recorder state. Histogram samples are *drained*
+    /// on each call, so a metric must be read from a single snapshot.
+    fn snapshot(snapshotter: &Snapshotter) -> Metrics {
+        snapshotter
+            .snapshot()
+            .into_vec()
+            .into_iter()
+            .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value))
+            .collect()
+    }
+
+    /// Run an async closure with a thread-local metrics recorder installed and
+    /// return the resulting metrics. Uses a current-thread runtime so all polls
+    /// happen on the thread that holds the recorder guard.
+    fn capture_metrics<F, Fut>(f: F) -> Metrics
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = ()>,
+    {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(f());
+        });
+        snapshot(&snapshotter)
+    }
+
+    fn key_matches(key: &metrics::Key, name: &str, labels: &[(&str, &str)]) -> bool {
+        if key.name() != name {
+            return false;
+        }
+        let actual: std::collections::HashSet<(&str, &str)> =
+            key.labels().map(|l| (l.key(), l.value())).collect();
+        labels.len() == actual.len() && labels.iter().all(|l| actual.contains(l))
+    }
+
+    fn counter_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> u64 {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Counter(v) = value
+            {
+                return *v;
+            }
+        }
+        0
+    }
+
+    fn histogram_count(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> usize {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Histogram(samples) = value
+            {
+                return samples.len();
+            }
+        }
+        0
+    }
+
+    fn gauge_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> f64 {
+        for (key, value) in metrics {
+            if key_matches(key, name, labels)
+                && let DebugValue::Gauge(v) = value
+            {
+                return v.0;
+            }
+        }
+        0.0
+    }
+
+    fn has_metric(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> bool {
+        metrics
+            .iter()
+            .any(|(key, _)| key_matches(key, name, labels))
+    }
+
+    #[test]
+    fn test_parse_base_label_mode() {
+        assert_eq!(parse_base_label_mode(None), BaseLabelMode::Scheme);
+        assert_eq!(parse_base_label_mode(Some("scheme")), BaseLabelMode::Scheme);
+        assert_eq!(parse_base_label_mode(Some("full")), BaseLabelMode::Full);
+        assert_eq!(parse_base_label_mode(Some("off")), BaseLabelMode::Off);
+        assert_eq!(parse_base_label_mode(Some("none")), BaseLabelMode::Off);
+        // Unrecognized values fall back to the conservative default.
+        assert_eq!(parse_base_label_mode(Some("bogus")), BaseLabelMode::Scheme);
+    }
+
+    #[test]
+    fn test_scoped_base() {
+        assert_eq!(
+            scoped_base(BaseLabelMode::Full, "s3$bucket").as_deref(),
+            Some("s3$bucket")
+        );
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "s3$bucket").as_deref(),
+            Some("s3")
+        );
+        // Azure keeps only the scheme even though its prefix carries the account.
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "az$container@account").as_deref(),
+            Some("az")
+        );
+        // A prefix without `$` (e.g. memory/file) is unchanged by scheme mode.
+        assert_eq!(
+            scoped_base(BaseLabelMode::Scheme, "memory").as_deref(),
+            Some("memory")
+        );
+        assert_eq!(scoped_base(BaseLabelMode::Off, "s3$bucket"), None);
+    }
+
+    #[test]
+    fn test_base_label_defaults_to_scheme() {
+        // No env var is set in the test process, so the default `scheme` mode
+        // applies: the full prefix collapses to just the scheme.
+        let recorded = capture_metrics(|| async {
+            let store = (Arc::new(InMemory::new()) as Arc<dyn object_store::ObjectStore>)
+                .metered("s3$my-bucket".into());
+            store.put(&Path::from("a"), payload(b"x")).await.unwrap();
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "put"), ("base", "s3")]
+            ),
+            1
+        );
+        // The full prefix is not emitted as the label under the default mode.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "put"), ("base", "s3$my-bucket")]
+            ),
+            0
+        );
+    }
+
+    #[test]
+    fn test_put_records_count_bytes_and_latency() {
+        let data = b"hello world";
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store
+                .put(&Path::from("a/b.bin"), payload(data))
+                .await
+                .unwrap();
+        });
+
+        let labels = [("operation", "put"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(
+            counter_value(&recorded, METRIC_BYTES, &labels),
+            data.len() as u64
+        );
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_get_records_count_and_bytes() {
+        let data = b"hello world";
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(data)).await.unwrap();
+            // The GET is only recorded once its body has been fully drained.
+            store.get(&path).await.unwrap().bytes().await.unwrap();
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(
+            counter_value(&recorded, METRIC_BYTES, &labels),
+            data.len() as u64
+        );
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_get_not_recorded_until_body_drained() {
+        let data = b"hello world";
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                let path = Path::from("a/b.bin");
+                store.put(&path, payload(data)).await.unwrap();
+
+                // Holding the result without reading the body records nothing yet.
+                let result = store.get(&path).await.unwrap();
+                assert_eq!(
+                    counter_value(&snapshot(&snapshotter), METRIC_REQUESTS, &labels),
+                    0
+                );
+
+                // Draining the body records the request with the true byte count.
+                let bytes = result.bytes().await.unwrap();
+                assert_eq!(bytes.len(), data.len());
+                let recorded = snapshot(&snapshotter);
+                assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+                assert_eq!(
+                    counter_value(&recorded, METRIC_BYTES, &labels),
+                    data.len() as u64
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_head_is_a_separate_operation() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello world")).await.unwrap();
+            store.head(&path).await.unwrap();
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "head"), ("base", "memory")]
+            ),
+            1
+        );
+        // The head call must not be counted as a get.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "get"), ("base", "memory")]
+            ),
+            0
+        );
+        // A HEAD transfers only metadata, so it records no payload bytes.
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_BYTES,
+                &[("operation", "head"), ("base", "memory")]
+            ),
+            0
+        );
+    }
+
+    #[test]
+    fn test_delete_records_one_request_per_call() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}.bin")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            // `delete` drives `delete_stream`; deleting three paths is still one
+            // logical delete request (a single batched request on real stores).
+            let paths =
+                futures::stream::iter((0..3).map(|i| Ok(Path::from(format!("a/{i}.bin"))))).boxed();
+            let _: Vec<_> = store.delete_stream(paths).collect().await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "delete"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_list_counts_one_request_not_per_item() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}.bin")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            let _: Vec<_> = store.list(Some(&Path::from("a"))).collect().await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "list"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_error_is_counted() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            // Getting a missing object errors.
+            let _ = store.get(&Path::from("does/not/exist")).await;
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1);
+        // A failed request is still counted as a request, with latency recorded.
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        // No bytes are transferred on a failed get.
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_get_ranges_sums_part_bytes_and_labels_get() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello world")).await.unwrap();
+            // Two disjoint ranges of 3 bytes each.
+            store.get_ranges(&path, &[2..5, 6..9]).await.unwrap();
+        });
+
+        let labels = [("operation", "get"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 6);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_copy_and_rename_record_zero_bytes() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store
+                .put(&Path::from("a/src"), payload(b"x"))
+                .await
+                .unwrap();
+            store
+                .copy(&Path::from("a/src"), &Path::from("a/copy"))
+                .await
+                .unwrap();
+            store
+                .rename(&Path::from("a/copy"), &Path::from("a/moved"))
+                .await
+                .unwrap();
+        });
+
+        for operation in ["copy", "rename"] {
+            let labels = [("operation", operation), ("base", "memory")];
+            assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+            assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+            assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        }
+    }
+
+    #[test]
+    fn test_list_with_delimiter_records_latency() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            store.put(&Path::from("a/b"), payload(b"x")).await.unwrap();
+            store
+                .list_with_delimiter(Some(&Path::from("a")))
+                .await
+                .unwrap();
+        });
+
+        let labels = [("operation", "list"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_list_with_offset_counts_one_request() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            for i in 0..3 {
+                store
+                    .put(&Path::from(format!("a/{i}")), payload(b"x"))
+                    .await
+                    .unwrap();
+            }
+            let _: Vec<_> = store
+                .list_with_offset(Some(&Path::from("a")), &Path::from("a/0"))
+                .collect()
+                .await;
+        });
+
+        assert_eq!(
+            counter_value(
+                &recorded,
+                METRIC_REQUESTS,
+                &[("operation", "list"), ("base", "memory")]
+            ),
+            1
+        );
+    }
+
+    #[test]
+    fn test_multipart_records_each_part_and_complete() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            upload.put_part(payload(b"hello")).await.unwrap(); // 5 bytes
+            upload.put_part(payload(b"world!!")).await.unwrap(); // 7 bytes
+            upload.complete().await.unwrap();
+        });
+
+        let part_labels = [("operation", "put_part"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &part_labels), 2);
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &part_labels), 12);
+        // Each part records its own latency sample, like a unary put.
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &part_labels), 2);
+        // A successful part upload records no error.
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &part_labels), 0);
+
+        // Completing the upload is its own metered request.
+        let complete_labels = [("operation", "complete_multipart"), ("base", "memory")];
+        assert_eq!(
+            counter_value(&recorded, METRIC_REQUESTS, &complete_labels),
+            1
+        );
+        assert_eq!(
+            histogram_count(&recorded, METRIC_DURATION, &complete_labels),
+            1
+        );
+    }
+
+    #[test]
+    fn test_multipart_abort_is_recorded() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            upload.put_part(payload(b"hello")).await.unwrap();
+            upload.abort().await.unwrap();
+        });
+
+        let labels = [("operation", "abort_multipart"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+    }
+
+    #[test]
+    fn test_multipart_part_error_is_counted() {
+        let recorded = capture_metrics(|| async {
+            let store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap();
+            let _ = upload.put_part(payload(b"data")).await;
+        });
+
+        let labels = [("operation", "put_part"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1);
+        assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1);
+        // A failed part transfers no counted bytes.
+        assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0);
+    }
+
+    #[test]
+    fn test_in_flight_guard_tracks_and_releases() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let g1 = InFlightGuard::new("memory", "get");
+            let g2 = InFlightGuard::new("memory", "get");
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                2.0
+            );
+            drop(g1);
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                1.0
+            );
+            drop(g2);
+            assert_eq!(
+                gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                0.0
+            );
+        });
+    }
+
+    #[test]
+    fn test_in_flight_gauge_is_wired_and_balances() {
+        let recorded = capture_metrics(|| async {
+            let store = metered_store();
+            let path = Path::from("a/b.bin");
+            store.put(&path, payload(b"hello")).await.unwrap();
+            store.get(&path).await.unwrap();
+        });
+
+        // The gauge is emitted for each operation (guard is wired in) and, once
+        // the operation completes, balances back to zero.
+        for operation in ["put", "get"] {
+            let labels = [("operation", operation), ("base", "memory")];
+            assert!(has_metric(&recorded, METRIC_IN_FLIGHT, &labels));
+            assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0);
+        }
+    }
+
+    #[test]
+    fn test_list_stream_holds_in_flight_until_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "list"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                store.put(&Path::from("a/x"), payload(b"x")).await.unwrap();
+
+                // Creating the stream raises the gauge; it stays raised until the
+                // stream is dropped, even before any items are drained.
+                let stream = store.list(Some(&Path::from("a")));
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(stream);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_delete_stream_holds_in_flight_until_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "delete"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let store = metered_store();
+                let locations = futures::stream::iter(vec![Ok(Path::from("a/b"))]).boxed();
+
+                // Like list, creating the delete stream raises the gauge and holds
+                // it until the stream is dropped, before any items are drained.
+                let stream = store.delete_stream(locations);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(stream);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_in_flight_released_when_operation_future_dropped() {
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let labels = [("operation", "get"), ("base", "memory")];
+        metrics::with_local_recorder(&recorder, || {
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .build()
+                .unwrap();
+            rt.block_on(async {
+                let started = Arc::new(tokio::sync::Notify::new());
+                // Never signalled: the request stays blocked mid-flight.
+                let release = Arc::new(tokio::sync::Notify::new());
+                let store = (Arc::new(BlockingStore {
+                    started: started.clone(),
+                    release,
+                }) as Arc<dyn object_store::ObjectStore>)
+                    .metered("memory".into());
+
+                let path = Path::from("a/b");
+                let mut fut = Box::pin(store.get(&path));
+                // Drive the request until it is blocked inside the inner store.
+                tokio::select! {
+                    _ = &mut fut => unreachable!("the blocking store never returns"),
+                    _ = started.notified() => {}
+                }
+
+                // The gauge is raised while the request is outstanding, and
+                // dropping the future before it completes releases it.
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    1.0
+                );
+                drop(fut);
+                assert_eq!(
+                    gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels),
+                    0.0
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn test_streaming_errors_are_counted() {
+        let recorded = capture_metrics(|| async {
+            let delete_store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let _ = delete_store.delete(&Path::from("a/b")).await;
+
+            let list_store = (Arc::new(FailingStreamStore) as Arc<dyn object_store::ObjectStore>)
+                .metered("memory".into());
+            let _: Vec<_> = list_store.list(None).collect().await;
+        });
+
+        // delete_stream counts the item and records an error when it fails.
+        let delete_labels = [("operation", "delete"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &delete_labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &delete_labels), 1);
+
+        // A list request is counted once; a failure while draining records an error.
+        let list_labels = [("operation", "list"), ("base", "memory")];
+        assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &list_labels), 1);
+        assert_eq!(counter_value(&recorded, METRIC_ERRORS, &list_labels), 1);
+    }
+
+    /// A store whose stream-producing operations always yield an error, used to
+    /// exercise the error branches of the streaming wrappers.
+    #[derive(Debug)]
+    struct FailingStreamStore;
+
+    impl std::fmt::Display for FailingStreamStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "FailingStreamStore")
+        }
+    }
+
+    fn test_error() -> object_store::Error {
+        object_store::Error::Generic {
+            store: "FailingStreamStore",
+            source: "injected failure".into(),
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl object_store::ObjectStore for FailingStreamStore {
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            Ok(Box::new(FailingUpload))
+        }
+
+        async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult<GetResult> {
+            unimplemented!()
+        }
+
+        fn delete_stream(
+            &self,
+            _locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            futures::stream::once(async { Err(test_error()) }).boxed()
+        }
+
+        fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            futures::stream::once(async { Err(test_error()) }).boxed()
+        }
+
+        fn list_with_offset(
+            &self,
+            _prefix: Option<&Path>,
+            _offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult<ListResult> {
+            unimplemented!()
+        }
+
+        async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> {
+            unimplemented!()
+        }
+
+        async fn rename_opts(
+            &self,
+            _from: &Path,
+            _to: &Path,
+            _opts: RenameOptions,
+        ) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+
+    /// A [`MultipartUpload`] whose part uploads always fail, used to exercise the
+    /// error branch of the metered `put_part`.
+    #[derive(Debug)]
+    struct FailingUpload;
+
+    #[async_trait::async_trait]
+    impl MultipartUpload for FailingUpload {
+        fn put_part(&mut self, _data: PutPayload) -> UploadPart {
+            async { Err(test_error()) }.boxed()
+        }
+
+        async fn complete(&mut self) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn abort(&mut self) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+
+    /// A store whose `get_opts` blocks after signalling `started`, so a request
+    /// can be observed mid-flight and then dropped before it completes.
+    #[derive(Debug)]
+    struct BlockingStore {
+        started: Arc<tokio::sync::Notify>,
+        release: Arc<tokio::sync::Notify>,
+    }
+
+    impl std::fmt::Display for BlockingStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "BlockingStore")
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl object_store::ObjectStore for BlockingStore {
+        async fn put_opts(
+            &self,
+            _location: &Path,
+            _bytes: PutPayload,
+            _opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            unimplemented!()
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            _location: &Path,
+            _opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            unimplemented!()
+        }
+
+        async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult<GetResult> {
+            self.started.notify_one();
+            self.release.notified().await;
+            unreachable!("release is never signalled in the test")
+        }
+
+        fn delete_stream(
+            &self,
+            _locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            unimplemented!()
+        }
+
+        fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        fn list_with_offset(
+            &self,
+            _prefix: Option<&Path>,
+            _offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            unimplemented!()
+        }
+
+        async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult<ListResult> {
+            unimplemented!()
+        }
+
+        async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> {
+            unimplemented!()
+        }
+
+        async fn rename_opts(
+            &self,
+            _from: &Path,
+            _to: &Path,
+            _opts: RenameOptions,
+        ) -> OSResult<()> {
+            unimplemented!()
+        }
+    }
+}
diff --git a/rust/lance-io/src/object_store/providers.rs b/rust/lance-io/src/object_store/providers.rs
index 20fa251a0c5..5e7963e5e97 100644
--- a/rust/lance-io/src/object_store/providers.rs
+++ b/rust/lance-io/src/object_store/providers.rs
@@ -24,6 +24,8 @@ pub mod aws;
 pub mod azure;
 #[cfg(feature = "gcp")]
 pub mod gcp;
+#[cfg(feature = "goosefs")]
+pub mod goosefs;
 #[cfg(feature = "huggingface")]
 pub mod huggingface;
 pub mod local;
@@ -33,6 +35,8 @@ pub mod oss;
 pub mod shared_memory;
 #[cfg(feature = "tencent")]
 pub mod tencent;
+#[cfg(feature = "tos")]
+pub mod tos;
 
 #[async_trait::async_trait]
 pub trait ObjectStoreProvider: std::fmt::Debug + Sync + Send {
@@ -46,8 +50,13 @@ pub trait ObjectStoreProvider: std::fmt::Debug + Sync + Send {
     /// Meanwhile, for a file store, the path is relative to the filesystem root.
     /// So a URL of `file:///path/to/file` would return `/path/to/file`.
     fn extract_path(&self, url: &Url) -> Result<Path> {
-        Path::parse(url.path())
-            .map_err(|_| Error::invalid_input(format!("Invalid path in URL: {}", url.path())))
+        // url.path() returns a percent-encoded string (per the WHATWG URL spec).
+        // Path::from_url_path decodes it first so the Path internal representation
+        // holds the raw UTF-8 string. This prevents double-encoding when the
+        // object store client later percent-encodes the path for HTTP requests.
+        Path::from_url_path(url.path()).map_err(|e| {
+            Error::invalid_input(format!("Invalid path in URL '{}': {}", url.path(), e))
+        })
     }
 
     /// Calculate the unique prefix that should be used for this object store.
@@ -95,6 +104,7 @@ pub struct ObjectStoreRegistryStats {
 /// - `s3+ddb`: An S3 object store with DynamoDB for metadata.
 /// - `az`: An Azure Blob Storage object store.
 /// - `gs`: A Google Cloud Storage object store.
+/// - `tos`: A Volcengine TOS object store.
 ///
 /// Use [`Self::empty()`] to create an empty registry, with no providers registered.
 ///
@@ -205,6 +215,12 @@ impl ObjectStoreRegistry {
         base_path: Url,
         params: &ObjectStoreParams,
     ) -> Result<Arc<ObjectStore>> {
+        // Base-scoped storage options (`base_<id>.<key>`) are directives for
+        // other registered base paths; resolve them away before building or
+        // caching a store for this location. Params already resolved for a
+        // base contain no scoped entries, so this is a no-op for them.
+        let params = params.scoped_to_base(None);
+        let params = params.as_ref();
         let scheme = base_path.scheme();
         let Some(provider) = self.get_provider(scheme) else {
             return Err(self.scheme_not_found_error(scheme));
@@ -249,6 +265,14 @@ impl ObjectStoreRegistry {
 
         store.inner = store.inner.traced();
 
+        #[cfg(feature = "metrics")]
+        {
+            // Label metrics by the store's unique prefix (e.g. `s3$bucket`,
+            // `az$container@account`) so multiple stores on one cloud differ.
+            use crate::object_store::metrics::ObjectStoreMetricsExt;
+            store.inner = store.inner.metered(cache_path.clone());
+        }
+
         if let Some(wrapper) = &params.object_store_wrapper {
             store.inner = wrapper.wrap(&cache_path, store.inner);
         }
@@ -324,12 +348,16 @@ impl Default for ObjectStoreRegistry {
         }
         #[cfg(feature = "gcp")]
         providers.insert("gs".into(), Arc::new(gcp::GcsStoreProvider));
+        #[cfg(feature = "goosefs")]
+        providers.insert("goosefs".into(), Arc::new(goosefs::GooseFsStoreProvider));
         #[cfg(feature = "oss")]
         providers.insert("oss".into(), Arc::new(oss::OssStoreProvider));
         #[cfg(feature = "tencent")]
         providers.insert("cos".into(), Arc::new(tencent::TencentStoreProvider));
         #[cfg(feature = "huggingface")]
         providers.insert("hf".into(), Arc::new(huggingface::HuggingfaceStoreProvider));
+        #[cfg(feature = "tos")]
+        providers.insert("tos".into(), Arc::new(tos::TosStoreProvider));
         Self {
             providers: RwLock::new(providers),
             active_stores: RwLock::new(HashMap::new()),
@@ -380,6 +408,36 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_get_store_resolves_base_scoped_options() {
+        use crate::object_store::StorageOptionsAccessor;
+
+        let registry = ObjectStoreRegistry::default();
+        let url = Url::parse("memory://test").unwrap();
+
+        let with_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("shared".to_string(), "value".to_string()),
+                    ("base_1.account_key".to_string(), "base1-key".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+        let without_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([("shared".to_string(), "value".to_string())]),
+            ))),
+            ..Default::default()
+        };
+
+        // Base-scoped entries are resolved away before the store is built and
+        // cached, so params with and without them yield the same cached store.
+        let store_scoped = registry.get_store(url.clone(), &with_scoped).await.unwrap();
+        let store_plain = registry.get_store(url, &without_scoped).await.unwrap();
+        assert!(Arc::ptr_eq(&store_scoped, &store_plain));
+    }
+
     #[test]
     fn test_calculate_object_store_scheme_not_found() {
         let registry = ObjectStoreRegistry::empty();
diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs
index a1549120c50..8cad196f087 100644
--- a/rust/lance-io/src/object_store/providers/aws.rs
+++ b/rust/lance-io/src/object_store/providers/aws.rs
@@ -73,6 +73,12 @@ impl AwsStoreProvider {
             s3_storage_options.insert(AmazonS3ConfigKey::S3Express, true.to_string());
         }
 
+        // Compute the metrics label before rewriting the url below, so it
+        // matches the prefix the registry uses to key this store.
+        #[cfg(feature = "metrics")]
+        let store_prefix =
+            self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?;
+
         // before creating the OSObjectStore we need to rewrite the url to drop ddb related parts
         base_path.set_scheme("s3").unwrap();
         base_path.set_query(None);
@@ -89,6 +95,13 @@ impl AwsStoreProvider {
             .with_retry(retry_config)
             .with_region(region);
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(store_prefix),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 
@@ -520,7 +533,7 @@ mod tests {
 
         let cases = [
             ("s3://bucket/path/to/file", "path/to/file"),
-            // for non ASCII string tests
+            // for non ASCII string tests: the URL encodes them, extract_path must decode back
             ("s3://bucket/测试path/to/file", "测试path/to/file"),
             ("s3://bucket/path/&to/file", "path/&to/file"),
             ("s3://bucket/path/=to/file", "path/=to/file"),
@@ -533,11 +546,34 @@ mod tests {
         for (uri, expected_path) in cases {
             let url = Url::parse(uri).unwrap();
             let path = provider.extract_path(&url).unwrap();
-            let expected_path = Path::from(expected_path);
+            // extract_path decodes url.path(), so the Path stores the raw (decoded)
+            // string. Path::parse keeps its input verbatim, matching that, whereas
+            // Path::from would percent-encode non-ASCII bytes and not match.
+            let expected_path = Path::parse(expected_path).unwrap();
             assert_eq!(path, expected_path)
         }
     }
 
+    // Regression test for https://github.com/lance-format/lance/issues/6643
+    // extract_path must NOT double-encode paths that contain non-ASCII characters.
+    // url.path() returns a percent-encoded string; we must decode it back to raw
+    // UTF-8 before storing it in a Path, so the object store HTTP client can apply
+    // a single, correct percent-encoding when building the request URL.
+    #[test]
+    fn test_s3_non_ascii_path_no_double_encoding() {
+        let provider = AwsStoreProvider;
+
+        // "s3://bucket/中文路径" → url.path() == "/%E4%B8%AD%E6%96%87%E8%B7%AF%E5%BE%84".
+        // The buggy Path::parse(url.path()) stored "%E4%B8%AD..." verbatim; the S3
+        // client then percent-encodes the '%' again, yielding "%25E4%25B8%25AD...".
+        // With Path::from_url_path the Path stores the decoded UTF-8 instead.
+        let url = Url::parse("s3://bucket/中文路径").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+
+        // The Path must hold the decoded UTF-8, not the percent-encoded form.
+        assert_eq!(path.as_ref(), "中文路径");
+    }
+
     #[test]
     fn test_is_s3_express() {
         let cases = [
diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs
index e61f3f3b364..9de866bff91 100644
--- a/rust/lance-io/src/object_store/providers/azure.rs
+++ b/rust/lance-io/src/object_store/providers/azure.rs
@@ -192,6 +192,15 @@ impl AzureBlobStoreProvider {
             builder = builder.with_credentials(credentials);
         }
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(
+                    self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?,
+                ),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 
diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs
index f7f0a7672ff..35f79b66804 100644
--- a/rust/lance-io/src/object_store/providers/gcp.rs
+++ b/rust/lance-io/src/object_store/providers/gcp.rs
@@ -89,6 +89,15 @@ impl GcsStoreProvider {
             builder = builder.with_credentials(credential_provider);
         }
 
+        #[cfg(feature = "metrics")]
+        {
+            builder = builder.with_http_connector(
+                crate::object_store::metrics::MeteringHttpConnector::new(
+                    self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?,
+                ),
+            );
+        }
+
         Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
     }
 }
diff --git a/rust/lance-io/src/object_store/providers/goosefs.rs b/rust/lance-io/src/object_store/providers/goosefs.rs
new file mode 100644
index 00000000000..5d2a648522b
--- /dev/null
+++ b/rust/lance-io/src/object_store/providers/goosefs.rs
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::GooseFs};
+use url::Url;
+
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+/// Default GooseFS Master gRPC port.
+const DEFAULT_GOOSEFS_PORT: u16 = 9200;
+
+/// GooseFS object store provider.
+///
+/// Uses OpenDAL's GooseFs service to access GooseFS via gRPC.
+/// URL format: `goosefs://host:port/path`
+///
+/// Where:
+/// - `host:port` is the GooseFS Master address (default port: 9200)
+/// - `/path` is the filesystem path within GooseFS
+///
+/// Path handling model (S3-style):
+/// - The OpenDAL `root` is fixed to `/` (or a user-supplied cluster-wide base)
+///   so that a single `Operator` can serve every dataset under the same
+///   master. This keeps the `ObjectStoreRegistry` cache correct: two URLs
+///   like `goosefs://host:9200/a.lance` and `goosefs://host:9200/b.lance`
+///   share one store and each request carries its own object key.
+/// - Path extraction relies on the default [`ObjectStoreProvider::extract_path`]
+///   implementation, which returns the URL path (percent-decoded) as the key
+///   passed to `ObjectStore::get`, `put`, etc. — mirroring how `s3://bucket/k`
+///   yields key `k`.
+///
+/// Supported configuration keys (via `storage_options` or environment variables,
+/// resolved with priority: `storage_options` > env var > URL authority > default):
+///
+/// | storage_options key       | env var                 | purpose                                                                                       |
+/// |---------------------------|-------------------------|-----------------------------------------------------------------------------------------------|
+/// | `goosefs_master_addr`     | `GOOSEFS_MASTER_ADDR`   | Master gRPC address, e.g. `host:9200`. Supports HA: `addr1:port,addr2:port`.                  |
+/// | `goosefs_root`            | `GOOSEFS_ROOT`          | Cluster-wide OpenDAL root shared by all datasets under the same master. Defaults to `/`.      |
+/// | `goosefs_write_type`      | `GOOSEFS_WRITE_TYPE`    | GooseFS write type (e.g. `MUST_CACHE`, `CACHE_THROUGH`, `THROUGH`, `ASYNC_THROUGH`).          |
+/// | `goosefs_block_size`      | `GOOSEFS_BLOCK_SIZE`    | GooseFS block size (bytes). Distinct from Lance's own `block_size`.                           |
+/// | `goosefs_chunk_size`      | `GOOSEFS_CHUNK_SIZE`    | GooseFS chunk size (bytes) used by the client.                                                |
+/// | `goosefs_auth_type`       | `GOOSEFS_AUTH_TYPE`     | Authentication mode: `nosasl` or `simple`.                                                    |
+/// | `goosefs_auth_username`   | `GOOSEFS_AUTH_USERNAME` | Username for `simple` auth mode.                                                              |
+///
+/// Note on `goosefs_root`: it is deliberately cluster-wide (not per-URL) so
+/// that many datasets under the same master share a single cached `Operator`.
+/// A custom root also participates in the `ObjectStoreRegistry` cache prefix,
+/// so stores rooted at different subtrees do not collide.
+#[derive(Default, Debug)]
+pub struct GooseFsStoreProvider;
+
+impl GooseFsStoreProvider {
+    /// Resolve the GooseFS Master address from storage_options, environment, or URL.
+    ///
+    /// Priority:
+    /// 1. `storage_options["goosefs_master_addr"]` (supports HA: "addr1:port,addr2:port")
+    /// 2. `GOOSEFS_MASTER_ADDR` environment variable
+    /// 3. URL authority (host:port from the URL)
+    fn resolve_master_addr(url: &Url, storage_options: &StorageOptions) -> Result<String> {
+        // 1. storage_options
+        if let Some(addr) = storage_options
+            .0
+            .get("goosefs_master_addr")
+            .filter(|v| !v.is_empty())
+        {
+            return Ok(addr.clone());
+        }
+
+        // 2. Environment variable
+        if let Ok(addr) = std::env::var("GOOSEFS_MASTER_ADDR")
+            && !addr.is_empty()
+        {
+            return Ok(addr);
+        }
+
+        // 3. URL authority
+        let host = url.host_str().ok_or_else(|| {
+            Error::invalid_input(
+                "GooseFS URL must contain a master address (host), e.g. goosefs://host:port/path",
+            )
+        })?;
+
+        let port = url.port().unwrap_or(DEFAULT_GOOSEFS_PORT);
+        Ok(format!("{}:{}", host, port))
+    }
+
+    /// Resolve a storage option from storage_options or environment variable.
+    fn resolve_option(
+        storage_options: &StorageOptions,
+        option_key: &str,
+        env_key: &str,
+    ) -> Option<String> {
+        storage_options
+            .0
+            .get(option_key)
+            .cloned()
+            .or_else(|| std::env::var(env_key).ok())
+            .filter(|v| !v.is_empty())
+    }
+
+    /// Resolve the OpenDAL `root` for this Operator. See the file-level docs on
+    /// [`GooseFsStoreProvider`] for the semantics of `goosefs_root`.
+    fn resolve_root(storage_options: &StorageOptions) -> String {
+        Self::resolve_option(storage_options, "goosefs_root", "GOOSEFS_ROOT")
+            .unwrap_or_else(|| "/".to_string())
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for GooseFsStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        // Resolve master address
+        let master_addr = Self::resolve_master_addr(&base_path, &storage_options)?;
+
+        // Resolve a stable cluster-wide root. The URL path is *not* used here
+        // because it varies per dataset; per-request keys are supplied by
+        // `extract_path` instead.
+        let root = Self::resolve_root(&storage_options);
+
+        // Build OpenDAL config map
+        let mut config_map: HashMap<String, String> = HashMap::new();
+        config_map.insert("master_addr".to_string(), master_addr);
+        config_map.insert("root".to_string(), root);
+
+        // Optional: write_type
+        if let Some(wt) =
+            Self::resolve_option(&storage_options, "goosefs_write_type", "GOOSEFS_WRITE_TYPE")
+        {
+            config_map.insert("write_type".to_string(), wt);
+        }
+
+        // Optional: block_size (for GooseFS, not Lance block_size)
+        if let Some(bs) =
+            Self::resolve_option(&storage_options, "goosefs_block_size", "GOOSEFS_BLOCK_SIZE")
+        {
+            config_map.insert("block_size".to_string(), bs);
+        }
+
+        // Optional: chunk_size
+        if let Some(cs) =
+            Self::resolve_option(&storage_options, "goosefs_chunk_size", "GOOSEFS_CHUNK_SIZE")
+        {
+            config_map.insert("chunk_size".to_string(), cs);
+        }
+
+        // Optional: auth_type (nosasl / simple)
+        if let Some(at) =
+            Self::resolve_option(&storage_options, "goosefs_auth_type", "GOOSEFS_AUTH_TYPE")
+        {
+            config_map.insert("auth_type".to_string(), at);
+        }
+
+        // Optional: auth_username (used in SIMPLE auth mode)
+        if let Some(au) = Self::resolve_option(
+            &storage_options,
+            "goosefs_auth_username",
+            "GOOSEFS_AUTH_USERNAME",
+        ) {
+            config_map.insert("auth_username".to_string(), au);
+        }
+
+        // Create OpenDAL Operator with GooseFS service
+        let operator = Operator::from_iter::<GooseFs>(config_map)
+            .map_err(|e| {
+                Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e))
+            })?
+            .finish();
+
+        // Wrap as object_store::ObjectStore via OpendalStore bridge
+        let opendal_store = Arc::new(OpendalStore::new(operator));
+
+        Ok(ObjectStore {
+            scheme: "goosefs".to_string(),
+            inner: opendal_store,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(false),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    // `extract_path` uses the default `ObjectStoreProvider` trait implementation:
+    // it percent-decodes the URL path and returns it as the object key, exactly
+    // like S3 does for `s3://bucket/key`. Overriding it here would only
+    // duplicate that behavior. See the file-level doc comment above for the
+    // full path-handling model.
+
+    /// Calculate the object store prefix used as the registry cache key.
+    ///
+    /// Format: `goosefs$host:port`. Because the OpenDAL root is now cluster-
+    /// wide (not per-URL), all datasets under the same master intentionally
+    /// share the same cached [`ObjectStore`]; the URL path is disambiguated
+    /// by [`Self::extract_path`] on each request. This is analogous to how
+    /// two `s3://bucket/a` and `s3://bucket/b` URLs share one store.
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        // If a custom `goosefs_root` is provided, include it in the prefix so
+        // that stores built with different roots don't accidentally collide.
+        let opts = StorageOptions(storage_options.cloned().unwrap_or_default());
+        let root = Self::resolve_root(&opts);
+        if root == "/" {
+            Ok(format!("{}${}", url.scheme(), url.authority()))
+        } else {
+            Ok(format!("{}${}#{}", url.scheme(), url.authority(), root))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_goosefs_extract_path_basic() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/data/embeddings.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "data/embeddings.lance");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_deep() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://master:9200/a/b/c/d.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "a/b/c/d.lance");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_percent_decoded() {
+        // The URL contains a percent-encoded space; extract_path must decode
+        // it once so the ObjectStore layer does not double-encode later.
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://master:9200/dir/with%20space/f.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "dir/with space/f.lance");
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_default_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "goosefs$10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_with_hostname() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://myhost:9200/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "goosefs$myhost:9200");
+    }
+
+    /// Regression test: two URLs pointing at different datasets under the
+    /// same master must produce the *same* cache prefix so they share one
+    /// Operator, and correctness must come from `extract_path` returning
+    /// distinct keys — never from a per-URL root baked into the prefix.
+    #[test]
+    fn test_prefix_shared_across_datasets_same_master() {
+        let provider = GooseFsStoreProvider;
+        let url_a = Url::parse("goosefs://10.0.0.1:9200/repro/a.lance").unwrap();
+        let url_b = Url::parse("goosefs://10.0.0.1:9200/repro/b.lance").unwrap();
+
+        let pa = provider
+            .calculate_object_store_prefix(&url_a, None)
+            .unwrap();
+        let pb = provider
+            .calculate_object_store_prefix(&url_b, None)
+            .unwrap();
+        assert_eq!(pa, pb, "same master must share one cache prefix");
+
+        // Extracted keys must differ so the shared Operator can route
+        // requests to the correct dataset.
+        assert_ne!(
+            provider.extract_path(&url_a).unwrap(),
+            provider.extract_path(&url_b).unwrap(),
+            "distinct URLs must yield distinct object keys",
+        );
+    }
+
+    /// Different masters must never share a cache entry.
+    #[test]
+    fn test_prefix_isolated_across_masters() {
+        let provider = GooseFsStoreProvider;
+        let u1 = Url::parse("goosefs://host-a:9200/x.lance").unwrap();
+        let u2 = Url::parse("goosefs://host-b:9200/x.lance").unwrap();
+        assert_ne!(
+            provider.calculate_object_store_prefix(&u1, None).unwrap(),
+            provider.calculate_object_store_prefix(&u2, None).unwrap(),
+        );
+    }
+
+    /// A user-supplied `goosefs_root` participates in the cache prefix so
+    /// stores rooted at different subtrees don't collide.
+    #[test]
+    fn test_prefix_includes_custom_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://host:9200/x.lance").unwrap();
+
+        let default_prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        let custom_opts: HashMap<String, String> =
+            HashMap::from([("goosefs_root".to_string(), "/tenant-a".to_string())]);
+        let custom_prefix = provider
+            .calculate_object_store_prefix(&url, Some(&custom_opts))
+            .unwrap();
+
+        assert_eq!(default_prefix, "goosefs$host:9200");
+        assert_eq!(custom_prefix, "goosefs$host:9200#/tenant-a");
+        assert_ne!(default_prefix, custom_prefix);
+    }
+
+    #[test]
+    fn test_resolve_master_addr_from_url() {
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let storage_options = StorageOptions(HashMap::new());
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_resolve_master_addr_default_port() {
+        let url = Url::parse("goosefs://10.0.0.1/data").unwrap();
+        let storage_options = StorageOptions(HashMap::new());
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_resolve_master_addr_from_storage_options() {
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let storage_options = StorageOptions(HashMap::from([(
+            "goosefs_master_addr".to_string(),
+            "10.0.0.2:9200,10.0.0.3:9200".to_string(),
+        )]));
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.2:9200,10.0.0.3:9200");
+    }
+
+    #[test]
+    fn test_resolve_root_defaults_to_slash() {
+        let opts = StorageOptions(HashMap::new());
+        assert_eq!(GooseFsStoreProvider::resolve_root(&opts), "/");
+    }
+
+    #[test]
+    fn test_resolve_root_from_storage_options() {
+        let opts = StorageOptions(HashMap::from([(
+            "goosefs_root".to_string(),
+            "/tenant-a".to_string(),
+        )]));
+        assert_eq!(GooseFsStoreProvider::resolve_root(&opts), "/tenant-a");
+    }
+}
diff --git a/rust/lance-io/src/object_store/providers/huggingface.rs b/rust/lance-io/src/object_store/providers/huggingface.rs
index 4522a6c156e..cfef0440068 100644
--- a/rust/lance-io/src/object_store/providers/huggingface.rs
+++ b/rust/lance-io/src/object_store/providers/huggingface.rs
@@ -223,7 +223,7 @@ impl ObjectStoreProvider for HuggingfaceStoreProvider {
 
     fn extract_path(&self, url: &Url) -> Result<Path> {
         let parsed = parse_hf_url(url)?;
-        Path::parse(&parsed.relative_path).map_err(|_| {
+        Path::from_url_path(&parsed.relative_path).map_err(|_| {
             Error::invalid_input(format!("Invalid path in Huggingface URL: {}", url.path()))
         })
     }
diff --git a/rust/lance-io/src/object_store/providers/local.rs b/rust/lance-io/src/object_store/providers/local.rs
index ac60fc919e9..9f0762916f7 100644
--- a/rust/lance-io/src/object_store/providers/local.rs
+++ b/rust/lance-io/src/object_store/providers/local.rs
@@ -43,7 +43,7 @@ impl ObjectStoreProvider for FileStoreProvider {
             return Ok(path);
         }
 
-        Path::parse(url.path()).map_err(|e| {
+        Path::from_url_path(url.path()).map_err(|e| {
             Error::invalid_input(format!("Failed to parse path '{}': {}", url.path(), e))
         })
     }
diff --git a/rust/lance-io/src/object_store/providers/memory.rs b/rust/lance-io/src/object_store/providers/memory.rs
index 4c68cea6260..dd72edc4627 100644
--- a/rust/lance-io/src/object_store/providers/memory.rs
+++ b/rust/lance-io/src/object_store/providers/memory.rs
@@ -42,6 +42,9 @@ impl ObjectStoreProvider for MemoryStoreProvider {
             output.push_str(domain);
         }
         output.push_str(url.path());
+        // The in-memory store uses the Path directly as a key with no HTTP layer,
+        // so there is no re-encoding step and thus no double-encoding to avoid.
+        // Path::from also tolerates the empty segments that local temp paths embed.
         Ok(Path::from(output))
     }
 
diff --git a/rust/lance-io/src/object_store/providers/tos.rs b/rust/lance-io/src/object_store/providers/tos.rs
new file mode 100644
index 00000000000..923186484c6
--- /dev/null
+++ b/rust/lance-io/src/object_store/providers/tos.rs
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Tos};
+use url::Url;
+
+use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct TosStoreProvider;
+
+impl TosStoreProvider {
+    fn tos_env_options_from_iter<I, K, V>(vars: I) -> HashMap<String, String>
+    where
+        I: IntoIterator<Item = (K, V)>,
+        K: Into<String>,
+        V: Into<String>,
+    {
+        let vars = vars
+            .into_iter()
+            .map(|(key, value)| (key.into(), value.into()))
+            .collect::<Vec<_>>();
+        let mut config_map = HashMap::new();
+
+        for prefix in ["VOLCENGINE_", "TOS_"] {
+            for (key, value) in &vars {
+                if let Some(stripped_key) = key.strip_prefix(prefix) {
+                    config_map.insert(stripped_key.to_ascii_lowercase(), value.clone());
+                }
+            }
+        }
+
+        config_map
+    }
+
+    fn base_tos_options(
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<HashMap<String, String>> {
+        let bucket = base_path
+            .host_str()
+            .ok_or_else(|| Error::invalid_input("TOS URL must contain bucket name"))?
+            .to_string();
+
+        let prefix = base_path.path().trim_start_matches('/').to_string();
+
+        let mut config_map = Self::tos_env_options_from_iter(std::env::vars());
+
+        config_map.extend(storage_options.0.clone());
+
+        config_map.insert("bucket".to_string(), bucket);
+        if prefix.is_empty() {
+            config_map.remove("root");
+        } else {
+            config_map.insert("root".to_string(), "/".to_string());
+        }
+
+        Ok(config_map)
+    }
+
+    /// Normalize TOS storage options, resolving aliases for well-known keys
+    /// while passing through all other options so that OpenDAL can use them.
+    fn normalize_tos_config(options: &HashMap<String, String>) -> Result<HashMap<String, String>> {
+        let mut config_map = options.clone();
+
+        let alias_groups: &[(&str, &[&str])] = &[
+            ("endpoint", &["tos_endpoint"]),
+            ("region", &["tos_region"]),
+            ("access_key_id", &["tos_access_key_id"]),
+            ("secret_access_key", &["tos_secret_access_key"]),
+            ("security_token", &["tos_security_token"]),
+        ];
+
+        for (canonical, aliases) in alias_groups {
+            for alias in *aliases {
+                if let Some(value) = config_map.remove(*alias) {
+                    config_map.insert(canonical.to_string(), value);
+                    break;
+                }
+            }
+        }
+
+        if !config_map.contains_key("endpoint") {
+            return Err(Error::invalid_input(
+                "TOS endpoint is required. Please provide 'tos_endpoint' in storage options or set TOS_ENDPOINT environment variable",
+            ));
+        }
+
+        Ok(config_map)
+    }
+
+    fn build_tos_store(config_map: HashMap<String, String>) -> Result<OpendalStore> {
+        let operator = Operator::from_iter::<Tos>(config_map)
+            .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))?
+            .finish();
+
+        Ok(OpendalStore::new(operator))
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for TosStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        let base_options = Self::base_tos_options(&base_path, &storage_options)?;
+        let accessor = params.get_accessor();
+
+        let inner: Arc<dyn OSObjectStore> =
+            if let Some(accessor) = accessor.filter(|a| a.has_provider()) {
+                Arc::new(
+                    DynamicOpenDalStore::new(
+                        format!("tos:{}", base_path),
+                        base_options,
+                        accessor,
+                        Self::normalize_tos_config,
+                        Self::build_tos_store,
+                    )
+                    .with_protected_keys(["bucket", "root"]),
+                )
+            } else {
+                Arc::new(Self::build_tos_store(Self::normalize_tos_config(
+                    &base_options,
+                )?)?)
+            };
+
+        let mut url = base_path;
+        if !url.path().ends_with('/') {
+            url.set_path(&format!("{}/", url.path()));
+        }
+
+        Ok(ObjectStore {
+            scheme: "tos".to_string(),
+            inner,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(true),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::TosStoreProvider;
+    use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+    use crate::object_store::{ObjectStoreProvider, StorageOptionsAccessor};
+    use url::Url;
+
+    #[test]
+    fn test_tos_store_path() {
+        let provider = TosStoreProvider;
+
+        let url = Url::parse("tos://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[test]
+    fn test_tos_env_options_normalize_supported_prefixes() {
+        let config = TosStoreProvider::tos_env_options_from_iter([
+            ("VOLCENGINE_ENDPOINT", "https://tos-cn-beijing.volces.com"),
+            ("TOS_ACCESS_KEY_ID", "tos-akid"),
+            ("TOS_SECRET_ACCESS_KEY", "tos-secret"),
+        ]);
+
+        assert_eq!(
+            config.get("endpoint").unwrap(),
+            "https://tos-cn-beijing.volces.com"
+        );
+        assert_eq!(config.get("access_key_id").unwrap(), "tos-akid");
+        assert_eq!(config.get("secret_access_key").unwrap(), "tos-secret");
+    }
+
+    #[test]
+    fn test_tos_alias_options_override_canonical_env_options() {
+        let config = TosStoreProvider::normalize_tos_config(&HashMap::from([
+            (
+                "endpoint".to_string(),
+                "https://env.example.com".to_string(),
+            ),
+            (
+                "tos_endpoint".to_string(),
+                "https://user.example.com".to_string(),
+            ),
+            ("region".to_string(), "env-region".to_string()),
+            ("tos_region".to_string(), "user-region".to_string()),
+            ("access_key_id".to_string(), "env-akid".to_string()),
+            ("tos_access_key_id".to_string(), "user-akid".to_string()),
+            ("secret_access_key".to_string(), "env-secret".to_string()),
+            (
+                "tos_secret_access_key".to_string(),
+                "user-secret".to_string(),
+            ),
+            ("security_token".to_string(), "env-token".to_string()),
+            ("tos_security_token".to_string(), "user-token".to_string()),
+            ("bucket".to_string(), "bucket".to_string()),
+        ]))
+        .unwrap();
+
+        assert_eq!(config.get("endpoint").unwrap(), "https://user.example.com");
+        assert_eq!(config.get("region").unwrap(), "user-region");
+        assert_eq!(config.get("access_key_id").unwrap(), "user-akid");
+        assert_eq!(config.get("secret_access_key").unwrap(), "user-secret");
+        assert_eq!(config.get("security_token").unwrap(), "user-token");
+        assert!(!config.contains_key("tos_endpoint"));
+        assert!(!config.contains_key("tos_secret_access_key"));
+        assert!(!config.contains_key("tos_security_token"));
+    }
+
+    #[test]
+    fn test_tos_url_bucket_and_root_are_authoritative() {
+        let storage_options = crate::object_store::StorageOptions(HashMap::from([
+            (
+                "tos_endpoint".to_string(),
+                "https://tos-cn-beijing.volces.com".to_string(),
+            ),
+            ("bucket".to_string(), "storage-options-bucket".to_string()),
+            ("root".to_string(), "/storage-options-root".to_string()),
+        ]));
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket/path").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = TosStoreProvider::normalize_tos_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert_eq!(config.get("root").unwrap(), "/");
+
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = TosStoreProvider::normalize_tos_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert!(!config.contains_key("root"));
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_opendal_tos_store_uses_provider_credentials() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([
+                    (
+                        "tos_endpoint".to_string(),
+                        "https://tos-cn-beijing.volces.com".to_string(),
+                    ),
+                    ("tos_region".to_string(), "cn-beijing".to_string()),
+                    ("tos_access_key_id".to_string(), "akid".to_string()),
+                    ("tos_secret_access_key".to_string(), "secret".to_string()),
+                    ("tos_security_token".to_string(), "token".to_string()),
+                ]),
+            },
+        )));
+
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket/path").unwrap(),
+            &crate::object_store::StorageOptions(HashMap::new()),
+        )
+        .unwrap();
+
+        let store = DynamicOpenDalStore::new(
+            "tos",
+            base_options,
+            accessor,
+            TosStoreProvider::normalize_tos_config,
+            TosStoreProvider::build_tos_store,
+        )
+        .with_protected_keys(["bucket", "root"]);
+
+        let current_store = store
+            .current_store()
+            .await
+            .expect("dynamic OpenDAL TOS store should build");
+
+        assert!(current_store.to_string().contains("Opendal"));
+    }
+}
diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs
index 640e0a26b1e..fa5f4995afd 100644
--- a/rust/lance-io/src/object_store/storage_options.rs
+++ b/rust/lance-io/src/object_store/storage_options.rs
@@ -79,6 +79,15 @@ pub trait StorageOptionsProvider: Send + Sync + fmt::Debug {
     /// If [`EXPIRES_AT_MILLIS_KEY`] is not provided, the options are considered to never expire.
     async fn fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>>;
 
+    /// Fetch fresh storage options, bypassing caches along the chain.
+    ///
+    /// Providers that serve from an upstream cache (e.g. base-scoped wrappers
+    /// reading through a parent accessor) override this to force the upstream
+    /// to re-fetch. Defaults to [`Self::fetch_storage_options`].
+    async fn force_fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        self.fetch_storage_options().await
+    }
+
     /// Return a human-readable unique identifier for this provider instance
     ///
     /// This is used for equality comparison and hashing in the object store registry.
@@ -155,6 +164,104 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider {
     }
 }
 
+/// Prefix marking a storage option as scoped to a single registered base path.
+///
+/// A key of the form `base_<id>.<key>` applies `<key>` only to the base path
+/// with manifest id `<id>`, overriding the shared (unscoped) options for that
+/// base. For example `base_1.account_key = abc` gives the base with id 1 the
+/// option `account_key = abc` while it inherits every unscoped option.
+pub const BASE_SCOPED_OPTION_PREFIX: &str = "base_";
+
+/// Parse a base-scoped storage option key of the form `base_<id>.<key>`.
+///
+/// Returns `Some((base_id, key))` only for keys that match the convention
+/// exactly: the `base_` prefix, an all-digit u32 base id, a `.` separator, and
+/// a non-empty remainder. Any other key (e.g. `base_url`, `base_x.key`,
+/// `base_1.`) is not base-scoped.
+pub fn parse_base_scoped_key(key: &str) -> Option<(u32, &str)> {
+    let rest = key.strip_prefix(BASE_SCOPED_OPTION_PREFIX)?;
+    let (id_str, scoped_key) = rest.split_once('.')?;
+    if scoped_key.is_empty() || id_str.is_empty() || !id_str.bytes().all(|b| b.is_ascii_digit()) {
+        return None;
+    }
+    let id = id_str.parse::<u32>().ok()?;
+    Some((id, scoped_key))
+}
+
+/// Returns true if any key in `options` is base-scoped (`base_<id>.<key>`).
+pub fn has_base_scoped_options(options: &HashMap<String, String>) -> bool {
+    options
+        .keys()
+        .any(|key| parse_base_scoped_key(key).is_some())
+}
+
+/// Resolve the effective storage options for one base path scope.
+///
+/// All base-scoped keys are removed from the result. When `base_id` is
+/// `Some(id)`, entries scoped to that id are overlaid on the unscoped options,
+/// adding or overriding keys. `None` resolves the default scope (the primary
+/// dataset base), which simply drops every base-scoped entry.
+pub fn resolve_base_scoped_options(
+    options: &HashMap<String, String>,
+    base_id: Option<u32>,
+) -> HashMap<String, String> {
+    let mut resolved = HashMap::with_capacity(options.len());
+    let mut overrides = Vec::new();
+    for (key, value) in options {
+        match parse_base_scoped_key(key) {
+            Some((id, scoped_key)) => {
+                if Some(id) == base_id {
+                    overrides.push((scoped_key.to_string(), value.clone()));
+                }
+            }
+            None => {
+                resolved.insert(key.clone(), value.clone());
+            }
+        }
+    }
+    resolved.extend(overrides);
+    resolved
+}
+
+/// A [`StorageOptionsProvider`] that resolves another accessor's options for a
+/// single base path scope.
+///
+/// Fetching through this provider first refreshes the parent accessor when its
+/// options have expired, then resolves the refreshed options for the scope. As
+/// a result, dynamically vended per-base credentials (e.g. a namespace server
+/// returning `base_<id>.<key>` entries in one flat map) stay fresh per base.
+#[derive(Debug)]
+pub struct BaseScopedStorageOptionsProvider {
+    inner: Arc<StorageOptionsAccessor>,
+    base_id: Option<u32>,
+}
+
+impl BaseScopedStorageOptionsProvider {
+    pub fn new(inner: Arc<StorageOptionsAccessor>, base_id: Option<u32>) -> Self {
+        Self { inner, base_id }
+    }
+}
+
+#[async_trait]
+impl StorageOptionsProvider for BaseScopedStorageOptionsProvider {
+    async fn fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        let options = self.inner.get_storage_options().await?;
+        Ok(Some(resolve_base_scoped_options(&options.0, self.base_id)))
+    }
+
+    async fn force_fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        let options = self.inner.refresh_storage_options().await?;
+        Ok(Some(resolve_base_scoped_options(&options.0, self.base_id)))
+    }
+
+    fn provider_id(&self) -> String {
+        match self.base_id {
+            Some(id) => format!("base-scoped[base_id={}]({})", id, self.inner.accessor_id()),
+            None => format!("base-scoped[default]({})", self.inner.accessor_id()),
+        }
+    }
+}
+
 /// Unified access to storage options with automatic caching and refresh
 ///
 /// This struct bundles static storage options with an optional dynamic provider,
@@ -184,6 +291,11 @@ pub struct StorageOptionsAccessor {
 
     /// Duration before expiry to trigger refresh
     refresh_offset: Duration,
+
+    /// True when this accessor was produced by [`Self::scoped_to_base`]; its
+    /// options are already resolved for one base path scope, so scoping again
+    /// is a no-op.
+    scope_resolved: bool,
 }
 
 impl fmt::Debug for StorageOptionsAccessor {
@@ -212,14 +324,33 @@ impl StorageOptionsAccessor {
             .unwrap_or(Duration::from_millis(DEFAULT_REFRESH_OFFSET_MILLIS))
     }
 
+    /// Effective expiration of a raw options map: the minimum of the unscoped
+    /// `expires_at_millis` and every `base_<id>.expires_at_millis` entry.
+    ///
+    /// A flat map may vend per-base credentials that expire before the shared
+    /// ones. Refreshing when the earliest credential is due keeps base-scoped
+    /// accessors (which refresh through this accessor) from re-resolving stale
+    /// per-base credentials out of a still-"valid" cache.
+    fn effective_expires_at_millis(options: &HashMap<String, String>) -> Option<u64> {
+        options
+            .iter()
+            .filter(|(key, _)| {
+                key.as_str() == EXPIRES_AT_MILLIS_KEY
+                    || matches!(
+                        parse_base_scoped_key(key),
+                        Some((_, scoped_key)) if scoped_key == EXPIRES_AT_MILLIS_KEY
+                    )
+            })
+            .filter_map(|(_, value)| value.parse::<u64>().ok())
+            .min()
+    }
+
     /// Create an accessor with only static options (no refresh capability)
     ///
     /// The returned accessor will always return the provided options.
     /// This is useful when credentials don't expire or are managed externally.
     pub fn with_static_options(options: HashMap<String, String>) -> Self {
-        let expires_at_millis = options
-            .get(EXPIRES_AT_MILLIS_KEY)
-            .and_then(|s| s.parse::<u64>().ok());
+        let expires_at_millis = Self::effective_expires_at_millis(&options);
         let refresh_offset = Self::extract_refresh_offset(&options);
 
         Self {
@@ -230,6 +361,7 @@ impl StorageOptionsAccessor {
                 expires_at_millis,
             }))),
             refresh_offset,
+            scope_resolved: false,
         }
     }
 
@@ -247,6 +379,7 @@ impl StorageOptionsAccessor {
             provider: Some(provider),
             cache: Arc::new(RwLock::new(None)),
             refresh_offset: Duration::from_millis(DEFAULT_REFRESH_OFFSET_MILLIS),
+            scope_resolved: false,
         }
     }
 
@@ -263,9 +396,7 @@ impl StorageOptionsAccessor {
         initial_options: HashMap<String, String>,
         provider: Arc<dyn StorageOptionsProvider>,
     ) -> Self {
-        let expires_at_millis = initial_options
-            .get(EXPIRES_AT_MILLIS_KEY)
-            .and_then(|s| s.parse::<u64>().ok());
+        let expires_at_millis = Self::effective_expires_at_millis(&initial_options);
         let refresh_offset = Self::extract_refresh_offset(&initial_options);
 
         Self {
@@ -276,6 +407,7 @@ impl StorageOptionsAccessor {
                 expires_at_millis,
             }))),
             refresh_offset,
+            scope_resolved: false,
         }
     }
 
@@ -306,8 +438,9 @@ impl StorageOptionsAccessor {
     /// Fetch fresh options from the provider and update the cache.
     ///
     /// This bypasses the cache for callers that need to validate provider-vended
-    /// credentials even when initial metadata has no expiration.
-    #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))]
+    /// credentials even when initial metadata has no expiration. The force
+    /// propagates through provider chains (e.g. base-scoped wrappers), so the
+    /// origin provider is re-queried even when intermediate caches are valid.
     pub(crate) async fn refresh_storage_options(&self) -> Result<super::StorageOptions> {
         let Some(provider) = &self.provider else {
             return self.get_storage_options().await;
@@ -318,7 +451,7 @@ impl StorageOptionsAccessor {
             provider.provider_id()
         );
 
-        let storage_options_map = provider.fetch_storage_options().await.map_err(|e| {
+        let storage_options_map = provider.force_fetch_storage_options().await.map_err(|e| {
             Error::io_source(Box::new(std::io::Error::other(format!(
                 "Failed to fetch storage options: {}",
                 e
@@ -336,9 +469,7 @@ impl StorageOptionsAccessor {
             return Ok(super::StorageOptions(HashMap::new()));
         };
 
-        let expires_at_millis = options
-            .get(EXPIRES_AT_MILLIS_KEY)
-            .and_then(|s| s.parse::<u64>().ok());
+        let expires_at_millis = Self::effective_expires_at_millis(&options);
 
         let mut cache = self.cache.write().await;
         *cache = Some(CachedStorageOptions {
@@ -409,9 +540,7 @@ impl StorageOptionsAccessor {
             return Ok(Some(super::StorageOptions(HashMap::new())));
         };
 
-        let expires_at_millis = options
-            .get(EXPIRES_AT_MILLIS_KEY)
-            .and_then(|s| s.parse::<u64>().ok());
+        let expires_at_millis = Self::effective_expires_at_millis(&options);
 
         if let Some(expires_at) = expires_at_millis {
             let now_ms = SystemTime::now()
@@ -493,6 +622,50 @@ impl StorageOptionsAccessor {
         }
     }
 
+    /// Resolve this accessor for a single base path scope.
+    ///
+    /// Storage options may carry base-scoped entries (`base_<id>.<key>`) that
+    /// apply only to one registered base path. The returned accessor resolves
+    /// options for `base_id`: entries scoped to that base overlay the unscoped
+    /// defaults, and all other scoped entries are dropped. `None` resolves the
+    /// default scope used for the primary dataset base.
+    ///
+    /// A static accessor whose options contain no base-scoped entries is
+    /// returned unchanged, preserving accessor identity (and thus object store
+    /// registry cache keys). A provider-backed accessor is always wrapped
+    /// through [`BaseScopedStorageOptionsProvider`] — fetched options may vend
+    /// base-scoped entries at any refresh, even when the initial options carry
+    /// none — so refreshed options are re-resolved for the scope on every
+    /// fetch. Accessors already produced by this method are returned unchanged.
+    pub fn scoped_to_base(self: &Arc<Self>, base_id: Option<u32>) -> Arc<Self> {
+        if self.scope_resolved {
+            return self.clone();
+        }
+        if self.has_provider() {
+            let provider = Arc::new(BaseScopedStorageOptionsProvider::new(self.clone(), base_id));
+            let mut scoped = match self.initial_storage_options() {
+                Some(initial) => Self::with_initial_and_provider(
+                    resolve_base_scoped_options(initial, base_id),
+                    provider,
+                ),
+                None => Self::with_provider(provider),
+            };
+            scoped.scope_resolved = true;
+            Arc::new(scoped)
+        } else {
+            match self.initial_storage_options() {
+                Some(initial) if has_base_scoped_options(initial) => {
+                    let mut scoped =
+                        Self::with_static_options(resolve_base_scoped_options(initial, base_id));
+                    scoped.scope_resolved = true;
+                    Arc::new(scoped)
+                }
+                // Static options never change, so there is nothing to scope.
+                _ => self.clone(),
+            }
+        }
+    }
+
     /// Check if this accessor has a dynamic provider
     pub fn has_provider(&self) -> bool {
         self.provider.is_some()
@@ -748,4 +921,322 @@ mod tests {
         accessor.get_storage_options().await.unwrap();
         assert_eq!(mock_provider.get_call_count().await, 1);
     }
+
+    #[test]
+    fn test_parse_base_scoped_key() {
+        assert_eq!(
+            parse_base_scoped_key("base_1.account_key"),
+            Some((1, "account_key"))
+        );
+        assert_eq!(
+            parse_base_scoped_key("base_12.headers.x-ms-version"),
+            Some((12, "headers.x-ms-version"))
+        );
+        assert_eq!(parse_base_scoped_key("base_0.region"), Some((0, "region")));
+
+        // Not base-scoped keys
+        assert_eq!(parse_base_scoped_key("account_key"), None);
+        assert_eq!(parse_base_scoped_key("base_url"), None);
+        assert_eq!(parse_base_scoped_key("base_hot.account_key"), None);
+        assert_eq!(parse_base_scoped_key("base_1x.account_key"), None);
+        assert_eq!(parse_base_scoped_key("base_+1.account_key"), None);
+        assert_eq!(parse_base_scoped_key("base_.account_key"), None);
+        assert_eq!(parse_base_scoped_key("base_1."), None);
+        assert_eq!(parse_base_scoped_key("base_1"), None);
+        // Overflows u32
+        assert_eq!(parse_base_scoped_key("base_4294967296.key"), None);
+    }
+
+    #[test]
+    fn test_resolve_base_scoped_options() {
+        let options = HashMap::from([
+            ("region".to_string(), "us-east-1".to_string()),
+            ("account_key".to_string(), "shared-key".to_string()),
+            ("base_1.account_key".to_string(), "base1-key".to_string()),
+            ("base_2.account_key".to_string(), "base2-key".to_string()),
+            ("base_2.endpoint".to_string(), "http://b2".to_string()),
+        ]);
+        assert!(has_base_scoped_options(&options));
+
+        let base1 = resolve_base_scoped_options(&options, Some(1));
+        assert_eq!(
+            base1,
+            HashMap::from([
+                ("region".to_string(), "us-east-1".to_string()),
+                ("account_key".to_string(), "base1-key".to_string()),
+            ])
+        );
+
+        let base2 = resolve_base_scoped_options(&options, Some(2));
+        assert_eq!(
+            base2,
+            HashMap::from([
+                ("region".to_string(), "us-east-1".to_string()),
+                ("account_key".to_string(), "base2-key".to_string()),
+                ("endpoint".to_string(), "http://b2".to_string()),
+            ])
+        );
+
+        // A base without scoped entries inherits only the unscoped options
+        let base3 = resolve_base_scoped_options(&options, Some(3));
+        assert_eq!(
+            base3,
+            HashMap::from([
+                ("region".to_string(), "us-east-1".to_string()),
+                ("account_key".to_string(), "shared-key".to_string()),
+            ])
+        );
+
+        // The default scope drops every scoped entry
+        let default = resolve_base_scoped_options(&options, None);
+        assert_eq!(default, base3);
+
+        assert!(!has_base_scoped_options(&HashMap::from([(
+            "account_key".to_string(),
+            "shared-key".to_string()
+        )])));
+    }
+
+    #[tokio::test]
+    async fn test_scoped_to_base_identity_and_idempotency() {
+        // Static accessors without scoped keys are returned unchanged.
+        let accessor = Arc::new(StorageOptionsAccessor::with_static_options(HashMap::from(
+            [("account_key".to_string(), "shared-key".to_string())],
+        )));
+        assert!(Arc::ptr_eq(&accessor.scoped_to_base(Some(1)), &accessor));
+        assert!(Arc::ptr_eq(&accessor.scoped_to_base(None), &accessor));
+
+        // Scoping an already-scoped accessor is a no-op (the registry choke
+        // point re-applies the default scope to every params it sees).
+        let scoped = Arc::new(StorageOptionsAccessor::with_static_options(HashMap::from(
+            [
+                ("account_key".to_string(), "shared-key".to_string()),
+                ("base_1.account_key".to_string(), "base1-key".to_string()),
+            ],
+        )))
+        .scoped_to_base(Some(1));
+        assert!(Arc::ptr_eq(&scoped.scoped_to_base(None), &scoped));
+
+        let provider_scoped = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            MockStorageOptionsProvider::new(None),
+        )))
+        .scoped_to_base(Some(1));
+        assert!(Arc::ptr_eq(
+            &provider_scoped.scoped_to_base(None),
+            &provider_scoped
+        ));
+    }
+
+    #[tokio::test]
+    async fn test_scoped_to_base_provider_only_resolves_vended_options() {
+        MockClock::set_system_time(Duration::from_secs(100_000));
+
+        // No initial options: scoped entries arrive only through the provider.
+        let provider = Arc::new(MockBaseScopedVendingProvider {
+            call_count: Arc::new(RwLock::new(0)),
+            expires_in_millis: 600_000,
+        });
+        let parent = Arc::new(StorageOptionsAccessor::with_provider(provider.clone()));
+
+        let base1 = parent.scoped_to_base(Some(1));
+        assert!(!Arc::ptr_eq(&base1, &parent));
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_1");
+        assert!(!result.0.contains_key("base_1.account_key"));
+
+        let default = parent.scoped_to_base(None);
+        let result = default.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "SHARED_1");
+        assert!(!result.0.contains_key("base_1.account_key"));
+
+        // Both scopes were served from one parent fetch.
+        assert_eq!(*provider.call_count.read().await, 1);
+    }
+
+    #[tokio::test]
+    async fn test_scoped_earlier_base_expiry_refreshes_parent() {
+        MockClock::set_system_time(Duration::from_secs(100_000));
+        let now_ms = MockClock::system_time().as_millis() as u64;
+
+        // Base 1 credentials expire before the shared ones; the parent must
+        // refresh when the earliest credential is due, or the scoped accessor
+        // would keep re-resolving stale base-1 credentials from a still-
+        // "valid" parent cache.
+        let provider = Arc::new(MockBaseScopedVendingProvider {
+            call_count: Arc::new(RwLock::new(0)),
+            expires_in_millis: 600_000,
+        });
+        let initial = HashMap::from([
+            ("account_key".to_string(), "SHARED_0".to_string()),
+            ("base_1.account_key".to_string(), "BASE1_0".to_string()),
+            (
+                EXPIRES_AT_MILLIS_KEY.to_string(),
+                (now_ms + 600_000).to_string(),
+            ),
+            (
+                format!("base_1.{}", EXPIRES_AT_MILLIS_KEY),
+                (now_ms + 120_000).to_string(),
+            ),
+        ]);
+        let parent = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+            initial,
+            provider.clone(),
+        ));
+
+        let base1 = parent.scoped_to_base(Some(1));
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_0");
+        assert_eq!(*provider.call_count.read().await, 0);
+
+        // Past the base-1 expiry but before the shared expiry: the parent's
+        // effective expiry is the earlier one, so the refresh chain fetches
+        // fresh credentials instead of re-serving BASE1_0.
+        MockClock::set_system_time(Duration::from_secs(100_000 + 121));
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_1");
+        assert_eq!(*provider.call_count.read().await, 1);
+    }
+
+    #[tokio::test]
+    async fn test_scoped_to_base_static() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_static_options(HashMap::from(
+            [
+                ("account_key".to_string(), "shared-key".to_string()),
+                ("base_1.account_key".to_string(), "base1-key".to_string()),
+            ],
+        )));
+
+        let base1 = accessor.scoped_to_base(Some(1));
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(
+            result.0,
+            HashMap::from([("account_key".to_string(), "base1-key".to_string())])
+        );
+        assert!(!base1.has_provider());
+
+        let default = accessor.scoped_to_base(None);
+        let result = default.get_storage_options().await.unwrap();
+        assert_eq!(
+            result.0,
+            HashMap::from([("account_key".to_string(), "shared-key".to_string())])
+        );
+
+        // Scoped accessor ids are stable across derivations and distinct per scope
+        assert_eq!(
+            accessor.scoped_to_base(Some(1)).accessor_id(),
+            base1.accessor_id()
+        );
+        assert_ne!(base1.accessor_id(), default.accessor_id());
+        assert_ne!(base1.accessor_id(), accessor.accessor_id());
+    }
+
+    #[derive(Debug)]
+    struct MockBaseScopedVendingProvider {
+        call_count: Arc<RwLock<usize>>,
+        expires_in_millis: u64,
+    }
+
+    #[async_trait]
+    impl StorageOptionsProvider for MockBaseScopedVendingProvider {
+        async fn fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+            let count = {
+                let mut c = self.call_count.write().await;
+                *c += 1;
+                *c
+            };
+            let now_ms = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as u64;
+            Ok(Some(HashMap::from([
+                ("account_key".to_string(), format!("SHARED_{}", count)),
+                ("base_1.account_key".to_string(), format!("BASE1_{}", count)),
+                (
+                    EXPIRES_AT_MILLIS_KEY.to_string(),
+                    (now_ms + self.expires_in_millis).to_string(),
+                ),
+            ])))
+        }
+
+        fn provider_id(&self) -> String {
+            "MockBaseScopedVendingProvider".to_string()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_scoped_to_base_refreshes_through_parent() {
+        MockClock::set_system_time(Duration::from_secs(100_000));
+        let now_ms = MockClock::system_time().as_millis() as u64;
+
+        let provider = Arc::new(MockBaseScopedVendingProvider {
+            call_count: Arc::new(RwLock::new(0)),
+            expires_in_millis: 600_000,
+        });
+        let initial = HashMap::from([
+            ("account_key".to_string(), "SHARED_0".to_string()),
+            ("base_1.account_key".to_string(), "BASE1_0".to_string()),
+            (
+                EXPIRES_AT_MILLIS_KEY.to_string(),
+                (now_ms + 600_000).to_string(),
+            ),
+        ]);
+        let parent = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+            initial,
+            provider.clone(),
+        ));
+
+        let base1 = parent.scoped_to_base(Some(1));
+        let default = parent.scoped_to_base(None);
+        assert!(base1.has_provider());
+
+        // Initial options are resolved per scope without fetching
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_0");
+        assert!(!result.0.contains_key("base_1.account_key"));
+        let result = default.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "SHARED_0");
+        assert_eq!(*provider.call_count.read().await, 0);
+
+        // Expire the vended options; the scoped accessor refreshes through the
+        // parent and re-resolves the refreshed options for its scope.
+        MockClock::set_system_time(Duration::from_secs(100_000 + 601));
+        let result = base1.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_1");
+        assert_eq!(*provider.call_count.read().await, 1);
+
+        // The parent refresh is shared: other scopes see it without refetching
+        let result = default.get_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "SHARED_1");
+        assert_eq!(*provider.call_count.read().await, 1);
+    }
+
+    #[tokio::test]
+    async fn test_scoped_forced_refresh_reaches_origin_provider() {
+        MockClock::set_system_time(Duration::from_secs(100_000));
+        let now_ms = MockClock::system_time().as_millis() as u64;
+
+        let provider = Arc::new(MockBaseScopedVendingProvider {
+            call_count: Arc::new(RwLock::new(0)),
+            expires_in_millis: 600_000,
+        });
+        let initial = HashMap::from([
+            ("account_key".to_string(), "SHARED_0".to_string()),
+            ("base_1.account_key".to_string(), "BASE1_0".to_string()),
+            (
+                EXPIRES_AT_MILLIS_KEY.to_string(),
+                (now_ms + 600_000).to_string(),
+            ),
+        ]);
+        let parent = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+            initial,
+            provider.clone(),
+        ));
+        let base1 = parent.scoped_to_base(Some(1));
+
+        // A forced refresh must reach the origin provider even though both the
+        // scoped and the parent caches are still valid.
+        let result = base1.refresh_storage_options().await.unwrap();
+        assert_eq!(result.0.get("account_key").unwrap(), "BASE1_1");
+        assert_eq!(*provider.call_count.read().await, 1);
+    }
 }
diff --git a/rust/lance-io/src/object_store/throttle.rs b/rust/lance-io/src/object_store/throttle.rs
index dc66f69cf90..bac1d3a538e 100644
--- a/rust/lance-io/src/object_store/throttle.rs
+++ b/rust/lance-io/src/object_store/throttle.rs
@@ -318,7 +318,7 @@ impl AimdThrottleConfig {
 
 struct TokenBucketState {
     tokens: f64,
-    last_refill: std::time::Instant,
+    last_refill: tokio::time::Instant,
     rate: f64,
 }
 
@@ -346,7 +346,7 @@ impl OperationThrottle {
             controller,
             bucket: Mutex::new(TokenBucketState {
                 tokens: burst_capacity,
-                last_refill: std::time::Instant::now(),
+                last_refill: tokio::time::Instant::now(),
                 rate: initial_rate,
             }),
             burst_capacity,
@@ -364,7 +364,7 @@ impl OperationThrottle {
     async fn acquire_token(&self) {
         let sleep_duration = {
             let mut bucket = self.bucket.lock().await;
-            let now = std::time::Instant::now();
+            let now = tokio::time::Instant::now();
             let elapsed = now.duration_since(bucket.last_refill).as_secs_f64();
             bucket.tokens = (bucket.tokens + elapsed * bucket.rate).min(self.burst_capacity);
             bucket.last_refill = now;
@@ -1176,12 +1176,15 @@ mod tests {
     }
 
     fn list_start_throttle_config() -> AimdThrottleConfig {
+        // Use a low rate (10 tokens/s) so that the token-acquisition sleep is
+        // 1/10 = 100 ms — well above the 50 ms timeout used in assertions,
+        // avoiding flakiness from coarse OS timer resolution (e.g. Windows ~16 ms).
         AimdThrottleConfig::default()
             .with_burst_capacity(0)
-            .with_list_aimd(AimdConfig::default().with_initial_rate(50.0))
+            .with_list_aimd(AimdConfig::default().with_initial_rate(10.0))
     }
 
-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
     async fn test_list_acquires_token_before_starting_underlying_stream() {
         let store = Arc::new(CountingListStartStore::default());
         store
@@ -1199,14 +1202,16 @@ mod tests {
 
         let mut stream = throttled.list(Some(&Path::from("prefix")));
         assert_eq!(store.list_calls(), 0);
+        // With rate=10 tokens/s and burst_capacity=0, the token acquisition
+        // sleeps for 100 ms. A 50 ms timeout must expire before that.
         assert!(
-            tokio::time::timeout(std::time::Duration::from_millis(5), stream.next())
+            tokio::time::timeout(std::time::Duration::from_millis(50), stream.next())
                 .await
                 .is_err()
         );
         assert_eq!(store.list_calls(), 0);
 
-        let item = tokio::time::timeout(std::time::Duration::from_millis(100), stream.next())
+        let item = tokio::time::timeout(std::time::Duration::from_millis(300), stream.next())
             .await
             .unwrap()
             .unwrap()
@@ -1215,7 +1220,7 @@ mod tests {
         assert_eq!(store.list_calls(), 1);
     }
 
-    #[tokio::test]
+    #[tokio::test(start_paused = true)]
     async fn test_list_with_offset_acquires_token_before_starting_underlying_stream() {
         let store = Arc::new(CountingListStartStore::default());
         store
@@ -1231,14 +1236,16 @@ mod tests {
         let mut stream =
             throttled.list_with_offset(Some(&Path::from("prefix")), &Path::from("prefix/a"));
         assert_eq!(store.offset_calls(), 0);
+        // With rate=10 tokens/s and burst_capacity=0, the token acquisition
+        // sleeps for 100 ms. A 50 ms timeout must expire before that.
         assert!(
-            tokio::time::timeout(std::time::Duration::from_millis(5), stream.next())
+            tokio::time::timeout(std::time::Duration::from_millis(50), stream.next())
                 .await
                 .is_err()
         );
         assert_eq!(store.offset_calls(), 0);
 
-        let item = tokio::time::timeout(std::time::Duration::from_millis(100), stream.next())
+        let item = tokio::time::timeout(std::time::Duration::from_millis(300), stream.next())
             .await
             .unwrap()
             .unwrap()
diff --git a/rust/lance-io/src/object_writer.rs b/rust/lance-io/src/object_writer.rs
index 4b9bb901446..6dd6c4d992f 100644
--- a/rust/lance-io/src/object_writer.rs
+++ b/rust/lance-io/src/object_writer.rs
@@ -47,8 +47,12 @@ fn max_conn_reset_retries() -> u16 {
     })
 }
 
-/// Maximum part size in GCS and S3: 5GB.
-const MAX_UPLOAD_PART_SIZE: usize = 1024 * 1024 * 1024 * 5;
+/// Maximum body size for a single S3 PUT: strictly less than 5 GiB.
+/// AWS rejects single-PUT bodies of exactly 5 GiB (= 5 * 1024^3) with
+/// `EntityTooLarge`, so we clamp `LANCE_INITIAL_UPLOAD_SIZE` one byte
+/// below that threshold to keep the buffer-fills-to-clamp single-PUT
+/// path safe. See lance#6750 for the related txn-file write fix.
+const MAX_UPLOAD_PART_SIZE: usize = 1024 * 1024 * 1024 * 5 - 1;
 
 /// Clamps a requested upload part size to the valid [5MB, 5GB] range.
 /// Returns the clamped value and whether clamping was necessary.
@@ -388,25 +392,23 @@ impl AsyncWrite for ObjectWriter {
                     let fut = Box::pin(async move { store.put_multipart(path.as_ref()).await });
                     self.state = UploadState::CreatingUpload(fut);
                 }
+                // TODO: Make max concurrency configurable from storage options.
                 UploadState::InProgress {
                     upload,
                     part_idx,
                     futures,
                     ..
-                } => {
-                    // TODO: Make max concurrency configurable from storage options.
-                    if futures.len() < max_upload_parallelism() {
-                        let data = Self::next_part_buffer(
-                            &mut mut_self.buffer,
-                            *part_idx,
-                            mut_self.use_constant_size_upload_parts,
-                        );
-                        futures.spawn(
-                            Self::put_part(upload.as_mut(), data, *part_idx, None)
-                                .instrument(tracing::Span::current()),
-                        );
-                        *part_idx += 1;
-                    }
+                } if futures.len() < max_upload_parallelism() => {
+                    let data = Self::next_part_buffer(
+                        &mut mut_self.buffer,
+                        *part_idx,
+                        mut_self.use_constant_size_upload_parts,
+                    );
+                    futures.spawn(
+                        Self::put_part(upload.as_mut(), data, *part_idx, None)
+                            .instrument(tracing::Span::current()),
+                    );
+                    *part_idx += 1;
                 }
                 _ => {}
             }
@@ -898,4 +900,17 @@ mod tests {
             (MAX_UPLOAD_PART_SIZE, true)
         );
     }
+
+    /// Regression for the foot-gun where `LANCE_INITIAL_UPLOAD_SIZE=5368709120`
+    /// (exactly 5 GiB, Pucheng's setting) caused a single-PUT of 5 GiB on
+    /// shutdown — which S3 rejects with `EntityTooLarge`. After tightening
+    /// `MAX_UPLOAD_PART_SIZE` to 5 GiB - 1, raw 5 GiB must clamp DOWN.
+    #[test]
+    fn clamp_initial_upload_size_at_5gib_clamps_down() {
+        let exactly_5_gib: usize = 5 * 1024 * 1024 * 1024;
+        assert_eq!(
+            clamp_initial_upload_size(exactly_5_gib),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+    }
 }
diff --git a/rust/lance-io/src/scheduler.rs b/rust/lance-io/src/scheduler.rs
index 5993b161497..c8b3a09e17c 100644
--- a/rust/lance-io/src/scheduler.rs
+++ b/rust/lance-io/src/scheduler.rs
@@ -15,6 +15,7 @@ use std::sync::{Arc, Mutex};
 use std::time::Instant;
 use tokio::sync::Notify;
 
+use lance_core::utils::io_stats::IoStatsRecorder;
 use lance_core::utils::parse::str_is_truthy;
 use lance_core::{Error, Result};
 
@@ -28,6 +29,7 @@ mod lite;
 const BACKPRESSURE_MIN: u64 = 5;
 // Don't log backpressure warnings more than once / minute
 const BACKPRESSURE_DEBOUNCE: u64 = 60;
+const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state";
 
 // Global counter of how many IOPS we have issued
 static IOPS_COUNTER: AtomicU64 = AtomicU64::new(0);
@@ -65,6 +67,10 @@ impl PrioritiesInFlight {
         self.in_flight.first().copied().unwrap_or(u128::MAX)
     }
 
+    fn contains(&self, prio: u128) -> bool {
+        self.in_flight.binary_search(&prio).is_ok()
+    }
+
     fn push(&mut self, prio: u128) {
         let pos = match self.in_flight.binary_search(&prio) {
             Ok(pos) => pos,
@@ -78,11 +84,23 @@ impl PrioritiesInFlight {
             self.in_flight.remove(pos);
         }
     }
+
+    fn len(&self) -> usize {
+        self.in_flight.len()
+    }
+
+    fn is_empty(&self) -> bool {
+        self.in_flight.is_empty()
+    }
 }
 
 struct IoQueueState {
+    // The configured number of IOPS that can be issued concurrently.
+    io_capacity: u32,
     // Number of IOPS we can issue concurrently before pausing I/O
     iops_avail: u32,
+    // The configured byte budget for unread I/O.
+    io_buffer_size: u64,
     // Number of bytes we are allowed to buffer in memory before pausing I/O
     //
     // This can dip below 0 due to I/O prioritization
@@ -105,7 +123,9 @@ struct IoQueueState {
 impl IoQueueState {
     fn new(io_capacity: u32, io_buffer_size: u64) -> Self {
         Self {
+            io_capacity,
             iops_avail: io_capacity,
+            io_buffer_size,
             bytes_avail: io_buffer_size as i64,
             pending_requests: BinaryHeap::new(),
             priorities_in_flight: PrioritiesInFlight::new(io_capacity),
@@ -116,6 +136,65 @@ impl IoQueueState {
         }
     }
 
+    fn scheduler_state_event(&self) -> Option<SchedulerStateEvent> {
+        if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) {
+            return None;
+        }
+
+        let pending_bytes = self
+            .pending_requests
+            .iter()
+            .map(IoTask::num_bytes)
+            .sum::<u64>();
+        let head_task = self.pending_requests.peek();
+        let min_in_flight_priority = if self.priorities_in_flight.is_empty() {
+            None
+        } else {
+            Some(self.priorities_in_flight.min_in_flight())
+        };
+        let head_task_priority_bypass = head_task.map(|task| {
+            self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight()
+        });
+        let head_task_blocked_by_iops = head_task.map(|_| self.iops_avail == 0);
+        let head_task_blocked_by_bytes = head_task.map(|task| {
+            let bypasses_bytes = self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight();
+            !bypasses_bytes && task.num_bytes() as i64 > self.bytes_avail
+        });
+        let head_task_can_deliver = head_task.map(|task| self.can_deliver_without_warning(task));
+        let head_task_bytes = head_task.map(IoTask::num_bytes);
+        let (head_task_priority_high, head_task_priority_low) =
+            split_priority(head_task.map(|task| task.priority));
+        let (min_in_flight_priority_high, min_in_flight_priority_low) =
+            split_priority(min_in_flight_priority);
+
+        Some(SchedulerStateEvent {
+            queue_kind: "standard",
+            io_capacity: u64::from(self.io_capacity),
+            iops_available: u64::from(self.iops_avail),
+            active_iops: u64::from(self.io_capacity.saturating_sub(self.iops_avail)),
+            pending_iops: self.pending_requests.len() as u64,
+            pending_bytes,
+            bytes_available: self.bytes_avail,
+            bytes_reserved: self.io_buffer_size as i64 - self.bytes_avail,
+            io_buffer_size_bytes: self.io_buffer_size,
+            priorities_in_flight: self.priorities_in_flight.len() as u64,
+            no_backpressure: self.no_backpressure,
+            head_task_bytes,
+            head_task_priority_high,
+            head_task_priority_low,
+            min_in_flight_priority_high,
+            min_in_flight_priority_low,
+            head_task_can_deliver,
+            head_task_priority_bypass,
+            head_task_blocked_by_iops,
+            head_task_blocked_by_bytes,
+        })
+    }
+
     fn warn_if_needed(&self) {
         let seconds_elapsed = self.start.elapsed().as_secs();
         let last_warn = self.last_warn.load(Ordering::Acquire);
@@ -135,18 +214,33 @@ impl IoQueueState {
     }
 
     fn can_deliver(&self, task: &IoTask) -> bool {
+        let can_deliver = self.can_deliver_without_warning(task);
+        if !can_deliver
+            && self.iops_avail > 0
+            && !(self.no_backpressure
+                || task.bypass_backpressure
+                || task.priority <= self.priorities_in_flight.min_in_flight())
+            && task.num_bytes() as i64 > self.bytes_avail
+        {
+            self.warn_if_needed();
+        }
+        can_deliver
+    }
+
+    fn can_deliver_without_warning(&self, task: &IoTask) -> bool {
         if self.iops_avail == 0 {
             false
         } else if self.no_backpressure
             || task.bypass_backpressure
             || task.priority <= self.priorities_in_flight.min_in_flight()
+            // Chunks from an admitted logical request must keep moving.  A
+            // higher-priority request may be scheduled later and remain
+            // unconsumed while the caller awaits this request.
+            || self.priorities_in_flight.contains(task.priority)
         {
             true
-        } else if task.num_bytes() as i64 > self.bytes_avail {
-            self.warn_if_needed();
-            false
         } else {
-            true
+            task.num_bytes() as i64 <= self.bytes_avail
         }
     }
 
@@ -182,13 +276,15 @@ struct IoQueue {
     state: Mutex<IoQueueState>,
     // Used to signal new I/O requests have arrived that might potentially be runnable
     notify: Notify,
+    stats: IoStats,
 }
 
 impl IoQueue {
-    fn new(io_capacity: u32, io_buffer_size: u64) -> Self {
+    fn new(io_capacity: u32, io_buffer_size: u64, stats: IoStats) -> Self {
         Self {
             state: Mutex::new(IoQueueState::new(io_capacity, io_buffer_size)),
             notify: Notify::new(),
+            stats,
         }
     }
 
@@ -199,9 +295,12 @@ impl IoQueue {
             task.priority >> 64,
             task.priority & 0xFFFFFFFFFFFFFFFF
         );
-        let mut state = self.state.lock().unwrap();
-        state.pending_requests.push(task);
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.pending_requests.push(task);
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
@@ -211,6 +310,9 @@ impl IoQueue {
             {
                 let mut state = self.state.lock().unwrap();
                 if let Some(task) = state.next_task() {
+                    let event = state.scheduler_state_event();
+                    drop(state);
+                    emit_scheduler_state_event(event, &self.stats);
                     return Some(task);
                 }
 
@@ -224,29 +326,39 @@ impl IoQueue {
     }
 
     fn on_iop_complete(&self) {
-        let mut state = self.state.lock().unwrap();
-        state.iops_avail += 1;
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.iops_avail += 1;
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
 
     fn on_bytes_consumed(&self, bytes: u64, priority: u128, num_reqs: usize) {
-        let mut state = self.state.lock().unwrap();
-        state.bytes_avail += bytes as i64;
-        for _ in 0..num_reqs {
-            state.priorities_in_flight.remove(priority);
-        }
-        drop(state);
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            state.bytes_avail += bytes as i64;
+            for _ in 0..num_reqs {
+                state.priorities_in_flight.remove(priority);
+            }
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
 
         self.notify.notify_one();
     }
 
     fn close(&self) {
-        let mut state = self.state.lock().unwrap();
-        state.done_scheduling = true;
-        let pending_requests = std::mem::take(&mut state.pending_requests);
-        drop(state);
+        let (pending_requests, event) = {
+            let mut state = self.state.lock().unwrap();
+            state.done_scheduling = true;
+            let pending_requests = std::mem::take(&mut state.pending_requests);
+            let event = state.scheduler_state_event();
+            (pending_requests, event)
+        };
+        emit_scheduler_state_event(event, &self.stats);
         for request in pending_requests {
             request.cancel();
         }
@@ -268,6 +380,9 @@ struct MutableBatch<F: FnOnce(Response) + Send> {
     err: Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
     // When true, report 0 bytes consumed so the backpressure budget is unaffected
     bypass_backpressure: bool,
+    // Queue the batch's backpressure reservation is refunded to once its response
+    // is delivered or discarded (see `Response`'s `Drop`).
+    io_queue: Arc<IoQueue>,
 }
 
 impl<F: FnOnce(Response) + Send> MutableBatch<F> {
@@ -277,6 +392,7 @@ impl<F: FnOnce(Response) + Send> MutableBatch<F> {
         priority: u128,
         num_reqs: usize,
         bypass_backpressure: bool,
+        io_queue: Arc<IoQueue>,
     ) -> Self {
         Self {
             when_done: Some(when_done),
@@ -286,6 +402,7 @@ impl<F: FnOnce(Response) + Send> MutableBatch<F> {
             num_reqs,
             err: None,
             bypass_backpressure,
+            io_queue,
         }
     }
 }
@@ -307,7 +424,8 @@ impl<F: FnOnce(Response) + Send> Drop for MutableBatch<F> {
         // We don't really care if no one is around to receive it, just let
         // the result go out of scope and get cleaned up
         let response = Response {
-            data: result,
+            data: Some(result),
+            io_queue: self.io_queue.clone(),
             // Report 0 bytes for bypass tasks so the backpressure budget is unaffected
             num_bytes: if self.bypass_backpressure {
                 0
@@ -475,8 +593,25 @@ impl StatsCollector {
             Ordering::Relaxed,
         );
     }
+
+    /// Add already-aggregated counts (e.g. a snapshot captured from another
+    /// scheduler) into these counters.
+    fn add(&self, iops: u64, requests: u64, bytes_read: u64) {
+        self.iops.fetch_add(iops, Ordering::Relaxed);
+        self.requests.fetch_add(requests, Ordering::Relaxed);
+        self.bytes_read.fetch_add(bytes_read, Ordering::Relaxed);
+    }
 }
 
+impl IoStatsRecorder for StatsCollector {
+    fn record_request(&self, request: &[Range<u64>]) {
+        // Inherent methods take precedence in resolution, so this delegates to
+        // the inherent `record_request` above rather than recursing.
+        Self::record_request(self, request)
+    }
+}
+
+#[derive(Debug, Clone, Copy, Default)]
 pub struct ScanStats {
     pub iops: u64,
     pub requests: u64,
@@ -493,6 +628,135 @@ impl ScanStats {
     }
 }
 
+fn split_priority(priority: Option<u128>) -> (Option<u64>, Option<u64>) {
+    priority
+        .map(|priority| ((priority >> 64) as u64, priority as u64))
+        .unzip()
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(super) struct SchedulerStateEvent {
+    pub(super) queue_kind: &'static str,
+    pub(super) io_capacity: u64,
+    pub(super) iops_available: u64,
+    pub(super) active_iops: u64,
+    pub(super) pending_iops: u64,
+    pub(super) pending_bytes: u64,
+    pub(super) bytes_available: i64,
+    pub(super) bytes_reserved: i64,
+    pub(super) io_buffer_size_bytes: u64,
+    pub(super) priorities_in_flight: u64,
+    pub(super) no_backpressure: bool,
+    pub(super) head_task_bytes: Option<u64>,
+    pub(super) head_task_priority_high: Option<u64>,
+    pub(super) head_task_priority_low: Option<u64>,
+    pub(super) min_in_flight_priority_high: Option<u64>,
+    pub(super) min_in_flight_priority_low: Option<u64>,
+    pub(super) head_task_can_deliver: Option<bool>,
+    pub(super) head_task_priority_bypass: Option<bool>,
+    pub(super) head_task_blocked_by_iops: Option<bool>,
+    pub(super) head_task_blocked_by_bytes: Option<bool>,
+}
+
+impl SchedulerStateEvent {
+    fn trace(self, stats: ScanStats) {
+        tracing::event!(
+            target: SCHEDULER_STATE_EVENT_TARGET,
+            tracing::Level::TRACE,
+            queue_kind = self.queue_kind,
+            scheduler_iops = stats.iops,
+            scheduler_requests = stats.requests,
+            scheduler_bytes_read = stats.bytes_read,
+            io_capacity = self.io_capacity,
+            iops_available = self.iops_available,
+            active_iops = self.active_iops,
+            pending_iops = self.pending_iops,
+            pending_bytes = self.pending_bytes,
+            bytes_available = self.bytes_available,
+            bytes_reserved = self.bytes_reserved,
+            io_buffer_size_bytes = self.io_buffer_size_bytes,
+            priorities_in_flight = self.priorities_in_flight,
+            no_backpressure = self.no_backpressure,
+            head_task_bytes_present = self.head_task_bytes.is_some(),
+            head_task_bytes = self.head_task_bytes.unwrap_or_default(),
+            head_task_priority_high_present = self.head_task_priority_high.is_some(),
+            head_task_priority_high = self.head_task_priority_high.unwrap_or_default(),
+            head_task_priority_low_present = self.head_task_priority_low.is_some(),
+            head_task_priority_low = self.head_task_priority_low.unwrap_or_default(),
+            min_in_flight_priority_high_present = self.min_in_flight_priority_high.is_some(),
+            min_in_flight_priority_high = self.min_in_flight_priority_high.unwrap_or_default(),
+            min_in_flight_priority_low_present = self.min_in_flight_priority_low.is_some(),
+            min_in_flight_priority_low = self.min_in_flight_priority_low.unwrap_or_default(),
+            head_task_can_deliver_present = self.head_task_can_deliver.is_some(),
+            head_task_can_deliver = self.head_task_can_deliver.unwrap_or(false),
+            head_task_priority_bypass_present = self.head_task_priority_bypass.is_some(),
+            head_task_priority_bypass = self.head_task_priority_bypass.unwrap_or(false),
+            head_task_blocked_by_iops_present = self.head_task_blocked_by_iops.is_some(),
+            head_task_blocked_by_iops = self.head_task_blocked_by_iops.unwrap_or(false),
+            head_task_blocked_by_bytes_present = self.head_task_blocked_by_bytes.is_some(),
+            head_task_blocked_by_bytes = self.head_task_blocked_by_bytes.unwrap_or(false),
+            "Scheduler state"
+        );
+    }
+}
+
+pub(super) fn emit_scheduler_state_event(event: Option<SchedulerStateEvent>, stats: &IoStats) {
+    if let Some(event) = event {
+        event.trace(stats.snapshot());
+    }
+}
+
+/// A shareable, cloneable handle to a set of cumulative I/O counters.
+///
+/// All clones share the same underlying counters.  This serves two purposes:
+///
+/// 1. It backs each [`ScanScheduler`]'s own running totals.
+/// 2. It can be attached to an individual [`FileScheduler`] (via
+///    [`FileScheduler::with_io_stats`]) as a *secondary* sink, so a caller can
+///    measure the exact bytes/IOPS performed through that file handle for a
+///    bounded scope (e.g. a single query) without disturbing the scheduler's
+///    global totals.  Read the result back with [`IoStats::snapshot`].
+#[derive(Debug, Clone)]
+pub struct IoStats(Arc<StatsCollector>);
+
+impl IoStats {
+    pub fn new() -> Self {
+        Self(Arc::new(StatsCollector::new()))
+    }
+
+    /// Record a single completed request.  `request` holds the byte ranges as
+    /// actually submitted to storage (post coalescing/splitting), so the counts
+    /// reflect physical I/O.
+    pub fn record_request(&self, request: &[Range<u64>]) {
+        self.0.record_request(request);
+    }
+
+    /// Take an immutable snapshot of the current cumulative counters.
+    pub fn snapshot(&self) -> ScanStats {
+        ScanStats::new(self.0.as_ref())
+    }
+
+    /// Return this handle as a type-erased [`IoStatsRecorder`], suitable for
+    /// attaching to a file reader (e.g. `FileReader::with_io_stats`).  The
+    /// returned recorder shares the same underlying counters as `self`.
+    pub fn recorder(&self) -> Arc<dyn IoStatsRecorder> {
+        self.0.clone()
+    }
+
+    /// Add a snapshot of already-aggregated statistics into this sink.  Used to
+    /// fold in I/O measured on a separate scheduler (e.g. the one-time reads
+    /// performed while opening an index).
+    pub fn add_scan_stats(&self, stats: &ScanStats) {
+        self.0.add(stats.iops, stats.requests, stats.bytes_read);
+    }
+}
+
+impl Default for IoStats {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 enum IoQueueType {
     Standard(Arc<IoQueue>),
     Lite(Arc<lite::IoQueue>),
@@ -509,7 +773,7 @@ enum IoQueueType {
 pub struct ScanScheduler {
     object_store: Arc<ObjectStore>,
     io_queue: IoQueueType,
-    stats: Arc<StatsCollector>,
+    stats: IoStats,
 }
 
 impl Debug for ScanScheduler {
@@ -521,12 +785,25 @@ impl Debug for ScanScheduler {
 }
 
 struct Response {
-    data: Result<Vec<Bytes>>,
+    // `Option` so the caller can take the data out while the response (and its
+    // backpressure refund on drop) stays intact.
+    data: Option<Result<Vec<Bytes>>>,
+    io_queue: Arc<IoQueue>,
     priority: u128,
     num_reqs: usize,
     num_bytes: u64,
 }
 
+// Refund the batch's backpressure reservation when the response is dropped, be
+// that on delivery or when a cancelled request's undelivered response is
+// discarded.  This releases the budget even if the caller drops the future early.
+impl Drop for Response {
+    fn drop(&mut self) {
+        self.io_queue
+            .on_bytes_consumed(self.num_bytes, self.priority, self.num_reqs);
+    }
+}
+
 #[derive(Debug, Clone, Copy)]
 pub struct SchedulerConfig {
     /// the # of bytes that can be buffered but not yet requested.
@@ -582,6 +859,7 @@ impl ScanScheduler {
     /// * config - configuration settings for the scheduler
     pub fn new(object_store: Arc<ObjectStore>, config: SchedulerConfig) -> Arc<Self> {
         let io_capacity = object_store.io_parallelism();
+        let stats = IoStats::new();
         let use_lite = config
             .use_lite_scheduler
             .unwrap_or_else(|| object_store.prefers_lite_scheduler());
@@ -589,12 +867,14 @@ impl ScanScheduler {
             let io_queue = Arc::new(lite::IoQueue::new(
                 io_capacity as u64,
                 config.io_buffer_size_bytes,
+                stats.clone(),
             ));
             IoQueueType::Lite(io_queue)
         } else {
             let io_queue = Arc::new(IoQueue::new(
                 io_capacity as u32,
                 config.io_buffer_size_bytes,
+                stats.clone(),
             ));
             let io_queue_clone = io_queue.clone();
             // Best we can do here is fire and forget.  If the I/O loop is still running when the scheduler is
@@ -606,7 +886,7 @@ impl ScanScheduler {
         Arc::new(Self {
             object_store,
             io_queue,
-            stats: Arc::new(StatsCollector::new()),
+            stats,
         })
     }
 
@@ -646,6 +926,7 @@ impl ScanScheduler {
             base_priority,
             max_iop_size,
             bypass_backpressure: false,
+            extra_stats: None,
         })
     }
 
@@ -660,6 +941,28 @@ impl ScanScheduler {
         self.open_file_with_priority(path, 0, file_size_bytes).await
     }
 
+    /// Open a [`FileScheduler`] over an already-open [`Reader`].
+    ///
+    /// Unlike [`Self::open_file`], this skips the path lookup and size probe and
+    /// schedules I/O against `reader` directly. This is useful when the reader
+    /// was produced outside the scheduler's object store (e.g. a spill file
+    /// opened via [`crate::spill::Spill::reader`]), since a bare `Reader`
+    /// cannot otherwise drive a v2 `FileReader` (which needs a scheduler).
+    ///
+    /// Uses a base priority of 0; chain [`FileScheduler::with_priority`] to set
+    /// a different one.
+    pub fn open_reader(self: &Arc<Self>, reader: Arc<dyn Reader>) -> FileScheduler {
+        FileScheduler {
+            reader,
+            block_size: self.object_store.block_size() as u64,
+            root: self.clone(),
+            base_priority: 0,
+            max_iop_size: self.object_store.max_iop_size(),
+            bypass_backpressure: false,
+            extra_stats: None,
+        }
+    }
+
     fn do_submit_request(
         &self,
         reader: Arc<dyn Reader>,
@@ -682,6 +985,7 @@ impl ScanScheduler {
             priority,
             request.len(),
             bypass_backpressure,
+            io_queue.clone(),
         ))));
 
         for (task_idx, iop) in request.into_iter().enumerate() {
@@ -720,14 +1024,11 @@ impl ScanScheduler {
 
         self.do_submit_request(reader, request, tx, priority, io_queue, bypass_backpressure);
 
-        let io_queue_clone = io_queue.clone();
-
-        rx.map(move |wrapped_rsp| {
-            // Right now, it isn't possible for I/O to be cancelled so a cancel error should
-            // not occur
-            let rsp = wrapped_rsp.unwrap();
-            io_queue_clone.on_bytes_consumed(rsp.num_bytes, rsp.priority, rsp.num_reqs);
-            rsp.data
+        rx.map(|wrapped_rsp| {
+            // A cancel error can't occur: the sender always sends before dropping.
+            // The reservation is refunded on `Response` drop, so just take the data.
+            let mut rsp = wrapped_rsp.unwrap();
+            rsp.data.take().unwrap()
         })
     }
 
@@ -791,7 +1092,7 @@ impl ScanScheduler {
     }
 
     pub fn stats(&self) -> ScanStats {
-        ScanStats::new(self.stats.as_ref())
+        self.stats.snapshot()
     }
 
     #[cfg(test)]
@@ -829,6 +1130,10 @@ pub struct FileScheduler {
     base_priority: u64,
     max_iop_size: u64,
     bypass_backpressure: bool,
+    /// Optional secondary statistics sink.  When set, every request submitted
+    /// through this handle is also recorded here, in addition to the
+    /// scheduler's global totals.  Used to measure per-scope I/O.
+    extra_stats: Option<Arc<dyn IoStatsRecorder>>,
 }
 
 fn is_close_together(range1: &Range<u64>, range2: &Range<u64>, block_size: u64) -> bool {
@@ -899,6 +1204,9 @@ impl FileScheduler {
         }
 
         self.root.stats.record_request(&updated_requests);
+        if let Some(extra_stats) = &self.extra_stats {
+            extra_stats.record_request(&updated_requests);
+        }
 
         let bytes_vec_fut = self.root.submit_request(
             self.reader.clone(),
@@ -964,6 +1272,23 @@ impl FileScheduler {
             max_iop_size: self.max_iop_size,
             base_priority: priority,
             bypass_backpressure: self.bypass_backpressure,
+            extra_stats: self.extra_stats.clone(),
+        }
+    }
+
+    /// Returns a copy of this scheduler that additionally records the I/O it
+    /// performs into `stats`, on top of the scheduler's global statistics.
+    ///
+    /// This is the mechanism for measuring exact per-scope (e.g. per-query) I/O:
+    /// attach a recorder here (e.g. via [`IoStats::recorder`]), perform the reads
+    /// through the returned handle, then read the totals back with
+    /// [`IoStats::snapshot`].  The returned handle is cheap to create (a few
+    /// `Arc` clones) and reuses the same underlying reader, so it does not
+    /// re-open the file.
+    pub fn with_io_stats(&self, stats: Arc<dyn IoStatsRecorder>) -> Self {
+        Self {
+            extra_stats: Some(stats),
+            ..self.clone()
         }
     }
 
@@ -1035,6 +1360,47 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_scheduler_state_event_fields() {
+        use tracing_mock::{expect, subscriber};
+
+        let event = expect::event()
+            .with_target(SCHEDULER_STATE_EVENT_TARGET)
+            .at_level(tracing::Level::TRACE)
+            .with_fields(
+                expect::field("queue_kind")
+                    .with_value(&"standard")
+                    .and(expect::field("scheduler_iops").with_value(&7u64))
+                    .and(expect::field("scheduler_requests").with_value(&3u64))
+                    .and(expect::field("scheduler_bytes_read").with_value(&4096u64))
+                    .and(expect::field("io_capacity").with_value(&4u64))
+                    .and(expect::field("pending_iops").with_value(&1u64))
+                    .and(expect::field("bytes_available").with_value(&128i64))
+                    .and(expect::field("head_task_bytes_present").with_value(&true))
+                    .and(expect::field("head_task_bytes").with_value(&1u64))
+                    .and(expect::field("head_task_can_deliver_present").with_value(&true))
+                    .and(expect::field("head_task_can_deliver").with_value(&true)),
+            );
+        let (subscriber, handle) = subscriber::mock().event(event).run_with_handle();
+
+        let stats = IoStats::new();
+        stats.add_scan_stats(&ScanStats {
+            iops: 7,
+            requests: 3,
+            bytes_read: 4096,
+        });
+        let mut state = IoQueueState::new(4, 192);
+        state.iops_avail = 2;
+        state.bytes_avail = 128;
+        state.pending_requests.push(make_task(1, false));
+
+        tracing::subscriber::with_default(subscriber, || {
+            emit_scheduler_state_event(state.scheduler_state_event(), &stats);
+        });
+
+        handle.assert_finished();
+    }
+
     #[test]
     fn test_iotask_ordering() {
         // Bypass tasks must come out of the heap before non-bypass tasks.
@@ -1099,6 +1465,31 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_open_reader_bridge() {
+        let tmp_file = TempObjFile::default();
+
+        let obj_store = Arc::new(ObjectStore::local());
+
+        const DATA_SIZE: u64 = 64 * 1024;
+        let mut some_data = vec![0; DATA_SIZE as usize];
+        rand::rng().fill_bytes(&mut some_data);
+        obj_store.put(&tmp_file, &some_data).await.unwrap();
+
+        let config = SchedulerConfig::default_for_testing();
+        let scheduler = ScanScheduler::new(obj_store.clone(), config);
+
+        // Open a bare Reader ourselves, then bridge it into a FileScheduler.
+        let reader: Arc<dyn Reader> = obj_store.open(&tmp_file).await.unwrap().into();
+        let file_scheduler = scheduler.open_reader(reader);
+
+        let bytes = file_scheduler
+            .submit_request(vec![0..DATA_SIZE], 0)
+            .await
+            .unwrap();
+        assert_eq!(bytes[0], some_data);
+    }
+
     #[tokio::test]
     async fn test_split_coalesce() {
         let tmp_file = TempObjFile::default();
@@ -1183,6 +1574,59 @@ mod tests {
         assert_eq!(11, scheduler.stats().iops);
     }
 
+    #[tokio::test]
+    async fn test_io_stats_sink() {
+        let tmp_file = TempObjFile::default();
+        let obj_store = Arc::new(ObjectStore::local());
+
+        const DATA_SIZE: u64 = 1024 * 1024;
+        let mut some_data = vec![0; DATA_SIZE as usize];
+        rand::rng().fill_bytes(&mut some_data);
+        obj_store.put(&tmp_file, &some_data).await.unwrap();
+
+        let scheduler = ScanScheduler::new(obj_store, SchedulerConfig::default_for_testing());
+
+        // Attach a per-scope sink to one file handle.
+        let sink = IoStats::new();
+        let file_scheduler = scheduler
+            .open_file(&tmp_file, &CachedFileSize::unknown())
+            .await
+            .unwrap()
+            .with_io_stats(sink.recorder());
+
+        // Three reads within 4KiB coalesce into a single physical IOP.  The sink
+        // and the scheduler's global totals must agree exactly, because both are
+        // recorded from the same post-coalescing request.
+        file_scheduler
+            .submit_request(vec![50_000..51_000, 52_000..53_000, 54_000..55_000], 0)
+            .await
+            .unwrap();
+
+        let global = scheduler.stats();
+        let scoped = sink.snapshot();
+        assert_eq!(1, scoped.iops);
+        assert_eq!(1, scoped.requests);
+        // Coalesced range 50_000..55_000 => 5000 physical bytes.
+        assert_eq!(5000, scoped.bytes_read);
+        assert_eq!(global.iops, scoped.iops);
+        assert_eq!(global.requests, scoped.requests);
+        assert_eq!(global.bytes_read, scoped.bytes_read);
+
+        // A sibling handle without the sink: the global totals advance but the
+        // sink stays put, proving per-scope isolation.
+        let other = scheduler
+            .open_file(&tmp_file, &CachedFileSize::unknown())
+            .await
+            .unwrap();
+        other.submit_request(vec![0..1000], 0).await.unwrap();
+
+        let global_after = scheduler.stats();
+        let scoped_after = sink.snapshot();
+        assert_eq!(global.bytes_read + 1000, global_after.bytes_read);
+        assert_eq!(scoped.bytes_read, scoped_after.bytes_read);
+        assert_eq!(scoped.iops, scoped_after.iops);
+    }
+
     #[tokio::test]
     async fn test_priority() {
         let some_path = Path::parse("foo").unwrap();
@@ -1272,6 +1716,136 @@ mod tests {
         assert!(second_fut.await.unwrap().unwrap().len() == 20);
     }
 
+    #[tokio::test]
+    async fn test_standard_scheduler_state_tracks_queue_state() {
+        let some_path = Path::parse("foo").unwrap();
+        let base_store = Arc::new(InMemory::new());
+        base_store
+            .put(&some_path, vec![0; 1000].into())
+            .await
+            .unwrap();
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+        let mut obj_store = MockObjectStore::default();
+        let semaphore_copy = semaphore.clone();
+        obj_store
+            .expect_get_opts()
+            .returning(move |location, options| {
+                let semaphore = semaphore.clone();
+                let base_store = base_store.clone();
+                let location = location.clone();
+                async move {
+                    semaphore.acquire().await.unwrap().forget();
+                    base_store.get_opts(&location, options).await
+                }
+                .boxed()
+            });
+        let obj_store = Arc::new(ObjectStore::new(
+            Arc::new(obj_store),
+            Url::parse("mem://").unwrap(),
+            Some(500),
+            None,
+            false,
+            false,
+            1,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+
+        let scheduler = ScanScheduler::new(
+            obj_store,
+            SchedulerConfig {
+                io_buffer_size_bytes: 1024 * 1024,
+                use_lite_scheduler: Some(false),
+            },
+        );
+        let file_scheduler = scheduler
+            .open_file(&Path::parse("foo").unwrap(), &CachedFileSize::new(1000))
+            .await
+            .unwrap();
+
+        let first_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..10, 0),
+        )
+        .boxed();
+        let second_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..20, 100),
+        )
+        .boxed();
+        let third_fut = timeout(
+            Duration::from_secs(10),
+            file_scheduler.submit_single(0..30, 0),
+        )
+        .boxed();
+
+        let io_queue = match &scheduler.io_queue {
+            IoQueueType::Standard(io_queue) => io_queue.clone(),
+            IoQueueType::Lite(_) => unreachable!("test forces the standard scheduler"),
+        };
+        let (
+            io_capacity,
+            iops_available,
+            pending_bytes,
+            bytes_reserved,
+            priorities_in_flight,
+            head_task_bytes,
+            head_task_blocked_by_iops,
+            head_task_blocked_by_bytes,
+        ) = timeout(Duration::from_secs(5), async {
+            loop {
+                let observed = {
+                    let state = io_queue.state.lock().unwrap();
+                    let active_iops = state.io_capacity.saturating_sub(state.iops_avail);
+                    if active_iops == 1 && state.pending_requests.len() == 2 {
+                        let pending_bytes = state
+                            .pending_requests
+                            .iter()
+                            .map(IoTask::num_bytes)
+                            .sum::<u64>();
+                        let head_task = state.pending_requests.peek().unwrap();
+                        let bypasses_bytes = state.no_backpressure
+                            || head_task.bypass_backpressure
+                            || head_task.priority <= state.priorities_in_flight.min_in_flight();
+                        Some((
+                            state.io_capacity,
+                            state.iops_avail,
+                            pending_bytes,
+                            state.io_buffer_size as i64 - state.bytes_avail,
+                            state.priorities_in_flight.len(),
+                            head_task.num_bytes(),
+                            state.iops_avail == 0,
+                            !bypasses_bytes && head_task.num_bytes() as i64 > state.bytes_avail,
+                        ))
+                    } else {
+                        None
+                    }
+                };
+                if let Some(observed) = observed {
+                    break observed;
+                }
+                tokio::task::yield_now().await;
+            }
+        })
+        .await
+        .unwrap();
+
+        assert_eq!(io_capacity, 1);
+        assert_eq!(iops_available, 0);
+        assert_eq!(pending_bytes, 50);
+        assert_eq!(bytes_reserved, 10);
+        assert_eq!(priorities_in_flight, 1);
+        assert_eq!(head_task_bytes, 30);
+        assert!(head_task_blocked_by_iops);
+        assert!(!head_task_blocked_by_bytes);
+
+        semaphore_copy.add_permits(3);
+        assert_eq!(first_fut.await.unwrap().unwrap().len(), 10);
+        assert_eq!(third_fut.await.unwrap().unwrap().len(), 30);
+        assert_eq!(second_fut.await.unwrap().unwrap().len(), 20);
+    }
+
     #[tokio::test(flavor = "multi_thread")]
     async fn test_backpressure() {
         let some_path = Path::parse("foo").unwrap();
@@ -1404,6 +1978,102 @@ mod tests {
         assert_eq!(second_fut.await.unwrap().len(), 10);
     }
 
+    #[derive(Debug)]
+    struct BlockingReader {
+        semaphore: Arc<tokio::sync::Semaphore>,
+        get_range_count: Arc<AtomicU64>,
+        path: Path,
+    }
+
+    impl lance_core::deepsize::DeepSizeOf for BlockingReader {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
+            0
+        }
+    }
+
+    impl Reader for BlockingReader {
+        fn path(&self) -> &Path {
+            &self.path
+        }
+
+        fn block_size(&self) -> usize {
+            4096
+        }
+
+        fn io_parallelism(&self) -> usize {
+            1
+        }
+
+        fn size(&self) -> futures::future::BoxFuture<'_, object_store::Result<usize>> {
+            Box::pin(async { Ok(1_000_000) })
+        }
+
+        fn get_range(
+            &self,
+            range: Range<usize>,
+        ) -> futures::future::BoxFuture<'static, object_store::Result<Bytes>> {
+            self.get_range_count.fetch_add(1, Ordering::Release);
+            let semaphore = self.semaphore.clone();
+            let num_bytes = range.end - range.start;
+            Box::pin(async move {
+                semaphore.acquire().await.unwrap().forget();
+                Ok(Bytes::from(vec![0u8; num_bytes]))
+            })
+        }
+
+        fn get_all(&self) -> futures::future::BoxFuture<'_, object_store::Result<Bytes>> {
+            Box::pin(async { Ok(Bytes::from(vec![0u8; 1_000_000])) })
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_same_priority_chunks_continue_after_higher_priority_request() {
+        let obj_store = Arc::new(ObjectStore::new(
+            Arc::new(InMemory::new()),
+            Url::parse("mem://").unwrap(),
+            Some(4096),
+            None,
+            false,
+            false,
+            1,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        let scheduler = ScanScheduler::new(
+            obj_store,
+            SchedulerConfig {
+                io_buffer_size_bytes: 10,
+                use_lite_scheduler: Some(false),
+            },
+        );
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+        let reader: Arc<dyn Reader> = Arc::new(BlockingReader {
+            semaphore: semaphore.clone(),
+            get_range_count: Arc::new(AtomicU64::new(0)),
+            path: Path::parse("test").unwrap(),
+        });
+
+        let low_priority =
+            scheduler.submit_request(reader.clone(), vec![0..6, 100..106], 10, false);
+        let high_priority = scheduler.submit_request(reader, vec![200..204], 0, false);
+
+        semaphore.add_permits(3);
+        let low_priority = timeout(Duration::from_secs(5), low_priority)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(
+            low_priority.iter().map(|bytes| bytes.len()).sum::<usize>(),
+            12
+        );
+
+        let high_priority = timeout(Duration::from_secs(5), high_priority)
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(high_priority[0].len(), 4);
+    }
+
     /// A Reader that tracks how many times get_range has been called.
     #[derive(Debug)]
     struct TrackingReader {
@@ -1411,8 +2081,8 @@ mod tests {
         path: Path,
     }
 
-    impl deepsize::DeepSizeOf for TrackingReader {
-        fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    impl lance_core::deepsize::DeepSizeOf for TrackingReader {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
             0
         }
     }
@@ -1686,4 +2356,105 @@ mod tests {
             .unwrap();
         assert_eq!(bytes_dispatched.load(Ordering::Acquire), 30);
     }
+
+    // Against a 100-byte budget: submit fut1 (50 bytes, priority 0), drop it while
+    // its read is still blocked in get_range, then submit fut2 (60 bytes, priority 1).
+    // fut2's priority can't win the priority-bypass, so it needs 60 of the budget --
+    // available only if fut1's dropped reservation was refunded. Returns whether fut2
+    // completed within 2s (false = the reservation leaked and fut2 deadlocked).
+    async fn run_caller_drop_scenario(use_lite_scheduler: bool) -> (bool, Duration) {
+        let obj_store = Arc::new(ObjectStore::new(
+            Arc::new(InMemory::new()),
+            Url::parse("mem://").unwrap(),
+            Some(4096),
+            None,
+            false,
+            false,
+            1,
+            DEFAULT_DOWNLOAD_RETRY_COUNT,
+            None,
+        ));
+        let scheduler = ScanScheduler::new(
+            obj_store,
+            SchedulerConfig {
+                io_buffer_size_bytes: 100,
+                use_lite_scheduler: Some(use_lite_scheduler),
+            },
+        );
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+        let get_range_count = Arc::new(AtomicU64::new(0));
+        let reader: Arc<dyn Reader> = Arc::new(BlockingReader {
+            semaphore: semaphore.clone(),
+            get_range_count: get_range_count.clone(),
+            path: Path::parse("test").unwrap(),
+        });
+
+        // Step 1: reserve 50 of the 100 budget bytes with a read we never consume.
+        // Spawn it so we can cancel the caller-side future while it is still parked
+        // waiting for the (blocked) read to finish.
+        let fut1 = scheduler.submit_request(reader.clone(), vec![0..50], 0, false);
+        let handle = tokio::spawn(async move {
+            let _ = fut1.await;
+        });
+
+        // Wait until the read is genuinely in flight (blocked on the semaphore).
+        // This guarantees the 50-byte reservation has been taken before we drop
+        // the caller, closing the race between the I/O loop and the abort.
+        while get_range_count.load(Ordering::Acquire) == 0 {
+            tokio::time::sleep(Duration::from_millis(1)).await;
+        }
+
+        // Step 2: drop the caller-side future while its `rx` is still pending.
+        handle.abort();
+        let _ = handle.await;
+
+        // Step 3: let the in-flight read finish. The reservation should be refunded
+        // now that the request is done, whether or not the caller is still around.
+        semaphore.add_permits(1);
+        // Give the read time to run to completion so the refund would already have
+        // happened.
+        tokio::time::sleep(Duration::from_millis(50)).await;
+
+        // Step 4: submit the follow-up. Add a permit up front so that, if it *is*
+        // admitted, its own read can complete rather than block on the semaphore.
+        semaphore.add_permits(1);
+        let fut2 = scheduler.submit_request(reader, vec![100..160], 1, false);
+
+        let start = std::time::Instant::now();
+        let outcome = timeout(Duration::from_secs(2), fut2).await;
+        let elapsed = start.elapsed();
+        match outcome {
+            Ok(res) => {
+                assert_eq!(res.unwrap().iter().map(|b| b.len()).sum::<usize>(), 60);
+                (true, elapsed)
+            }
+            Err(_) => (false, elapsed),
+        }
+    }
+
+    /// Dropping a standard-scheduler request future while its read is in flight must
+    /// still refund the backpressure reservation, so a later request that needs the
+    /// budget does not deadlock.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn standard_scheduler_refunds_reservation_on_caller_drop() {
+        let (completed, elapsed) = run_caller_drop_scenario(false).await;
+        assert!(
+            completed,
+            "standard scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \
+             the dropped request's reservation was not refunded"
+        );
+    }
+
+    /// Same guarantee for the lite scheduler: dropping a request future mid-read
+    /// releases its reservation via the `TaskHandle` drop path.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn lite_scheduler_refunds_reservation_on_caller_drop() {
+        let (completed, elapsed) = run_caller_drop_scenario(true).await;
+        assert!(
+            completed,
+            "lite scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \
+             the dropped request's reservation was not refunded"
+        );
+    }
 }
diff --git a/rust/lance-io/src/scheduler/lite.rs b/rust/lance-io/src/scheduler/lite.rs
index 338a4e2a46f..b7a06e11bb7 100644
--- a/rust/lance-io/src/scheduler/lite.rs
+++ b/rust/lance-io/src/scheduler/lite.rs
@@ -33,7 +33,10 @@ use std::{
 use bytes::Bytes;
 use lance_core::{Error, Result};
 
-use super::{BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN};
+use super::{
+    BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN, IoStats, SCHEDULER_STATE_EVENT_TARGET,
+    SchedulerStateEvent, emit_scheduler_state_event,
+};
 
 type RunFn = Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send>> + Send>;
 
@@ -67,6 +70,26 @@ enum TaskState {
     },
 }
 
+impl TaskState {
+    fn backpressure_reservation(&self) -> Option<BackpressureReservation> {
+        match self {
+            Self::Reserved {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Running {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Finished {
+                backpressure_reservation,
+                ..
+            } => Some(*backpressure_reservation),
+            Self::Initial { .. } | Self::Broken => None,
+        }
+    }
+}
+
 /// A custom error type that might have a backpressure reservation
 ///
 /// This is used instead of Lance's standard error type so we can ensure
@@ -85,25 +108,14 @@ impl BrokenTaskError {
     // This will capture any backpressure reservation the task has and put it into the
     // error so we make sure to release it when returning the error.
     fn new(task_state: TaskState, message: String) -> Self {
-        match task_state {
-            TaskState::Reserved {
-                backpressure_reservation,
-                ..
-            }
-            | TaskState::Running {
-                backpressure_reservation,
-                ..
-            }
-            | TaskState::Finished {
-                backpressure_reservation,
-                ..
-            } => Self {
+        match task_state.backpressure_reservation() {
+            None => Self {
                 message,
-                backpressure_reservation: Some(backpressure_reservation),
+                backpressure_reservation: None,
             },
-            TaskState::Broken | TaskState::Initial { .. } => Self {
+            Some(reservation) => Self {
                 message,
-                backpressure_reservation: None,
+                backpressure_reservation: Some(reservation),
             },
         }
     }
@@ -237,6 +249,7 @@ trait BackpressureThrottle: Send {
     /// Unconditionally acquire a zero-cost reservation, tracking only the priority.
     /// Used for bypass tasks that must never be blocked by backpressure.
     fn force_acquire(&mut self, priority: u128) -> BackpressureReservation;
+    fn state(&self) -> BackpressureState;
 }
 
 // We want to allow requests that have a lower priority than any
@@ -262,6 +275,10 @@ impl PrioritiesInFlight {
         self.in_flight.first().copied().unwrap_or(u128::MAX)
     }
 
+    fn contains(&self, prio: u128) -> bool {
+        self.in_flight.binary_search(&prio).is_ok()
+    }
+
     fn push(&mut self, prio: u128) {
         let pos = match self.in_flight.binary_search(&prio) {
             Ok(pos) => pos,
@@ -275,9 +292,22 @@ impl PrioritiesInFlight {
             self.in_flight.remove(pos);
         }
     }
+
+    fn len(&self) -> usize {
+        self.in_flight.len()
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BackpressureState {
+    max_bytes: u64,
+    bytes_available: i64,
+    priorities_in_flight: u64,
+    no_backpressure: bool,
 }
 
 struct SimpleBackpressureThrottle {
+    max_bytes: u64,
     start: Instant,
     last_warn: AtomicU64,
     bytes_available: i64,
@@ -293,6 +323,7 @@ impl SimpleBackpressureThrottle {
             panic!("Max bytes must be less than {}", i64::MAX);
         }
         Self {
+            max_bytes,
             start: Instant::now(),
             last_warn: AtomicU64::new(0),
             bytes_available: max_bytes as i64,
@@ -325,6 +356,10 @@ impl BackpressureThrottle for SimpleBackpressureThrottle {
         if self.no_backpressure
             || self.bytes_available >= num_bytes as i64
             || self.priorities_in_flight.min_in_flight() >= priority
+            // Chunks from an admitted logical request must keep moving.  A
+            // higher-priority request may be scheduled later and remain
+            // unconsumed while the caller awaits this request.
+            || self.priorities_in_flight.contains(priority)
         {
             self.bytes_available -= num_bytes as i64;
             self.priorities_in_flight.push(priority);
@@ -350,6 +385,15 @@ impl BackpressureThrottle for SimpleBackpressureThrottle {
             priority,
         }
     }
+
+    fn state(&self) -> BackpressureState {
+        BackpressureState {
+            max_bytes: self.max_bytes,
+            bytes_available: self.bytes_available,
+            priorities_in_flight: self.priorities_in_flight.len() as u64,
+            no_backpressure: self.no_backpressure,
+        }
+    }
 }
 
 struct TaskEntry {
@@ -419,6 +463,48 @@ impl IoQueueState {
             Ok(())
         }
     }
+
+    fn scheduler_state_event(&self) -> Option<SchedulerStateEvent> {
+        if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) {
+            return None;
+        }
+
+        let backpressure = self.backpressure_throttle.state();
+        let pending_bytes = self
+            .pending_tasks
+            .iter()
+            .filter_map(|entry| self.tasks.get(&entry.task_id))
+            .map(|task| task.num_bytes)
+            .sum::<u64>();
+        let active_iops = self
+            .tasks
+            .values()
+            .filter(|task| matches!(task.state, TaskState::Running { .. }))
+            .count() as u64;
+
+        Some(SchedulerStateEvent {
+            queue_kind: "lite",
+            io_capacity: 0,
+            iops_available: 0,
+            active_iops,
+            pending_iops: self.pending_tasks.len() as u64,
+            pending_bytes,
+            bytes_available: backpressure.bytes_available,
+            bytes_reserved: backpressure.max_bytes as i64 - backpressure.bytes_available,
+            io_buffer_size_bytes: backpressure.max_bytes,
+            priorities_in_flight: backpressure.priorities_in_flight,
+            no_backpressure: backpressure.no_backpressure,
+            head_task_bytes: None,
+            head_task_priority_high: None,
+            head_task_priority_low: None,
+            min_in_flight_priority_high: None,
+            min_in_flight_priority_low: None,
+            head_task_can_deliver: None,
+            head_task_priority_bypass: None,
+            head_task_blocked_by_iops: None,
+            head_task_blocked_by_bytes: None,
+        })
+    }
 }
 
 /// A queue of I/O tasks to be shared between the I/O scheduler and the I/O decoder.
@@ -441,12 +527,14 @@ impl IoQueueState {
 /// day as well)
 pub(super) struct IoQueue {
     state: Arc<Mutex<IoQueueState>>,
+    stats: IoStats,
 }
 
 impl IoQueue {
-    pub fn new(max_concurrency: u64, max_bytes: u64) -> Self {
+    pub fn new(max_concurrency: u64, max_bytes: u64, stats: IoStats) -> Self {
         Self {
             state: Arc::new(Mutex::new(IoQueueState::new(max_concurrency, max_bytes))),
+            stats,
         }
     }
 
@@ -463,6 +551,9 @@ impl IoQueue {
             state.handle_result(task.reserve(reservation))?;
             state.handle_result(task.start())?;
             state.tasks.insert(task_id, task);
+            let event = state.scheduler_state_event();
+            drop(state);
+            emit_scheduler_state_event(event, &self.stats);
             return Ok(());
         }
 
@@ -472,6 +563,9 @@ impl IoQueue {
             reserved: task.is_reserved(),
         });
         state.tasks.insert(task_id, task);
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
         Ok(())
     }
 
@@ -510,34 +604,42 @@ impl IoQueue {
 
     // When a task completes we should check to see if any other tasks are now runnable
     fn on_task_complete(&self, mut state: MutexGuard<IoQueueState>) -> Result<()> {
-        let state_ref = &mut *state;
-        let mut task_result = TaskResult::Ok(());
-        while !state_ref.pending_tasks.is_empty() {
-            // Unwrap safe here since we just checked the queue is not empty
-            let next_task = state_ref.pending_tasks.peek().unwrap();
-            let Some(task) = state_ref.tasks.get_mut(&next_task.task_id) else {
-                log::warn!("Task with id {} was lost", next_task.task_id);
-                continue;
-            };
-            if !task.is_reserved() {
-                let Some(reservation) = state_ref
-                    .backpressure_throttle
-                    .try_acquire(task.num_bytes, task.priority)
-                else {
-                    break;
+        let result = {
+            let state_ref = &mut *state;
+            let mut task_result = TaskResult::Ok(());
+            while !state_ref.pending_tasks.is_empty() {
+                // Unwrap safe here since we just checked the queue is not empty
+                let task_id = state_ref.pending_tasks.peek().unwrap().task_id;
+                let Some(task) = state_ref.tasks.get_mut(&task_id) else {
+                    // The caller dropped this task's handle (see `abandon`); discard the
+                    // stale queue entry instead of spinning on it.
+                    state_ref.pending_tasks.pop();
+                    continue;
                 };
-                if let Err(e) = task.reserve(reservation) {
+                if !task.is_reserved() {
+                    let Some(reservation) = state_ref
+                        .backpressure_throttle
+                        .try_acquire(task.num_bytes, task.priority)
+                    else {
+                        break;
+                    };
+                    if let Err(e) = task.reserve(reservation) {
+                        task_result = Err(e);
+                        break;
+                    }
+                }
+                state_ref.pending_tasks.pop();
+                if let Err(e) = task.start() {
                     task_result = Err(e);
                     break;
                 }
             }
-            state_ref.pending_tasks.pop();
-            if let Err(e) = task.start() {
-                task_result = Err(e);
-                break;
-            }
-        }
-        state_ref.handle_result(task_result)
+            state_ref.handle_result(task_result)
+        };
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
+        result
     }
 
     fn poll(&self, task_id: u64, cx: &mut Context<'_>) -> Poll<Result<Bytes>> {
@@ -565,10 +667,34 @@ impl IoQueue {
     }
 
     pub(super) fn close(&self) {
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            for task in std::mem::take(&mut state.tasks).values_mut() {
+                task.cancel();
+            }
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
+    }
+
+    // Called when a caller drops a task's handle before the task finishes.  Removes
+    // the task and returns any backpressure reservation it holds to the budget, then
+    // re-checks the queue so newly-affordable tasks can start.  Unlike the standard
+    // release path (`poll`), this runs without the task being polled to completion,
+    // so a cancelled read does not leak its reservation.
+    fn abandon(&self, task_id: u64) {
         let mut state = self.state.lock().unwrap();
-        for task in std::mem::take(&mut state.tasks).values_mut() {
-            task.cancel();
+        let Some(task) = state.tasks.remove(&task_id) else {
+            // Already consumed by `poll`; nothing to release.
+            return;
+        };
+
+        if let Some(reservation) = task.state.backpressure_reservation() {
+            state.backpressure_throttle.release(reservation);
         }
+        // Freed budget may make queued tasks runnable; there is no caller to surface
+        // an error to here.
+        let _ = self.on_task_complete(state);
     }
 }
 
@@ -584,6 +710,12 @@ impl Future for TaskHandle {
     }
 }
 
+impl Drop for TaskHandle {
+    fn drop(&mut self) {
+        self.queue.abandon(self.task_id);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -592,7 +724,7 @@ mod tests {
     #[tokio::test]
     async fn test_priority_ordering() {
         // Backpressure budget of 10 bytes: only one 10-byte task runs at a time.
-        let queue = Arc::new(IoQueue::new(128, 10));
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
 
         // Records the priority of each task when its run_fn is invoked (i.e. when
         // the task transitions to Running).
@@ -700,7 +832,7 @@ mod tests {
     async fn test_zero_buffer_bypasses_backpressure() {
         // Budget = 0 sets no_backpressure = true, so all tasks start immediately
         // regardless of how many bytes are "outstanding".
-        let queue = Arc::new(IoQueue::new(128, 0));
+        let queue = Arc::new(IoQueue::new(128, 0, IoStats::default()));
         let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
 
         let make_run_fn =
@@ -742,7 +874,7 @@ mod tests {
     async fn test_bypass_flag_proceeds_past_exhausted_budget() {
         // Budget of 10 bytes. A blocker task fills it. A task with bypass=true starts
         // immediately despite the exhausted budget; a normal task stays queued.
-        let queue = Arc::new(IoQueue::new(128, 10));
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
         let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
 
         let make_run_fn =
@@ -802,4 +934,22 @@ mod tests {
         normal_tx.send(Bytes::from_static(b"done")).unwrap();
         normal.await.unwrap();
     }
+
+    #[test]
+    fn test_same_priority_reservation_continues_after_higher_priority() {
+        let mut throttle = SimpleBackpressureThrottle::new(10, 128);
+
+        let low_priority_first = throttle.try_acquire(6, 10).unwrap();
+        let high_priority = throttle.try_acquire(4, 0).unwrap();
+        let low_priority_next = throttle.try_acquire(6, 10);
+
+        assert!(
+            low_priority_next.is_some(),
+            "chunks from an already admitted logical request should continue"
+        );
+
+        throttle.release(low_priority_first);
+        throttle.release(high_priority);
+        throttle.release(low_priority_next.unwrap());
+    }
 }
diff --git a/rust/lance-io/src/spill.rs b/rust/lance-io/src/spill.rs
new file mode 100644
index 00000000000..16b4c10f144
--- /dev/null
+++ b/rust/lance-io/src/spill.rs
@@ -0,0 +1,543 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Reclaimable scratch storage.
+//!
+//! A [`SpillStore`] hands out scratch space for temporary state that is too
+//! large to keep in memory and is read back later in the same process (for
+//! example, posting lists or shuffle runs accumulated while building an index).
+//! The backing storage is reclaimed automatically when the handle is dropped.
+//!
+//! [`SpillStore::new_spill`] returns a [`Writer`] paired with a [`Spill`]
+//! handle: the writer is the byte sink (feed it to `FileWriter::try_new`, or
+//! write to it directly); the [`Spill`] reads the bytes back (via
+//! [`crate::scheduler::ScanScheduler::open_reader`] for a v2 `FileReader`) and
+//! owns the file's lifetime.
+//!
+//! # Lifecycle
+//!
+//! - **Write-once.** The only way to obtain a writer is `new_spill`, and each
+//!   call allocates a fresh unit of storage, so a single spill cannot be
+//!   written twice — there is no second-writer path to guard against.
+//! - **Write-before-read.** [`Spill::reader`] fails until the writer has been
+//!   shut down, so partially written bytes are never read back.
+//! - **RAII.** Dropping the [`Spill`] deletes the file and releases its bytes
+//!   back to the store's disk budget. The store's temp directory is the
+//!   backstop for anything leaked if a handle is forgotten.
+//!
+//! # Disk cap
+//!
+//! [`LocalSpillStore::with_cap`] enforces a byte budget shared across all live
+//! handles, returning a typed [`lance_core::Error::DiskCapExceeded`] rather than
+//! silently filling the disk. Accounting is reserve-on-write + release-on-drop
+//! (by stat), which is exact for the write-once contract. Two minor
+//! inexactnesses are not engineered around: a write aborted at the cap leaks its
+//! reservation until the store is dropped, and a file whose size cannot be
+//! stat-ed on drop is not released.
+
+use std::io;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+
+use async_trait::async_trait;
+use object_store::path::Path;
+use tokio::io::AsyncWrite;
+
+use lance_core::{Error, Result};
+
+use crate::object_store::ObjectStore;
+use crate::object_writer::WriteResult;
+use crate::traits::{Reader, Writer};
+
+/// A factory for scratch storage.
+///
+/// The trait is object-safe and `Send + Sync` so it can be held behind an
+/// `Arc<dyn SpillStore>` (e.g. inside a `Session`). Implementations need not be
+/// backed by local files (e.g. in-memory buffers, remote object stores).
+#[async_trait]
+pub trait SpillStore: Send + Sync + 'static {
+    /// Allocate a unit of scratch storage.
+    ///
+    /// Returns the byte sink to write it with and a [`Spill`] handle to read it
+    /// back. For a capped store, writes that would exceed the cap fail with
+    /// [`lance_core::Error::DiskCapExceeded`]. The storage is reclaimed when the
+    /// [`Spill`] is dropped.
+    async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)>;
+}
+
+/// The readable half of a spill, and the owner of its backing storage.
+///
+/// Dropping it reclaims the storage. The trait is object-safe so it can be
+/// returned as `Box<dyn Spill>` from [`SpillStore::new_spill`].
+#[async_trait]
+pub trait Spill: Send + Sync {
+    /// Open a reader over the spilled bytes.
+    ///
+    /// Fails until the paired writer has been shut down, since the bytes are not
+    /// complete before then.
+    async fn reader(&self) -> Result<Box<dyn Reader>>;
+}
+
+/// A shared, cloneable byte budget.
+///
+/// Cloning produces another handle to the *same* underlying counter, so a quota
+/// shared across many writers enforces a single combined cap.
+#[derive(Debug, Clone)]
+struct DiskQuota {
+    cap_bytes: u64,
+    used: Arc<Mutex<u64>>,
+}
+
+impl DiskQuota {
+    fn new(cap_bytes: u64) -> Self {
+        Self {
+            cap_bytes,
+            used: Arc::new(Mutex::new(0)),
+        }
+    }
+
+    /// Try to reserve `n` bytes, failing with [`Error::DiskCapExceeded`] if the
+    /// reservation would push total usage past the cap.
+    fn try_reserve(&self, n: u64) -> Result<()> {
+        // The lock is held only for a couple of arithmetic ops and never across
+        // an `.await`, so a std `Mutex` is the simplest correct choice.
+        let mut used = self.used.lock().unwrap();
+        let next = used.saturating_add(n);
+        if next > self.cap_bytes {
+            return Err(Error::disk_cap_exceeded(self.cap_bytes, *used));
+        }
+        *used = next;
+        Ok(())
+    }
+
+    /// Release `n` previously reserved bytes back to the budget.
+    fn release(&self, n: u64) {
+        // Saturating sub keeps a stray double-release from underflowing.
+        let mut used = self.used.lock().unwrap();
+        *used = used.saturating_sub(n);
+    }
+}
+
+/// The byte sink handed out by [`SpillStore::new_spill`].
+///
+/// It optionally reserves a [`DiskQuota`] as bytes are written (keeping cap
+/// enforcement inside the spill store rather than in [`ObjectStore`], and
+/// working for any backend the store opens), and flips a shared `finished` flag
+/// on shutdown so the paired [`Spill`] knows the bytes are complete.
+struct SpillWriter {
+    inner: Box<dyn Writer>,
+    quota: Option<DiskQuota>,
+    finished: Arc<AtomicBool>,
+}
+
+impl AsyncWrite for SpillWriter {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        let this = self.get_mut();
+        let Some(quota) = &this.quota else {
+            return Pin::new(this.inner.as_mut()).poll_write(cx, buf);
+        };
+        // Reserve up-front for the bytes we intend to write, then release the
+        // remainder the inner writer did not accept so the reservation tracks
+        // bytes actually buffered (and, for a write-once file, the file size).
+        if let Err(e) = quota.try_reserve(buf.len() as u64) {
+            return Poll::Ready(Err(io::Error::other(e)));
+        }
+        let poll = Pin::new(this.inner.as_mut()).poll_write(cx, buf);
+        match &poll {
+            Poll::Ready(Ok(n)) => quota.release((buf.len() - *n) as u64),
+            _ => quota.release(buf.len() as u64),
+        }
+        poll
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        Pin::new(self.get_mut().inner.as_mut()).poll_flush(cx)
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        let this = self.get_mut();
+        let poll = Pin::new(this.inner.as_mut()).poll_shutdown(cx);
+        if matches!(poll, Poll::Ready(Ok(()))) {
+            // Mirrors `Writer::shutdown` so the flag is set whichever shutdown
+            // surface the consumer drives (`AsyncWrite` vs the `Writer` trait).
+            this.finished.store(true, Ordering::Relaxed);
+        }
+        poll
+    }
+}
+
+#[async_trait]
+impl Writer for SpillWriter {
+    async fn tell(&mut self) -> Result<usize> {
+        self.inner.tell().await
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        let result = self.inner.shutdown().await?;
+        // Signal the paired `Spill` that the bytes are now complete. `Relaxed`
+        // is sufficient: this only flags that shutdown happened; the file
+        // contents are synchronized through the filesystem, not this flag.
+        self.finished.store(true, Ordering::Relaxed);
+        Ok(result)
+    }
+}
+
+/// A [`SpillStore`] that writes temporary files to a local temp directory.
+///
+/// By default there is no disk cap. Use [`LocalSpillStore::with_cap`] to
+/// configure one shared across every handle this store produces.
+///
+/// The temp directory is deleted when the store is dropped, cleaning up any
+/// files whose handles have already been dropped.
+pub struct LocalSpillStore {
+    store: Arc<ObjectStore>,
+    /// Backstop cleanup: removes the whole scratch directory on drop.
+    temp_dir: Arc<tempfile::TempDir>,
+    file_counter: Arc<AtomicU64>,
+    /// Byte budget shared across every handle, enforced while writing.
+    quota: Option<DiskQuota>,
+}
+
+impl LocalSpillStore {
+    /// Create a store with no disk cap.
+    pub fn new() -> Result<Self> {
+        Ok(Self {
+            store: Arc::new(ObjectStore::local()),
+            temp_dir: Arc::new(tempfile::tempdir()?),
+            file_counter: Arc::new(AtomicU64::new(0)),
+            quota: None,
+        })
+    }
+
+    /// Create a store that returns [`lance_core::Error::DiskCapExceeded`] once
+    /// total bytes written across all live handles would exceed `cap_bytes`.
+    pub fn with_cap(cap_bytes: u64) -> Result<Self> {
+        Ok(Self {
+            store: Arc::new(ObjectStore::local()),
+            temp_dir: Arc::new(tempfile::tempdir()?),
+            file_counter: Arc::new(AtomicU64::new(0)),
+            quota: Some(DiskQuota::new(cap_bytes)),
+        })
+    }
+}
+
+impl Default for LocalSpillStore {
+    fn default() -> Self {
+        Self::new().expect("failed to create temp directory for LocalSpillStore")
+    }
+}
+
+#[async_trait]
+impl SpillStore for LocalSpillStore {
+    async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)> {
+        let idx = self.file_counter.fetch_add(1, Ordering::Relaxed);
+        let fs_path = self.temp_dir.path().join(format!("spill_{idx:06}.bin"));
+        let os_path = Path::from_absolute_path(&fs_path)?;
+        let finished = Arc::new(AtomicBool::new(false));
+
+        let writer = Box::new(SpillWriter {
+            inner: self.store.create(&os_path).await?,
+            quota: self.quota.clone(),
+            finished: finished.clone(),
+        });
+        let spill = Box::new(LocalSpill {
+            store: self.store.clone(),
+            os_path,
+            fs_path,
+            quota: self.quota.clone(),
+            finished,
+            _temp_dir: self.temp_dir.clone(),
+        });
+        Ok((writer, spill))
+    }
+}
+
+/// The readable half of a [`LocalSpillStore`] spill; reclaims the file on drop.
+struct LocalSpill {
+    store: Arc<ObjectStore>,
+    os_path: Path,
+    fs_path: PathBuf,
+    quota: Option<DiskQuota>,
+    /// Set by the paired [`SpillWriter`] once it has been shut down.
+    finished: Arc<AtomicBool>,
+    /// Keep the store's temp directory alive for at least this file's lifetime.
+    _temp_dir: Arc<tempfile::TempDir>,
+}
+
+#[async_trait]
+impl Spill for LocalSpill {
+    async fn reader(&self) -> Result<Box<dyn Reader>> {
+        // `Relaxed` is sufficient: the flag only gates "has the writer shut
+        // down"; the bytes themselves are synchronized through the filesystem,
+        // not this load.
+        if !self.finished.load(Ordering::Relaxed) {
+            return Err(Error::invalid_input(
+                "spill reader requested before the writer was shut down",
+            ));
+        }
+        self.store.open(&self.os_path).await
+    }
+}
+
+impl Drop for LocalSpill {
+    fn drop(&mut self) {
+        // Release the bytes this file occupied back to the budget. We stat the
+        // persisted file rather than tracking writes, which is exact for the
+        // write-once contract.
+        if let Some(quota) = &self.quota
+            && let Ok(metadata) = std::fs::metadata(&self.fs_path)
+        {
+            quota.release(metadata.len());
+        }
+        // Best-effort removal; the temp dir is the backstop.
+        let _ = std::fs::remove_file(&self.fs_path);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tokio::io::AsyncWriteExt;
+
+    /// Write `data` to a fresh writer and shut it down.
+    async fn finish_writer(mut writer: Box<dyn Writer>, data: &[u8]) -> Result<()> {
+        writer.write_all(data).await?;
+        Writer::shutdown(writer.as_mut()).await?;
+        Ok(())
+    }
+
+    #[test]
+    fn test_disk_quota_reserve_release() {
+        let quota = DiskQuota::new(100);
+        quota.try_reserve(60).unwrap();
+        assert!(quota.try_reserve(60).is_err());
+        quota.release(60);
+        quota.try_reserve(60).unwrap();
+        // Reserving exactly up to the cap succeeds; one byte past it fails.
+        quota.try_reserve(40).unwrap();
+        assert!(quota.try_reserve(1).is_err());
+    }
+
+    #[tokio::test]
+    async fn test_write_then_read() {
+        let store = LocalSpillStore::new().unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+
+        let data = b"hello spill world";
+        finish_writer(writer, data).await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        let read_back = reader.get_all().await.unwrap();
+        assert_eq!(read_back.as_ref(), data);
+    }
+
+    #[tokio::test]
+    async fn test_reader_requires_finished_writer() {
+        let store = LocalSpillStore::new().unwrap();
+        let (mut writer, spill) = store.new_spill().await.unwrap();
+        writer.write_all(b"partial").await.unwrap();
+
+        // Reading before the writer is shut down is rejected.
+        let Err(err) = spill.reader().await else {
+            panic!("reader before shutdown should be rejected");
+        };
+        assert!(
+            matches!(err, Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+
+        // After shutdown the reader sees the bytes.
+        Writer::shutdown(writer.as_mut()).await.unwrap();
+        let reader = spill.reader().await.unwrap();
+        assert_eq!(reader.get_all().await.unwrap().as_ref(), b"partial");
+    }
+
+    #[tokio::test]
+    async fn test_reader_ready_after_async_shutdown() {
+        // Shutting down through the `AsyncWrite` surface (not the `Writer`
+        // trait) must also mark the spill readable — covers poll_shutdown's
+        // flag set, the path the `Writer::shutdown` tests don't reach.
+        let store = LocalSpillStore::new().unwrap();
+        let (mut writer, spill) = store.new_spill().await.unwrap();
+        writer.write_all(b"async").await.unwrap();
+        AsyncWriteExt::shutdown(&mut writer).await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        assert_eq!(reader.get_all().await.unwrap().as_ref(), b"async");
+    }
+
+    #[tokio::test]
+    async fn test_empty_spill() {
+        // A spill written with no bytes round-trips empty, and the capped path
+        // handles the zero-byte reserve/stat without error.
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+        finish_writer(writer, b"").await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        assert!(reader.get_all().await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_raii_cleanup() {
+        let store = LocalSpillStore::new().unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+        finish_writer(writer, b"some bytes").await.unwrap();
+
+        // The first spill gets a deterministic name under the store's temp dir.
+        let path = store.temp_dir.path().join("spill_000000.bin");
+        assert!(path.exists());
+        drop(spill);
+        assert!(!path.exists(), "spill file should be deleted on drop");
+    }
+
+    #[tokio::test]
+    async fn test_cap_exceeded() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer, _spill) = store.new_spill().await.unwrap();
+        let err = finish_writer(writer, &[0u8; 101]).await.unwrap_err();
+        assert!(
+            matches!(err, Error::DiskCapExceeded { cap_bytes: 100, .. }),
+            "expected DiskCapExceeded, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cap_shared_across_files() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer_a, _spill_a) = store.new_spill().await.unwrap();
+        let (writer_b, _spill_b) = store.new_spill().await.unwrap();
+
+        finish_writer(writer_a, &[0u8; 60]).await.unwrap();
+        // 60 already reserved by `a`; writing 60 more would reach 120 > 100.
+        let err = finish_writer(writer_b, &[0u8; 60]).await.unwrap_err();
+        assert!(
+            matches!(err, Error::DiskCapExceeded { cap_bytes: 100, .. }),
+            "expected DiskCapExceeded, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cap_freed_on_drop() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+
+        {
+            let (writer, spill) = store.new_spill().await.unwrap();
+            finish_writer(writer, &[0u8; 80]).await.unwrap();
+            // `spill` drops at the end of this block, releasing its 80 bytes.
+            drop(spill);
+        }
+
+        let (writer, _spill) = store.new_spill().await.unwrap();
+        // Succeeds because the cap is no longer under pressure.
+        finish_writer(writer, &[0u8; 80]).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_custom_implementation() {
+        // A custom store can satisfy the traits without a local file.
+        struct MemStore;
+        struct MemSpill;
+
+        #[async_trait]
+        impl Spill for MemSpill {
+            async fn reader(&self) -> Result<Box<dyn Reader>> {
+                ObjectStore::memory().open(&Path::from("/mem")).await
+            }
+        }
+
+        #[async_trait]
+        impl SpillStore for MemStore {
+            async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)> {
+                let writer = ObjectStore::memory().create(&Path::from("/mem")).await?;
+                Ok((writer, Box::new(MemSpill)))
+            }
+        }
+
+        let store = MemStore;
+        // Exercise the factory + trait objects; the in-memory store is a fresh
+        // instance per call so we don't round-trip data here.
+        let (_writer, _spill) = store.new_spill().await.unwrap();
+    }
+
+    /// A [`Writer`] whose `poll_write` accepts a fixed number of bytes per call,
+    /// or fails, so we can drive the [`SpillWriter`] release arms that the local
+    /// backend (which accepts every write in full) never hits.
+    struct ControlledWriter {
+        outcome: Poll<io::Result<usize>>,
+    }
+
+    impl AsyncWrite for ControlledWriter {
+        fn poll_write(
+            self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            buf: &[u8],
+        ) -> Poll<io::Result<usize>> {
+            match &self.outcome {
+                Poll::Ready(Ok(n)) => Poll::Ready(Ok((*n).min(buf.len()))),
+                Poll::Ready(Err(e)) => Poll::Ready(Err(io::Error::new(e.kind(), e.to_string()))),
+                Poll::Pending => Poll::Pending,
+            }
+        }
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[async_trait]
+    impl Writer for ControlledWriter {
+        async fn tell(&mut self) -> Result<usize> {
+            Ok(0)
+        }
+        async fn shutdown(&mut self) -> Result<WriteResult> {
+            Ok(WriteResult::default())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_spill_writer_releases_unaccepted_bytes() {
+        // Short write: the inner writer accepts only 10 of the 40 reserved bytes,
+        // so the 30-byte remainder must be returned to the budget.
+        let quota = DiskQuota::new(100);
+        let mut writer = SpillWriter {
+            inner: Box::new(ControlledWriter {
+                outcome: Poll::Ready(Ok(10)),
+            }),
+            quota: Some(quota.clone()),
+            finished: Arc::new(AtomicBool::new(false)),
+        };
+        let n = writer.write(&[0u8; 40]).await.unwrap();
+        assert_eq!(n, 10);
+        assert_eq!(
+            *quota.used.lock().unwrap(),
+            10,
+            "only the accepted bytes should remain reserved"
+        );
+
+        // Failed write: the full reservation must be released.
+        let quota = DiskQuota::new(100);
+        let mut writer = SpillWriter {
+            inner: Box::new(ControlledWriter {
+                outcome: Poll::Ready(Err(io::Error::other("boom"))),
+            }),
+            quota: Some(quota.clone()),
+            finished: Arc::new(AtomicBool::new(false)),
+        };
+        writer.write(&[0u8; 40]).await.unwrap_err();
+        assert_eq!(
+            *quota.used.lock().unwrap(),
+            0,
+            "a failed write should release its entire reservation"
+        );
+    }
+}
diff --git a/rust/lance-io/src/traits.rs b/rust/lance-io/src/traits.rs
index eb83cf1a2ba..6a40171b6e0 100644
--- a/rust/lance-io/src/traits.rs
+++ b/rust/lance-io/src/traits.rs
@@ -5,8 +5,8 @@ use std::ops::Range;
 
 use async_trait::async_trait;
 use bytes::Bytes;
-use deepsize::DeepSizeOf;
 use futures::{StreamExt, future::BoxFuture, stream::BoxStream};
+use lance_core::deepsize::DeepSizeOf;
 use object_store::path::Path;
 use prost::Message;
 use tokio::io::{AsyncWrite, AsyncWriteExt};
diff --git a/rust/lance-io/src/uring/current_thread.rs b/rust/lance-io/src/uring/current_thread.rs
index bc09af058e6..abac772218b 100644
--- a/rust/lance-io/src/uring/current_thread.rs
+++ b/rust/lance-io/src/uring/current_thread.rs
@@ -14,10 +14,10 @@ use crate::traits::Reader;
 use crate::uring::DEFAULT_URING_QUEUE_DEPTH;
 use crate::utils::tracking_store::IOTracker;
 use bytes::{Bytes, BytesMut};
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
 use futures::{FutureExt, TryFutureExt};
 use io_uring::{IoUring, opcode, types};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use object_store::path::Path;
 
@@ -258,7 +258,7 @@ pub struct UringCurrentThreadReader {
 }
 
 impl DeepSizeOf for UringCurrentThreadReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // Skip file handle (just a system resource)
         // Only count the path's deep size
         self.handle.path.as_ref().deep_size_of_children(context)
diff --git a/rust/lance-io/src/uring/reader.rs b/rust/lance-io/src/uring/reader.rs
index 0e7b0101ba8..a948e6c63dc 100644
--- a/rust/lance-io/src/uring/reader.rs
+++ b/rust/lance-io/src/uring/reader.rs
@@ -12,9 +12,9 @@ use crate::traits::Reader;
 use crate::uring::requests::RequestState;
 use crate::utils::tracking_store::IOTracker;
 use bytes::{Bytes, BytesMut};
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
 use futures::{FutureExt, TryFutureExt};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use object_store::path::Path;
 use std::fs::File;
@@ -109,7 +109,7 @@ pub struct UringReader {
 }
 
 impl DeepSizeOf for UringReader {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // Skip file handle (just a system resource)
         // Only count the path's deep size
         self.handle.path.as_ref().deep_size_of_children(context)
diff --git a/rust/lance-io/src/utils.rs b/rust/lance-io/src/utils.rs
index 48d28526a5e..a36aadec9d9 100644
--- a/rust/lance-io/src/utils.rs
+++ b/rust/lance-io/src/utils.rs
@@ -10,8 +10,8 @@ use arrow_array::{
 use arrow_schema::DataType;
 use byteorder::{ByteOrder, LittleEndian};
 use bytes::Bytes;
-use deepsize::DeepSizeOf;
 use lance_arrow::*;
+use lance_core::deepsize::DeepSizeOf;
 use prost::Message;
 use serde::{Deserialize, Serialize};
 
@@ -84,7 +84,11 @@ pub async fn read_fixed_stride_array(
 /// followed by the message itself.
 pub async fn read_message<M: Message + Default>(reader: &dyn Reader, pos: usize) -> Result<M> {
     let file_size = reader.size().await?;
-    if pos > file_size {
+    // A message is a u32 length prefix followed by its body; both must lie before
+    // the end. A `pos` too close to the end means the reader size is too small
+    // (e.g. a stale cached size). Reject it rather than slice a short buffer and
+    // panic.
+    if pos + 4 > file_size {
         return Err(Error::io("file size is too small".to_string()));
     }
 
@@ -96,7 +100,9 @@ pub async fn read_message<M: Message + Default>(reader: &dyn Reader, pos: usize)
         let remaining_range = range.end..min(4 + pos + msg_len, file_size);
         let remaining_bytes = reader.get_range(remaining_range).await?;
         let buf = [buf, remaining_bytes].concat();
-        assert!(buf.len() >= msg_len + 4);
+        if buf.len() < msg_len + 4 {
+            return Err(Error::io("file size is too small".to_string()));
+        }
         Ok(M::decode(&buf[4..4 + msg_len])?)
     } else {
         Ok(M::decode(&buf[4..4 + msg_len])?)
diff --git a/rust/lance-io/tests/goosefs_integration.rs b/rust/lance-io/tests/goosefs_integration.rs
new file mode 100644
index 00000000000..fd9015bcda0
--- /dev/null
+++ b/rust/lance-io/tests/goosefs_integration.rs
@@ -0,0 +1,271 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! GooseFS integration tests via OpenDAL.
+//!
+//! Covers Stage 2 (OpenDAL direct), Stage 3 (Lance ObjectStore I/O),
+//! and diagnostic tests (OpenDAL via lance-io ObjectStore).
+//!
+//! Run:
+//!   cargo test -p lance-io --features "goosefs goosefs-test" --test goosefs_integration -- --ignored --nocapture --test-threads=1
+#![cfg(feature = "goosefs-test")]
+#![allow(clippy::print_stderr)]
+
+use std::sync::Arc;
+
+use futures::TryStreamExt;
+use object_store::ObjectStoreExt;
+use opendal::{Operator, services::GooseFs};
+use std::collections::HashMap;
+
+fn get_operator() -> Operator {
+    let addr = std::env::var("GOOSEFS_MASTER_ADDR").unwrap_or("127.0.0.1:9200".into());
+    let auth_type = std::env::var("GOOSEFS_AUTH_TYPE").unwrap_or("simple".into());
+    let mut cfg = HashMap::new();
+    cfg.insert("master_addr".to_string(), addr);
+    cfg.insert("root".to_string(), "/lance-test/opendal".to_string());
+    cfg.insert("auth_type".to_string(), auth_type);
+    Operator::from_iter::<GooseFs>(cfg).unwrap().finish()
+}
+
+// ============================================================
+// Stage 2: OpenDAL GooseFs Service tests
+// ============================================================
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_opendal_write_read() {
+    let op = get_operator();
+    // Cleanup any leftover from previous runs
+    let _ = op.delete("hello.txt").await;
+    op.write("hello.txt", "Hello from OpenDAL").await.unwrap();
+    let data = op.read("hello.txt").await.unwrap();
+    assert_eq!(data.to_vec(), b"Hello from OpenDAL");
+    op.delete("hello.txt").await.unwrap();
+}
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_opendal_list() {
+    let op = get_operator();
+    // Write files directly (GooseFS may have h2 issues with newly-created subdirs)
+    let _ = op.delete("list_a.txt").await;
+    let _ = op.delete("list_b.txt").await;
+    op.write("list_a.txt", "aaa").await.unwrap();
+    op.write("list_b.txt", "bbb").await.unwrap();
+    let entries: Vec<_> = op.list("/").await.unwrap();
+    let names: Vec<String> = entries.iter().map(|e| e.name().to_string()).collect();
+    eprintln!("Listed entries: {:?}", names);
+    assert!(
+        entries.len() >= 2,
+        "Expected at least 2 entries, got {}",
+        entries.len()
+    );
+    op.delete("list_a.txt").await.unwrap();
+    op.delete("list_b.txt").await.unwrap();
+}
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_opendal_stat() {
+    let op = get_operator();
+    // Cleanup leftover from previous runs
+    let _ = op.delete("stat_test.txt").await;
+    op.write("stat_test.txt", "12345").await.unwrap();
+    let meta = op.stat("stat_test.txt").await.unwrap();
+    assert_eq!(meta.content_length(), 5);
+    op.delete("stat_test.txt").await.unwrap();
+}
+
+// ============================================================
+// Stage 3: Lance ObjectStore I/O tests
+// ============================================================
+
+use lance_io::object_store::ObjectStore;
+
+async fn get_lance_store() -> Arc<ObjectStore> {
+    let addr = std::env::var("GOOSEFS_MASTER_ADDR").unwrap_or("127.0.0.1:9200".into());
+    let uri = format!("goosefs://{}/lance-test/lance-io", addr);
+    ObjectStore::from_uri(&uri).await.unwrap().0
+}
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_lance_objectstore_put_get() {
+    let store = get_lance_store().await;
+    let path = object_store::path::Path::from("test_put_get.bin");
+
+    // Cleanup
+    let _ = store.inner.delete(&path).await;
+
+    // Write
+    store
+        .inner
+        .put(&path, (&b"lance-goosefs-test"[..]).into())
+        .await
+        .unwrap();
+
+    // Read
+    let result = store.inner.get(&path).await.unwrap();
+    let bytes = result.bytes().await.unwrap();
+    assert_eq!(&bytes[..], b"lance-goosefs-test");
+
+    // Cleanup
+    store.inner.delete(&path).await.unwrap();
+}
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_lance_objectstore_list() {
+    let store = get_lance_store().await;
+
+    let file_a = object_store::path::Path::from("list_a.bin");
+    let file_b = object_store::path::Path::from("list_b.bin");
+
+    // Cleanup leftovers
+    let _ = store.inner.delete(&file_a).await;
+    let _ = store.inner.delete(&file_b).await;
+
+    store
+        .inner
+        .put(&file_a, (&b"aaa"[..]).into())
+        .await
+        .unwrap();
+    store
+        .inner
+        .put(&file_b, (&b"bbb"[..]).into())
+        .await
+        .unwrap();
+
+    let entries: Vec<_> = store.inner.list(None).try_collect().await.unwrap();
+    eprintln!("Listed {} entries", entries.len());
+    assert!(
+        entries.len() >= 2,
+        "Expected at least 2 entries, got {}",
+        entries.len()
+    );
+
+    store.inner.delete(&file_a).await.unwrap();
+    store.inner.delete(&file_b).await.unwrap();
+}
+
+#[ignore = "Requires GooseFS cluster"]
+#[tokio::test]
+async fn test_lance_objectstore_large_file() {
+    let store = get_lance_store().await;
+    let path = object_store::path::Path::from("large_file.bin");
+    let _ = store.inner.delete(&path).await;
+
+    // Write 5MB file
+    let data = vec![42u8; 5 * 1024 * 1024];
+    store.inner.put(&path, data.clone().into()).await.unwrap();
+
+    let result = store.inner.get(&path).await.unwrap();
+    let bytes = result.bytes().await.unwrap();
+    assert_eq!(bytes.len(), 5 * 1024 * 1024);
+    assert_eq!(&bytes[..10], &[42u8; 10]);
+
+    store.inner.delete(&path).await.unwrap();
+}
+
+// ============================================================
+// Diagnostic: lance-io ObjectStore advanced write modes
+// ============================================================
+
+use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};
+
+#[tokio::test]
+#[ignore = "Requires GooseFS cluster"]
+async fn test_diag_lance_io_write_modes() {
+    let addr = std::env::var("GOOSEFS_MASTER_ADDR").unwrap_or_else(|_| "127.0.0.1:9200".into());
+    let ts = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_millis();
+    let root = format!("goosefs://{}/lance-test/lance_io_direct_{}", addr, ts);
+
+    eprintln!("[DIAG] Creating ObjectStore at: {}", root);
+
+    let params = ObjectStoreParams::default();
+    let registry = Arc::new(ObjectStoreRegistry::default());
+    let (object_store, _path) = ObjectStore::from_uri_and_params(registry, &root, &params)
+        .await
+        .expect("Failed to create ObjectStore");
+
+    // Test 1: Basic put + get
+    let test_path = object_store::path::Path::parse("test_file.txt").unwrap();
+    let test_data = bytes::Bytes::from("Hello from lance-io ObjectStore!");
+
+    eprintln!(
+        "[DIAG] Writing test_file.txt ({} bytes)...",
+        test_data.len()
+    );
+    match object_store
+        .inner
+        .put(&test_path, test_data.clone().into())
+        .await
+    {
+        Ok(_) => eprintln!("[DIAG] Write succeeded! ✅"),
+        Err(e) => {
+            eprintln!("[DIAG] Write FAILED: {:?}", e);
+            eprintln!("[DIAG] Error source: {:?}", std::error::Error::source(&e));
+            return;
+        }
+    }
+
+    eprintln!("[DIAG] Reading test_file.txt...");
+    match object_store.inner.get(&test_path).await {
+        Ok(result) => {
+            let bytes = result.bytes().await.unwrap();
+            let content = String::from_utf8_lossy(&bytes);
+            eprintln!("[DIAG] Read content: '{}' ({} bytes)", content, bytes.len());
+            assert_eq!(bytes, test_data);
+        }
+        Err(e) => eprintln!("[DIAG] Read FAILED: {:?}", e),
+    }
+
+    // Test 2: PutMode::Create (if_not_exists)
+    eprintln!("[DIAG] Writing with PutMode::Create (if_not_exists)...");
+    match object_store
+        .inner
+        .put_opts(
+            &object_store::path::Path::parse("test_create.txt").unwrap(),
+            bytes::Bytes::from("conditional write!").into(),
+            object_store::PutOptions {
+                mode: object_store::PutMode::Create,
+                ..Default::default()
+            },
+        )
+        .await
+    {
+        Ok(_) => eprintln!("[DIAG] PutMode::Create succeeded! ✅"),
+        Err(e) => {
+            eprintln!("[DIAG] PutMode::Create FAILED: {:?}", e);
+        }
+    }
+
+    // Test 3: rename_if_not_exists
+    eprintln!("[DIAG] Testing rename_if_not_exists...");
+    let tmp_path = object_store::path::Path::parse("_tmp_rename.txt").unwrap();
+    let dest_path = object_store::path::Path::parse("renamed.txt").unwrap();
+    match object_store
+        .inner
+        .put(&tmp_path, bytes::Bytes::from("rename me!").into())
+        .await
+    {
+        Ok(_) => {
+            eprintln!("[DIAG] Tmp file written ✅");
+            match object_store
+                .inner
+                .rename_if_not_exists(&tmp_path, &dest_path)
+                .await
+            {
+                Ok(_) => eprintln!("[DIAG] rename_if_not_exists succeeded! ✅"),
+                Err(e) => eprintln!("[DIAG] rename_if_not_exists FAILED: {:?}", e),
+            }
+        }
+        Err(e) => eprintln!("[DIAG] Tmp file write FAILED: {:?}", e),
+    }
+
+    eprintln!("[DIAG] lance-io direct write test complete ✅");
+}
diff --git a/rust/lance-io/tests/tos_integration.rs b/rust/lance-io/tests/tos_integration.rs
new file mode 100644
index 00000000000..97eca30090e
--- /dev/null
+++ b/rust/lance-io/tests/tos_integration.rs
@@ -0,0 +1,81 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+//! These integration tests can only be run against a real Volcengine TOS bucket.
+
+#![cfg(feature = "tos-test")]
+
+use futures::TryStreamExt;
+use lance_io::object_store::ObjectStore;
+use object_store::ObjectStoreExt;
+use object_store::path::Path;
+use tokio::io::AsyncWriteExt;
+
+fn tos_bucket() -> String {
+    std::env::var("TOS_BUCKET").expect("TOS_BUCKET must be set")
+}
+
+async fn delete_prefix(store: &ObjectStore, prefix: &str) {
+    let prefix_path = Path::from(prefix);
+    let locations = store
+        .inner
+        .list(Some(&prefix_path))
+        .map_ok(|meta| meta.location)
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap_or_default();
+
+    for location in locations {
+        let _ = store.inner.delete(&location).await;
+    }
+}
+
+#[ignore = "Must be run manually on Volcengine TOS"]
+#[tokio::test]
+async fn test_tos_write_read_list_delete() {
+    let prefix = format!("lance-tos-{}-{}", std::process::id(), rand::random::<u64>());
+    let bucket = tos_bucket();
+    let (store, base_path) = ObjectStore::from_uri(&format!("tos://{bucket}/{prefix}"))
+        .await
+        .unwrap();
+    assert_eq!(base_path, Path::from(prefix.as_str()));
+
+    let path = Path::from(format!("{prefix}/small.txt"));
+    delete_prefix(&store, &prefix).await;
+
+    let result: Result<(), Box<dyn std::error::Error>> = async {
+        let mut writer = store.create(&path).await?;
+        writer.write_all(b"hello").await?;
+        writer.write_all(b" tos").await?;
+        writer.shutdown().await?;
+
+        let meta = store.inner.head(&path).await?;
+        if meta.size != 9 {
+            return Err(format!("expected object size 9, got {}", meta.size).into());
+        }
+
+        let data = store.inner.get(&path).await?.bytes().await?;
+        if data.as_ref() != b"hello tos" {
+            return Err("downloaded TOS object content did not match".into());
+        }
+
+        let listed = store
+            .inner
+            .list(Some(&Path::from(prefix.as_str())))
+            .try_collect::<Vec<_>>()
+            .await?;
+        if !listed.iter().any(|meta| meta.location == path) {
+            return Err("uploaded TOS object was not returned by list".into());
+        }
+
+        store.inner.delete(&path).await?;
+        if store.exists(&path).await? {
+            return Err("deleted TOS object still exists".into());
+        }
+
+        Ok(())
+    }
+    .await;
+
+    delete_prefix(&store, &prefix).await;
+    result.unwrap();
+}
diff --git a/rust/lance-linalg/Cargo.toml b/rust/lance-linalg/Cargo.toml
index 19d7ad4da47..d94efa7b2fa 100644
--- a/rust/lance-linalg/Cargo.toml
+++ b/rust/lance-linalg/Cargo.toml
@@ -13,18 +13,19 @@ categories = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
 arrow-schema = { workspace = true }
-deepsize = { workspace = true }
 half = { workspace = true }
 lance-arrow = { workspace = true }
 lance-core = { workspace = true }
 num-traits = { workspace = true }
 rand = { workspace = true }
+rayon = { workspace = true }
 
 [dev-dependencies]
 approx = { workspace = true }
 criterion = { workspace = true }
 lance-testing = { path = "../lance-testing" }
 proptest.workspace = true
+rstest.workspace = true
 
 [build-dependencies]
 cc = "1.0.83"
@@ -51,10 +52,6 @@ harness = false
 name = "cosine"
 harness = false
 
-[[bench]]
-name = "hamming"
-harness = false
-
 [[bench]]
 name = "norm_l2"
 harness = false
diff --git a/rust/lance-linalg/benches/cosine.rs b/rust/lance-linalg/benches/cosine.rs
index 223299a934c..49e816b76df 100644
--- a/rust/lance-linalg/benches/cosine.rs
+++ b/rust/lance-linalg/benches/cosine.rs
@@ -40,7 +40,7 @@ where
     T::Native: Cosine,
 {
     const DIMENSION: usize = 1024;
-    const TOTAL: usize = 1024 * 1024; // 1M vectors
+    const TOTAL: usize = 512 * 1024;
 
     let type_name = std::any::type_name::<T::Native>();
     let key = generate_random_array_with_seed::<T>(DIMENSION, [0; 32]);
diff --git a/rust/lance-linalg/benches/hamming.rs b/rust/lance-linalg/benches/hamming.rs
deleted file mode 100644
index 9af3bf4614b..00000000000
--- a/rust/lance-linalg/benches/hamming.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-use std::iter::repeat_with;
-
-use std::hint::black_box;
-
-use criterion::{Criterion, criterion_group, criterion_main};
-use lance_linalg::distance::hamming::{hamming, hamming_scalar};
-use rand::Rng;
-
-const DIMENSION: usize = 1024;
-const TOTAL: usize = 1024 * 1024; // 1M vectors
-
-fn bench_hamming(c: &mut Criterion) {
-    let mut rng = rand::rng();
-
-    let key = repeat_with(|| rng.random::<u8>())
-        .take(DIMENSION)
-        .collect::<Vec<_>>();
-    let target = repeat_with(|| rng.random::<u8>())
-        .take(TOTAL * DIMENSION)
-        .collect::<Vec<_>>();
-
-    c.bench_function("hamming,scalar", |b| {
-        b.iter(|| {
-            black_box(
-                target
-                    .chunks_exact(DIMENSION)
-                    .map(|tgt| hamming_scalar(&key, tgt))
-                    .sum::<f32>(),
-            );
-        })
-    });
-
-    c.bench_function("hamming,auto_vec", |b| {
-        b.iter(|| {
-            black_box(
-                target
-                    .chunks_exact(DIMENSION)
-                    .map(|tgt| hamming(&key, tgt))
-                    .sum::<f32>(),
-            );
-        })
-    });
-}
-
-criterion_group!(
-    name=benches;
-    config = Criterion::default().significance_level(0.1).sample_size(10);
-    targets = bench_hamming);
-criterion_main!(benches);
diff --git a/rust/lance-linalg/build.rs b/rust/lance-linalg/build.rs
index 06e1439c77a..407f2a589ea 100644
--- a/rust/lance-linalg/build.rs
+++ b/rust/lance-linalg/build.rs
@@ -16,7 +16,9 @@ fn main() -> Result<(), String> {
     }
 
     // Let clippy know about our custom cfg attribute
-    println!("cargo::rustc-check-cfg=cfg(kernel_support, values(\"avx512\"))");
+    println!(
+        "cargo::rustc-check-cfg=cfg(kernel_support, values(\"avx512_f16\", \"avx512_bf16\", \"avx512_dist_table\"))"
+    );
 
     println!("cargo:rerun-if-changed=src/simd/f16.c");
     println!("cargo:rerun-if-changed=src/simd/bf16.c");
@@ -58,10 +60,10 @@ fn main() -> Result<(), String> {
                 "cargo:warning=Skipping build of AVX-512 fp16 kernels. Error: {}",
                 err
             );
-        } else {
+        } else if cfg!(feature = "fp16kernels") {
             // We create a special cfg so that we can detect we have in fact
             // generated the AVX512 version of the f16 kernels.
-            println!("cargo:rustc-cfg=kernel_support=\"avx512\"");
+            println!("cargo:rustc-cfg=kernel_support=\"avx512_f16\"");
         };
         // Build AVX-512 bf16 kernels (sapphirerapids has native vdpbf16ps)
         if let Err(err) =
@@ -71,16 +73,16 @@ fn main() -> Result<(), String> {
                 "cargo:warning=Skipping build of AVX-512 bf16 kernels. Error: {}",
                 err
             );
-        } else {
-            println!("cargo:rustc-cfg=kernel_support=\"avx512\"");
+        } else if cfg!(feature = "fp16kernels") {
+            println!("cargo:rustc-cfg=kernel_support=\"avx512_bf16\"");
         };
-        if let Err(err) = build_dist_table_with_flags("avx512", &["-march=native"]) {
+        if let Err(err) = build_dist_table_with_flags("avx512", &["-march=sapphirerapids"]) {
             println!(
                 "cargo:warning=Skipping build of AVX-512 dist_table. Error: {}",
                 err
             );
         } else {
-            println!("cargo:rustc-cfg=kernel_support=\"avx512\"");
+            println!("cargo:rustc-cfg=kernel_support=\"avx512_dist_table\"");
         };
         // Build a version with AVX
         // While GCC doesn't have support for _Float16 until GCC 12, clang
diff --git a/rust/lance-linalg/proptest-regressions/distance/cosine.txt b/rust/lance-linalg/proptest-regressions/distance/cosine.txt
new file mode 100644
index 00000000000..04e6dcb967f
--- /dev/null
+++ b/rust/lance-linalg/proptest-regressions/distance/cosine.txt
@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 679c764b02e2726ec2d943b3abdc7d7d3565e168d4081b155860c1e69b616fce # shrinks to (x, y) = ([-1.2933521e-15, -1.1194652e-8, 7.447341e-32, -0.0, -14247490.0, -8.55e-43, -4576.872, 0.009181444], [4.926e-42, -9.267065e-15, -0.0, 7e-45, -0.00036999694, -0.0, -5.7810876e-8, 1.9273644e-21])
diff --git a/rust/lance-linalg/src/distance.rs b/rust/lance-linalg/src/distance.rs
index 9dd57edb3f9..142ec1303b4 100644
--- a/rust/lance-linalg/src/distance.rs
+++ b/rust/lance-linalg/src/distance.rs
@@ -25,11 +25,107 @@ pub mod l2;
 pub mod l2_u8;
 pub mod norm_l2;
 
+/// What a per-batch distance kernel yields.
+///
+/// The two batch paths produce different shapes: the build-baseline path maps
+/// lazily over the batch and allocates nothing, while a `#[target_feature]`
+/// kernel must collect eagerly because it cannot be inlined into a lazy
+/// closure. A concrete enum keeps both statically dispatched.
+///
+/// Only sub-AVX2 builds need this. On an AVX2-baseline build the batch methods
+/// return the bare `Map` instead, because any wrapper — trait object or enum —
+/// loses `TrustedLen` (so `.collect()` stops preallocating) and loses
+/// `Map::fold`'s inlined loop. Benchmarks showed that costing 2.5x on the dim-8
+/// batch, far more than the per-vector dispatch it was meant to remove.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+pub(crate) enum BatchIter<L> {
+    /// Lazy per-vector map. No allocation.
+    Lazy(L),
+    /// Eagerly collected by a `#[target_feature]` kernel.
+    Eager(std::vec::IntoIter<f32>),
+}
+
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+impl<L: Iterator<Item = f32>> Iterator for BatchIter<L> {
+    type Item = f32;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        match self {
+            Self::Lazy(iter) => iter.next(),
+            Self::Eager(iter) => iter.next(),
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        match self {
+            Self::Lazy(iter) => iter.size_hint(),
+            Self::Eager(iter) => iter.size_hint(),
+        }
+    }
+
+    /// Delegated, not defaulted. `Map` overrides `fold` to drive the underlying
+    /// `ChunksExact` in one inlined, auto-vectorized loop; the default `fold`
+    /// would instead call `next()` per element, paying an enum branch and
+    /// losing that loop. On the dim-8 batch that costs ~2.5x.
+    #[inline]
+    fn fold<B, F>(self, init: B, f: F) -> B
+    where
+        F: FnMut(B, Self::Item) -> B,
+    {
+        match self {
+            Self::Lazy(iter) => iter.fold(init, f),
+            Self::Eager(iter) => iter.fold(init, f),
+        }
+    }
+
+    /// `for_each`, `sum` and `collect` all route through `fold`, so delegating
+    /// it covers them too. (`try_fold` cannot be overridden on stable: its
+    /// `Try` bound is unstable.)
+    #[inline]
+    fn for_each<F>(self, f: F)
+    where
+        F: FnMut(Self::Item),
+    {
+        match self {
+            Self::Lazy(iter) => iter.for_each(f),
+            Self::Eager(iter) => iter.for_each(f),
+        }
+    }
+}
+
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+impl<L: ExactSizeIterator<Item = f32>> ExactSizeIterator for BatchIter<L> {
+    #[inline]
+    fn len(&self) -> usize {
+        match self {
+            Self::Lazy(iter) => iter.len(),
+            Self::Eager(iter) => iter.len(),
+        }
+    }
+}
+
 pub use cosine::*;
-use deepsize::DeepSizeOf;
 pub use dot::*;
-use hamming::hamming_distance_arrow_batch;
+pub use hamming::{
+    BinaryHashValues, Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges,
+    cluster_pairwise_result, extract_binary_hashes_from_fixed_list, extract_hashes_from_fixed_list,
+    hamming_distance_arrow_batch, hamming_u64, pairwise_hamming_distance,
+    pairwise_hamming_distance_binary, pairwise_hamming_distance_binary_parallel,
+    pairwise_hamming_distance_parallel,
+};
 pub use l2::*;
+use lance_core::deepsize::DeepSizeOf;
 pub use norm_l2::*;
 
 use crate::Result;
diff --git a/rust/lance-linalg/src/distance/cosine.rs b/rust/lance-linalg/src/distance/cosine.rs
index be5bf436344..457c0c0dd6a 100644
--- a/rust/lance-linalg/src/distance/cosine.rs
+++ b/rust/lance-linalg/src/distance/cosine.rs
@@ -17,12 +17,12 @@ use arrow_array::{
 use arrow_schema::DataType;
 use half::{bf16, f16};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+#[allow(unused_imports)]
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 
 use super::{Dot, norm_l2::norm_l2};
 use super::{Normalize, dot::dot};
+#[allow(unused_imports)]
 use crate::simd::{
     FloatSimd, SIMD,
     f32::{f32x8, f32x16},
@@ -82,7 +82,7 @@ mod bf16_kernel {
         #[cfg(target_arch = "aarch64")]
         pub fn cosine_bf16_neon(x: *const bf16, x_norm: f32, y: *const bf16, dimension: u32)
         -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_bf16", target_arch = "x86_64"))]
         pub fn cosine_bf16_avx512(
             x: *const bf16,
             x_norm: f32,
@@ -109,7 +109,7 @@ impl Cosine for bf16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_bf16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -127,6 +127,10 @@ impl Cosine for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::cosine_bf16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels in `bf16_kernel::*` are compiled with
+            // `-march=haswell` minimum (which requires AVX2), so they cannot
+            // run on AVX-only or AVX+FMA hosts. Scalar is the correct route.
             _ => cosine_scalar(x, x_norm, y),
         }
     }
@@ -141,7 +145,7 @@ mod kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn cosine_f16_neon(x: *const f16, x_norm: f32, y: *const f16, dimension: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_f16", target_arch = "x86_64"))]
         pub fn cosine_f16_avx512(x: *const f16, x_norm: f32, y: *const f16, dimension: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn cosine_f16_avx2(x: *const f16, x_norm: f32, y: *const f16, dimension: u32) -> f32;
@@ -161,7 +165,7 @@ impl Cosine for f16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_f16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -179,88 +183,348 @@ impl Cosine for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::cosine_f16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => cosine_scalar(x, x_norm, y),
         }
     }
 }
 
-/// f32 kernels for Cosine
+/// f32 single-vector cosine helpers used by `cosine_batch` for fixed
+/// dimensions 8 and 16.
+///
+/// These were previously a single generic `cosine_once<S, N>` but the
+/// monomorphizations have to dispatch on `SIMD_SUPPORT` for the SIMD path
+/// to stay correct under any compile baseline. Splitting them into two
+/// concrete entry points keeps the dispatch site flat and lets each width
+/// route to a `#[target_feature]` AVX2 inner function.
 mod f32 {
     use super::*;
 
-    // TODO: how can we explicitly infer N?
     #[inline]
-    pub(super) fn cosine_once<S: SIMD<f32, N>, const N: usize>(
+    pub(super) fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                    cosine_once_x86::cosine_once_8_avx512(x, x_norm, y)
+                },
+                SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                    cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y)
+                },
+                SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) },
+                _ => cosine_once_8_scalar(x, x_norm, y),
+            }
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            cosine_once_8_other(x, x_norm, y)
+        }
+    }
+
+    #[inline]
+    pub(super) fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                    cosine_once_x86::cosine_once_16_avx512(x, x_norm, y)
+                },
+                SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                    cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y)
+                },
+                SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) },
+                _ => cosine_once_16_scalar(x, x_norm, y),
+            }
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            cosine_once_16_other(x, x_norm, y)
+        }
+    }
+
+    /// Portable scalar `cosine_once` for length-8 vectors. Matches the SIMD
+    /// path modulo summation order.
+    #[cfg(target_arch = "x86_64")]
+    #[inline]
+    pub(super) fn cosine_once_8_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let mut xy = 0.0f32;
+        let mut y2 = 0.0f32;
+        for i in 0..8 {
+            xy += x[i] * y[i];
+            y2 += y[i] * y[i];
+        }
+        1.0 - xy / x_norm / y2.sqrt()
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[inline]
+    pub(super) fn cosine_once_16_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let mut xy = 0.0f32;
+        let mut y2 = 0.0f32;
+        for i in 0..16 {
+            xy += x[i] * y[i];
+            y2 += y[i] * y[i];
+        }
+        1.0 - xy / x_norm / y2.sqrt()
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    pub(super) mod cosine_once_x86 {
+        use std::arch::x86_64::*;
+
+        use super::{f32x8, f32x16};
+        use crate::simd::SIMD;
+
+        /// AVX + FMA path for 8-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+        #[target_feature(enable = "avx,fma")]
+        pub unsafe fn cosine_once_8_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX + FMA path for 16-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+        #[target_feature(enable = "avx,fma")]
+        pub unsafe fn cosine_once_16_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-only path for 8-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge.
+        #[target_feature(enable = "avx")]
+        pub unsafe fn cosine_once_8_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-only path for 16-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge.
+        #[target_feature(enable = "avx")]
+        pub unsafe fn cosine_once_16_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        }
+
+        /// AVX-512 path for 8-lane cosine: masked load into a `__m512` lower half, reduce.
+        #[target_feature(enable = "avx512f")]
+        pub unsafe fn cosine_once_8_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            // mask 0x00FF: load the lower 8 f32 lanes, zero the upper 8.
+            let mask: __mmask16 = 0x00FF;
+            let xv = _mm512_maskz_loadu_ps(mask, x.as_ptr());
+            let yv = _mm512_maskz_loadu_ps(mask, y.as_ptr());
+            let xy = _mm512_mul_ps(xv, yv);
+            let y2 = _mm512_mul_ps(yv, yv);
+            let xy_sum = _mm512_reduce_add_ps(xy);
+            let y2_sum = _mm512_reduce_add_ps(y2);
+            1.0 - xy_sum / x_norm / y2_sum.sqrt()
+        }
+
+        /// AVX-512 path for 16-lane cosine: single full-width `__m512` load (16 f32 fits one `zmm`).
+        #[target_feature(enable = "avx512f")]
+        pub unsafe fn cosine_once_16_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+            let xv = _mm512_loadu_ps(x.as_ptr());
+            let yv = _mm512_loadu_ps(y.as_ptr());
+            let xy = _mm512_mul_ps(xv, yv);
+            let y2 = _mm512_mul_ps(yv, yv);
+            let xy_sum = _mm512_reduce_add_ps(xy);
+            let y2_sum = _mm512_reduce_add_ps(y2);
+            1.0 - xy_sum / x_norm / y2_sum.sqrt()
+        }
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[inline]
+    fn cosine_once_8_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let xv = unsafe { f32x8::load_unaligned(x.as_ptr()) };
+        let yv = unsafe { f32x8::load_unaligned(y.as_ptr()) };
+        let y2 = yv * yv;
+        let xy = xv * yv;
+        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+    }
+
+    #[cfg(not(target_arch = "x86_64"))]
+    #[inline]
+    fn cosine_once_16_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        let xv = unsafe { f32x16::load_unaligned(x.as_ptr()) };
+        let yv = unsafe { f32x16::load_unaligned(y.as_ptr()) };
+        let y2 = yv * yv;
+        let xy = xv * yv;
+        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+    }
+
+    /// Batch-level SIMD dispatch: the tier is chosen once by the caller, and the
+    /// whole `chunks_exact` loop runs inside one `#[target_feature]` context so
+    /// the per-vector `cosine_once_*` / `cosine_fast` kernels inline (no
+    /// per-vector `*SIMD_SUPPORT` branch, no per-vector call boundary). Used for
+    /// the AVX-512 path on the default wheel and for all tiers on sub-AVX2 builds.
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn cosine_batch_avx_fma(
         x: &[f32],
         x_norm: f32,
-        y: &[f32],
-    ) -> f32 {
-        let x = unsafe { S::load_unaligned(x.as_ptr()) };
-        let y = unsafe { S::load_unaligned(y.as_ptr()) };
-        let y2 = y * y;
-        let xy = x * y;
-        1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx_fma(x, x_norm, y) })
+                .collect(),
+        }
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn cosine_batch_avx512(
+        x: &[f32],
+        x_norm: f32,
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx512(x, x_norm, y) })
+                .collect(),
+        }
+    }
+
+    #[cfg(target_arch = "x86_64")]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn cosine_batch_avx(
+        x: &[f32],
+        x_norm: f32,
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        match dimension {
+            8 => batch
+                .chunks_exact(8)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) })
+                .collect(),
+            16 => batch
+                .chunks_exact(16)
+                .map(|y| unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) })
+                .collect(),
+            _ => batch
+                .chunks_exact(dimension)
+                .map(|y| unsafe { super::f32_x86::cosine_fast_avx(x, x_norm, y) })
+                .collect(),
+        }
     }
 }
 
-impl Cosine for f32 {
+/// Inlined f32 cosine kernels for builds whose baseline already guarantees AVX2
+/// (the default `haswell` wheel). No `#[target_feature]`, no runtime dispatch:
+/// under `target-feature=+avx2,+fma` these compile to AVX2 and inline into the
+/// batch loop exactly like the pre-PR code, so the modern path is not taxed by
+/// the runtime-dispatch machinery (only needed below the AVX2 baseline).
+#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
+mod f32_baseline {
+    use super::{dot, f32x8, f32x16, norm_l2};
+    use crate::simd::{FloatSimd, SIMD};
+
     #[inline]
-    fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 {
-        let dim = x.len();
-        let unrolled_len = dim / 16 * 16;
-        let mut y_norm16 = f32x16::zeros();
-        let mut xy16 = f32x16::zeros();
-        for i in (0..unrolled_len).step_by(16) {
-            unsafe {
-                let x = f32x16::load_unaligned(x.as_ptr().add(i));
-                let y = f32x16::load_unaligned(other.as_ptr().add(i));
-                xy16.multiply_add(x, y);
-                y_norm16.multiply_add(y, y);
-            }
+    pub fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr());
+            let yv = f32x8::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
         }
-        let aligned_len = dim / 8 * 8;
-        let mut y_norm8 = f32x8::zeros();
-        let mut xy8 = f32x8::zeros();
-        for i in (unrolled_len..aligned_len).step_by(8) {
-            unsafe {
-                let x = f32x8::load_unaligned(x.as_ptr().add(i));
-                let y = f32x8::load_unaligned(other.as_ptr().add(i));
-                xy8.multiply_add(x, y);
-                y_norm8.multiply_add(y, y);
-            }
+    }
+
+    #[inline]
+    pub fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr());
+            let yv = f32x16::load_unaligned(y.as_ptr());
+            let y2 = yv * yv;
+            let xy = xv * yv;
+            1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt()
         }
-        let y_norm =
-            y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
-        let xy =
-            xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
-        1.0 - xy / x_norm / y_norm.sqrt()
     }
 
     #[inline]
-    fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self {
-        let dim = x.len();
-        let unrolled_len = dim / 16 * 16;
-        let mut xy16 = f32x16::zeros();
-        for i in (0..unrolled_len).step_by(16) {
-            unsafe {
-                let x = f32x16::load_unaligned(x.as_ptr().add(i));
-                let y = f32x16::load_unaligned(y.as_ptr().add(i));
-                xy16.multiply_add(x, y);
+    pub fn cosine_fast(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        unsafe {
+            let dim = x.len();
+            let unrolled_len = dim / 16 * 16;
+            let mut y_norm16 = f32x16::zeros();
+            let mut xy16 = f32x16::zeros();
+            for i in (0..unrolled_len).step_by(16) {
+                let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+                let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+                xy16.multiply_add(xv, yv);
+                y_norm16.multiply_add(yv, yv);
             }
-        }
-        let aligned_len = dim / 8 * 8;
-        let mut xy8 = f32x8::zeros();
-        for i in (unrolled_len..aligned_len).step_by(8) {
-            unsafe {
-                let x = f32x8::load_unaligned(x.as_ptr().add(i));
-                let y = f32x8::load_unaligned(y.as_ptr().add(i));
-                xy8.multiply_add(x, y);
+            let aligned_len = dim / 8 * 8;
+            let mut y_norm8 = f32x8::zeros();
+            let mut xy8 = f32x8::zeros();
+            for i in (unrolled_len..aligned_len).step_by(8) {
+                let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+                let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+                xy8.multiply_add(xv, yv);
+                y_norm8.multiply_add(yv, yv);
             }
+            let y_norm = y_norm16.reduce_sum()
+                + y_norm8.reduce_sum()
+                + norm_l2(&other[aligned_len..]).powi(2);
+            let xy = xy16.reduce_sum()
+                + xy8.reduce_sum()
+                + dot(&x[aligned_len..], &other[aligned_len..]);
+            1.0 - xy / x_norm / y_norm.sqrt()
         }
-        let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
-        1.0 - xy / x_norm / y_norm
+    }
+}
+
+impl Cosine for f32 {
+    #[inline]
+    fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 {
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_fast_f32_dispatched(x, x_norm, other)
     }
 
+    #[inline]
+    fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self {
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_with_norms_f32_dispatched(x, x_norm, y_norm, y)
+    }
+
+    #[allow(unreachable_code)]
     fn cosine_batch<'a>(
         x: &'a [Self],
         batch: &'a [Self],
@@ -268,16 +532,83 @@ impl Cosine for f32 {
     ) -> Box<dyn Iterator<Item = f32> + 'a> {
         let x_norm = norm_l2(x);
 
+        // On a build whose baseline already guarantees AVX2 (the default
+        // `haswell` wheel), avoid the per-vector runtime dispatch + `#[target_feature]`
+        // wrapping that taxes the modern path. Dispatch ONCE per batch: AVX-512
+        // hosts get the wide kernel; everyone else uses the inlined AVX2 baseline
+        // path (base-equivalent). The runtime-dispatch path below is only
+        // compiled/reached when the baseline is below AVX2 (pre-Haswell builds).
+        #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
+        {
+            // dim 8/16 always use the inlined AVX2 baseline: AVX-512 gives no
+            // benefit for such tiny vectors (a masked 512-bit load is slower than
+            // a plain AVX2 load) and only adds dispatch + eager-collect overhead.
+            // Only the larger-dim path routes to AVX-512 on capable hosts — that's
+            // where the wider lanes actually pay off.
+            return match dimension {
+                8 => Box::new(
+                    batch
+                        .chunks_exact(8)
+                        .map(move |y| f32_baseline::cosine_once_8(x, x_norm, y)),
+                ),
+                16 => Box::new(
+                    batch
+                        .chunks_exact(16)
+                        .map(move |y| f32_baseline::cosine_once_16(x, x_norm, y)),
+                ),
+                _ => {
+                    if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) {
+                        Box::new(
+                            unsafe { f32::cosine_batch_avx512(x, x_norm, batch, dimension) }
+                                .into_iter(),
+                        )
+                    } else {
+                        Box::new(
+                            batch
+                                .chunks_exact(dimension)
+                                .map(move |y| f32_baseline::cosine_fast(x, x_norm, y)),
+                        )
+                    }
+                }
+            };
+        }
+
+        // Sub-AVX2 / non-x86 build: hoisted per-batch runtime dispatch.
+        #[cfg(target_arch = "x86_64")]
+        {
+            match *SIMD_SUPPORT {
+                SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx512(x, x_norm, batch, dimension) }
+                            .into_iter(),
+                    );
+                }
+                SimdSupport::Avx2 | SimdSupport::AvxFma => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx_fma(x, x_norm, batch, dimension) }
+                            .into_iter(),
+                    );
+                }
+                SimdSupport::Avx => {
+                    return Box::new(
+                        unsafe { f32::cosine_batch_avx(x, x_norm, batch, dimension) }.into_iter(),
+                    );
+                }
+                _ => {}
+            }
+        }
+
+        // Scalar / non-x86 fallback.
         match dimension {
             8 => Box::new(
                 batch
                     .chunks_exact(dimension)
-                    .map(move |y| f32::cosine_once::<f32x8, 8>(x, x_norm, y)),
+                    .map(move |y| f32::cosine_once_8(x, x_norm, y)),
             ),
             16 => Box::new(
                 batch
                     .chunks_exact(dimension)
-                    .map(move |y| f32::cosine_once::<f32x16, 16>(x, x_norm, y)),
+                    .map(move |y| f32::cosine_once_16(x, x_norm, y)),
             ),
             _ => Box::new(
                 batch
@@ -291,34 +622,102 @@ impl Cosine for f32 {
 impl Cosine for f64 {
     #[inline]
     fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 {
-        use crate::simd::f64::{f64x4, f64x8};
-        use crate::simd::{FloatSimd, SIMD};
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback.
+        cosine_fast_f64_dispatched(x, x_norm, y)
+    }
+}
 
+/// Fast cosine for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`.
+#[inline]
+fn cosine_fast_f64_dispatched(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f64_x86::cosine_fast_avx512(x, x_norm, y)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f64_x86::cosine_fast_avx_fma(x, x_norm, y)
+            },
+            SimdSupport::Avx => unsafe { f64_x86::cosine_fast_avx(x, x_norm, y) },
+            _ => cosine_scalar(x, x_norm, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_fast_f64_simd_other(x, x_norm, y)
+    }
+}
+
+/// AVX2 + FMA implementation of the f64 cosine_fast kernel.
+///
+/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives
+/// in `crate::simd::f64` (which use raw AVX intrinsics) inline correctly
+/// even when the compile baseline does not have AVX2 enabled. Caller must
+/// ensure the host supports AVX2 + FMA.
+#[cfg(target_arch = "x86_64")]
+mod f64_x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_pd;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX-512 path for f64 fast cosine: 8-wide `__m512d` xy/yy with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_fast_avx512(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc_xy = _mm512_setzero_pd();
+        let mut acc_yy = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let xv = _mm512_loadu_pd(x.as_ptr().add(i));
+            let yv = _mm512_loadu_pd(y.as_ptr().add(i));
+            acc_xy = _mm512_fmadd_pd(xv, yv, acc_xy);
+            acc_yy = _mm512_fmadd_pd(yv, yv, acc_yy);
+        }
+
+        let mut xy = _mm512_reduce_add_pd(acc_xy);
+        let mut yy = _mm512_reduce_add_pd(acc_yy);
+        for i in unrolled_len..dim {
+            xy += x[i] * y[i];
+            yy += y[i] * y[i];
+        }
+
+        let y_norm_sq = yy as f32;
+        let xy_f32 = xy as f32;
+        1.0 - xy_f32 / x_norm / y_norm_sq.sqrt()
+    }
+
+    /// AVX + FMA path for f64 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_fast_avx_fma(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
         let dim = x.len();
         let unrolled_len = dim / 8 * 8;
         let mut y_norm8 = f64x8::zeros();
         let mut xy8 = f64x8::zeros();
         for i in (0..unrolled_len).step_by(8) {
-            unsafe {
-                let xv = f64x8::load_unaligned(x.as_ptr().add(i));
-                let yv = f64x8::load_unaligned(y.as_ptr().add(i));
-                xy8.multiply_add(xv, yv);
-                y_norm8.multiply_add(yv, yv);
-            }
+            let xv = f64x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
         }
         let aligned_len = dim / 4 * 4;
         let mut y_norm4 = f64x4::zeros();
         let mut xy4 = f64x4::zeros();
         for i in (unrolled_len..aligned_len).step_by(4) {
-            unsafe {
-                let xv = f64x4::load_unaligned(x.as_ptr().add(i));
-                let yv = f64x4::load_unaligned(y.as_ptr().add(i));
-                xy4.multiply_add(xv, yv);
-                y_norm4.multiply_add(yv, yv);
-            }
+            let xv = f64x4::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x4::load_unaligned(y.as_ptr().add(i));
+            xy4.multiply_add(xv, yv);
+            y_norm4.multiply_add(yv, yv);
         }
-        let tail_y_norm: Self = y[aligned_len..].iter().map(|&v| v * v).sum();
-        let tail_xy: Self = x[aligned_len..]
+        let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+        let tail_xy: f64 = x[aligned_len..]
             .iter()
             .zip(y[aligned_len..].iter())
             .map(|(&a, &b)| a * b)
@@ -328,6 +727,336 @@ impl Cosine for f64 {
         let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32;
         1.0 - xy / x_norm / y_norm_sq.sqrt()
     }
+
+    /// AVX-only path for f64 fast cosine (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration; tail handled inline.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_fast_avx(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 4 * 4;
+
+        let mut acc_xy = _mm256_setzero_pd();
+        let mut acc_yy = _mm256_setzero_pd();
+        for i in (0..aligned_len).step_by(4) {
+            let xv = _mm256_loadu_pd(x.as_ptr().add(i));
+            let yv = _mm256_loadu_pd(y.as_ptr().add(i));
+            acc_xy = _mm256_add_pd(acc_xy, _mm256_mul_pd(xv, yv));
+            acc_yy = _mm256_add_pd(acc_yy, _mm256_mul_pd(yv, yv));
+        }
+
+        let xy_main = hsum256_pd(acc_xy);
+        let yy_main = hsum256_pd(acc_yy);
+
+        let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+        let tail_xy: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        let y_norm_sq = (yy_main + tail_y_norm) as f32;
+        let xy = (xy_main + tail_xy) as f32;
+        1.0 - xy / x_norm / y_norm_sq.sqrt()
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_fast_f64_simd_other(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::{FloatSimd, SIMD};
+
+    let dim = x.len();
+    let unrolled_len = dim / 8 * 8;
+    let mut y_norm8 = f64x8::zeros();
+    let mut xy8 = f64x8::zeros();
+    for i in (0..unrolled_len).step_by(8) {
+        unsafe {
+            let xv = f64x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+    }
+    let aligned_len = dim / 4 * 4;
+    let mut y_norm4 = f64x4::zeros();
+    let mut xy4 = f64x4::zeros();
+    for i in (unrolled_len..aligned_len).step_by(4) {
+        unsafe {
+            let xv = f64x4::load_unaligned(x.as_ptr().add(i));
+            let yv = f64x4::load_unaligned(y.as_ptr().add(i));
+            xy4.multiply_add(xv, yv);
+            y_norm4.multiply_add(yv, yv);
+        }
+    }
+    let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum();
+    let tail_xy: f64 = x[aligned_len..]
+        .iter()
+        .zip(y[aligned_len..].iter())
+        .map(|(&a, &b)| a * b)
+        .sum();
+
+    let y_norm_sq = (y_norm8.reduce_sum() + y_norm4.reduce_sum() + tail_y_norm) as f32;
+    let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32;
+    1.0 - xy / x_norm / y_norm_sq.sqrt()
+}
+
+/// Cosine for f32 with known norms, runtime-dispatched via `SIMD_SUPPORT`
+/// on x86_64 (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses
+/// the auto-vectorised scalar loop.
+#[inline]
+fn cosine_with_norms_f32_dispatched(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f32_x86::cosine_with_norms_avx512(x, x_norm, y_norm, y)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f32_x86::cosine_with_norms_avx_fma(x, x_norm, y_norm, y)
+            },
+            SimdSupport::Avx => unsafe { f32_x86::cosine_with_norms_avx(x, x_norm, y_norm, y) },
+            _ => cosine_scalar_fast(x, x_norm, y, y_norm),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_with_norms_f32_simd_other(x, x_norm, y_norm, y)
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_with_norms_f32_simd_other(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+    let dim = x.len();
+    let unrolled_len = dim / 16 * 16;
+    let mut xy16 = f32x16::zeros();
+    for i in (0..unrolled_len).step_by(16) {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(y.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+        }
+    }
+    let aligned_len = dim / 8 * 8;
+    let mut xy8 = f32x8::zeros();
+    for i in (unrolled_len..aligned_len).step_by(8) {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+        }
+    }
+    let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
+    1.0 - xy / x_norm / y_norm
+}
+
+/// Fast cosine for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// `simd::f32` primitives, unconditionally backed by NEON / LSX.
+#[inline]
+fn cosine_fast_f32_dispatched(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                f32_x86::cosine_fast_avx512(x, x_norm, other)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe {
+                f32_x86::cosine_fast_avx_fma(x, x_norm, other)
+            },
+            SimdSupport::Avx => unsafe { f32_x86::cosine_fast_avx(x, x_norm, other) },
+            _ => cosine_scalar(x, x_norm, other),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        cosine_fast_f32_simd_other(x, x_norm, other)
+    }
+}
+
+/// AVX2 + FMA implementation of the f32 fast cosine kernel.
+///
+/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives
+/// in `crate::simd::f32` (which use raw AVX intrinsics) inline correctly
+/// even when the compile baseline does not have AVX2 enabled. Caller must
+/// ensure the host supports AVX2 + FMA.
+#[cfg(target_arch = "x86_64")]
+mod f32_x86 {
+    use std::arch::x86_64::*;
+
+    use super::{dot, f32x8, f32x16, norm_l2};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX + FMA path for f32 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_fast_avx_fma(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+        let mut y_norm16 = f32x16::zeros();
+        let mut xy16 = f32x16::zeros();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+            y_norm16.multiply_add(yv, yv);
+        }
+        let aligned_len = dim / 8 * 8;
+        let mut y_norm8 = f32x8::zeros();
+        let mut xy8 = f32x8::zeros();
+        for i in (unrolled_len..aligned_len).step_by(8) {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+        let y_norm =
+            y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
+        let xy =
+            xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
+        1.0 - xy / x_norm / y_norm.sqrt()
+    }
+
+    /// AVX-only path for f32 fast cosine (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`/`norm_l2`.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_fast_avx(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 8 * 8;
+
+        let mut acc_xy = _mm256_setzero_ps();
+        let mut acc_yy = _mm256_setzero_ps();
+        for i in (0..aligned_len).step_by(8) {
+            let xv = _mm256_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm256_loadu_ps(other.as_ptr().add(i));
+            acc_xy = _mm256_add_ps(acc_xy, _mm256_mul_ps(xv, yv));
+            acc_yy = _mm256_add_ps(acc_yy, _mm256_mul_ps(yv, yv));
+        }
+
+        let xy_main = hsum256_ps(acc_xy);
+        let yy_main = hsum256_ps(acc_yy);
+
+        let y_norm = yy_main + norm_l2(&other[aligned_len..]).powi(2);
+        let xy = xy_main + dot(&x[aligned_len..], &other[aligned_len..]);
+        1.0 - xy / x_norm / y_norm.sqrt()
+    }
+
+    /// AVX-512 path for f32 fast cosine: 16-wide `__m512` xy/yy with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_fast_avx512(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc_xy = _mm512_setzero_ps();
+        let mut acc_yy = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = _mm512_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm512_loadu_ps(other.as_ptr().add(i));
+            acc_xy = _mm512_fmadd_ps(xv, yv, acc_xy);
+            acc_yy = _mm512_fmadd_ps(yv, yv, acc_yy);
+        }
+
+        let mut xy = _mm512_reduce_add_ps(acc_xy);
+        let mut yy = _mm512_reduce_add_ps(acc_yy);
+        for i in unrolled_len..dim {
+            xy += x[i] * other[i];
+            yy += other[i] * other[i];
+        }
+
+        1.0 - xy / x_norm / yy.sqrt()
+    }
+
+    /// AVX-512 path for f32 cosine with known norms: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn cosine_with_norms_avx512(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = _mm512_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm512_loadu_ps(y.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(xv, yv, acc);
+        }
+
+        let mut xy = _mm512_reduce_add_ps(acc);
+        for i in unrolled_len..dim {
+            xy += x[i] * y[i];
+        }
+
+        1.0 - xy / x_norm / y_norm
+    }
+
+    /// AVX + FMA path for f32 cosine with known norms. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn cosine_with_norms_avx_fma(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+        let mut xy16 = f32x16::zeros();
+        for i in (0..unrolled_len).step_by(16) {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(y.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+        }
+        let aligned_len = dim / 8 * 8;
+        let mut xy8 = f32x8::zeros();
+        for i in (unrolled_len..aligned_len).step_by(8) {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(y.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+        }
+        let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]);
+        1.0 - xy / x_norm / y_norm
+    }
+
+    /// AVX-only path for f32 cosine with known norms (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn cosine_with_norms_avx(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 {
+        let dim = x.len();
+        let aligned_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..aligned_len).step_by(8) {
+            let xv = _mm256_loadu_ps(x.as_ptr().add(i));
+            let yv = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(xv, yv));
+        }
+
+        let xy_main = hsum256_ps(acc);
+        let xy = xy_main + dot(&x[aligned_len..], &y[aligned_len..]);
+        1.0 - xy / x_norm / y_norm
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn cosine_fast_f32_simd_other(x: &[f32], x_norm: f32, other: &[f32]) -> f32 {
+    let dim = x.len();
+    let unrolled_len = dim / 16 * 16;
+    let mut y_norm16 = f32x16::zeros();
+    let mut xy16 = f32x16::zeros();
+    for i in (0..unrolled_len).step_by(16) {
+        unsafe {
+            let xv = f32x16::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x16::load_unaligned(other.as_ptr().add(i));
+            xy16.multiply_add(xv, yv);
+            y_norm16.multiply_add(yv, yv);
+        }
+    }
+    let aligned_len = dim / 8 * 8;
+    let mut y_norm8 = f32x8::zeros();
+    let mut xy8 = f32x8::zeros();
+    for i in (unrolled_len..aligned_len).step_by(8) {
+        unsafe {
+            let xv = f32x8::load_unaligned(x.as_ptr().add(i));
+            let yv = f32x8::load_unaligned(other.as_ptr().add(i));
+            xy8.multiply_add(xv, yv);
+            y_norm8.multiply_add(yv, yv);
+        }
+    }
+    let y_norm =
+        y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2);
+    let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]);
+    1.0 - xy / x_norm / y_norm.sqrt()
 }
 
 /// Fallback non-SIMD implementation
@@ -437,6 +1166,25 @@ pub fn cosine_distance_arrow_batch(
     }
 }
 
+/// Portable scalar reference cosine over f64 inputs. Used by parity tests
+/// to compare against every dispatched per-tier inner kernel. Computes
+/// `1 - xy / (x_norm * y_norm_sq.sqrt())` in f64 then casts to f32, matching
+/// the reduction order of the dispatched kernels.
+#[cfg(test)]
+fn cosine_fast_scalar(x: &[f64], x_norm: f32, y: &[f64]) -> f32 {
+    let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum();
+    let y_norm_sq: f64 = y.iter().map(|&v| v * v).sum();
+    1.0 - (xy as f32) / x_norm / (y_norm_sq as f32).sqrt()
+}
+
+/// Portable scalar reference cosine when both norms are known. Mirrors
+/// `cosine_with_norms_f32_dispatched` for parity testing.
+#[cfg(test)]
+fn cosine_with_norms_scalar(x: &[f64], x_norm: f32, y_norm: f32, y: &[f64]) -> f32 {
+    let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum();
+    1.0 - (xy as f32) / x_norm / y_norm
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -559,5 +1307,447 @@ mod tests {
             prop_assume!(norm_l2(&y) > 1e-20);
             do_cosine_test(&x, &y)?;
         }
+
+        /// Cross-backend parity for the f32 cosine_fast kernel. Exercises the
+        /// scalar fallback (`cosine_scalar`) against the dispatched SIMD path
+        /// so the runtime fallback is exercised even on AVX2-capable CI hosts.
+        #[test]
+        fn test_cosine_fast_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = <f32 as Cosine>::cosine_fast(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f32 cosine_fast kernel. Early-returns
+        /// on hosts without AVX-512F so the test stays portable.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx512 = unsafe { f32_x86::cosine_fast_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f32 cosine_fast kernel. Covers
+        /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns
+        /// on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx_fma = unsafe { f32_x86::cosine_fast_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f32 cosine_fast kernel. Covers
+        /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on
+        /// hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_scalar(&x, x_norm, &y);
+            let avx = unsafe { f32_x86::cosine_fast_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Cross-backend parity for the f32 cosine_with_norms kernel.
+        /// Exercises the scalar fallback (`cosine_scalar_fast`) against the
+        /// dispatched SIMD path so the runtime fallback is exercised even on
+        /// AVX2-capable CI hosts.
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_with_norms_scalar(&x_f64, x_norm, y_norm, &y_f64);
+            let simd = <f32 as Cosine>::cosine_with_norms(&x, x_norm, y_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f32 cosine_with_norms kernel.
+        /// Early-returns on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx512 = unsafe { f32_x86::cosine_with_norms_avx512(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f32 cosine_with_norms kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx_fma = unsafe { f32_x86::cosine_with_norms_avx_fma(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f32 cosine_with_norms kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_with_norms_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let y_norm = norm_l2(&y);
+            let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm);
+            let avx = unsafe { f32_x86::cosine_with_norms_avx(&x, x_norm, y_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Cross-backend parity for the f64 cosine_fast kernel. Uses the
+        /// hand-rolled `cosine_fast_scalar` (not the trait-routed
+        /// `cosine_scalar`, which would itself dispatch through `dot::<f64>`)
+        /// so the reference stays free of any AVX path on AVX2-capable hosts.
+        #[test]
+        fn test_cosine_fast_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let simd = <f64 as Cosine>::cosine_fast(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity for the f64 cosine_fast kernel. Early-returns
+        /// on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx512 = unsafe { f64_x86::cosine_fast_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the f64 cosine_fast kernel. Covers
+        /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns
+        /// on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx_fma = unsafe { f64_x86::cosine_fast_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the f64 cosine_fast kernel. Covers
+        /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on
+        /// hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_fast_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-20);
+            prop_assume!(norm_l2(&y) > 1e-20);
+            let x_norm = norm_l2(&x);
+            let scalar = cosine_fast_scalar(&x, x_norm, &y);
+            let avx = unsafe { f64_x86::cosine_fast_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// Parity check for `cosine_once_8` (despecialised 8-lane width).
+        ///
+        /// The `epsilon = 1e-6` clause handles the case where the proptest
+        /// generator produces inputs with extreme dynamic range (e.g., mixing
+        /// `1e-43` with `1e7` in the same vector). When the dot product is
+        /// dominated by one large term and the cosine result is near zero,
+        /// the f32-precision SIMD path and the f64-precision scalar reference
+        /// can legitimately differ by more than `max_relative = 1e-3` of the
+        /// (near-zero) result. The absolute epsilon catches these without
+        /// masking real bugs (where the absolute error would be > 1e-6).
+        #[test]
+        fn test_cosine_once_8_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = f32::cosine_once_8(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6));
+        }
+
+        /// Parity check for `cosine_once_16` (despecialised 16-lane width).
+        /// See `test_cosine_once_8_scalar_simd_parity` for `epsilon` rationale.
+        #[test]
+        fn test_cosine_once_16_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+            let simd = f32::cosine_once_16(&x, x_norm, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6));
+        }
+
+        /// AVX-512-direct parity for the 8-lane cosine_once kernel. Verifies
+        /// the masked-load (mask 0x00FF) AVX-512 implementation produces
+        /// the same result as the scalar reference. Early-returns on hosts
+        /// without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx512 =
+                unsafe { super::f32::cosine_once_x86::cosine_once_8_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX-512-direct parity for the 16-lane cosine_once kernel. Verifies
+        /// the full-width `__m512` load implementation produces the same
+        /// result as the scalar reference. Early-returns on hosts without
+        /// AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx512 =
+                unsafe { super::f32::cosine_once_x86::cosine_once_16_avx512(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the 8-lane cosine_once kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx_fma =
+                unsafe { super::f32::cosine_once_x86::cosine_once_8_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX + FMA-direct parity for the 16-lane cosine_once kernel.
+        /// Covers the AMD Piledriver / Steamroller / FX-7500 tier.
+        /// Early-returns on hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx_fma =
+                unsafe { super::f32::cosine_once_x86::cosine_once_16_avx_fma(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the 8-lane cosine_once kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_8_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y);
+            let avx = unsafe { super::f32::cosine_once_x86::cosine_once_8_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+
+        /// AVX-only-direct parity for the 16-lane cosine_once kernel.
+        /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_cosine_once_16_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            prop_assume!(norm_l2(&x) > 1e-10);
+            prop_assume!(norm_l2(&y) > 1e-10);
+            let x_norm = norm_l2(&x);
+            let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y);
+            let avx = unsafe { super::f32::cosine_once_x86::cosine_once_16_avx(&x, x_norm, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5));
+        }
+    }
+
+    /// Asserts a batch-level f32 cosine SIMD kernel matches the scalar
+    /// `cosine_fast` reference for every vector in a multi-vector batch. Runs
+    /// each of the kernel's three internal dimension arms (8, 16, and the
+    /// general `chunks_exact` path). The batch kernels only run at runtime on
+    /// sub-AVX2 builds, so a direct call is the only way they get covered.
+    #[cfg(target_arch = "x86_64")]
+    fn check_cosine_batch_kernel(kernel: unsafe fn(&[f32], f32, &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let x_norm = norm_l2(&x);
+            let num_vectors = 3;
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, x_norm, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let y_f64: Vec<f64> = chunk.iter().map(|&v| v as f64).collect();
+                let expected = cosine_fast_scalar(&x_f64, x_norm, &y_f64);
+                assert_relative_eq!(g, expected, max_relative = 1e-3, epsilon = 1e-6);
+            }
+        }
+    }
+
+    /// AVX + FMA batch kernel parity (AVX2 / AVX+FMA tiers). Runs on any
+    /// Haswell-or-newer host; early-returns without AVX and FMA.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx_fma_matches_scalar() {
+        if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx_fma);
+    }
+
+    /// AVX-only batch kernel parity (Sandy Bridge / Ivy Bridge tier).
+    /// Early-returns on hosts without AVX.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx);
+    }
+
+    /// AVX-512 batch kernel parity. Early-returns on hosts without AVX-512F.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_cosine_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_cosine_batch_kernel(super::f32::cosine_batch_avx512);
     }
 }
diff --git a/rust/lance-linalg/src/distance/cosine_u8.rs b/rust/lance-linalg/src/distance/cosine_u8.rs
index b2d06d35c31..59833e020a7 100644
--- a/rust/lance-linalg/src/distance/cosine_u8.rs
+++ b/rust/lance-linalg/src/distance/cosine_u8.rs
@@ -199,6 +199,10 @@ fn select_backend() -> CosineU8AccumFn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::cosine_u8_accum_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor
+        // AVX+FMA provides.
     }
 
     cosine_u8_accum_scalar
diff --git a/rust/lance-linalg/src/distance/dot.rs b/rust/lance-linalg/src/distance/dot.rs
index cf045b1996a..3411d7e8dce 100644
--- a/rust/lance-linalg/src/distance/dot.rs
+++ b/rust/lance-linalg/src/distance/dot.rs
@@ -14,12 +14,16 @@ use arrow_schema::DataType;
 use half::{bf16, f16};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
 use lance_core::assume_eq;
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+#[allow(unused_imports)]
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 use num_traits::{AsPrimitive, Num, real::Real};
 
 use crate::Result;
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+use crate::distance::BatchIter;
 
 /// Default implementation of dot product.
 ///
@@ -111,6 +115,24 @@ pub fn dot_distance<T: Dot>(from: &[T], to: &[T]) -> f32 {
 pub trait Dot: Num {
     /// Dot product.
     fn dot(x: &[Self], y: &[Self]) -> f32;
+
+    /// Dot product of `x` against each `dimension`-sized vector in `batch`.
+    ///
+    /// The default calls [`Dot::dot`] per vector. `f32` overrides it so the
+    /// SIMD tier is chosen once for the whole batch instead of once per
+    /// vector — on a build whose baseline already implies AVX2, per-vector
+    /// dispatch costs more than the kernel it selects.
+    ///
+    /// Returns `impl Iterator` rather than a trait object: hot consumers drive
+    /// this one element at a time, so a `Box<dyn Iterator>` would cost a
+    /// virtual call per element and an allocation per batch.
+    fn dot_batch<'a>(
+        x: &'a [Self],
+        batch: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = f32> + 'a {
+        batch.chunks_exact(dimension).map(move |y| Self::dot(x, y))
+    }
 }
 
 #[cfg(feature = "fp16kernels")]
@@ -122,7 +144,7 @@ mod bf16_kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn dot_bf16_neon(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_bf16", target_arch = "x86_64"))]
         pub fn dot_bf16_avx512(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn dot_bf16_avx2(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
@@ -143,7 +165,7 @@ impl Dot for bf16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_bf16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -161,6 +183,9 @@ impl Dot for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::dot_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => dot_scalar::<Self, f32, 32>(x, y),
         }
     }
@@ -175,7 +200,7 @@ mod kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn dot_f16_neon(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_f16", target_arch = "x86_64"))]
         pub fn dot_f16_avx512(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn dot_f16_avx2(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
@@ -196,7 +221,7 @@ impl Dot for f16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_f16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -214,6 +239,9 @@ impl Dot for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::dot_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => dot_scalar::<Self, f32, 32>(x, y),
         }
     }
@@ -222,10 +250,128 @@ impl Dot for f16 {
 impl Dot for f32 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
-        dot_scalar::<Self, Self, 16>(x, y)
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable
+        // scalar fallback. Same shape as the f64 sibling and the existing
+        // u8 distance kernels in `dot_u8.rs`.
+        dot_f32_dispatched(x, y)
+    }
+
+    fn dot_batch<'a>(
+        x: &'a [Self],
+        batch: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = Self> + 'a {
+        // Exactly one arm compiles. Keeping each a tail expression (rather than
+        // an early `return` guarded by `cfg`) mirrors `dot_f32_dispatched` and
+        // avoids an unreachable tail on AVX2-baseline builds.
+        // AVX2-baseline build (the default `haswell` wheel). Hoist the tier
+        // choice out of the loop, but keep the SIMD kernel: the baseline already
+        // guarantees avx2+fma, so call the AVX+FMA kernel directly rather than
+        // re-checking per vector. Falling back to the scalar kernel here would
+        // lose ~4x at small dimensions, which is where batch calls live (PQ
+        // sub-vectors are 8 wide).
+        //
+        // The iterator is a bare `Map`: `Map<ChunksExact, _>` is `TrustedLen`,
+        // so `.collect()` preallocates, and `Map::fold` drives `ChunksExact` in
+        // one inlined loop. Any wrapper — trait object or enum — loses both.
+        #[cfg(all(
+            target_arch = "x86_64",
+            target_feature = "avx2",
+            target_feature = "fma"
+        ))]
+        {
+            // See `L2::l2_batch` for f32: below 16 lanes `dot_scalar`'s chunking
+            // degenerates to a scalar remainder loop, so the explicit AVX kernel
+            // wins big; above it the autovectorizer is already good and the
+            // 8-wide kernel can lose, so keep the pre-dispatch kernel exactly.
+            //
+            // SAFETY: avx2+fma are enabled for the whole crate by the build
+            // baseline, so the kernel's `#[target_feature]` contract holds
+            // statically.
+            let narrow = dimension <= 16;
+            batch.chunks_exact(dimension).map(move |y| {
+                if narrow {
+                    unsafe { x86::dot_f32_avx_fma(x, y) }
+                } else {
+                    dot_f32_scalar(x, y)
+                }
+            })
+        }
+        #[cfg(all(
+            target_arch = "x86_64",
+            not(all(target_feature = "avx2", target_feature = "fma"))
+        ))]
+        {
+            dot_batch_f32_runtime_dispatch(x, batch, dimension)
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            batch.chunks_exact(dimension).map(move |y| Self::dot(x, y))
+        }
+    }
+}
+
+/// Sub-AVX2 builds: the scalar kernel cannot reach the wide registers, so pick
+/// a `#[target_feature]` kernel — once for the batch, not once per vector.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+#[inline]
+fn dot_batch_f32_runtime_dispatch<'a>(
+    x: &'a [f32],
+    batch: &'a [f32],
+    dimension: usize,
+) -> impl Iterator<Item = f32> + 'a {
+    // SAFETY: each kernel is entered only under its matching runtime detection.
+    match *SIMD_SUPPORT {
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx512(x, batch, dimension) }.into_iter())
+        }
+        SimdSupport::Avx2 | SimdSupport::AvxFma => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx_fma(x, batch, dimension) }.into_iter())
+        }
+        SimdSupport::Avx => {
+            BatchIter::Eager(unsafe { x86::dot_batch_f32_avx(x, batch, dimension) }.into_iter())
+        }
+        _ => BatchIter::Lazy(
+            batch
+                .chunks_exact(dimension)
+                .map(move |y| dot_f32_scalar(x, y)),
+        ),
+    }
+}
+
+/// Dot product for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn dot_f32_dispatched(x: &[f32], y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f32_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f32_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::dot_f32_avx(x, y) },
+            _ => dot_f32_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        dot_f32_scalar(x, y)
     }
 }
 
+/// Portable scalar dot product for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn dot_f32_scalar(x: &[f32], y: &[f32]) -> f32 {
+    dot_scalar::<f32, f32, 16>(x, y)
+}
+
 impl Dot for f64 {
     #[inline]
     fn dot(x: &[Self], y: &[Self]) -> f32 {
@@ -233,9 +379,246 @@ impl Dot for f64 {
     }
 }
 
-/// Explicit SIMD dot product for f64.
+/// Dot product for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX.
 #[inline]
 fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f64_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f64_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::dot_f64_avx(x, y) },
+            _ => dot_f64_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        dot_f64_simd_other(x, y)
+    }
+}
+
+/// Portable scalar dot product for f64. Used as the x86_64 fallback when no
+/// AVX2 is detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn dot_f64_scalar(x: &[f64], y: &[f64]) -> f32 {
+    x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::<f64>() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// Dot product of `x` against every `dimension`-sized vector in `batch`,
+    /// entering the AVX-512 tier once for the whole batch rather than once per
+    /// vector.
+    ///
+    /// # Safety
+    /// The host must support AVX-512F.
+    ///
+    /// Only compiled for builds whose baseline is below avx2+fma; at or above
+    /// that baseline `dot_batch` inlines the kernel directly and never runtime-
+    /// dispatches, so this wrapper would be dead code (see `dot_batch`).
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn dot_batch_f32_avx512(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx512(x, y) })
+            .collect()
+    }
+
+    /// As [`dot_batch_f32_avx512`], for the AVX2 and AVX+FMA tiers.
+    ///
+    /// # Safety
+    /// The host must support AVX and FMA.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn dot_batch_f32_avx_fma(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx_fma(x, y) })
+            .collect()
+    }
+
+    /// As [`dot_batch_f32_avx512`], for the AVX-without-FMA tier.
+    ///
+    /// # Safety
+    /// The host must support AVX.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn dot_batch_f32_avx(x: &[f32], batch: &[f32], dimension: usize) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { dot_f32_avx(x, y) })
+            .collect()
+    }
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn dot_f64_avx512(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm512_loadu_pd(x.as_ptr().add(i));
+            let b = _mm512_loadu_pd(y.as_ptr().add(i));
+            acc = _mm512_fmadd_pd(a, b, acc);
+        }
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (_mm512_reduce_add_pd(acc) + tail) as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn dot_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            acc8.multiply_add(a, b);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            acc4.multiply_add(a, b);
+        }
+
+        let tail: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn dot_f64_avx(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let a = _mm256_loadu_pd(x.as_ptr().add(i));
+            let b = _mm256_loadu_pd(y.as_ptr().add(i));
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(a, b));
+        }
+
+        // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        (acc_sum + tail) as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn dot_f32_avx512(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let a = _mm512_loadu_ps(x.as_ptr().add(i));
+            let b = _mm512_loadu_ps(y.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(a, b, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        _mm512_reduce_add_ps(acc) + tail
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn dot_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_fmadd_ps(a, b, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+
+    /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn dot_f32_avx(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(a, b));
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| a * b)
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn dot_f64_simd_other(x: &[f64], y: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -285,7 +668,7 @@ pub fn dot_distance_batch<'a, T: Dot>(
 ) -> Box<dyn Iterator<Item = f32> + 'a> {
     assume_eq!(from.len(), dimension);
     assume_eq!(to.len() % dimension, 0);
-    Box::new(to.chunks_exact(dimension).map(|v| dot_distance(from, v)))
+    Box::new(T::dot_batch(from, to, dimension).map(|d| 1.0 - d))
 }
 
 fn do_dot_distance_arrow_batch<T: ArrowFloatType>(
@@ -309,10 +692,9 @@ where
                 to.value_type()
             )))?;
 
-    let dists = to_values
-        .as_slice()
-        .chunks_exact(dimension)
-        .map(|v| dot_distance(from.as_slice(), v));
+    // Route through `dot_distance_batch` rather than mapping `dot_distance` per
+    // vector, so this entry point gets the same hoisted dispatch.
+    let dists = dot_distance_batch(from.as_slice(), to_values.as_slice(), dimension);
 
     Ok(Arc::new(Float32Array::new(
         dists.collect(),
@@ -480,5 +862,264 @@ mod tests {
         fn test_dot_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){
             do_dot_test(&x, &y)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `dot_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            let scalar = dot_f64_scalar(&x, &y);
+            let simd = dot_f64_simd(&x, &y);
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error));
+        }
+
+        /// Parity check for `dot_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar dot path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep
+        /// this test architecture-agnostic (the x86_64-only `dot_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_dot_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let scalar = x_f64
+                .iter()
+                .zip(y_f64.iter())
+                .map(|(&a, &b)| a * b)
+                .sum::<f64>() as f32;
+            let simd = <f32 as Dot>::dot(&x, &y);
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error));
+        }
+
+        /// AVX-512-direct parity for f32: explicitly compares the scalar
+        /// fallback against the native f32 AVX-512 inner kernel on
+        /// AVX-512F-capable hosts. Early-returns on hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx512 = unsafe { x86::dot_f32_avx512(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error));
+        }
+
+        /// AVX + FMA-direct parity for the f32 dot kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx_fma = unsafe { x86::dot_f32_avx_fma(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error));
+        }
+
+        /// AVX-only-direct parity for the f32 dot kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = dot_f32_scalar(&x, &y);
+            let avx = unsafe { x86::dot_f32_avx(&x, &y) };
+            let x_f64: Vec<f64> = x.iter().map(|&v| v as f64).collect();
+            let y_f64: Vec<f64> = y.iter().map(|&v| v as f64).collect();
+            let max_error = max_error::<f32>(&x_f64, &y_f64);
+            prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx512 = unsafe { x86::dot_f64_avx512(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error));
+        }
+
+        /// AVX + FMA-direct parity for the f64 dot kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx_fma = unsafe { x86::dot_f64_avx_fma(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error));
+        }
+
+        /// AVX-only-direct parity for the f64 dot kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier (AVX without FMA). Early-returns
+        /// on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_dot_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = dot_f64_scalar(&x, &y);
+            let avx = unsafe { x86::dot_f64_avx(&x, &y) };
+            let max_error = max_error::<f64>(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error));
+        }
+    }
+
+    /// `dot_batch` must agree with the per-vector `dot` it replaced, on every
+    /// build: AVX2-baseline, hoisted-dispatch, and portable fallback all
+    /// funnel through here.
+    #[rstest::rstest]
+    #[case::dim_8(8)]
+    #[case::dim_16(16)]
+    #[case::dim_32(32)]
+    #[case::dim_1024(1024)]
+    fn test_dot_batch_f32_matches_per_vector_dot(#[case] dimension: usize) {
+        let num_vectors = 5;
+        let x: Vec<f32> = (0..dimension)
+            .map(|i| ((i % 13) as f32) * 0.25 + 1.0)
+            .collect();
+        let batch: Vec<f32> = (0..dimension * num_vectors)
+            .map(|i| ((i % 11) as f32) * 0.5 - 2.0)
+            .collect();
+
+        let got: Vec<f32> = f32::dot_batch(&x, &batch, dimension).collect();
+        let want: Vec<f32> = batch
+            .chunks_exact(dimension)
+            .map(|y| f32::dot(&x, y))
+            .collect();
+
+        assert_eq!(got.len(), num_vectors);
+        for (g, w) in got.iter().zip(want.iter()) {
+            assert!(
+                approx::relative_eq!(g, w, epsilon = 1e-4),
+                "dim {dimension}: batch {g} != per-vector {w}"
+            );
+        }
+    }
+
+    /// `dot_distance_batch` still yields `1.0 - dot`, unchanged by the hoist.
+    #[test]
+    fn test_dot_distance_batch_preserves_distance_semantics() {
+        let dimension = 32;
+        let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.1).collect();
+        let batch: Vec<f32> = (0..dimension * 3).map(|i| (i as f32) * 0.05).collect();
+
+        let got: Vec<f32> = dot_distance_batch(&x, &batch, dimension).collect();
+        for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+            assert!(approx::relative_eq!(
+                g,
+                1.0 - f32::dot(&x, chunk),
+                epsilon = 1e-5
+            ));
+        }
+    }
+
+    /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2
+    /// builds or AVX-512 hosts, so call them directly to cover them.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    fn check_dot_batch_kernel(kernel: unsafe fn(&[f32], &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let num_vectors = 3;
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let want = dot_scalar::<f32, f32, 16>(&x, chunk);
+                assert!(
+                    approx::relative_eq!(g, want, epsilon = 1e-4),
+                    "dim {dimension}: kernel {g} != scalar {want}"
+                );
+            }
+        }
+    }
+
+    // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds
+    // (see `x86::dot_batch_f32_avx512`), so gate their tests the same way.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx_fma_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx_fma);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_dot_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_dot_batch_kernel(x86::dot_batch_f32_avx512);
     }
 }
diff --git a/rust/lance-linalg/src/distance/dot_u8.rs b/rust/lance-linalg/src/distance/dot_u8.rs
index de5522cddfe..7b2e335f094 100644
--- a/rust/lance-linalg/src/distance/dot_u8.rs
+++ b/rust/lance-linalg/src/distance/dot_u8.rs
@@ -134,6 +134,10 @@ fn select_backend() -> DotU8Fn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::dot_u8_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor
+        // AVX+FMA provides.
     }
 
     dot_u8_scalar
diff --git a/rust/lance-linalg/src/distance/hamming.rs b/rust/lance-linalg/src/distance/hamming.rs
index d8fd60f4054..1eeb8e42c38 100644
--- a/rust/lance-linalg/src/distance/hamming.rs
+++ b/rust/lance-linalg/src/distance/hamming.rs
@@ -2,14 +2,24 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 //! Hamming distance.
+//!
+//! This module provides hamming distance computation for binary vectors,
+//! including SIMD-accelerated pairwise hamming distance for binary hashes.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
-use crate::{Error, Result};
+use arrow_array::builder::{ListBuilder, UInt64Builder};
 use arrow_array::cast::AsArray;
 use arrow_array::types::UInt8Type;
-use arrow_array::{Array, FixedSizeListArray, Float32Array};
-use arrow_schema::DataType;
+use arrow_array::{
+    Array, ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator,
+    RecordBatchReader, UInt32Array, UInt64Array,
+};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use rayon::prelude::*;
+
+use crate::{Error, Result};
 
 pub trait Hamming {
     /// Hamming distance between two vectors.
@@ -86,9 +96,960 @@ pub fn hamming_distance_arrow_batch(
     )))
 }
 
+/// Compute hamming distance between two 64-bit values using POPCNT.
+#[inline(always)]
+pub fn hamming_u64(a: u64, b: u64) -> u32 {
+    (a ^ b).count_ones()
+}
+
+/// Binary hash values stored as 64-bit lanes in lane-major order.
+///
+/// For a hash width of `N` bytes, `N` must be a positive multiple of 8 and the
+/// number of lanes is `N / 8`. Lane-major layout keeps each lane contiguous for
+/// all rows, which allows the pairwise path to reuse the SIMD `u64` batch
+/// implementation for wider hashes.
+#[derive(Debug, Clone)]
+pub struct BinaryHashValues {
+    lane_values: Vec<u64>,
+    num_rows: usize,
+    byte_width: usize,
+}
+
+impl BinaryHashValues {
+    /// Create hash values from lane-major `u64` values.
+    pub fn try_new(lane_values: Vec<u64>, num_rows: usize, byte_width: usize) -> Result<Self> {
+        let num_lanes = validate_hash_byte_width(byte_width)?;
+        let expected_values = checked_lane_value_count(num_rows, num_lanes)?;
+        if lane_values.len() != expected_values {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected {} lane values for {} rows and {} byte hashes, got {}",
+                expected_values,
+                num_rows,
+                byte_width,
+                lane_values.len()
+            )));
+        }
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    /// Extract binary hash values from a `FixedSizeList<UInt8, N>` Arrow array.
+    pub fn from_fixed_size_list(array: &FixedSizeListArray) -> Result<Self> {
+        let byte_width = usize::try_from(array.value_length()).map_err(|_| {
+            Error::InvalidArgumentError(format!(
+                "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}",
+                array.value_length()
+            ))
+        })?;
+        let num_lanes = validate_hash_byte_width(byte_width)?;
+
+        let values = array
+            .values()
+            .as_any()
+            .downcast_ref::<arrow_array::UInt8Array>()
+            .ok_or_else(|| {
+                Error::InvalidArgumentError(
+                    "Expected UInt8Array values in FixedSizeList".to_string(),
+                )
+            })?;
+
+        let num_rows = array.len();
+        let value_count = checked_lane_value_count(num_rows, num_lanes)?;
+        let expected_bytes = checked_hash_byte_count(num_rows, byte_width)?;
+        let bytes = values.values();
+        if bytes.len() != expected_bytes {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected {} bytes for {} rows and {} byte hashes, got {}",
+                expected_bytes,
+                num_rows,
+                byte_width,
+                bytes.len()
+            )));
+        }
+
+        let mut lane_values = vec![0u64; value_count];
+        for row in 0..num_rows {
+            let row_start = row * byte_width;
+            for lane in 0..num_lanes {
+                let start = row_start + lane * 8;
+                let mut arr = [0u8; 8];
+                arr.copy_from_slice(&bytes[start..start + 8]);
+                lane_values[lane * num_rows + row] = u64::from_le_bytes(arr);
+            }
+        }
+
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    /// Concatenate chunks with the same hash width into a single lane-major set.
+    pub fn concat(chunks: &[Self]) -> Result<Self> {
+        let Some(first) = chunks.first() else {
+            return Err(Error::InvalidArgumentError(
+                "Cannot concatenate zero binary hash chunks".to_string(),
+            ));
+        };
+
+        let byte_width = first.byte_width;
+        let num_lanes = first.num_lanes();
+        let mut num_rows = 0usize;
+        for chunk in chunks {
+            if chunk.byte_width != byte_width {
+                return Err(Error::InvalidArgumentError(format!(
+                    "Cannot concatenate binary hash chunks with different widths: {} and {} bytes",
+                    byte_width, chunk.byte_width
+                )));
+            }
+            num_rows = num_rows.checked_add(chunk.num_rows).ok_or_else(|| {
+                Error::InvalidArgumentError(
+                    "Binary hash row count overflowed while concatenating chunks".to_string(),
+                )
+            })?;
+        }
+
+        let value_count = checked_lane_value_count(num_rows, num_lanes)?;
+        let mut lane_values = vec![0u64; value_count];
+        let mut row_offset = 0;
+        for chunk in chunks {
+            for lane in 0..num_lanes {
+                let dest_start = lane * num_rows + row_offset;
+                let dest_end = dest_start + chunk.num_rows;
+                lane_values[dest_start..dest_end].copy_from_slice(chunk.lane(lane));
+            }
+            row_offset += chunk.num_rows;
+        }
+
+        Ok(Self {
+            lane_values,
+            num_rows,
+            byte_width,
+        })
+    }
+
+    pub fn len(&self) -> usize {
+        self.num_rows
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.num_rows == 0
+    }
+
+    pub fn byte_width(&self) -> usize {
+        self.byte_width
+    }
+
+    pub fn num_lanes(&self) -> usize {
+        self.byte_width / 8
+    }
+
+    pub fn lane(&self, lane: usize) -> &[u64] {
+        let start = lane * self.num_rows;
+        &self.lane_values[start..start + self.num_rows]
+    }
+
+    pub fn into_u64_values(self) -> Result<Vec<u64>> {
+        if self.num_lanes() != 1 {
+            return Err(Error::InvalidArgumentError(format!(
+                "Expected 8-byte binary hashes, got {} byte hashes",
+                self.byte_width
+            )));
+        }
+        Ok(self.lane_values)
+    }
+}
+
+fn checked_lane_value_count(num_rows: usize, num_lanes: usize) -> Result<usize> {
+    num_rows.checked_mul(num_lanes).ok_or_else(|| {
+        Error::InvalidArgumentError(format!(
+            "Binary hash lane value count overflowed for {} rows and {} lanes",
+            num_rows, num_lanes
+        ))
+    })
+}
+
+fn checked_hash_byte_count(num_rows: usize, byte_width: usize) -> Result<usize> {
+    num_rows.checked_mul(byte_width).ok_or_else(|| {
+        Error::InvalidArgumentError(format!(
+            "Binary hash byte count overflowed for {} rows and {} byte hashes",
+            num_rows, byte_width
+        ))
+    })
+}
+
+fn validate_hash_byte_width(byte_width: usize) -> Result<usize> {
+    if byte_width == 0 || !byte_width.is_multiple_of(8) {
+        return Err(Error::InvalidArgumentError(format!(
+            "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}",
+            byte_width
+        )));
+    }
+    Ok(byte_width / 8)
+}
+
+/// Result of pairwise hamming distance computation.
+#[derive(Debug, Clone)]
+pub struct PairwiseResult {
+    pub row_id_a: Vec<u64>,
+    pub row_id_b: Vec<u64>,
+    pub distances: Vec<u32>,
+}
+
+impl PairwiseResult {
+    pub fn new() -> Self {
+        Self {
+            row_id_a: Vec::new(),
+            row_id_b: Vec::new(),
+            distances: Vec::new(),
+        }
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            row_id_a: Vec::with_capacity(capacity),
+            row_id_b: Vec::with_capacity(capacity),
+            distances: Vec::with_capacity(capacity),
+        }
+    }
+
+    pub fn push(&mut self, a: u64, b: u64, dist: u32) {
+        self.row_id_a.push(a);
+        self.row_id_b.push(b);
+        self.distances.push(dist);
+    }
+
+    pub fn len(&self) -> usize {
+        self.row_id_a.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.row_id_a.is_empty()
+    }
+
+    pub fn extend(&mut self, other: Self) {
+        self.row_id_a.extend(other.row_id_a);
+        self.row_id_b.extend(other.row_id_b);
+        self.distances.extend(other.distances);
+    }
+
+    /// Convert to Arrow RecordBatch, consuming self.
+    pub fn into_record_batch(self) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("row_id_a", DataType::UInt64, false),
+            Field::new("row_id_b", DataType::UInt64, false),
+            Field::new("distance", DataType::UInt32, false),
+        ]));
+
+        let row_id_a = Arc::new(UInt64Array::from(self.row_id_a));
+        let row_id_b = Arc::new(UInt64Array::from(self.row_id_b));
+        let distances = Arc::new(UInt32Array::from(self.distances));
+
+        RecordBatch::try_new(schema, vec![row_id_a, row_id_b, distances])
+            .expect("Failed to create RecordBatch")
+    }
+}
+
+impl Default for PairwiseResult {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Compute hamming distances for a query against multiple targets.
+/// Uses SIMD acceleration when available.
+#[inline]
+pub fn hamming_batch_u64(query: u64, targets: &[u64], results: &mut [u32]) {
+    debug_assert_eq!(targets.len(), results.len());
+    hamming_batch_simd(query, targets, results);
+}
+
+/// SIMD-accelerated batch hamming distance computation.
+#[inline]
+fn hamming_batch_simd(query: u64, targets: &[u64], results: &mut [u32]) {
+    #[cfg(target_arch = "x86_64")]
+    {
+        if is_x86_feature_detected!("avx512vpopcntdq") && is_x86_feature_detected!("avx512f") {
+            unsafe {
+                hamming_batch_avx512(query, targets, results);
+            }
+            return;
+        }
+        if is_x86_feature_detected!("avx2") {
+            unsafe {
+                hamming_batch_avx2(query, targets, results);
+            }
+            return;
+        }
+    }
+
+    // Scalar fallback (LLVM auto-vectorizes well on Apple Silicon)
+    hamming_batch_scalar(query, targets, results);
+}
+
+/// Scalar fallback using count_ones() which compiles to POPCNT.
+#[inline]
+fn hamming_batch_scalar(query: u64, targets: &[u64], results: &mut [u32]) {
+    // Unroll for better auto-vectorization
+    let n = targets.len();
+    let chunks = n / 8;
+    let mut i = 0;
+
+    for _ in 0..chunks {
+        results[i] = (query ^ targets[i]).count_ones();
+        results[i + 1] = (query ^ targets[i + 1]).count_ones();
+        results[i + 2] = (query ^ targets[i + 2]).count_ones();
+        results[i + 3] = (query ^ targets[i + 3]).count_ones();
+        results[i + 4] = (query ^ targets[i + 4]).count_ones();
+        results[i + 5] = (query ^ targets[i + 5]).count_ones();
+        results[i + 6] = (query ^ targets[i + 6]).count_ones();
+        results[i + 7] = (query ^ targets[i + 7]).count_ones();
+        i += 8;
+    }
+
+    // Handle remainder
+    while i < n {
+        results[i] = (query ^ targets[i]).count_ones();
+        i += 1;
+    }
+}
+
+/// AVX-512 VPOPCNTDQ: Process 8 x 64-bit values at once.
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx512f", enable = "avx512vpopcntdq")]
+unsafe fn hamming_batch_avx512(query: u64, targets: &[u64], results: &mut [u32]) {
+    use std::arch::x86_64::*;
+
+    let n = targets.len();
+    let query_vec = _mm512_set1_epi64(query as i64);
+
+    let chunks = n / 8;
+    let remainder = n % 8;
+
+    for i in 0..chunks {
+        let offset = i * 8;
+        let targets_ptr = targets.as_ptr().add(offset) as *const __m512i;
+        let target_vec = _mm512_loadu_si512(targets_ptr);
+
+        let xor_result = _mm512_xor_si512(query_vec, target_vec);
+        let popcount = _mm512_popcnt_epi64(xor_result);
+        let popcount_32 = _mm512_cvtepi64_epi32(popcount);
+
+        _mm256_storeu_si256(
+            results.as_mut_ptr().add(offset) as *mut __m256i,
+            popcount_32,
+        );
+    }
+
+    if remainder > 0 {
+        let offset = chunks * 8;
+        for j in 0..remainder {
+            results[offset + j] = (query ^ targets[offset + j]).count_ones();
+        }
+    }
+}
+
+/// AVX2 popcount using lookup table (Harley-Seal / PSHUFB method).
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn hamming_batch_avx2(query: u64, targets: &[u64], results: &mut [u32]) {
+    use std::arch::x86_64::*;
+
+    let n = targets.len();
+
+    let lookup = _mm256_setr_epi8(
+        0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3,
+        3, 4,
+    );
+    let low_mask = _mm256_set1_epi8(0x0f);
+    let query_vec = _mm256_set1_epi64x(query as i64);
+
+    let chunks = n / 4;
+    let remainder = n % 4;
+
+    for i in 0..chunks {
+        let offset = i * 4;
+        let targets_ptr = targets.as_ptr().add(offset) as *const __m256i;
+        let target_vec = _mm256_loadu_si256(targets_ptr);
+
+        let xor_result = _mm256_xor_si256(query_vec, target_vec);
+
+        // Popcount using nibble lookup
+        let lo = _mm256_and_si256(xor_result, low_mask);
+        let hi = _mm256_and_si256(_mm256_srli_epi16(xor_result, 4), low_mask);
+        let popcnt_lo = _mm256_shuffle_epi8(lookup, lo);
+        let popcnt_hi = _mm256_shuffle_epi8(lookup, hi);
+        let popcnt_bytes = _mm256_add_epi8(popcnt_lo, popcnt_hi);
+        let popcount = _mm256_sad_epu8(popcnt_bytes, _mm256_setzero_si256());
+
+        let results_ptr = results.as_mut_ptr().add(offset);
+        *results_ptr = _mm256_extract_epi32::<0>(popcount) as u32;
+        *results_ptr.add(1) = _mm256_extract_epi32::<2>(popcount) as u32;
+        *results_ptr.add(2) = _mm256_extract_epi32::<4>(popcount) as u32;
+        *results_ptr.add(3) = _mm256_extract_epi32::<6>(popcount) as u32;
+    }
+
+    if remainder > 0 {
+        let offset = chunks * 4;
+        for j in 0..remainder {
+            results[offset + j] = (query ^ targets[offset + j]).count_ones();
+        }
+    }
+}
+
+/// Compute pairwise hamming distances for all pairs of hashes.
+///
+/// Returns pairs where distance <= threshold (if provided).
+///
+/// # Arguments
+/// * `hashes` - Vector of 64-bit hash values
+/// * `row_ids` - Optional row IDs (defaults to indices if None)
+/// * `threshold` - Optional maximum distance to include in results
+pub fn pairwise_hamming_distance(
+    hashes: &[u64],
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let num_pairs = n * (n - 1) / 2;
+    let mut result = PairwiseResult::with_capacity(num_pairs.min(1_000_000));
+
+    for i in 0..n {
+        for j in (i + 1)..n {
+            let dist = hamming_u64(hashes[i], hashes[j]);
+            if dist <= threshold {
+                let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
+/// Compute pairwise hamming distances in parallel using rayon + SIMD.
+///
+/// Uses chunked parallelization for balanced workload distribution.
+pub fn pairwise_hamming_distance_parallel(
+    hashes: &[u64],
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let total_pairs = n * (n - 1) / 2;
+
+    // For small datasets, use sequential to avoid thread overhead
+    if total_pairs < 10_000 {
+        return pairwise_hamming_distance(hashes, row_ids, Some(threshold));
+    }
+
+    let threads = rayon::current_num_threads();
+    let pairs_per_chunk = total_pairs.div_ceil(threads);
+    let chunks = compute_balanced_chunks(n, pairs_per_chunk);
+
+    let results: Vec<PairwiseResult> = chunks
+        .into_par_iter()
+        .map(|(start_row, end_row)| {
+            process_row_range(hashes, row_ids, threshold, start_row, end_row)
+        })
+        .collect();
+
+    let mut combined = PairwiseResult::new();
+    for r in results {
+        combined.extend(r);
+    }
+    combined
+}
+
+/// Compute pairwise hamming distances for all pairs of fixed-width binary hashes.
+///
+/// This supports any hash width that is a positive multiple of 8 bytes. For
+/// 8-byte hashes this delegates to the existing `u64` implementation.
+pub fn pairwise_hamming_distance_binary(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    if hashes.num_lanes() == 1 {
+        return pairwise_hamming_distance(hashes.lane(0), row_ids, threshold);
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let num_pairs = n * (n - 1) / 2;
+    let mut result = PairwiseResult::with_capacity(num_pairs.min(1_000_000));
+
+    for i in 0..n {
+        for j in (i + 1)..n {
+            let mut dist = 0;
+            for lane in 0..hashes.num_lanes() {
+                let lane_values = hashes.lane(lane);
+                dist += hamming_u64(lane_values[i], lane_values[j]);
+            }
+            if dist <= threshold {
+                let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
+/// Compute pairwise hamming distances in parallel for fixed-width binary hashes.
+///
+/// Wider hashes reuse the SIMD `u64` batch implementation lane-by-lane.
+pub fn pairwise_hamming_distance_binary_parallel(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: Option<u32>,
+) -> PairwiseResult {
+    let n = hashes.len();
+    if n < 2 {
+        return PairwiseResult::new();
+    }
+
+    if hashes.num_lanes() == 1 {
+        return pairwise_hamming_distance_parallel(hashes.lane(0), row_ids, threshold);
+    }
+
+    let threshold = threshold.unwrap_or(u32::MAX);
+    let total_pairs = n * (n - 1) / 2;
+
+    if total_pairs < 10_000 {
+        return pairwise_hamming_distance_binary(hashes, row_ids, Some(threshold));
+    }
+
+    let threads = rayon::current_num_threads();
+    let pairs_per_chunk = total_pairs.div_ceil(threads);
+    let chunks = compute_balanced_chunks(n, pairs_per_chunk);
+
+    let results: Vec<PairwiseResult> = chunks
+        .into_par_iter()
+        .map(|(start_row, end_row)| {
+            process_row_range_binary(hashes, row_ids, threshold, start_row, end_row)
+        })
+        .collect();
+
+    let mut combined = PairwiseResult::new();
+    for r in results {
+        combined.extend(r);
+    }
+    combined
+}
+
+/// Compute balanced chunks for parallel processing.
+fn compute_balanced_chunks(n: usize, target_pairs_per_chunk: usize) -> Vec<(usize, usize)> {
+    let mut chunks = Vec::new();
+    let mut current_start = 0;
+    let mut current_pairs = 0;
+
+    for i in 0..n {
+        let pairs_for_row = n - i - 1;
+        current_pairs += pairs_for_row;
+
+        if current_pairs >= target_pairs_per_chunk || i == n - 1 {
+            chunks.push((current_start, i + 1));
+            current_start = i + 1;
+            current_pairs = 0;
+        }
+    }
+
+    chunks
+}
+
+/// Process a range of rows for pairwise comparison using SIMD.
+fn process_row_range(
+    hashes: &[u64],
+    row_ids: Option<&[u64]>,
+    threshold: u32,
+    start_row: usize,
+    end_row: usize,
+) -> PairwiseResult {
+    let n = hashes.len();
+    let mut result = PairwiseResult::new();
+
+    for i in start_row..end_row {
+        let remaining = n - i - 1;
+        if remaining == 0 {
+            continue;
+        }
+
+        let mut distances = vec![0u32; remaining];
+        hamming_batch_u64(hashes[i], &hashes[i + 1..], &mut distances);
+
+        let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+        for (j_offset, &dist) in distances.iter().enumerate() {
+            if dist <= threshold {
+                let j = i + 1 + j_offset;
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
+/// Process a range of rows for pairwise comparison of wider binary hashes.
+fn process_row_range_binary(
+    hashes: &BinaryHashValues,
+    row_ids: Option<&[u64]>,
+    threshold: u32,
+    start_row: usize,
+    end_row: usize,
+) -> PairwiseResult {
+    let n = hashes.len();
+    let mut result = PairwiseResult::new();
+    let mut distances = Vec::new();
+    let mut lane_distances = Vec::new();
+
+    for i in start_row..end_row {
+        let remaining = n - i - 1;
+        if remaining == 0 {
+            continue;
+        }
+
+        distances.clear();
+        distances.resize(remaining, 0);
+
+        let first_lane = hashes.lane(0);
+        hamming_batch_u64(
+            first_lane[i],
+            &first_lane[i + 1..],
+            distances.as_mut_slice(),
+        );
+
+        for lane in 1..hashes.num_lanes() {
+            lane_distances.clear();
+            lane_distances.resize(remaining, 0);
+            let lane_values = hashes.lane(lane);
+            hamming_batch_u64(
+                lane_values[i],
+                &lane_values[i + 1..],
+                lane_distances.as_mut_slice(),
+            );
+            for (distance, lane_distance) in distances.iter_mut().zip(&lane_distances) {
+                *distance += *lane_distance;
+            }
+        }
+
+        let id_a = row_ids.map_or(i as u64, |ids| ids[i]);
+        for (j_offset, &dist) in distances.iter().enumerate() {
+            if dist <= threshold {
+                let j = i + 1 + j_offset;
+                let id_b = row_ids.map_or(j as u64, |ids| ids[j]);
+                result.push(id_a, id_b, dist);
+            }
+        }
+    }
+
+    result
+}
+
+/// Extract u64 hashes from a FixedSizeList<UInt8, 8> Arrow array.
+pub fn extract_hashes_from_fixed_list(array: &FixedSizeListArray) -> Result<Vec<u64>> {
+    let hashes = extract_binary_hashes_from_fixed_list(array)?;
+    if hashes.byte_width() != 8 {
+        return Err(Error::InvalidArgumentError(format!(
+            "Expected FixedSizeList with size 8, got size {}",
+            hashes.byte_width()
+        )));
+    }
+    hashes.into_u64_values()
+}
+
+/// Extract binary hashes from a `FixedSizeList<UInt8, N>` Arrow array where
+/// `N` is a positive multiple of 8 bytes.
+pub fn extract_binary_hashes_from_fixed_list(
+    array: &FixedSizeListArray,
+) -> Result<BinaryHashValues> {
+    BinaryHashValues::from_fixed_size_list(array)
+}
+
+/// Union-Find data structure with path compression for clustering.
+pub struct UnionFind {
+    parent: HashMap<u64, u64>,
+    rank: HashMap<u64, u32>,
+}
+
+impl UnionFind {
+    pub fn new() -> Self {
+        Self {
+            parent: HashMap::new(),
+            rank: HashMap::new(),
+        }
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            parent: HashMap::with_capacity(capacity),
+            rank: HashMap::with_capacity(capacity),
+        }
+    }
+
+    /// Find the root of a node with path compression.
+    pub fn find(&mut self, x: u64) -> u64 {
+        if let std::collections::hash_map::Entry::Vacant(e) = self.parent.entry(x) {
+            e.insert(x);
+            self.rank.insert(x, 0);
+            return x;
+        }
+
+        let mut current = x;
+        let mut path = Vec::new();
+
+        while self.parent[&current] != current {
+            path.push(current);
+            current = self.parent[&current];
+        }
+        let root = current;
+
+        for node in path {
+            self.parent.insert(node, root);
+        }
+
+        root
+    }
+
+    /// Union two nodes, using union by rank.
+    pub fn union(&mut self, a: u64, b: u64) -> bool {
+        let root_a = self.find(a);
+        let root_b = self.find(b);
+
+        if root_a == root_b {
+            return false;
+        }
+
+        let rank_a = self.rank[&root_a];
+        let rank_b = self.rank[&root_b];
+
+        if rank_a < rank_b {
+            self.parent.insert(root_a, root_b);
+        } else if rank_a > rank_b {
+            self.parent.insert(root_b, root_a);
+        } else if root_a < root_b {
+            self.parent.insert(root_b, root_a);
+            *self.rank.get_mut(&root_a).unwrap() += 1;
+        } else {
+            self.parent.insert(root_a, root_b);
+            *self.rank.get_mut(&root_b).unwrap() += 1;
+        }
+
+        true
+    }
+
+    pub fn nodes(&self) -> impl Iterator<Item = &u64> {
+        self.parent.keys()
+    }
+
+    pub fn len(&self) -> usize {
+        self.parent.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.parent.is_empty()
+    }
+}
+
+impl Default for UnionFind {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// A cluster with representative and duplicates.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Cluster {
+    /// The representative row ID (smallest in the cluster).
+    pub representative: u64,
+    /// List of duplicate row IDs (excludes the representative).
+    pub duplicates: Vec<u64>,
+}
+
+impl Cluster {
+    pub fn size(&self) -> usize {
+        1 + self.duplicates.len()
+    }
+}
+
+/// Result of the clustering operation.
+#[derive(Debug, Clone)]
+pub struct ClusteringResult {
+    /// List of clusters, each with a representative and duplicates.
+    pub clusters: Vec<Cluster>,
+}
+
+impl ClusteringResult {
+    pub fn num_clusters(&self) -> usize {
+        self.clusters.len()
+    }
+
+    pub fn num_duplicates(&self) -> usize {
+        self.clusters.iter().map(|c| c.duplicates.len()).sum()
+    }
+
+    pub fn num_unique(&self) -> usize {
+        self.clusters.len()
+    }
+
+    /// Get the schema for clustering result batches.
+    pub fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            Field::new("representative", DataType::UInt64, false),
+            Field::new(
+                "duplicates",
+                DataType::List(Arc::new(Field::new("item", DataType::UInt64, true))),
+                false,
+            ),
+        ]))
+    }
+
+    /// Convert to Arrow RecordBatch with columns:
+    /// - `representative`: `UInt64`
+    /// - `duplicates`: `List<UInt64>`
+    pub fn to_record_batch(&self) -> RecordBatch {
+        let schema = Self::schema();
+
+        let mut representatives = Vec::with_capacity(self.clusters.len());
+        let mut duplicates_builder = ListBuilder::new(UInt64Builder::new());
+
+        for cluster in &self.clusters {
+            representatives.push(cluster.representative);
+            for &dup in &cluster.duplicates {
+                duplicates_builder.values().append_value(dup);
+            }
+            duplicates_builder.append(true);
+        }
+
+        let representative_array: ArrayRef = Arc::new(UInt64Array::from(representatives));
+        let duplicates_array: ArrayRef = Arc::new(duplicates_builder.finish());
+
+        RecordBatch::try_new(schema, vec![representative_array, duplicates_array])
+            .expect("Failed to create RecordBatch")
+    }
+
+    /// Convert to a RecordBatchReader that yields batches of the specified size.
+    ///
+    /// # Arguments
+    /// * `batch_size` - Number of clusters per batch (default: 10000)
+    pub fn into_reader(self, batch_size: Option<usize>) -> Box<dyn RecordBatchReader + Send> {
+        let batch_size = batch_size.unwrap_or(10_000);
+        let schema = Self::schema();
+
+        if self.clusters.is_empty() {
+            // Return empty reader
+            let batches: Vec<std::result::Result<RecordBatch, arrow_schema::ArrowError>> = vec![];
+            return Box::new(RecordBatchIterator::new(batches, schema));
+        }
+
+        let batches: Vec<std::result::Result<RecordBatch, arrow_schema::ArrowError>> = self
+            .clusters
+            .chunks(batch_size)
+            .map(|chunk| {
+                let mut representatives = Vec::with_capacity(chunk.len());
+                let mut duplicates_builder = ListBuilder::new(UInt64Builder::new());
+
+                for cluster in chunk {
+                    representatives.push(cluster.representative);
+                    for &dup in &cluster.duplicates {
+                        duplicates_builder.values().append_value(dup);
+                    }
+                    duplicates_builder.append(true);
+                }
+
+                let representative_array: ArrayRef = Arc::new(UInt64Array::from(representatives));
+                let duplicates_array: ArrayRef = Arc::new(duplicates_builder.finish());
+
+                RecordBatch::try_new(Self::schema(), vec![representative_array, duplicates_array])
+            })
+            .collect();
+
+        Box::new(RecordBatchIterator::new(batches, schema))
+    }
+}
+
+/// Cluster edges using union-find algorithm.
+///
+/// Takes a list of edges (row_id_a, row_id_b) and groups connected nodes
+/// into clusters. Each cluster has a representative (smallest row ID)
+/// and a list of duplicates.
+pub fn cluster_edges<I>(edges: I) -> ClusteringResult
+where
+    I: IntoIterator<Item = (u64, u64)>,
+{
+    let mut uf = UnionFind::new();
+
+    for (a, b) in edges {
+        uf.union(a, b);
+    }
+
+    let mut clusters_map: HashMap<u64, Vec<u64>> = HashMap::new();
+    let nodes: Vec<u64> = uf.nodes().copied().collect();
+
+    for node in nodes {
+        let root = uf.find(node);
+        clusters_map.entry(root).or_default().push(node);
+    }
+
+    let mut clusters = Vec::new();
+    for (_root, mut members) in clusters_map {
+        members.sort_unstable();
+
+        if members.len() > 1 {
+            let representative = *members.iter().min().unwrap();
+            let duplicates: Vec<u64> = members
+                .into_iter()
+                .filter(|&m| m != representative)
+                .collect();
+
+            clusters.push(Cluster {
+                representative,
+                duplicates,
+            });
+        }
+    }
+
+    clusters.sort_by_key(|c| c.representative);
+
+    ClusteringResult { clusters }
+}
+
+/// Cluster edges from PairwiseResult.
+pub fn cluster_pairwise_result(result: &PairwiseResult) -> ClusteringResult {
+    let edges = result
+        .row_id_a
+        .iter()
+        .zip(result.row_id_b.iter())
+        .map(|(&a, &b)| (a, b));
+
+    cluster_edges(edges)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use lance_arrow::FixedSizeListArrayExt;
 
     #[test]
     fn test_hamming() {
@@ -102,4 +1063,792 @@ mod tests {
         let y = vec![0b1101_1010, 0b1010_1010, 0b1010_1001];
         assert_eq!(hamming(&x, &y), 2.0);
     }
+
+    #[test]
+    fn test_hamming_u64() {
+        assert_eq!(hamming_u64(0, 0), 0);
+        assert_eq!(hamming_u64(0, 1), 1);
+        assert_eq!(hamming_u64(0b1111, 0b0000), 4);
+        assert_eq!(hamming_u64(u64::MAX, 0), 64);
+        assert_eq!(hamming_u64(0xAAAAAAAAAAAAAAAA, 0x5555555555555555), 64);
+    }
+
+    #[test]
+    fn test_hamming_batch_u64() {
+        let query = 0u64;
+        let targets: Vec<u64> = (0..128).collect();
+        let mut results = vec![0u32; 128];
+
+        hamming_batch_u64(query, &targets, &mut results);
+
+        assert_eq!(results[0], 0);
+        assert_eq!(results[1], 1);
+        assert_eq!(results[3], 2); // 0b11 has 2 bits set
+        assert_eq!(results[7], 3); // 0b111 has 3 bits set
+    }
+
+    #[test]
+    fn test_pairwise_basic() {
+        let hashes = vec![0b0000u64, 0b0001, 0b0011, 0b0111];
+        let result = pairwise_hamming_distance(&hashes, None, None);
+
+        assert_eq!(result.len(), 6); // C(4,2) = 6 pairs
+        assert!(result.distances.iter().all(|&d| d <= 3));
+    }
+
+    #[test]
+    fn test_pairwise_with_threshold() {
+        let hashes = vec![0b0000u64, 0b0001, 0b1111];
+        let result = pairwise_hamming_distance(&hashes, None, Some(1));
+
+        assert_eq!(result.len(), 1);
+        assert_eq!(result.row_id_a[0], 0);
+        assert_eq!(result.row_id_b[0], 1);
+        assert_eq!(result.distances[0], 1);
+    }
+
+    #[test]
+    fn test_pairwise_with_row_ids() {
+        let hashes = vec![0b0000u64, 0b0001];
+        let row_ids = vec![100u64, 200u64];
+        let result = pairwise_hamming_distance(&hashes, Some(&row_ids), None);
+
+        assert_eq!(result.len(), 1);
+        assert_eq!(result.row_id_a[0], 100);
+        assert_eq!(result.row_id_b[0], 200);
+    }
+
+    #[test]
+    fn test_pairwise_parallel() {
+        let hashes: Vec<u64> = (0..100).collect();
+        let result_seq = pairwise_hamming_distance(&hashes, None, None);
+        let result_par = pairwise_hamming_distance_parallel(&hashes, None, None);
+
+        assert_eq!(result_seq.len(), result_par.len());
+    }
+
+    #[test]
+    fn test_union_find_basic() {
+        let mut uf = UnionFind::new();
+
+        assert_eq!(uf.find(1), 1);
+        assert_eq!(uf.find(2), 2);
+        assert_eq!(uf.find(3), 3);
+
+        assert!(uf.union(1, 2));
+        assert_eq!(uf.find(1), uf.find(2));
+
+        assert!(uf.union(2, 3));
+        assert_eq!(uf.find(1), uf.find(3));
+
+        assert!(!uf.union(1, 3));
+    }
+
+    #[test]
+    fn test_cluster_edges_simple() {
+        let edges = vec![(1, 2), (2, 3), (4, 5)];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 2);
+
+        let c1 = result
+            .clusters
+            .iter()
+            .find(|c| c.representative == 1)
+            .unwrap();
+        assert_eq!(c1.duplicates.len(), 2);
+        assert!(c1.duplicates.contains(&2));
+        assert!(c1.duplicates.contains(&3));
+
+        let c2 = result
+            .clusters
+            .iter()
+            .find(|c| c.representative == 4)
+            .unwrap();
+        assert_eq!(c2.duplicates.len(), 1);
+        assert!(c2.duplicates.contains(&5));
+    }
+
+    #[test]
+    fn test_cluster_pairwise_result() {
+        let hashes = vec![0b0000u64, 0b0001, 0b0011]; // distances: (0,1)=1, (0,2)=2, (1,2)=1
+        let pairwise = pairwise_hamming_distance(&hashes, None, Some(1)); // threshold 1
+
+        // Only pairs with distance <= 1: (0,1) and (1,2)
+        assert_eq!(pairwise.len(), 2);
+
+        let clustering = cluster_pairwise_result(&pairwise);
+        // All three should be in one cluster since 0-1-2 are connected
+        assert_eq!(clustering.num_clusters(), 1);
+        assert_eq!(clustering.clusters[0].representative, 0);
+        assert_eq!(clustering.clusters[0].duplicates.len(), 2);
+    }
+
+    #[test]
+    fn test_into_record_batch() {
+        let hashes = vec![0b0000u64, 0b0001, 0b0011];
+        let result = pairwise_hamming_distance(&hashes, None, None);
+        let batch = result.into_record_batch();
+
+        assert_eq!(batch.num_rows(), 3);
+        assert_eq!(batch.num_columns(), 3);
+        assert_eq!(batch.schema().field(0).name(), "row_id_a");
+        assert_eq!(batch.schema().field(1).name(), "row_id_b");
+        assert_eq!(batch.schema().field(2).name(), "distance");
+    }
+
+    // =========================================================================
+    // Additional tests from pairwise-hamming reference implementation
+    // =========================================================================
+
+    /// Reference implementation for validation - simple O(n²) nested loop
+    fn reference_pairwise(hashes: &[u64], threshold: Option<u32>) -> Vec<(usize, usize, u32)> {
+        let threshold = threshold.unwrap_or(u32::MAX);
+        let mut results = Vec::new();
+        for i in 0..hashes.len() {
+            for j in (i + 1)..hashes.len() {
+                let dist = (hashes[i] ^ hashes[j]).count_ones();
+                if dist <= threshold {
+                    results.push((i, j, dist));
+                }
+            }
+        }
+        results
+    }
+
+    /// Convert PairwiseResult to sorted vec for comparison
+    fn result_to_sorted_vec(result: &PairwiseResult) -> Vec<(u64, u64, u32)> {
+        let mut v: Vec<_> = result
+            .row_id_a
+            .iter()
+            .zip(result.row_id_b.iter())
+            .zip(result.distances.iter())
+            .map(|((&a, &b), &d)| (a, b, d))
+            .collect();
+        v.sort();
+        v
+    }
+
+    #[test]
+    fn test_extract_binary_hashes_from_fixed_list_128() {
+        use arrow_array::UInt8Array;
+
+        let rows = [
+            (0x0102_0304_0506_0708u64, 0x1112_1314_1516_1718u64),
+            (0x2122_2324_2526_2728u64, 0x3132_3334_3536_3738u64),
+        ];
+        let bytes: Vec<u8> = rows
+            .iter()
+            .flat_map(|(lo, hi)| lo.to_le_bytes().into_iter().chain(hi.to_le_bytes()))
+            .collect();
+        let array = FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 16).unwrap();
+
+        let hashes = extract_binary_hashes_from_fixed_list(&array).unwrap();
+        assert_eq!(hashes.len(), 2);
+        assert_eq!(hashes.byte_width(), 16);
+        assert_eq!(hashes.num_lanes(), 2);
+        assert_eq!(hashes.lane(0), &[rows[0].0, rows[1].0]);
+        assert_eq!(hashes.lane(1), &[rows[0].1, rows[1].1]);
+    }
+
+    #[test]
+    fn test_extract_binary_hashes_rejects_non_u64_multiple() {
+        use arrow_array::UInt8Array;
+
+        let array =
+            FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0u8; 24]), 12).unwrap();
+        let err = extract_binary_hashes_from_fixed_list(&array).unwrap_err();
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+    }
+
+    #[test]
+    fn test_binary_hash_values_rejects_width_not_divisible_by_8() {
+        let err = BinaryHashValues::try_new(Vec::new(), 0, 12).unwrap_err();
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_128() {
+        let hashes = BinaryHashValues::try_new(
+            vec![
+                0,
+                0,
+                1,
+                u64::MAX, // lane 0
+                0,
+                1,
+                1,
+                u64::MAX, // lane 1
+            ],
+            4,
+            16,
+        )
+        .unwrap();
+
+        let seq = pairwise_hamming_distance_binary(&hashes, None, Some(1));
+        let par = pairwise_hamming_distance_binary_parallel(&hashes, None, Some(1));
+        let expected = vec![(0, 1, 1), (1, 2, 1)];
+        assert_eq!(result_to_sorted_vec(&seq), expected);
+        assert_eq!(result_to_sorted_vec(&par), expected);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_parallel_128_matches_sequential() {
+        let rows: Vec<(u64, u64)> = (0..80)
+            .flat_map(|i| {
+                let lo = (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
+                let hi = !lo.rotate_left(17);
+                [(lo, hi), (lo ^ 1, hi)]
+            })
+            .collect();
+        let mut lane_values = Vec::with_capacity(rows.len() * 2);
+        lane_values.extend(rows.iter().map(|(lo, _)| *lo));
+        lane_values.extend(rows.iter().map(|(_, hi)| *hi));
+        let hashes = BinaryHashValues::try_new(lane_values, rows.len(), 16).unwrap();
+        let row_ids: Vec<u64> = (0..rows.len()).map(|i| 10_000 + i as u64).collect();
+
+        let seq = pairwise_hamming_distance_binary(&hashes, Some(&row_ids), Some(1));
+        let par = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(1));
+
+        assert_eq!(result_to_sorted_vec(&par), result_to_sorted_vec(&seq));
+        assert!(par.len() >= 80);
+    }
+
+    #[test]
+    fn test_pairwise_binary_hashes_32_with_row_ids() {
+        let hashes = BinaryHashValues::try_new(
+            vec![
+                0,
+                0,
+                1, // lane 0
+                0,
+                1,
+                1, // lane 1
+                7,
+                7,
+                7, // lane 2
+                u64::MAX,
+                u64::MAX,
+                u64::MAX - 1, // lane 3
+            ],
+            3,
+            32,
+        )
+        .unwrap();
+        let row_ids = [10, 20, 30];
+
+        let result = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(2));
+        assert_eq!(
+            result_to_sorted_vec(&result),
+            vec![(10, 20, 1), (20, 30, 2)]
+        );
+    }
+
+    #[test]
+    fn test_pairwise_correctness_small() {
+        // Deterministic hashes with known distances
+        let hashes = vec![
+            0b0000_0000u64, // 0
+            0b0000_0001u64, // 1 bit from 0
+            0b0000_0011u64, // 2 bits from 0, 1 bit from 1
+            0b0000_0111u64, // 3 bits from 0, 2 bits from 1, 1 bit from 2
+            0b0000_1111u64, // 4 bits from 0, 3 bits from 1, 2 bits from 2, 1 bit from 3
+        ];
+
+        let result = pairwise_hamming_distance(&hashes, None, None);
+        let reference = reference_pairwise(&hashes, None);
+
+        assert_eq!(result.len(), reference.len());
+        assert_eq!(result.len(), 10); // C(5,2) = 10 pairs
+
+        // Verify specific distances
+        let result_vec = result_to_sorted_vec(&result);
+        for (i, j, expected_dist) in &reference {
+            let found = result_vec
+                .iter()
+                .find(|(a, b, _)| *a == *i as u64 && *b == *j as u64);
+            assert!(found.is_some(), "Missing pair ({}, {})", i, j);
+            assert_eq!(
+                found.unwrap().2,
+                *expected_dist,
+                "Wrong distance for pair ({}, {})",
+                i,
+                j
+            );
+        }
+    }
+
+    #[test]
+    fn test_pairwise_correctness_1000_deterministic() {
+        // Generate deterministic hashes using simple linear pattern
+        let hashes: Vec<u64> = (0u64..1000)
+            .map(|i| i.wrapping_mul(0x123456789ABCDEF))
+            .collect();
+
+        let result_seq = pairwise_hamming_distance(&hashes, None, Some(10));
+        let result_par = pairwise_hamming_distance_parallel(&hashes, None, Some(10));
+        let reference = reference_pairwise(&hashes, Some(10));
+
+        // Both implementations should match reference
+        assert_eq!(
+            result_seq.len(),
+            reference.len(),
+            "Sequential result count mismatch"
+        );
+        assert_eq!(
+            result_par.len(),
+            reference.len(),
+            "Parallel result count mismatch"
+        );
+
+        // Verify all pairs match
+        let seq_sorted = result_to_sorted_vec(&result_seq);
+        let par_sorted = result_to_sorted_vec(&result_par);
+
+        for (i, j, dist) in &reference {
+            let seq_found = seq_sorted
+                .iter()
+                .find(|(a, b, _)| *a == *i as u64 && *b == *j as u64);
+            let par_found = par_sorted
+                .iter()
+                .find(|(a, b, _)| *a == *i as u64 && *b == *j as u64);
+
+            assert!(
+                seq_found.is_some(),
+                "Sequential missing pair ({}, {})",
+                i,
+                j
+            );
+            assert!(par_found.is_some(), "Parallel missing pair ({}, {})", i, j);
+            assert_eq!(seq_found.unwrap().2, *dist);
+            assert_eq!(par_found.unwrap().2, *dist);
+        }
+    }
+
+    #[test]
+    fn test_pairwise_correctness_10000_deterministic() {
+        // Larger test with 10K hashes
+        let hashes: Vec<u64> = (0u64..10_000)
+            .map(|i| {
+                // Mix bits using a simple hash-like transformation
+                let x = i.wrapping_mul(0xDEADBEEFCAFEBABE);
+                x ^ (x >> 17) ^ (x << 13)
+            })
+            .collect();
+
+        let result_seq = pairwise_hamming_distance(&hashes, None, Some(5));
+        let result_par = pairwise_hamming_distance_parallel(&hashes, None, Some(5));
+
+        // Both should find the same number of pairs
+        assert_eq!(
+            result_seq.len(),
+            result_par.len(),
+            "10K test: sequential found {} pairs, parallel found {} pairs",
+            result_seq.len(),
+            result_par.len()
+        );
+
+        // Verify they contain the same pairs (sorted comparison)
+        let seq_sorted = result_to_sorted_vec(&result_seq);
+        let par_sorted = result_to_sorted_vec(&result_par);
+        assert_eq!(seq_sorted, par_sorted, "10K test: pair contents differ");
+    }
+
+    #[test]
+    fn test_pairwise_total_pairs_count() {
+        // Without threshold, should return exactly n*(n-1)/2 pairs
+        for n in [10, 50, 100, 500] {
+            let hashes: Vec<u64> = (0..n).map(|i| i as u64).collect();
+            let result = pairwise_hamming_distance_parallel(&hashes, None, None);
+            let expected = n * (n - 1) / 2;
+            assert_eq!(
+                result.len(),
+                expected,
+                "n={}: expected {} pairs, got {}",
+                n,
+                expected,
+                result.len()
+            );
+        }
+    }
+
+    #[test]
+    fn test_pairwise_threshold_filtering() {
+        // All identical hashes should have distance 0
+        let hashes = vec![0xABCDEF0123456789u64; 100];
+        let result = pairwise_hamming_distance_parallel(&hashes, None, Some(0));
+
+        // All pairs should be included (distance 0)
+        assert_eq!(result.len(), 100 * 99 / 2);
+        assert!(result.distances.iter().all(|&d| d == 0));
+
+        // With threshold 0 and all different hashes, should find fewer pairs
+        let different_hashes: Vec<u64> = (0u64..100).collect();
+        let result2 = pairwise_hamming_distance_parallel(&different_hashes, None, Some(0));
+        // Only pairs with identical values should match (none in this case except 0^0)
+        assert!(result2.len() < 100 * 99 / 2);
+    }
+
+    #[test]
+    fn test_pairwise_row_ids_preserved() {
+        let hashes: Vec<u64> = (0u64..100).collect();
+        let row_ids: Vec<u64> = (1000u64..1100).collect(); // offset row IDs
+
+        let result = pairwise_hamming_distance_parallel(&hashes, Some(&row_ids), Some(5));
+
+        // All row IDs should be in range [1000, 1100)
+        for &id in &result.row_id_a {
+            assert!((1000..1100).contains(&id), "row_id_a {} out of range", id);
+        }
+        for &id in &result.row_id_b {
+            assert!((1000..1100).contains(&id), "row_id_b {} out of range", id);
+        }
+        // row_id_a should always be less than row_id_b (upper triangular)
+        for (&a, &b) in result.row_id_a.iter().zip(result.row_id_b.iter()) {
+            assert!(a < b, "Expected row_id_a < row_id_b, got {} >= {}", a, b);
+        }
+    }
+
+    #[test]
+    fn test_pairwise_distance_bounds() {
+        // All distances should be in [0, 64] for u64 hashes
+        let hashes: Vec<u64> = (0u64..1000).map(|i| i.wrapping_mul(0x123456789)).collect();
+
+        let result = pairwise_hamming_distance_parallel(&hashes, None, None);
+
+        for &d in &result.distances {
+            assert!(d <= 64, "Distance {} exceeds maximum 64", d);
+        }
+    }
+
+    #[test]
+    fn test_pairwise_symmetry() {
+        // Hamming distance is symmetric: d(a,b) = d(b,a)
+        let hashes: Vec<u64> = vec![
+            0x0000000000000000,
+            0xFFFFFFFFFFFFFFFF,
+            0xAAAAAAAAAAAAAAAA,
+            0x5555555555555555,
+            0x123456789ABCDEF0,
+        ];
+
+        let result = pairwise_hamming_distance(&hashes, None, None);
+
+        // For each pair (i,j), verify distance matches manual calculation
+        for idx in 0..result.len() {
+            let i = result.row_id_a[idx] as usize;
+            let j = result.row_id_b[idx] as usize;
+            let dist = result.distances[idx];
+
+            let expected = (hashes[i] ^ hashes[j]).count_ones();
+            assert_eq!(dist, expected, "Distance mismatch for pair ({}, {})", i, j);
+        }
+    }
+
+    #[test]
+    fn test_balanced_chunks() {
+        // Verify chunks are reasonably balanced
+        let n = 10000;
+        let total_pairs = n * (n - 1) / 2;
+        let target_per_chunk = total_pairs / 16;
+
+        let chunks = compute_balanced_chunks(n, target_per_chunk);
+
+        // Should have roughly 16 chunks
+        assert!(
+            chunks.len() >= 14 && chunks.len() <= 18,
+            "Expected ~16 chunks, got {}",
+            chunks.len()
+        );
+
+        // Each chunk should have roughly equal work
+        for (start, end) in &chunks {
+            let mut chunk_pairs = 0usize;
+            for i in *start..*end {
+                chunk_pairs += n - i - 1;
+            }
+            // Allow 20% deviation from target
+            let lower = target_per_chunk * 80 / 100;
+            // last chunk may be smaller
+            assert!(
+                chunk_pairs >= lower || *end == n,
+                "Chunk [{}, {}) has {} pairs, expected ~{}",
+                start,
+                end,
+                chunk_pairs,
+                target_per_chunk
+            );
+        }
+
+        // Chunks should cover all rows without gaps
+        assert_eq!(chunks[0].0, 0);
+        assert_eq!(chunks.last().unwrap().1, n);
+        for i in 1..chunks.len() {
+            assert_eq!(chunks[i].0, chunks[i - 1].1, "Gap between chunks");
+        }
+    }
+
+    // =========================================================================
+    // SIMD-specific tests
+    // =========================================================================
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_avx2_popcount() {
+        if !is_x86_feature_detected!("avx2") {
+            return;
+        }
+
+        let query = 0u64;
+        let targets = vec![0u64, 1, 3, 7, 15, 31, 63, 127];
+        let mut results = vec![0u32; 8];
+
+        unsafe {
+            hamming_batch_avx2(query, &targets, &mut results);
+        }
+
+        assert_eq!(results[0], 0); // 0 ^ 0 = 0 bits
+        assert_eq!(results[1], 1); // 0 ^ 1 = 1 bit
+        assert_eq!(results[2], 2); // 0 ^ 3 = 2 bits
+        assert_eq!(results[3], 3); // 0 ^ 7 = 3 bits
+        assert_eq!(results[4], 4); // 0 ^ 15 = 4 bits
+        assert_eq!(results[5], 5); // 0 ^ 31 = 5 bits
+        assert_eq!(results[6], 6); // 0 ^ 63 = 6 bits
+        assert_eq!(results[7], 7); // 0 ^ 127 = 7 bits
+    }
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_avx2_max_distance() {
+        if !is_x86_feature_detected!("avx2") {
+            return;
+        }
+
+        let query = 0u64;
+        let targets = vec![u64::MAX; 4];
+        let mut results = vec![0u32; 4];
+
+        unsafe {
+            hamming_batch_avx2(query, &targets, &mut results);
+        }
+
+        for &r in &results {
+            assert_eq!(r, 64);
+        }
+    }
+
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_avx512_popcount() {
+        if !is_x86_feature_detected!("avx512vpopcntdq") || !is_x86_feature_detected!("avx512f") {
+            return;
+        }
+
+        let query = 0u64;
+        let targets = vec![0u64, 1, 3, 7, 15, 31, 63, 127];
+        let mut results = vec![0u32; 8];
+
+        unsafe {
+            hamming_batch_avx512(query, &targets, &mut results);
+        }
+
+        assert_eq!(results[0], 0);
+        assert_eq!(results[1], 1);
+        assert_eq!(results[2], 2);
+        assert_eq!(results[3], 3);
+        assert_eq!(results[4], 4);
+        assert_eq!(results[5], 5);
+        assert_eq!(results[6], 6);
+        assert_eq!(results[7], 7);
+    }
+
+    // =========================================================================
+    // Additional clustering tests
+    // =========================================================================
+
+    #[test]
+    fn test_union_find_path_compression() {
+        let mut uf = UnionFind::new();
+
+        // Create a chain: 1 -> 2 -> 3 -> 4 -> 5
+        uf.union(4, 5);
+        uf.union(3, 4);
+        uf.union(2, 3);
+        uf.union(1, 2);
+
+        // All should have the same root
+        let root = uf.find(1);
+        assert_eq!(uf.find(2), root);
+        assert_eq!(uf.find(3), root);
+        assert_eq!(uf.find(4), root);
+        assert_eq!(uf.find(5), root);
+    }
+
+    #[test]
+    fn test_cluster_edges_single_cluster() {
+        // All connected: 1-2-3-4-5
+        let edges = vec![(1, 2), (2, 3), (3, 4), (4, 5)];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 1);
+        let cluster = &result.clusters[0];
+        assert_eq!(cluster.representative, 1);
+        assert_eq!(cluster.duplicates.len(), 4);
+        assert_eq!(cluster.size(), 5);
+    }
+
+    #[test]
+    fn test_cluster_edges_no_duplicates() {
+        // No edges means no clusters
+        let edges: Vec<(u64, u64)> = vec![];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 0);
+        assert_eq!(result.num_duplicates(), 0);
+    }
+
+    #[test]
+    fn test_cluster_edges_self_loop() {
+        // Self-loop shouldn't create a cluster (size 1)
+        let edges = vec![(1, 1), (2, 3)];
+        let result = cluster_edges(edges);
+
+        // Only {2,3} should be a cluster
+        assert_eq!(result.num_clusters(), 1);
+        assert_eq!(result.clusters[0].representative, 2);
+    }
+
+    #[test]
+    fn test_cluster_edges_duplicate_edges() {
+        // Duplicate edges should be handled correctly
+        let edges = vec![(1, 2), (1, 2), (2, 3), (2, 3), (3, 1)];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 1);
+        assert_eq!(result.clusters[0].size(), 3);
+    }
+
+    #[test]
+    fn test_cluster_edges_large() {
+        // Create 100 clusters of size 10 each
+        let mut edges = Vec::new();
+        for cluster_id in 0..100u64 {
+            let base = cluster_id * 10;
+            for i in 0..9 {
+                edges.push((base + i, base + i + 1));
+            }
+        }
+
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 100);
+        for cluster in &result.clusters {
+            assert_eq!(cluster.size(), 10);
+            assert_eq!(cluster.duplicates.len(), 9);
+        }
+    }
+
+    #[test]
+    fn test_cluster_edges_random_order() {
+        // Same edges in different order should produce same result
+        let edges1 = vec![(1, 2), (2, 3), (4, 5), (3, 4)];
+        let edges2 = vec![(4, 5), (1, 2), (3, 4), (2, 3)];
+        let edges3 = vec![(3, 4), (4, 5), (2, 3), (1, 2)];
+
+        let r1 = cluster_edges(edges1);
+        let r2 = cluster_edges(edges2);
+        let r3 = cluster_edges(edges3);
+
+        // All should produce the same single cluster
+        assert_eq!(r1.num_clusters(), 1);
+        assert_eq!(r2.num_clusters(), 1);
+        assert_eq!(r3.num_clusters(), 1);
+
+        assert_eq!(r1.clusters[0].representative, 1);
+        assert_eq!(r2.clusters[0].representative, 1);
+        assert_eq!(r3.clusters[0].representative, 1);
+
+        assert_eq!(r1.clusters[0].size(), 5);
+        assert_eq!(r2.clusters[0].size(), 5);
+        assert_eq!(r3.clusters[0].size(), 5);
+    }
+
+    #[test]
+    fn test_cluster_edges_non_contiguous_ids() {
+        // Row IDs don't need to be contiguous
+        let edges = vec![(100, 200), (200, 500), (1000, 2000)];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 2);
+
+        let c1 = result
+            .clusters
+            .iter()
+            .find(|c| c.representative == 100)
+            .unwrap();
+        assert_eq!(c1.duplicates, vec![200, 500]);
+
+        let c2 = result
+            .clusters
+            .iter()
+            .find(|c| c.representative == 1000)
+            .unwrap();
+        assert_eq!(c2.duplicates, vec![2000]);
+    }
+
+    #[test]
+    fn test_cluster_representative_is_minimum() {
+        // Representative should always be the minimum row ID in cluster
+        let edges = vec![
+            (5, 3),
+            (3, 7),
+            (7, 1), // 1 is minimum
+            (100, 50),
+            (50, 75), // 50 is minimum
+        ];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 2);
+
+        let c1 = result
+            .clusters
+            .iter()
+            .find(|c| c.duplicates.contains(&7))
+            .unwrap();
+        assert_eq!(c1.representative, 1);
+
+        let c2 = result
+            .clusters
+            .iter()
+            .find(|c| c.duplicates.contains(&100))
+            .unwrap();
+        assert_eq!(c2.representative, 50);
+    }
+
+    #[test]
+    fn test_cluster_duplicates_sorted() {
+        // Duplicates should be sorted
+        let edges = vec![(1, 5), (1, 3), (1, 7), (1, 2)];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 1);
+        assert_eq!(result.clusters[0].representative, 1);
+        assert_eq!(result.clusters[0].duplicates, vec![2, 3, 5, 7]);
+    }
+
+    #[test]
+    fn test_clustering_result_stats() {
+        let edges = vec![
+            (1, 2),
+            (2, 3), // cluster of 3
+            (10, 20),
+            (20, 30),
+            (30, 40), // cluster of 4
+        ];
+        let result = cluster_edges(edges);
+
+        assert_eq!(result.num_clusters(), 2);
+        assert_eq!(result.num_duplicates(), 5); // 2 + 3
+        assert_eq!(result.num_unique(), 2);
+    }
 }
diff --git a/rust/lance-linalg/src/distance/l2.rs b/rust/lance-linalg/src/distance/l2.rs
index 9aa5de6b9c5..7ee2f4bc537 100644
--- a/rust/lance-linalg/src/distance/l2.rs
+++ b/rust/lance-linalg/src/distance/l2.rs
@@ -18,19 +18,43 @@ use arrow_schema::DataType;
 use half::{bf16, f16};
 use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray};
 use lance_core::assume_eq;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
+// Named tiers are only matched on x86_64, or by the fp16 kernels on the other
+// architectures; without either, nothing below names a `SimdSupport` variant.
+#[cfg(any(feature = "fp16kernels", target_arch = "x86_64"))]
 use lance_core::utils::cpu::SimdSupport;
 use num_traits::{AsPrimitive, Num};
 
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+use crate::distance::BatchIter;
+
 /// Calculate the L2 distance between two vectors.
 ///
 pub trait L2: Num {
     /// Calculate the L2 distance between two vectors.
     fn l2(x: &[Self], y: &[Self]) -> f32;
 
-    fn l2_batch(x: &[Self], y: &[Self], dimension: usize) -> impl Iterator<Item = f32> {
-        y.chunks_exact(dimension).map(|v| Self::l2(x, v))
+    /// L2 distance from `x` to each `dimension`-sized vector in `y`.
+    ///
+    /// The default calls [`L2::l2`] per vector. `f32` overrides it so the SIMD
+    /// tier is chosen once for the whole batch instead of once per vector —
+    /// on a build whose baseline already implies AVX2, per-vector dispatch
+    /// costs more than the kernel it selects.
+    ///
+    /// Returns `impl Iterator` rather than a trait object: the k-means
+    /// assignment loop drives this one element at a time, so a
+    /// `Box<dyn Iterator>` would cost a virtual call per element and an
+    /// allocation per batch.
+    fn l2_batch<'a>(
+        x: &'a [Self],
+        y: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = f32> + 'a {
+        y.chunks_exact(dimension).map(move |v| Self::l2(x, v))
     }
 }
 
@@ -51,7 +75,6 @@ pub fn l2<T: L2>(from: &[T], to: &[T]) -> f32 {
 pub fn l2_f32(x: &[f32], y: &[f32]) -> f32 {
     #[cfg(target_arch = "x86_64")]
     {
-        use lance_core::utils::cpu::SimdSupport;
         if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) {
             // SAFETY: guarded by the runtime AVX-512 detection above.
             return unsafe { l2_f32_avx512(x, y) };
@@ -151,7 +174,7 @@ mod bf16_kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn l2_bf16_neon(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_bf16", target_arch = "x86_64"))]
         pub fn l2_bf16_avx512(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn l2_bf16_avx2(ptr1: *const bf16, ptr2: *const bf16, len: u32) -> f32;
@@ -172,7 +195,7 @@ impl L2 for bf16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_bf16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -190,6 +213,9 @@ impl L2 for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::l2_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => l2_scalar::<Self, f32, 16>(x, y),
         }
     }
@@ -204,7 +230,7 @@ mod kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn l2_f16_neon(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_f16", target_arch = "x86_64"))]
         pub fn l2_f16_avx512(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn l2_f16_avx2(ptr1: *const f16, ptr2: *const f16, len: u32) -> f32;
@@ -225,7 +251,7 @@ impl L2 for f16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_f16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -243,6 +269,9 @@ impl L2 for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::l2_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => l2_scalar::<Self, f32, 16>(x, y),
         }
     }
@@ -251,12 +280,114 @@ impl L2 for f16 {
 impl L2 for f32 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
-        // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32))
-        // See https://github.com/lance-format/lance/pull/2450.
-        l2_scalar::<Self, Self, 16>(x, y)
+        // Trait methods cannot carry `#[target_feature]` attributes, so the body
+        // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT`
+        // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable
+        // scalar fallback.
+        l2_f32_dispatched(x, y)
+    }
+
+    fn l2_batch<'a>(
+        x: &'a [Self],
+        y: &'a [Self],
+        dimension: usize,
+    ) -> impl Iterator<Item = Self> + 'a {
+        // Exactly one arm compiles; see `Dot::dot_batch` for f32.
+        // See `Dot::dot_batch` for f32.
+        #[cfg(all(
+            target_arch = "x86_64",
+            target_feature = "avx2",
+            target_feature = "fma"
+        ))]
+        {
+            // `l2_scalar::<_, _, 16>` chunks the vector by 16 lanes. At or below
+            // that width the chunking degenerates to its scalar remainder loop
+            // and vectorizes nothing, so the explicit AVX kernel is worth ~40%.
+            // Above it the autovectorizer already does well and the 8-wide
+            // kernel can lose, so keep the exact kernel the pre-dispatch code
+            // used and stay non-regressing by construction.
+            //
+            // SAFETY: the build baseline enables avx2+fma, which imply avx+fma,
+            // so the kernel's `#[target_feature]` contract is met statically.
+            let narrow = dimension <= 16;
+            y.chunks_exact(dimension).map(move |v| {
+                if narrow {
+                    unsafe { x86::l2_f32_avx_fma(x, v) }
+                } else {
+                    l2_f32_scalar(x, v)
+                }
+            })
+        }
+        #[cfg(all(
+            target_arch = "x86_64",
+            not(all(target_feature = "avx2", target_feature = "fma"))
+        ))]
+        {
+            l2_batch_f32_runtime_dispatch(x, y, dimension)
+        }
+        #[cfg(not(target_arch = "x86_64"))]
+        {
+            y.chunks_exact(dimension).map(move |v| Self::l2(x, v))
+        }
+    }
+}
+
+/// Sub-AVX2 builds: pick a `#[target_feature]` kernel once for the batch.
+#[cfg(all(
+    target_arch = "x86_64",
+    not(all(target_feature = "avx2", target_feature = "fma"))
+))]
+#[inline]
+fn l2_batch_f32_runtime_dispatch<'a>(
+    x: &'a [f32],
+    y: &'a [f32],
+    dimension: usize,
+) -> impl Iterator<Item = f32> + 'a {
+    // SAFETY: each kernel is entered only under its matching runtime detection.
+    match *SIMD_SUPPORT {
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16 => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx512(x, y, dimension) }.into_iter())
+        }
+        SimdSupport::Avx2 | SimdSupport::AvxFma => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx_fma(x, y, dimension) }.into_iter())
+        }
+        SimdSupport::Avx => {
+            BatchIter::Eager(unsafe { x86::l2_batch_f32_avx(x, y, dimension) }.into_iter())
+        }
+        _ => BatchIter::Lazy(y.chunks_exact(dimension).map(move |v| l2_f32_scalar(x, v))),
     }
 }
 
+/// L2 distance for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn l2_f32_dispatched(x: &[f32], y: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f32_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f32_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::l2_f32_avx(x, y) },
+            _ => l2_f32_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        l2_f32_scalar(x, y)
+    }
+}
+
+/// Portable scalar L2 distance for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn l2_f32_scalar(x: &[f32], y: &[f32]) -> f32 {
+    // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32))
+    // See https://github.com/lance-format/lance/pull/2450.
+    l2_scalar::<f32, f32, 16>(x, y)
+}
+
 impl L2 for f64 {
     #[inline]
     fn l2(x: &[Self], y: &[Self]) -> f32 {
@@ -264,9 +395,277 @@ impl L2 for f64 {
     }
 }
 
-/// Explicit SIMD L2 distance for f64.
+/// L2 distance for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD
+/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX.
 #[inline]
 fn l2_f64_simd(x: &[f64], y: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f64_avx512(x, y) },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f64_avx_fma(x, y) },
+            SimdSupport::Avx => unsafe { x86::l2_f64_avx(x, y) },
+            _ => l2_f64_scalar(x, y),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        l2_f64_simd_other(x, y)
+    }
+}
+
+/// Portable scalar L2 distance for f64. Used as the x86_64 fallback when no
+/// AVX2 is detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn l2_f64_scalar(x: &[f64], y: &[f64]) -> f32 {
+    x.iter()
+        .zip(y.iter())
+        .map(|(&a, &b)| {
+            let diff = a - b;
+            diff * diff
+        })
+        .sum::<f64>() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// L2 distance from `x` to every `dimension`-sized vector in `batch`, with
+    /// the AVX-512 tier entered once for the whole batch rather than once per
+    /// vector.
+    ///
+    /// # Safety
+    /// The host must support AVX-512F.
+    ///
+    /// Only compiled for builds whose baseline is below avx2+fma; at or above
+    /// that baseline `l2_batch` inlines the kernel directly and never runtime-
+    /// dispatches, so this wrapper would be dead code (see `l2_batch`).
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx512f")]
+    pub(super) unsafe fn l2_batch_f32_avx512(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx512(x, y) })
+            .collect()
+    }
+
+    /// As [`l2_batch_f32_avx512`], for the AVX+FMA and AVX2 tiers.
+    ///
+    /// # Safety
+    /// The host must support AVX and FMA.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx,fma")]
+    pub(super) unsafe fn l2_batch_f32_avx_fma(
+        x: &[f32],
+        batch: &[f32],
+        dimension: usize,
+    ) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx_fma(x, y) })
+            .collect()
+    }
+
+    /// As [`l2_batch_f32_avx512`], for the AVX-without-FMA tier.
+    ///
+    /// # Safety
+    /// The host must support AVX.
+    #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))]
+    #[target_feature(enable = "avx")]
+    pub(super) unsafe fn l2_batch_f32_avx(x: &[f32], batch: &[f32], dimension: usize) -> Vec<f32> {
+        batch
+            .chunks_exact(dimension)
+            .map(|y| unsafe { l2_f32_avx(x, y) })
+            .collect()
+    }
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vsubpd` + `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn l2_f64_avx512(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm512_loadu_pd(x.as_ptr().add(i));
+            let b = _mm512_loadu_pd(y.as_ptr().add(i));
+            let diff = _mm512_sub_pd(a, b);
+            acc = _mm512_fmadd_pd(diff, diff, acc);
+        }
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (_mm512_reduce_add_pd(acc) + tail) as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn l2_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = f64x8::load_unaligned(x.as_ptr().add(i));
+            let b = f64x8::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc8.multiply_add(diff, diff);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let a = f64x4::load_unaligned(x.as_ptr().add(i));
+            let b = f64x4::load_unaligned(y.as_ptr().add(i));
+            let diff = a - b;
+            acc4.multiply_add(diff, diff);
+        }
+
+        let tail: f64 = x[aligned_len..]
+            .iter()
+            .zip(y[aligned_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): squared diff via `_mm256_mul_pd` + `_mm256_add_pd` for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn l2_f64_avx(x: &[f64], y: &[f64]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let a = _mm256_loadu_pd(x.as_ptr().add(i));
+            let b = _mm256_loadu_pd(y.as_ptr().add(i));
+            let diff = _mm256_sub_pd(a, b);
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(diff, diff));
+        }
+
+        // Horizontal sum of __m256d -> f64.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        (acc_sum + tail) as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vsubps` + `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn l2_f32_avx512(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let a = _mm512_loadu_ps(x.as_ptr().add(i));
+            let b = _mm512_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm512_sub_ps(a, b);
+            acc = _mm512_fmadd_ps(diff, diff, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        _mm512_reduce_add_ps(acc) + tail
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn l2_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm256_sub_ps(a, b);
+            acc = _mm256_fmadd_ps(diff, diff, acc);
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+
+    /// AVX-only path for f32 (no FMA): squared diff via `_mm256_mul_ps` + `_mm256_add_ps` for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn l2_f32_avx(x: &[f32], y: &[f32]) -> f32 {
+        let dim = x.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let a = _mm256_loadu_ps(x.as_ptr().add(i));
+            let b = _mm256_loadu_ps(y.as_ptr().add(i));
+            let diff = _mm256_sub_ps(a, b);
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(diff, diff));
+        }
+
+        let tail: f32 = x[unrolled_len..]
+            .iter()
+            .zip(y[unrolled_len..].iter())
+            .map(|(&a, &b)| {
+                let diff = a - b;
+                diff * diff
+            })
+            .sum();
+
+        hsum256_ps(acc) + tail
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn l2_f64_simd_other(x: &[f64], y: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -331,7 +730,7 @@ fn accumulate_l2_dimension(q: f32, row: &[f32], result: &mut [f32]) {
 /// sub-vector codebooks (e.g. 256 centroids × 16 dims = 16 KB).
 /// For large target sets the SoA layout causes L1 thrashing and
 /// [`l2_distance_batch`] with its AoS per-target locality is faster.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, DeepSizeOf)]
 pub struct L2Prepared {
     transposed: Vec<f32>,
     dimension: usize,
@@ -670,6 +1069,136 @@ mod tests {
         fn test_l2_distance_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){
             do_l2_test(&x, &y)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `l2_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            let scalar = l2_f64_scalar(&x, &y);
+            let simd = l2_f64_simd(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6));
+        }
+
+        /// Parity check for `l2_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar L2 path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep
+        /// this test architecture-agnostic (the x86_64-only `l2_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_l2_f32_scalar_simd_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            let scalar = x
+                .iter()
+                .zip(y.iter())
+                .map(|(&a, &b)| ((a as f64) - (b as f64)).powi(2))
+                .sum::<f64>() as f32;
+            let simd = <f32 as L2>::l2(&x, &y);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx512 = unsafe { x86::l2_f64_avx512(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6));
+        }
+
+        /// AVX + FMA-direct parity for the f64 L2 kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx_fma = unsafe { x86::l2_f64_avx_fma(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6));
+        }
+
+        /// AVX-only-direct parity for the f64 L2 kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f64_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = l2_f64_scalar(&x, &y);
+            let avx = unsafe { x86::l2_f64_avx(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6));
+        }
+
+        /// AVX-512-direct parity for f32: explicitly compares the scalar
+        /// fallback against the native f32 AVX-512 inner kernel on
+        /// AVX-512F-capable hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx512_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx512 = unsafe { x86::l2_f32_avx512(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3));
+        }
+
+        /// AVX + FMA-direct parity for the f32 L2 kernel. Covers the AMD
+        /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts
+        /// without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx_fma_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx_fma = unsafe { x86::l2_f32_avx_fma(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3));
+        }
+
+        /// AVX-only-direct parity for the f32 L2 kernel. Covers the Intel
+        /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without
+        /// AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_f32_scalar_vs_avx_parity(
+            (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = l2_f32_scalar(&x, &y);
+            let avx = unsafe { x86::l2_f32_avx(&x, &y) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3));
+        }
     }
 
     #[test]
@@ -786,4 +1315,100 @@ mod tests {
         assert_relative_eq!(d1[0], 0.0); // q1 == target[0]
         assert_relative_eq!(d2[1], 0.0); // q2 == target[1]
     }
+
+    /// `l2_batch` must agree with the per-vector `l2` it replaced, on every
+    /// build: the AVX2-baseline path, the hoisted-dispatch path, and the
+    /// portable fallback all funnel through here.
+    #[rstest::rstest]
+    #[case::dim_8(8)]
+    #[case::dim_16(16)]
+    #[case::dim_32(32)]
+    #[case::dim_1024(1024)]
+    fn test_l2_batch_f32_matches_per_vector_l2(#[case] dimension: usize) {
+        let num_vectors = 5;
+        let x: Vec<f32> = (0..dimension)
+            .map(|i| ((i % 13) as f32) * 0.25 + 1.0)
+            .collect();
+        let batch: Vec<f32> = (0..dimension * num_vectors)
+            .map(|i| ((i % 11) as f32) * 0.5 - 2.0)
+            .collect();
+
+        let got: Vec<f32> = f32::l2_batch(&x, &batch, dimension).collect();
+        let want: Vec<f32> = batch
+            .chunks_exact(dimension)
+            .map(|y| f32::l2(&x, y))
+            .collect();
+
+        assert_eq!(got.len(), num_vectors);
+        for (g, w) in got.iter().zip(want.iter()) {
+            assert!(
+                approx::relative_eq!(g, w, epsilon = 1e-4),
+                "dim {dimension}: batch {g} != per-vector {w}"
+            );
+        }
+    }
+
+    /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2
+    /// builds or AVX-512 hosts, so call them directly to cover them.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    fn check_l2_batch_kernel(kernel: unsafe fn(&[f32], &[f32], usize) -> Vec<f32>) {
+        for dimension in [8_usize, 16, 40] {
+            let num_vectors = 3;
+            let x: Vec<f32> = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect();
+            let batch: Vec<f32> = (0..dimension * num_vectors)
+                .map(|i| ((i % 7) as f32) + 1.0)
+                .collect();
+
+            let got = unsafe { kernel(&x, &batch, dimension) };
+            assert_eq!(got.len(), num_vectors);
+            for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) {
+                let want = l2_scalar::<f32, f32, 16>(&x, chunk);
+                assert!(
+                    approx::relative_eq!(g, want, epsilon = 1e-4),
+                    "dim {dimension}: kernel {g} != scalar {want}"
+                );
+            }
+        }
+    }
+
+    // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds
+    // (see `x86::l2_batch_f32_avx512`), so gate their tests the same way.
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx_fma_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx_fma);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx);
+    }
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        not(all(target_feature = "avx2", target_feature = "fma"))
+    ))]
+    #[test]
+    fn test_l2_batch_avx512_matches_scalar() {
+        if !std::is_x86_feature_detected!("avx512f") {
+            return;
+        }
+        check_l2_batch_kernel(x86::l2_batch_f32_avx512);
+    }
 }
diff --git a/rust/lance-linalg/src/distance/l2_u8.rs b/rust/lance-linalg/src/distance/l2_u8.rs
index 1b111f91338..5fbf8a3af55 100644
--- a/rust/lance-linalg/src/distance/l2_u8.rs
+++ b/rust/lance-linalg/src/distance/l2_u8.rs
@@ -143,6 +143,10 @@ fn select_backend() -> L2U8Fn {
         if is_x86_feature_detected!("avx2") {
             return |a, b| unsafe { x86::l2_u8_avx2(a, b) };
         }
+        // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy
+        // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses
+        // AVX2 integer ops (`vpsubusb` / `vpmaddwd`) which neither AVX nor
+        // AVX+FMA provides.
     }
 
     l2_u8_scalar
diff --git a/rust/lance-linalg/src/distance/norm_l2.rs b/rust/lance-linalg/src/distance/norm_l2.rs
index cd604ac8c05..8d126d6ed57 100644
--- a/rust/lance-linalg/src/distance/norm_l2.rs
+++ b/rust/lance-linalg/src/distance/norm_l2.rs
@@ -9,9 +9,7 @@ use arrow_array::types::{Float16Type, Float32Type, Float64Type};
 use arrow_schema::DataType;
 use half::{bf16, f16};
 #[allow(unused_imports)]
-use lance_core::utils::cpu::SIMD_SUPPORT;
-#[cfg(feature = "fp16kernels")]
-use lance_core::utils::cpu::SimdSupport;
+use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 use num_traits::{AsPrimitive, Float, Num};
 
 /// L2 normalization
@@ -29,7 +27,7 @@ mod kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn norm_l2_f16_neon(ptr: *const f16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_f16", target_arch = "x86_64"))]
         pub fn norm_l2_f16_avx512(ptr: *const f16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn norm_l2_f16_avx2(ptr: *const f16, len: u32) -> f32;
@@ -57,7 +55,7 @@ impl Normalize for f16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_f16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -75,6 +73,9 @@ impl Normalize for f16 {
             SimdSupport::Lsx => unsafe {
                 kernel::norm_l2_f16_lsx(vector.as_ptr(), vector.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the f16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => norm_l2_impl::<Self, f32, 32>(vector),
         }
     }
@@ -87,7 +88,7 @@ mod bf16_kernel {
     unsafe extern "C" {
         #[cfg(target_arch = "aarch64")]
         pub fn norm_l2_bf16_neon(ptr: *const bf16, len: u32) -> f32;
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+        #[cfg(all(kernel_support = "avx512_bf16", target_arch = "x86_64"))]
         pub fn norm_l2_bf16_avx512(ptr: *const bf16, len: u32) -> f32;
         #[cfg(target_arch = "x86_64")]
         pub fn norm_l2_bf16_avx2(ptr: *const bf16, len: u32) -> f32;
@@ -108,7 +109,7 @@ impl Normalize for bf16 {
             },
             #[cfg(all(
                 feature = "fp16kernels",
-                kernel_support = "avx512",
+                kernel_support = "avx512_bf16",
                 target_arch = "x86_64"
             ))]
             SimdSupport::Avx512FP16 => unsafe {
@@ -126,6 +127,9 @@ impl Normalize for bf16 {
             SimdSupport::Lsx => unsafe {
                 bf16_kernel::norm_l2_bf16_lsx(vector.as_ptr(), vector.len() as u32)
             },
+            // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+            // the bf16 C kernels are compiled with `-march=haswell` minimum
+            // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts.
             _ => norm_l2_impl::<Self, f32, 32>(vector),
         }
     }
@@ -134,10 +138,40 @@ impl Normalize for bf16 {
 impl Normalize for f32 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
-        norm_l2_impl::<Self, Self, 16>(vector)
+        norm_l2_f32_dispatched(vector)
+    }
+}
+
+/// L2 norm for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64
+/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the
+/// auto-vectorised scalar loop.
+#[inline]
+fn norm_l2_f32_dispatched(vector: &[f32]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                x86::norm_l2_f32_avx512(vector)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f32_avx_fma(vector) },
+            SimdSupport::Avx => unsafe { x86::norm_l2_f32_avx(vector) },
+            _ => norm_l2_f32_scalar(vector),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        norm_l2_f32_scalar(vector)
     }
 }
 
+/// Portable scalar L2 norm for f32. Used as the x86_64 fallback when no
+/// AVX2 is detected, and as the only path on non-x86 architectures. The
+/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above.
+#[inline]
+fn norm_l2_f32_scalar(vector: &[f32]) -> f32 {
+    norm_l2_impl::<f32, f32, 16>(vector)
+}
+
 impl Normalize for f64 {
     #[inline]
     fn norm_l2(vector: &[Self]) -> f32 {
@@ -145,11 +179,166 @@ impl Normalize for f64 {
     }
 }
 
-/// Explicit SIMD implementation of L2 norm for f64.
+/// L2 norm for f64. Runtime-dispatched to the best available backend.
 ///
-/// Two-level unrolling: f64x8 main loop, f64x4 remainder, scalar tail.
+/// On x86_64, dispatches via `SIMD_SUPPORT` to a native AVX-512 inner kernel
+/// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4), an AVX2 + FMA kernel
+/// (Haswell+), an AVX + FMA kernel (AMD Piledriver / Steamroller), an
+/// AVX-only kernel (Intel Sandy Bridge / Ivy Bridge), or a portable scalar
+/// fallback. The per-tier inner functions each carry their own
+/// `#[target_feature]` so they stay correct under any compile baseline.
+/// On aarch64 and loongarch64, the SIMD primitives in `crate::simd::f64`
+/// are unconditionally backed by NEON / LSX-LASX respectively, so no
+/// runtime gate is required.
 #[inline]
 pub fn norm_l2_f64_simd(vector: &[f64]) -> f32 {
+    #[cfg(target_arch = "x86_64")]
+    {
+        match *SIMD_SUPPORT {
+            SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+                x86::norm_l2_f64_avx512(vector)
+            },
+            SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f64_avx_fma(vector) },
+            SimdSupport::Avx => unsafe { x86::norm_l2_f64_avx(vector) },
+            _ => norm_l2_f64_scalar(vector),
+        }
+    }
+    #[cfg(not(target_arch = "x86_64"))]
+    {
+        norm_l2_f64_simd_other(vector)
+    }
+}
+
+/// Portable scalar L2 norm. Used as the x86_64 fallback when no AVX2 is
+/// detected, and exposed for cross-backend parity testing.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn norm_l2_f64_scalar(vector: &[f64]) -> f32 {
+    vector.iter().map(|v| v * v).sum::<f64>().sqrt() as f32
+}
+
+#[cfg(target_arch = "x86_64")]
+mod x86 {
+    use std::arch::x86_64::*;
+
+    use crate::simd::f64::{f64x4, f64x8};
+    use crate::simd::x86::hsum256_ps;
+    use crate::simd::{FloatSimd, SIMD};
+
+    /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn norm_l2_f64_avx512(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm512_setzero_pd();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm512_loadu_pd(vector.as_ptr().add(i));
+            acc = _mm512_fmadd_pd(v, v, acc);
+        }
+
+        let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (_mm512_reduce_add_pd(acc) + tail).sqrt() as f32
+    }
+
+    /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn norm_l2_f64_avx_fma(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc8 = f64x8::zeros();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = f64x8::load_unaligned(vector.as_ptr().add(i));
+            acc8.multiply_add(v, v);
+        }
+
+        let aligned_len = dim / 4 * 4;
+        let mut acc4 = f64x4::zeros();
+        for i in (unrolled_len..aligned_len).step_by(4) {
+            let v = f64x4::load_unaligned(vector.as_ptr().add(i));
+            acc4.multiply_add(v, v);
+        }
+
+        let tail: f64 = vector[aligned_len..].iter().map(|&v| v * v).sum();
+        (acc8.reduce_sum() + acc4.reduce_sum() + tail).sqrt() as f32
+    }
+
+    /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn norm_l2_f64_avx(vector: &[f64]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 4 * 4;
+
+        let mut acc = _mm256_setzero_pd();
+        for i in (0..unrolled_len).step_by(4) {
+            let v = _mm256_loadu_pd(vector.as_ptr().add(i));
+            acc = _mm256_add_pd(acc, _mm256_mul_pd(v, v));
+        }
+
+        // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes.
+        let lo = _mm256_castpd256_pd128(acc);
+        let hi = _mm256_extractf128_pd(acc, 1);
+        let sum128 = _mm_add_pd(lo, hi);
+        let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+        let acc_sum = _mm_cvtsd_f64(sum64);
+
+        let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (acc_sum + tail).sqrt() as f32
+    }
+
+    /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration.
+    #[target_feature(enable = "avx512f")]
+    pub unsafe fn norm_l2_f32_avx512(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 16 * 16;
+
+        let mut acc = _mm512_setzero_ps();
+        for i in (0..unrolled_len).step_by(16) {
+            let v = _mm512_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm512_fmadd_ps(v, v, acc);
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (_mm512_reduce_add_ps(acc) + tail).sqrt()
+    }
+
+    /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics).
+    #[target_feature(enable = "avx,fma")]
+    pub unsafe fn norm_l2_f32_avx_fma(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm256_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm256_fmadd_ps(v, v, acc);
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (hsum256_ps(acc) + tail).sqrt()
+    }
+
+    /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge.
+    #[target_feature(enable = "avx")]
+    pub unsafe fn norm_l2_f32_avx(vector: &[f32]) -> f32 {
+        let dim = vector.len();
+        let unrolled_len = dim / 8 * 8;
+
+        let mut acc = _mm256_setzero_ps();
+        for i in (0..unrolled_len).step_by(8) {
+            let v = _mm256_loadu_ps(vector.as_ptr().add(i));
+            acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v));
+        }
+
+        let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum();
+        (hsum256_ps(acc) + tail).sqrt()
+    }
+}
+
+#[cfg(not(target_arch = "x86_64"))]
+#[inline]
+fn norm_l2_f64_simd_other(vector: &[f64]) -> f32 {
     use crate::simd::f64::{f64x4, f64x8};
     use crate::simd::{FloatSimd, SIMD};
 
@@ -291,5 +480,133 @@ mod tests {
         fn test_l2_norm_f64(data in prop::collection::vec(arbitrary_f64(), 4..4048)){
             do_norm_l2_test(&data)?;
         }
+
+        /// Cross-backend parity: scalar fallback must match the dispatched
+        /// SIMD path within numerical tolerance. Exercises `norm_l2_f64_scalar`
+        /// directly so the runtime fallback is exercised even on AVX2-capable
+        /// CI hosts.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_simd_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            let scalar = norm_l2_f64_scalar(&data);
+            let simd = norm_l2_f64_simd(&data);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6));
+        }
+
+        /// Parity check for `norm_l2_f32_dispatched` (Branch B exclusive: the
+        /// auto-vectorised scalar L2-norm path). The dispatched kernel must
+        /// agree with a portable f64-precision scalar reference within
+        /// numerical tolerance. The reference is hand-rolled here to keep this
+        /// test architecture-agnostic (the x86_64-only `norm_l2_f64_scalar`
+        /// helper is gated above).
+        #[test]
+        fn test_l2_norm_f32_scalar_simd_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            let scalar = data.iter().map(|&v| (v as f64).powi(2)).sum::<f64>().sqrt() as f32;
+            let simd = <f32 as Normalize>::norm_l2(&data);
+            prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3));
+        }
+
+        /// AVX-512-direct parity: explicitly compares the scalar fallback
+        /// against the native AVX-512 inner kernel on AVX-512F-capable hosts
+        /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on
+        /// hosts without AVX-512F so the test stays portable; CI runners with
+        /// AVX-512F exercise the `_mm512_*` path.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx512_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx512 = unsafe { x86::norm_l2_f64_avx512(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6));
+        }
+
+        /// AVX + FMA-direct parity for the f64 L2-norm kernel. Covers the
+        /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on
+        /// hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx_fma_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx_fma = unsafe { x86::norm_l2_f64_avx_fma(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6));
+        }
+
+        /// AVX-only-direct parity for the f64 L2-norm kernel. Covers the
+        /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA).
+        /// Early-returns on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_l2_norm_f64_scalar_vs_avx_parity(
+            data in prop::collection::vec(arbitrary_f64(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f64_scalar(&data);
+            let avx = unsafe { x86::norm_l2_f64_avx(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6));
+        }
+
+        /// AVX-512-direct parity for the f32 L2-norm kernel. Explicitly
+        /// compares the scalar fallback against the native AVX-512 inner
+        /// kernel on AVX-512F-capable hosts. Early-returns on hosts without
+        /// AVX-512F; CI runners with AVX-512F exercise the `_mm512_*` path.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx512_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx512f") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx512 = unsafe { x86::norm_l2_f32_avx512(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3));
+        }
+
+        /// AVX + FMA-direct parity for the f32 L2-norm kernel. Covers the
+        /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on
+        /// hosts without both AVX and FMA.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx_fma_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx_fma = unsafe { x86::norm_l2_f32_avx_fma(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3));
+        }
+
+        /// AVX-only-direct parity for the f32 L2-norm kernel. Covers the
+        /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA).
+        /// Early-returns on hosts without AVX.
+        #[cfg(target_arch = "x86_64")]
+        #[test]
+        fn test_norm_l2_f32_scalar_vs_avx_parity(
+            data in prop::collection::vec(arbitrary_f32(), 4..4048)
+        ) {
+            if !std::is_x86_feature_detected!("avx") {
+                return Ok(());
+            }
+            let scalar = norm_l2_f32_scalar(&data);
+            let avx = unsafe { x86::norm_l2_f32_avx(&data) };
+            prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3));
+        }
     }
 }
diff --git a/rust/lance-linalg/src/simd.rs b/rust/lance-linalg/src/simd.rs
index 91dc1c6959d..6722eafd768 100644
--- a/rust/lance-linalg/src/simd.rs
+++ b/rust/lance-linalg/src/simd.rs
@@ -19,6 +19,8 @@ pub mod f32;
 pub mod f64;
 pub mod i32;
 pub mod u8;
+#[cfg(target_arch = "x86_64")]
+pub(crate) mod x86;
 
 use num_traits::{Float, Num};
 use u8::u8x16;
diff --git a/rust/lance-linalg/src/simd/dist_table.c b/rust/lance-linalg/src/simd/dist_table.c
index 9e7fc2b2205..e8be8e52068 100644
--- a/rust/lance-linalg/src/simd/dist_table.c
+++ b/rust/lance-linalg/src/simd/dist_table.c
@@ -23,8 +23,15 @@ void sum_4bit_dist_table_32bytes_batch_avx512(const uint8_t *codes,
   __m512i accu3 = _mm512_setzero_si512();
 
   for (size_t i = 0; i < code_length; i += 64) {
-    c = _mm512_loadu_si512(&codes[i]);
-    lut = _mm512_loadu_si512(&dist_table[i]);
+    const size_t remaining = code_length - i;
+    if (remaining >= 64) {
+      c = _mm512_loadu_si512(&codes[i]);
+      lut = _mm512_loadu_si512(&dist_table[i]);
+    } else {
+      const __mmask64 load_mask = (UINT64_C(1) << remaining) - 1;
+      c = _mm512_maskz_loadu_epi8(load_mask, &codes[i]);
+      lut = _mm512_maskz_loadu_epi8(load_mask, &dist_table[i]);
+    }
     lo = _mm512_and_si512(c, lo_mask);
     hi = _mm512_and_si512(_mm512_srli_epi16(c, 4), lo_mask);
 
@@ -50,4 +57,4 @@ void sum_4bit_dist_table_32bytes_batch_avx512(const uint8_t *codes,
   ret = _mm512_add_epi16(ret, _mm512_shuffle_i64x2(ret1, ret2, 0xDD));
 
   _mm512_storeu_si512(dists, ret);
-}
\ No newline at end of file
+}
diff --git a/rust/lance-linalg/src/simd/dist_table.rs b/rust/lance-linalg/src/simd/dist_table.rs
index 66c30c75427..e337953ec10 100644
--- a/rust/lance-linalg/src/simd/dist_table.rs
+++ b/rust/lance-linalg/src/simd/dist_table.rs
@@ -10,7 +10,7 @@ use std::arch::x86_64::*;
 use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport};
 
 pub const PERM0: [usize; 16] = [0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15];
-pub const PERM0_INVERSE: [usize; 16] = [0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15];
+pub const PERM0_INVERSE: [usize; 16] = [0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15];
 pub const BATCH_SIZE: usize = 32;
 
 // This function is used to sum the distance table for 4-bit codes.
@@ -37,18 +37,22 @@ pub fn sum_4bit_dist_table(
     debug_assert!(n.is_multiple_of(BATCH_SIZE));
 
     match *SIMD_SUPPORT {
-        #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
-        SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe {
+        #[cfg(all(kernel_support = "avx512_dist_table", target_arch = "x86_64"))]
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16
+            if std::arch::is_x86_feature_detected!("avx512bw") =>
+        {
             for i in (0..n).step_by(BATCH_SIZE) {
                 let codes = &codes[i * code_len..(i + BATCH_SIZE) * code_len];
-                sum_4bit_dist_table_32bytes_batch_avx512(
-                    codes.as_ptr(),
-                    codes.len(),
-                    dist_table.as_ptr(),
-                    dists[i..i + BATCH_SIZE].as_mut_ptr(),
-                )
+                unsafe {
+                    sum_4bit_dist_table_32bytes_batch_avx512(
+                        codes.as_ptr(),
+                        codes.len(),
+                        dist_table.as_ptr(),
+                        dists[i..i + BATCH_SIZE].as_mut_ptr(),
+                    )
+                }
             }
-        },
+        }
         #[cfg(target_arch = "x86_64")]
         SimdSupport::Avx2 => unsafe {
             for i in (0..n).step_by(BATCH_SIZE) {
@@ -69,6 +73,10 @@ pub fn sum_4bit_dist_table(
                 )
             }
         },
+        // SimdSupport::AvxFma and SimdSupport::Avx fall through here:
+        // the AVX2 inner uses `_mm256_shuffle_epi8` / `_mm256_and_si256` /
+        // `_mm256_srli_epi16` / `_mm256_add_epi16` integer ops which
+        // neither AVX nor AVX+FMA provides. Scalar is the correct route.
         _ => sum_4bit_dist_table_scalar(code_len, codes, dist_table, dists),
     }
 }
@@ -109,6 +117,274 @@ pub fn sum_4bit_dist_table_scalar(
     }
 }
 
+#[inline]
+#[allow(unused)]
+pub fn sum_4bit_dist_table_u16(
+    n: usize,
+    code_len: usize,
+    codes: &[u8],
+    dist_table: &[u16],
+    dists: &mut [u32],
+) {
+    debug_assert!(n.is_multiple_of(BATCH_SIZE));
+    debug_assert!(dists.len() >= n);
+    debug_assert!(codes.len() >= n * code_len);
+    sum_4bit_dist_table_u16_scalar(
+        code_len,
+        &codes[..n * code_len],
+        dist_table,
+        &mut dists[..n],
+    );
+}
+
+#[inline]
+pub fn transfer_4bit_dist_table_u16(dist_table: &[u16], hacc_dist_table: &mut Vec<u8>) {
+    debug_assert!(dist_table.len().is_multiple_of(32));
+
+    let num_tables = dist_table.len() / 16;
+    hacc_dist_table.clear();
+    hacc_dist_table.resize(dist_table.len() * 2, 0);
+
+    for table_idx in 0..num_tables {
+        let table = &dist_table[table_idx * 16..(table_idx + 1) * 16];
+        let low_offset = (table_idx / 2) * 64 + (table_idx % 2) * 16;
+        let high_offset = low_offset + 32;
+        for (code, value) in table.iter().enumerate() {
+            hacc_dist_table[low_offset + code] = *value as u8;
+            hacc_dist_table[high_offset + code] = (value >> 8) as u8;
+        }
+    }
+}
+
+#[inline]
+pub fn sum_4bit_hacc_dist_table(
+    n: usize,
+    code_len: usize,
+    codes: &[u8],
+    hacc_dist_table: &[u8],
+    dists: &mut [u32],
+) {
+    debug_assert!(n.is_multiple_of(BATCH_SIZE));
+    debug_assert!(dists.len() >= n);
+    debug_assert!(codes.len() >= n * code_len);
+    debug_assert!(hacc_dist_table.len() >= code_len * 64);
+
+    match *SIMD_SUPPORT {
+        #[cfg(target_arch = "x86_64")]
+        SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx2
+            if std::arch::is_x86_feature_detected!("avx2") =>
+        {
+            sum_4bit_hacc_dist_table_avx2(n, code_len, codes, hacc_dist_table, dists);
+        }
+        _ => sum_4bit_hacc_dist_table_scalar(code_len, codes, hacc_dist_table, dists),
+    }
+}
+
+#[inline]
+#[allow(unused)]
+pub fn sum_4bit_hacc_dist_table_scalar(
+    code_len: usize,
+    codes: &[u8],
+    hacc_dist_table: &[u8],
+    dists: &mut [u32],
+) {
+    let num_full_vectors = codes.len() / (BATCH_SIZE * code_len) * BATCH_SIZE;
+    dists[..num_full_vectors].fill(0);
+
+    for (vec_block_idx, blocks) in codes.chunks_exact(BATCH_SIZE * code_len).enumerate() {
+        for (sub_vec_idx, block) in blocks.chunks_exact(BATCH_SIZE).enumerate() {
+            let table_offset = sub_vec_idx * 64;
+            let current_low = &hacc_dist_table[table_offset..table_offset + 16];
+            let next_low = &hacc_dist_table[table_offset + 16..table_offset + 32];
+            let current_high = &hacc_dist_table[table_offset + 32..table_offset + 48];
+            let next_high = &hacc_dist_table[table_offset + 48..table_offset + 64];
+
+            for j in 0..16 {
+                let low_current_code = (block[j] & 0x0F) as usize;
+                let high_current_code = (block[j] >> 4) as usize;
+                let low_next_code = (block[j + 16] & 0x0F) as usize;
+                let high_next_code = (block[j + 16] >> 4) as usize;
+
+                let lower_id = vec_block_idx * BATCH_SIZE + PERM0[j];
+                let higher_id = lower_id + 16;
+                dists[lower_id] += ((current_high[low_current_code] as u32) << 8)
+                    + current_low[low_current_code] as u32
+                    + ((next_high[low_next_code] as u32) << 8)
+                    + next_low[low_next_code] as u32;
+                dists[higher_id] += ((current_high[high_current_code] as u32) << 8)
+                    + current_low[high_current_code] as u32
+                    + ((next_high[high_next_code] as u32) << 8)
+                    + next_low[high_next_code] as u32;
+            }
+        }
+    }
+}
+
+#[inline]
+#[allow(unused)]
+pub fn sum_4bit_dist_table_u16_scalar(
+    code_len: usize,
+    codes: &[u8],
+    dist_table: &[u16],
+    dists: &mut [u32],
+) {
+    let num_full_vectors = codes.len() / (BATCH_SIZE * code_len) * BATCH_SIZE;
+    dists[..num_full_vectors].fill(0);
+
+    for (vec_block_idx, blocks) in codes.chunks_exact(BATCH_SIZE * code_len).enumerate() {
+        for (sub_vec_idx, block) in blocks.chunks_exact(BATCH_SIZE).enumerate() {
+            let current_dist_table = &dist_table[sub_vec_idx * 2 * 16..(sub_vec_idx * 2 + 1) * 16];
+            let next_dist_table =
+                &dist_table[(sub_vec_idx * 2 + 1) * 16..(sub_vec_idx * 2 + 2) * 16];
+
+            for j in 0..16 {
+                let low_current_code = (block[j] & 0x0F) as usize;
+                let high_current_code = (block[j] >> 4) as usize;
+                let low_next_code = (block[j + 16] & 0x0F) as usize;
+                let high_next_code = (block[j + 16] >> 4) as usize;
+
+                let lower_id = vec_block_idx * BATCH_SIZE + PERM0[j];
+                let higher_id = lower_id + 16;
+                dists[lower_id] += current_dist_table[low_current_code] as u32
+                    + next_dist_table[low_next_code] as u32;
+                dists[higher_id] += current_dist_table[high_current_code] as u32
+                    + next_dist_table[high_next_code] as u32;
+            }
+        }
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn sum_4bit_hacc_dist_table_avx2(
+    n: usize,
+    code_len: usize,
+    codes: &[u8],
+    hacc_dist_table: &[u8],
+    dists: &mut [u32],
+) {
+    const SAFE_CODE_LEN: usize = 128;
+
+    for i in (0..n).step_by(BATCH_SIZE) {
+        let batch_codes = &codes[i * code_len..(i + BATCH_SIZE) * code_len];
+        let batch_dists = &mut dists[i..i + BATCH_SIZE];
+        batch_dists.fill(0);
+
+        for code_start in (0..code_len).step_by(SAFE_CODE_LEN) {
+            let code_end = (code_start + SAFE_CODE_LEN).min(code_len);
+            let code_range = code_start * BATCH_SIZE..code_end * BATCH_SIZE;
+            let table_range = code_start * 64..code_end * 64;
+            if code_start == 0 && code_end == code_len {
+                unsafe {
+                    sum_hacc_dist_table_32bytes_batch_avx2(
+                        &batch_codes[code_range],
+                        &hacc_dist_table[table_range],
+                        batch_dists,
+                    );
+                }
+            } else {
+                let mut chunk_dists = [0u32; BATCH_SIZE];
+                unsafe {
+                    sum_hacc_dist_table_32bytes_batch_avx2(
+                        &batch_codes[code_range],
+                        &hacc_dist_table[table_range],
+                        &mut chunk_dists,
+                    );
+                }
+                batch_dists
+                    .iter_mut()
+                    .zip(chunk_dists.iter())
+                    .for_each(|(dist, chunk_dist)| *dist += *chunk_dist);
+            }
+        }
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+#[inline]
+#[allow(unused)]
+unsafe fn sum_hacc_dist_table_32bytes_batch_avx2(
+    codes: &[u8],
+    hacc_dist_table: &[u8],
+    dists: &mut [u32],
+) {
+    let low_mask = _mm256_set1_epi8(0x0f);
+    let mut low_accu0 = _mm256_setzero_si256();
+    let mut low_accu1 = _mm256_setzero_si256();
+    let mut low_accu2 = _mm256_setzero_si256();
+    let mut low_accu3 = _mm256_setzero_si256();
+    let mut high_accu0 = _mm256_setzero_si256();
+    let mut high_accu1 = _mm256_setzero_si256();
+    let mut high_accu2 = _mm256_setzero_si256();
+    let mut high_accu3 = _mm256_setzero_si256();
+
+    for code_offset in (0..codes.len()).step_by(BATCH_SIZE) {
+        let table_offset = code_offset * 2;
+        let c = _mm256_loadu_si256(codes.as_ptr().add(code_offset) as *const __m256i);
+        let lo = _mm256_and_si256(c, low_mask);
+        let hi = _mm256_and_si256(_mm256_srli_epi16(c, 4), low_mask);
+
+        let low_lut =
+            _mm256_loadu_si256(hacc_dist_table.as_ptr().add(table_offset) as *const __m256i);
+        let low_res_lo = _mm256_shuffle_epi8(low_lut, lo);
+        let low_res_hi = _mm256_shuffle_epi8(low_lut, hi);
+        low_accu0 = _mm256_add_epi16(low_accu0, low_res_lo);
+        low_accu1 = _mm256_add_epi16(low_accu1, _mm256_srli_epi16(low_res_lo, 8));
+        low_accu2 = _mm256_add_epi16(low_accu2, low_res_hi);
+        low_accu3 = _mm256_add_epi16(low_accu3, _mm256_srli_epi16(low_res_hi, 8));
+
+        let high_lut =
+            _mm256_loadu_si256(hacc_dist_table.as_ptr().add(table_offset + 32) as *const __m256i);
+        let high_res_lo = _mm256_shuffle_epi8(high_lut, lo);
+        let high_res_hi = _mm256_shuffle_epi8(high_lut, hi);
+        high_accu0 = _mm256_add_epi16(high_accu0, high_res_lo);
+        high_accu1 = _mm256_add_epi16(high_accu1, _mm256_srli_epi16(high_res_lo, 8));
+        high_accu2 = _mm256_add_epi16(high_accu2, high_res_hi);
+        high_accu3 = _mm256_add_epi16(high_accu3, _mm256_srli_epi16(high_res_hi, 8));
+    }
+
+    low_accu0 = _mm256_sub_epi16(low_accu0, _mm256_slli_epi16(low_accu1, 8));
+    let low_dis0 = _mm256_add_epi16(
+        _mm256_permute2f128_si256(low_accu0, low_accu1, 0x21),
+        _mm256_blend_epi32(low_accu0, low_accu1, 0xF0),
+    );
+    low_accu2 = _mm256_sub_epi16(low_accu2, _mm256_slli_epi16(low_accu3, 8));
+    let low_dis1 = _mm256_add_epi16(
+        _mm256_permute2f128_si256(low_accu2, low_accu3, 0x21),
+        _mm256_blend_epi32(low_accu2, low_accu3, 0xF0),
+    );
+
+    high_accu0 = _mm256_sub_epi16(high_accu0, _mm256_slli_epi16(high_accu1, 8));
+    let high_dis0 = _mm256_add_epi16(
+        _mm256_permute2f128_si256(high_accu0, high_accu1, 0x21),
+        _mm256_blend_epi32(high_accu0, high_accu1, 0xF0),
+    );
+    high_accu2 = _mm256_sub_epi16(high_accu2, _mm256_slli_epi16(high_accu3, 8));
+    let high_dis1 = _mm256_add_epi16(
+        _mm256_permute2f128_si256(high_accu2, high_accu3, 0x21),
+        _mm256_blend_epi32(high_accu2, high_accu3, 0xF0),
+    );
+
+    let low0 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(low_dis0));
+    let low1 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(low_dis0, 1));
+    let high0 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(high_dis0));
+    let high1 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(high_dis0, 1));
+    let res0 = _mm256_add_epi32(low0, _mm256_slli_epi32(high0, 8));
+    let res1 = _mm256_add_epi32(low1, _mm256_slli_epi32(high1, 8));
+    _mm256_storeu_si256(dists.as_mut_ptr() as *mut __m256i, res0);
+    _mm256_storeu_si256(dists.as_mut_ptr().add(8) as *mut __m256i, res1);
+
+    let low2 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(low_dis1));
+    let low3 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(low_dis1, 1));
+    let high2 = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(high_dis1));
+    let high3 = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(high_dis1, 1));
+    let res2 = _mm256_add_epi32(low2, _mm256_slli_epi32(high2, 8));
+    let res3 = _mm256_add_epi32(low3, _mm256_slli_epi32(high3, 8));
+    _mm256_storeu_si256(dists.as_mut_ptr().add(16) as *mut __m256i, res2);
+    _mm256_storeu_si256(dists.as_mut_ptr().add(24) as *mut __m256i, res3);
+}
+
 #[cfg(target_arch = "x86_64")]
 #[target_feature(enable = "avx2")]
 #[inline]
@@ -145,6 +421,10 @@ unsafe fn sum_dist_table_32bytes_batch_avx2(codes: &[u8], dist_table: &[u8], dis
         accu2 = _mm256_add_epi16(accu2, res_hi);
         accu3 = _mm256_add_epi16(accu3, _mm256_srli_epi16(res_hi, 8));
 
+        if i + 32 >= codes.len() {
+            continue;
+        }
+
         // load the left 32 bytes of codes and lut
         c = _mm256_loadu_si256(codes.as_ptr().add(i + 32) as *const __m256i);
         lut_vec = _mm256_loadu_si256(dist_table.as_ptr().add(i + 32) as *const __m256i);
@@ -253,7 +533,7 @@ unsafe fn sum_dist_table_32bytes_batch_neon(codes: &[u8], dist_table: &[u8], dis
 // We implement the AVX512 version in C because AVX512 is not stable yet in Rust,
 // implement it in Rust once we upgrade rust to 1.89.0.
 unsafe extern "C" {
-    #[cfg(all(kernel_support = "avx512", target_arch = "x86_64"))]
+    #[cfg(all(kernel_support = "avx512_dist_table", target_arch = "x86_64"))]
     pub fn sum_4bit_dist_table_32bytes_batch_avx512(
         codes: *const u8,
         code_length: usize,
@@ -266,6 +546,13 @@ unsafe extern "C" {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_perm0_inverse_matches_perm0() {
+        for (idx, &value) in PERM0.iter().enumerate() {
+            assert_eq!(PERM0_INVERSE[value], idx);
+        }
+    }
+
     #[test]
     fn test_sum_4bit_dist_table_basic() {
         // we have 32 vectors
@@ -323,6 +610,101 @@ mod tests {
         assert!(actual.iter().all(|dist| *dist != u16::MAX));
     }
 
+    #[test]
+    fn test_sum_4bit_dist_table_u16_basic() {
+        let n = BATCH_SIZE;
+        let code_len = 2;
+        let codes = [
+            0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66,
+            0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, 0x00, 0x12, 0x34, 0x56, 0x78,
+            0x9a, 0xbc, 0xde, 0xf0,
+        ];
+        let codes = codes.repeat(n * code_len / codes.len());
+        let dist_table: Vec<u16> = (0..16 * 4).map(|idx| (idx % 16 + 1) as u16).collect();
+
+        let mut dists = vec![0u32; n];
+        sum_4bit_dist_table_u16(n, code_len, &codes, &dist_table, &mut dists);
+
+        assert_eq!(dists[1], 38);
+    }
+
+    #[test]
+    fn test_transfer_4bit_dist_table_u16_layout() {
+        let dist_table: Vec<u16> = (0..32).map(|idx| 0x1200 + idx as u16).collect();
+        let mut hacc_dist_table = Vec::new();
+        transfer_4bit_dist_table_u16(&dist_table, &mut hacc_dist_table);
+
+        assert_eq!(hacc_dist_table.len(), 64);
+        for code in 0..16 {
+            assert_eq!(hacc_dist_table[code], dist_table[code] as u8);
+            assert_eq!(hacc_dist_table[16 + code], dist_table[16 + code] as u8);
+            assert_eq!(hacc_dist_table[32 + code], (dist_table[code] >> 8) as u8);
+            assert_eq!(
+                hacc_dist_table[48 + code],
+                (dist_table[16 + code] >> 8) as u8
+            );
+        }
+    }
+
+    #[test]
+    fn test_sum_4bit_dist_table_u16_matches_reference_multi_batch() {
+        use rand::{Rng, SeedableRng};
+        let mut rng = rand::rngs::StdRng::seed_from_u64(99);
+
+        for code_len in [1, 3, 16, 191, 192, 1024] {
+            let n = BATCH_SIZE * 4;
+            let codes: Vec<u8> = (0..n * code_len).map(|_| rng.random::<u8>()).collect();
+            let dist_table: Vec<u16> = (0..BATCH_SIZE * code_len)
+                .map(|_| rng.random::<u16>())
+                .collect();
+
+            let mut expected = vec![0u32; n];
+            sum_4bit_dist_table_u16_scalar(code_len, &codes, &dist_table, &mut expected);
+
+            let mut actual = vec![u32::MAX; n];
+            sum_4bit_dist_table_u16(n, code_len, &codes, &dist_table, &mut actual);
+
+            assert_eq!(
+                actual,
+                expected,
+                "u16 dist-table mismatch for code_len={} (DIM={})",
+                code_len,
+                code_len * 8,
+            );
+        }
+    }
+
+    #[test]
+    fn test_sum_4bit_hacc_dist_table_matches_u16_reference_multi_batch() {
+        use rand::{Rng, SeedableRng};
+        let mut rng = rand::rngs::StdRng::seed_from_u64(101);
+
+        for code_len in [1, 3, 16, 191, 192, 1024] {
+            let n = BATCH_SIZE * 4;
+            let codes: Vec<u8> = (0..n * code_len).map(|_| rng.random::<u8>()).collect();
+            let dist_table: Vec<u16> = (0..BATCH_SIZE * code_len)
+                .map(|_| rng.random::<u16>())
+                .collect();
+
+            let mut hacc_dist_table = Vec::new();
+            transfer_4bit_dist_table_u16(&dist_table, &mut hacc_dist_table);
+
+            let mut expected = vec![0u32; n];
+            sum_4bit_dist_table_u16_scalar(code_len, &codes, &dist_table, &mut expected);
+
+            let mut actual = vec![u32::MAX; n];
+            sum_4bit_hacc_dist_table(n, code_len, &codes, &hacc_dist_table, &mut actual);
+
+            assert_eq!(
+                actual,
+                expected,
+                "hacc dist-table mismatch for code_len={} (DIM={})",
+                code_len,
+                code_len * 8,
+            );
+        }
+    }
+
     /// Test that the SIMD path (NEON on ARM, AVX2 on x86) produces identical
     /// results to the scalar reference across a range of dimensions, including
     /// very large ones (up to DIM=65536).
@@ -341,7 +723,7 @@ mod tests {
         // directly since that's what the function sees.
         // code_len=16 → DIM=128, code_len=192 → DIM=1536,
         // code_len=512 → DIM=4096, code_len=8192 → DIM=65536
-        for code_len in [2, 16, 96, 192, 512, 1024, 8192] {
+        for code_len in [1, 2, 3, 16, 95, 96, 192, 512, 1024, 8192] {
             let n = BATCH_SIZE; // 32 vectors per batch
 
             // Each code byte produces 2 lookups; cap values so
@@ -375,7 +757,7 @@ mod tests {
         use rand::{Rng, SeedableRng};
         let mut rng = rand::rngs::StdRng::seed_from_u64(123);
 
-        for code_len in [16, 192, 1024] {
+        for code_len in [1, 3, 16, 191, 192, 1024] {
             let n = BATCH_SIZE * 10; // 320 vectors = 10 batches
 
             let max_val = (u16::MAX as usize / (2 * code_len)).min(255) as u8;
diff --git a/rust/lance-linalg/src/simd/f32.rs b/rust/lance-linalg/src/simd/f32.rs
index 78042997121..4ce7f64706d 100644
--- a/rust/lance-linalg/src/simd/f32.rs
+++ b/rust/lance-linalg/src/simd/f32.rs
@@ -46,14 +46,42 @@ impl std::fmt::Debug for f32x8 {
 }
 
 impl f32x8 {
+    /// Gather 8 f32 values from `slice` at the offsets in `indices`.
+    ///
+    /// On x86_64 this uses the AVX2 `vgatherdps` instruction when the host
+    /// supports it (gated at runtime via `is_x86_feature_detected!`). On
+    /// other architectures (and on x86_64 hosts without AVX2) the function
+    /// falls back to a per-index scalar load followed by `Self::from(&out)`,
+    /// which goes through `load_unaligned` (NEON / LASX / `_mm256_loadu_ps`
+    /// depending on platform). Per-tier macro stamping (e.g., the
+    /// `multiversion` crate) was considered but doesn't fit here: the function
+    /// returns `Self` and `_mm256_i32gather_ps::<4>` requires the const-generic
+    /// stride to be a compile-time literal — neither composes with the macro.
+    ///
+    /// # Panics
+    ///
+    /// If any index is negative or lands outside `slice`.
     #[inline]
     pub fn gather(slice: &[f32], indices: &[i32; 8]) -> Self {
-        #[cfg(target_arch = "x86_64")]
-        unsafe {
-            use super::i32::i32x8;
+        // Every backend below reads without bounds checking: `vgatherdps` does
+        // none, and the NEON / LASX arms offset a raw pointer. Check once here
+        // so an out-of-range index panics on every host rather than reading out
+        // of bounds on some and panicking on others.
+        for &i in indices {
+            assert!(
+                (i as usize) < slice.len(),
+                "gather index {i} is out of bounds for a slice of length {}",
+                slice.len()
+            );
+        }
 
-            let idx = i32x8::from(indices);
-            Self(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0))
+        #[cfg(target_arch = "x86_64")]
+        {
+            if is_x86_feature_detected!("avx2") {
+                unsafe { gather_avx2(slice, indices) }
+            } else {
+                gather_scalar_x86(slice, indices)
+            }
         }
 
         #[cfg(target_arch = "aarch64")]
@@ -94,6 +122,30 @@ impl f32x8 {
     }
 }
 
+/// AVX2 gather. Caller must ensure the host supports AVX2 (gated by
+/// the `is_x86_feature_detected!("avx2")` check in `f32x8::gather`).
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn gather_avx2(slice: &[f32], indices: &[i32; 8]) -> f32x8 {
+    use super::i32::i32x8;
+
+    let idx = i32x8::from(indices);
+    f32x8(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0))
+}
+
+/// Portable scalar gather for x86_64 hosts without AVX2.
+///
+/// Indexes the slice rather than offsetting a raw pointer: this is the slow
+/// path already, so an out-of-range index should panic instead of reading out
+/// of bounds.
+#[cfg(target_arch = "x86_64")]
+#[inline]
+fn gather_scalar_x86(slice: &[f32], indices: &[i32; 8]) -> f32x8 {
+    let values = indices.map(|i| slice[i as usize]);
+    // SAFETY: `values` is eight contiguous, initialized `f32`.
+    unsafe { f32x8::load_unaligned(values.as_ptr()) }
+}
+
 impl From<&[f32]> for f32x8 {
     fn from(value: &[f32]) -> Self {
         unsafe { Self::load_unaligned(value.as_ptr()) }
@@ -439,15 +491,18 @@ impl Mul for f32x8 {
     }
 }
 
-/// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible.
+/// 16 of 32-bit `f32` values. Stored as a pair of 256-bit AVX vectors on
+/// x86_64. Originally there was a sibling AVX-512 variant gated on
+/// `target_feature = "avx512f"`, but no project CI configuration enables
+/// `+avx512f` globally (and one of the avx512 arms still contained `todo!()`),
+/// so the variant was dead code. Removed in the runtime-SIMD-dispatch
+/// retrofit; per-tier dispatch happens in the kernel functions in
+/// `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier
+/// `#[target_feature(enable = "...")]` inner functions.
 #[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+#[cfg(target_arch = "x86_64")]
 #[derive(Clone, Copy)]
 pub struct f32x16(__m256, __m256);
-#[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-#[derive(Clone, Copy)]
-pub struct f32x16(__m512);
 
 /// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible.
 #[allow(non_camel_case_types)]
@@ -486,11 +541,7 @@ impl<'a> From<&'a [f32; 16]> for f32x16 {
 impl SIMD<f32, 16> for f32x16 {
     #[inline]
     fn splat(val: f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_set1_ps(val))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_set1_ps(val), _mm256_set1_ps(val))
         }
@@ -514,11 +565,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn zeros() -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_setzero_ps())
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_setzero_ps(), _mm256_setzero_ps())
         }
@@ -534,14 +581,10 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn load(ptr: *const f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_load_ps(ptr), _mm256_load_ps(ptr.add(8)))
         }
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_load_ps(ptr))
-        }
         #[cfg(target_arch = "aarch64")]
         {
             Self::load_unaligned(ptr)
@@ -557,14 +600,10 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn load_unaligned(ptr: *const f32) -> Self {
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_loadu_ps(ptr), _mm256_loadu_ps(ptr.add(8)))
         }
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_loadu_ps(ptr))
-        }
         #[cfg(target_arch = "aarch64")]
         {
             Self(vld1q_f32_x4(ptr))
@@ -580,11 +619,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn store(&self, ptr: *mut f32) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_store_ps(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_store_ps(ptr, self.0);
             _mm256_store_ps(ptr.add(8), self.1);
@@ -602,11 +637,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     unsafe fn store_unaligned(&self, ptr: *mut f32) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_storeu_ps(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_storeu_ps(ptr, self.0);
             _mm256_storeu_ps(ptr.add(8), self.1);
@@ -622,12 +653,9 @@ impl SIMD<f32, 16> for f32x16 {
         }
     }
 
+    #[inline]
     fn reduce_sum(&self) -> f32 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_add_ps(0xFFFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let mut sum = _mm256_add_ps(self.0, self.1);
             // Shift and add vector, until only 1 value left.
@@ -657,11 +685,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn reduce_min(&self) -> f32 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_min_ps(0xFFFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let mut m1 = _mm256_min_ps(self.0, self.1);
             let mut m2 = _mm256_permute2f128_ps(m1, m1, 1);
@@ -695,11 +719,7 @@ impl SIMD<f32, 16> for f32x16 {
 
     #[inline]
     fn min(&self, rhs: &Self) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_min_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_min_ps(self.0, rhs.0), _mm256_min_ps(self.1, rhs.1))
         }
@@ -718,21 +738,15 @@ impl SIMD<f32, 16> for f32x16 {
         }
     }
 
+    #[inline]
     fn find(&self, val: f32) -> Option<i32> {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            // let tgt = _mm512_set1_ps(val);
-            // let mask = _mm512_cmpeq_ps_mask(self.0, tgt);
-            // if mask != 0 {
-            //     return Some(mask.trailing_zeros() as i32);
-            // }
-            todo!()
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
-            // _mm256_cmpeq_ps_mask requires "avx512l".
+            // _mm256_cmpeq_ps_mask requires AVX-512 (avx512f); use a scalar scan here
+            // since we only require AVX2.
+            let arr = self.as_array();
             for i in 0..16 {
-                if self.as_array().get_unchecked(i) == &val {
+                if arr.get_unchecked(i) == &val {
                     return Some(i as i32);
                 }
             }
@@ -774,11 +788,7 @@ impl SIMD<f32, 16> for f32x16 {
 impl FloatSimd<f32, 16> for f32x16 {
     #[inline]
     fn multiply_add(&mut self, a: Self, b: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_fmadd_ps(a.0, b.0, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_fmadd_ps(a.0, b.0, self.0);
             self.1 = _mm256_fmadd_ps(a.1, b.1, self.1);
@@ -803,11 +813,7 @@ impl Add for f32x16 {
 
     #[inline]
     fn add(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_add_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_add_ps(self.0, rhs.0), _mm256_add_ps(self.1, rhs.1))
         }
@@ -830,11 +836,7 @@ impl Add for f32x16 {
 impl AddAssign for f32x16 {
     #[inline]
     fn add_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_add_ps(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_add_ps(self.0, rhs.0);
             self.1 = _mm256_add_ps(self.1, rhs.1);
@@ -859,11 +861,7 @@ impl Mul for f32x16 {
 
     #[inline]
     fn mul(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_mul_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_mul_ps(self.0, rhs.0), _mm256_mul_ps(self.1, rhs.1))
         }
@@ -888,11 +886,7 @@ impl Sub for f32x16 {
 
     #[inline]
     fn sub(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_sub_ps(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_sub_ps(self.0, rhs.0), _mm256_sub_ps(self.1, rhs.1))
         }
@@ -915,11 +909,7 @@ impl Sub for f32x16 {
 impl SubAssign for f32x16 {
     #[inline]
     fn sub_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_sub_ps(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_sub_ps(self.0, rhs.0);
             self.1 = _mm256_sub_ps(self.1, rhs.1);
@@ -943,9 +933,17 @@ impl SubAssign for f32x16 {
 mod tests {
 
     use super::*;
+    use rstest::rstest;
 
     #[test]
     fn test_basic_ops() {
+        // Load / store / arithmetic on `f32x8` lower to AVX intrinsics, and
+        // `multiply_add` lowers to `_mm256_fmadd_ps`, which needs FMA. Both
+        // are present from the AvxFma tier up.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
         let b = (10..18).map(|f| f as f32).collect::<Vec<_>>();
 
@@ -983,6 +981,11 @@ mod tests {
 
     #[test]
     fn test_f32x8_cmp_ops() {
+        // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0];
         let b = [2.0_f32, 1.0, 4.0, 5.0, 9.0, 5.0, 6.0, 2.0];
         let c = [2.0_f32, 1.0, 4.0, 5.0, 7.0, 3.0, 2.0, 1.0];
@@ -1007,6 +1010,11 @@ mod tests {
 
     #[test]
     fn test_basic_f32x16_ops() {
+        // `f32x16` is a pair of `__m256`; `multiply_add` needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = (0..16).map(|f| f as f32).collect::<Vec<_>>();
         let b = (10..26).map(|f| f as f32).collect::<Vec<_>>();
 
@@ -1041,6 +1049,11 @@ mod tests {
 
     #[test]
     fn test_f32x16_cmp_ops() {
+        // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [
             1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0, -0.5, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0,
         ];
@@ -1074,9 +1087,59 @@ mod tests {
 
     #[test]
     fn test_f32x8_gather() {
+        // `f32x8::gather` does its own runtime AVX2 detection and falls back
+        // to a scalar gather, so this test only needs whatever reading the
+        // `__m256`-backed result costs: AVX, for `reduce_sum`.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = (0..256).map(|f| f as f32).collect::<Vec<_>>();
         let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29];
         let v = f32x8::gather(&a, &idx);
         assert_eq!(v.reduce_sum(), 113.0);
     }
+
+    /// Directly exercises `gather_scalar_x86`, the per-index scalar fallback
+    /// `f32x8::gather` takes on x86_64 hosts without AVX2. Runtime AVX2 hosts
+    /// route through `gather_avx2` instead, so the fallback is otherwise never
+    /// hit under coverage. Reading the `__m256`-backed result needs AVX, so
+    /// skip on hosts without it.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    fn test_gather_scalar_x86() {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let a = (0..256).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29];
+        let v = gather_scalar_x86(&a, &idx);
+        let expected = idx.map(|i| a[i as usize]);
+        assert_eq!(v.as_array(), expected);
+    }
+
+    /// An index past the end of the slice panics rather than reading out of
+    /// bounds. The bounds check fires before any AVX instruction, so this
+    /// case runs on every x86_64 host.
+    #[cfg(target_arch = "x86_64")]
+    #[test]
+    #[should_panic(expected = "index out of bounds")]
+    fn test_gather_scalar_x86_rejects_out_of_range_index() {
+        let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 1, 2, 3, 4, 5, 6, 99];
+        let _ = gather_scalar_x86(&a, &idx);
+    }
+
+    /// `gather` validates before dispatching, so every backend — `vgatherdps`,
+    /// the x86 scalar fallback, and the NEON / LASX raw-pointer arms — rejects
+    /// a bad index identically instead of reading out of bounds.
+    #[rstest]
+    #[case::past_end(99)]
+    #[case::negative(-1)]
+    #[should_panic(expected = "out of bounds")]
+    fn test_gather_rejects_invalid_index(#[case] bad_index: i32) {
+        let a = (0..8).map(|f| f as f32).collect::<Vec<_>>();
+        let idx = [0_i32, 1, 2, 3, 4, 5, 6, bad_index];
+        let _ = f32x8::gather(&a, &idx);
+    }
 }
diff --git a/rust/lance-linalg/src/simd/f64.rs b/rust/lance-linalg/src/simd/f64.rs
index 32c0d389e5b..1276f54da56 100644
--- a/rust/lance-linalg/src/simd/f64.rs
+++ b/rust/lance-linalg/src/simd/f64.rs
@@ -355,14 +355,15 @@ impl Mul for f64x4 {
 // f64x8: 8 × f64 values (512-bit SIMD or 2 × 256-bit)
 // ---------------------------------------------------------------------------
 
-/// 8 of 64-bit `f64` values. Uses 512-bit SIMD if possible.
+/// 8 of 64-bit `f64` values. Stored as a pair of 256-bit AVX vectors on
+/// x86_64. Originally there was a sibling AVX-512 variant gated on
+/// `target_feature = "avx512f"`, but no project CI configuration enables
+/// `+avx512f` globally, so the variant was dead code. Removed in the
+/// runtime-SIMD-dispatch retrofit; per-tier dispatch happens in the kernel
+/// functions in `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier
+/// `#[target_feature(enable = "...")]` inner functions.
 #[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-#[derive(Clone, Copy)]
-pub struct f64x8(__m512d);
-
-#[allow(non_camel_case_types)]
-#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+#[cfg(target_arch = "x86_64")]
 #[derive(Clone, Copy)]
 pub struct f64x8(__m256d, __m256d);
 
@@ -401,11 +402,7 @@ impl<'a> From<&'a [f64; 8]> for f64x8 {
 impl SIMD<f64, 8> for f64x8 {
     #[inline]
     fn splat(val: f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_set1_pd(val))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_set1_pd(val), _mm256_set1_pd(val))
         }
@@ -423,11 +420,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn zeros() -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_setzero_pd())
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_setzero_pd(), _mm256_setzero_pd())
         }
@@ -443,11 +436,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn load(ptr: *const f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_load_pd(ptr))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_load_pd(ptr), _mm256_load_pd(ptr.add(4)))
         }
@@ -466,11 +455,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn load_unaligned(ptr: *const f64) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_loadu_pd(ptr))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_loadu_pd(ptr), _mm256_loadu_pd(ptr.add(4)))
         }
@@ -489,11 +474,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn store(&self, ptr: *mut f64) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_store_pd(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_store_pd(ptr, self.0);
             _mm256_store_pd(ptr.add(4), self.1);
@@ -512,11 +493,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     unsafe fn store_unaligned(&self, ptr: *mut f64) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_storeu_pd(ptr, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             _mm256_storeu_pd(ptr, self.0);
             _mm256_storeu_pd(ptr.add(4), self.1);
@@ -533,12 +510,9 @@ impl SIMD<f64, 8> for f64x8 {
         }
     }
 
+    #[inline]
     fn reduce_sum(&self) -> f64 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_add_pd(0xFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let sum = _mm256_add_pd(self.0, self.1);
             let hi = _mm256_permute2f128_pd(sum, sum, 1);
@@ -561,11 +535,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn reduce_min(&self) -> f64 {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            _mm512_mask_reduce_min_pd(0xFF, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             let m = _mm256_min_pd(self.0, self.1);
             let hi = _mm256_permute2f128_pd(m, m, 1);
@@ -592,11 +562,7 @@ impl SIMD<f64, 8> for f64x8 {
 
     #[inline]
     fn min(&self, rhs: &Self) -> Self {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_min_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_min_pd(self.0, rhs.0), _mm256_min_pd(self.1, rhs.1))
         }
@@ -613,6 +579,7 @@ impl SIMD<f64, 8> for f64x8 {
         }
     }
 
+    #[inline]
     fn find(&self, val: f64) -> Option<i32> {
         unsafe {
             for i in 0..8 {
@@ -628,11 +595,7 @@ impl SIMD<f64, 8> for f64x8 {
 impl FloatSimd<f64, 8> for f64x8 {
     #[inline]
     fn multiply_add(&mut self, a: Self, b: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_fmadd_pd(a.0, b.0, self.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_fmadd_pd(a.0, b.0, self.0);
             self.1 = _mm256_fmadd_pd(a.1, b.1, self.1);
@@ -657,11 +620,7 @@ impl Add for f64x8 {
 
     #[inline]
     fn add(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_add_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_add_pd(self.0, rhs.0), _mm256_add_pd(self.1, rhs.1))
         }
@@ -682,11 +641,7 @@ impl Add for f64x8 {
 impl AddAssign for f64x8 {
     #[inline]
     fn add_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_add_pd(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_add_pd(self.0, rhs.0);
             self.1 = _mm256_add_pd(self.1, rhs.1);
@@ -711,11 +666,7 @@ impl Mul for f64x8 {
 
     #[inline]
     fn mul(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_mul_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_mul_pd(self.0, rhs.0), _mm256_mul_pd(self.1, rhs.1))
         }
@@ -738,11 +689,7 @@ impl Sub for f64x8 {
 
     #[inline]
     fn sub(self, rhs: Self) -> Self::Output {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            Self(_mm512_sub_pd(self.0, rhs.0))
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             Self(_mm256_sub_pd(self.0, rhs.0), _mm256_sub_pd(self.1, rhs.1))
         }
@@ -763,11 +710,7 @@ impl Sub for f64x8 {
 impl SubAssign for f64x8 {
     #[inline]
     fn sub_assign(&mut self, rhs: Self) {
-        #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))]
-        unsafe {
-            self.0 = _mm512_sub_pd(self.0, rhs.0)
-        }
-        #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))]
+        #[cfg(target_arch = "x86_64")]
         unsafe {
             self.0 = _mm256_sub_pd(self.0, rhs.0);
             self.1 = _mm256_sub_pd(self.1, rhs.1);
@@ -793,6 +736,12 @@ mod tests {
 
     #[test]
     fn test_f64x4_basic_ops() {
+        // The `f64x4` constructor / load / store / arithmetic paths all lower
+        // to AVX intrinsics on x86_64; none of them need AVX2.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f64, 2.0, 3.0, 4.0];
         let b = [5.0_f64, 6.0, 7.0, 8.0];
 
@@ -814,6 +763,11 @@ mod tests {
 
     #[test]
     fn test_f64x4_fma() {
+        // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a = [1.0_f64, 2.0, 3.0, 4.0];
         let b = [2.0_f64, 3.0, 4.0, 5.0];
 
@@ -826,6 +780,11 @@ mod tests {
 
     #[test]
     fn test_f64x4_min() {
+        // `min` / `reduce_min` are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a = [1.0_f64, 5.0, 2.0, 8.0];
         let b = [3.0_f64, 2.0, 4.0, 1.0];
         let simd_a: f64x4 = (&a).into();
@@ -838,6 +797,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_basic_ops() {
+        // `f64x8` is a pair of `__m256d`; add / reduce are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
         let b: [f64; 8] = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0];
 
@@ -856,6 +820,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_fma() {
+        // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") {
+            return;
+        }
         let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
         let b: [f64; 8] = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0];
 
@@ -869,6 +838,11 @@ mod tests {
 
     #[test]
     fn test_f64x8_min() {
+        // `min` / `reduce_min` are AVX intrinsics.
+        #[cfg(target_arch = "x86_64")]
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
         let a: [f64; 8] = [5.0, 1.0, 8.0, 3.0, 9.0, 2.0, 7.0, 4.0];
         let b: [f64; 8] = [2.0, 6.0, 3.0, 7.0, 1.0, 8.0, 4.0, 9.0];
         let simd_a: f64x8 = (&a).into();
diff --git a/rust/lance-linalg/src/simd/x86.rs b/rust/lance-linalg/src/simd/x86.rs
new file mode 100644
index 00000000000..437018669e2
--- /dev/null
+++ b/rust/lance-linalg/src/simd/x86.rs
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Reduction helpers shared by the x86_64 AVX kernels in [`crate::distance`].
+//!
+//! Each distance kernel accumulates into a 256-bit register and folds it down
+//! to a scalar once, after the main loop. The fold is identical across
+//! `cosine`, `dot`, `l2` and `norm_l2`, so it lives here instead of being
+//! copied into each kernel's private `mod x86`.
+//!
+//! The module itself is `pub(crate)`, which is what keeps these helpers off the
+//! public API; the items are `pub` rather than `pub(crate)` only because
+//! `clippy::redundant_pub_crate` fires on the narrower visibility.
+
+use std::arch::x86_64::*;
+
+/// Horizontal sum of the eight `f32` lanes of an `__m256`.
+///
+/// Folds the upper 128-bit lane into the lower one, then reduces the
+/// remaining four lanes pairwise. Uses `movehl`/`shuffle` plus scalar adds
+/// rather than two `vhaddps`, which is one fewer uop on most cores.
+///
+/// # Safety
+///
+/// The host must support AVX. Callers are `#[target_feature]`-annotated
+/// kernels that the runtime dispatcher only selects after checking.
+#[inline]
+#[target_feature(enable = "avx")]
+pub unsafe fn hsum256_ps(v: __m256) -> f32 {
+    let lo = _mm256_castps256_ps128(v);
+    let hi = _mm256_extractf128_ps(v, 1);
+    let sum128 = _mm_add_ps(lo, hi);
+    let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128));
+    // 0x55 broadcasts lane 1 into lane 0, so the scalar add below lands the
+    // last of the four partial sums.
+    let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55));
+    _mm_cvtss_f32(sum32)
+}
+
+/// Horizontal sum of the four `f64` lanes of an `__m256d`.
+///
+/// Folds the upper 128-bit lane into the lower one, then adds the remaining
+/// pair.
+///
+/// # Safety
+///
+/// The host must support AVX. Callers are `#[target_feature]`-annotated
+/// kernels that the runtime dispatcher only selects after checking.
+#[inline]
+#[target_feature(enable = "avx")]
+pub unsafe fn hsum256_pd(v: __m256d) -> f64 {
+    let lo = _mm256_castpd256_pd128(v);
+    let hi = _mm256_extractf128_pd(v, 1);
+    let sum128 = _mm_add_pd(lo, hi);
+    let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128));
+    _mm_cvtsd_f64(sum64)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::ascending([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], 36.0)]
+    #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5, 0.0, -0.5, 1.0, 2.5], 5.0)]
+    #[case::zeros([0.0; 8], 0.0)]
+    #[case::cancelling([1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0, -4.0], 0.0)]
+    fn hsum256_ps_sums_every_lane(#[case] lanes: [f32; 8], #[case] expected: f32) {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let sum = unsafe { hsum256_ps(_mm256_loadu_ps(lanes.as_ptr())) };
+        assert_eq!(sum, expected);
+    }
+
+    #[rstest]
+    #[case::ascending([1.0, 2.0, 3.0, 4.0], 10.0)]
+    #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5], 2.0)]
+    #[case::zeros([0.0; 4], 0.0)]
+    #[case::cancelling([1.0, -1.0, 2.0, -2.0], 0.0)]
+    fn hsum256_pd_sums_every_lane(#[case] lanes: [f64; 4], #[case] expected: f64) {
+        if !std::is_x86_feature_detected!("avx") {
+            return;
+        }
+        let sum = unsafe { hsum256_pd(_mm256_loadu_pd(lanes.as_ptr())) };
+        assert_eq!(sum, expected);
+    }
+}
diff --git a/rust/lance-namespace-datafusion/src/catalog.rs b/rust/lance-namespace-datafusion/src/catalog.rs
index 4fe57f63c9b..ce699037ba0 100755
--- a/rust/lance-namespace-datafusion/src/catalog.rs
+++ b/rust/lance-namespace-datafusion/src/catalog.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::collections::HashSet;
 use std::sync::Arc;
 
@@ -53,10 +52,6 @@ impl LanceCatalogProviderList {
 }
 
 impl CatalogProviderList for LanceCatalogProviderList {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     /// Adds a new catalog to this catalog list.
     /// If a catalog of the same name existed before, it is replaced in the list and returned.
     fn register_catalog(
@@ -116,10 +111,6 @@ impl LanceCatalogProvider {
 }
 
 impl CatalogProvider for LanceCatalogProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema_names(&self) -> Vec<String> {
         self.schemas
             .iter()
diff --git a/rust/lance-namespace-datafusion/src/schema.rs b/rust/lance-namespace-datafusion/src/schema.rs
index 9acf30a97bf..194346001c3 100755
--- a/rust/lance-namespace-datafusion/src/schema.rs
+++ b/rust/lance-namespace-datafusion/src/schema.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -51,10 +50,6 @@ impl LanceSchemaProvider {
 
 #[async_trait]
 impl SchemaProvider for LanceSchemaProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn table_names(&self) -> Vec<String> {
         self.tables
             .iter()
diff --git a/rust/lance-namespace-datafusion/tests/sql.rs b/rust/lance-namespace-datafusion/tests/sql.rs
index e49cd7e58e3..5332e831cb6 100755
--- a/rust/lance-namespace-datafusion/tests/sql.rs
+++ b/rust/lance-namespace-datafusion/tests/sql.rs
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+#![recursion_limit = "256"]
+
 use std::sync::Arc;
 
 use arrow_array::{Int32Array, Int64Array, RecordBatch, RecordBatchIterator, StringArray};
diff --git a/rust/lance-namespace-impls/BENCHMARK.md b/rust/lance-namespace-impls/BENCHMARK.md
new file mode 100644
index 00000000000..074ec303347
--- /dev/null
+++ b/rust/lance-namespace-impls/BENCHMARK.md
@@ -0,0 +1,73 @@
+# `__manifest` commit benchmark
+
+Measures how fast the copy-on-write directory catalog commits `__manifest` mutations as
+the manifest scales, with the inline scalar indices on or off.
+
+The catalog commits every mutation by rewriting the whole `__manifest` (copy-on-write)
+and atomically writing a new manifest version. This benchmark characterises:
+
+- **Continuous commit** — a single process commits `N` times into a manifest already
+  holding `rows` entries (per-commit latency + throughput).
+- **Concurrent commit** — `C` processes commit continuously for a fixed duration against
+  a manifest of `rows` entries (steady, contended TPS).
+
+## Binary: `examples/manifest_bench.rs`
+
+```
+manifest_bench seed-large --root <uri> --count <rows> --inline-optimization <true|false> \
+    [--storage-option aws_region=us-east-1]
+manifest_bench run --root <uri> --operation write-create-namespace \
+    --concurrency 1 --operations 100 --initial-entries <rows> --inline-optimization <bool>   # continuous
+manifest_bench run --root <uri> --operation write-create-namespace \
+    --concurrency 50 --duration-secs 30 --initial-entries <rows> --inline-optimization <bool> # concurrent
+```
+
+- `seed-large` bootstraps a manifest to `count` rows by writing the Lance dataset
+  directly (O(rows) once) and then triggering one CoW rewrite so the on-disk state
+  matches the steady catalog form (single fragment; inline indices when enabled).
+- `run` spawns `--concurrency` worker subprocesses. With `--operations` it runs a fixed
+  commit budget (continuous); with `--duration-secs` each worker commits until the
+  deadline (steady TPS). It prints one JSON `BenchResult` per concurrency level with
+  throughput and p50/p90/p99 latency.
+- The committed operation (`--operation`) defaults to `write-create-namespace`, the
+  cheapest pure-`__manifest` mutation (no table data). `write-create-table` /
+  `write-declare-table` are also available.
+
+S3 requires the default `dir-aws` feature (on by default) and AWS credentials in the
+environment; pass `--storage-option aws_region=<region>`.
+
+## Sweep panel: `benches/manifest_commit_sweep.sh`
+
+Runs the full panel — sizes × {inline index, no index} × {continuous, concurrent×C} —
+with per-run S3-copy isolation (each run starts at exactly the bootstrapped size),
+JSONL results, a `summary.csv`, and resume support.
+
+```bash
+cargo build --release --example manifest_bench -p lance-namespace-impls
+S3_BASE=s3://<bucket>/manifest-cow-bench/$(date -u +%Y%m%dT%H%M%SZ) \
+  rust/lance-namespace-impls/benches/manifest_commit_sweep.sh
+```
+
+Default panel (override via env): `SIZES="1000 2000 5000 10000 20000 50000 100000 200000
+500000 1000000"`, `CONCURRENCY="10 20 50 100 120 150 200"`, `INLINE_VARIANTS="true false"`,
+`CONT_OPS=100`, `CONC_DURATION_SECS=30`. Results land in `$OUT_DIR` (default
+`~/manifest_cow_bench_<RUN_ID>`).
+
+## Representative results
+
+EC2 `c7i.48xlarge`, S3 `us-east-1`, op `write-create-namespace`. The catalog is a
+single-writer-throughput system: per-commit cost scales ~O(rows) and throughput does **not**
+scale with concurrency (every commit is a serialized `__manifest` version bump).
+
+Continuous (1 process, 100 commits), ops/s — inline index vs no index:
+
+| rows | inline | no index |
+|---:|---:|---:|
+| 1,000 | 2.0 | 3.5 |
+| 100,000 | 1.1 | 2.1 |
+| 1,000,000 | 0.34 | 0.53 |
+
+Concurrent steady TPS is flat across C=10..200 (e.g. inline @100k ≈ 1.4–1.5 ops/s at every C;
+@1M ≈ 0.3 ops/s). Conflicts that exceed the retry budget surface as errors and grow with C
+(≈0 at C≤20, climbing at C≥100) — the contention ceiling, not data loss. No-index commits run
+~1.5–2× faster (no per-commit index build) at the cost of unindexed reads.
diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml
index 963edf5e8ca..c5d8e120740 100644
--- a/rust/lance-namespace-impls/Cargo.toml
+++ b/rust/lance-namespace-impls/Cargo.toml
@@ -21,6 +21,7 @@ dir-aws = ["lance-io/aws", "lance/aws"]
 dir-azure = ["lance-io/azure", "lance/azure"]
 dir-oss = ["lance-io/oss", "lance/oss"]
 dir-huggingface = ["lance-io/huggingface", "lance/huggingface"]
+dir-goosefs = ["lance-io/goosefs", "lance/goosefs"]
 # Credential vending features
 credential-vendor-aws = ["dep:aws-sdk-sts", "dep:aws-config", "dep:sha2", "dep:base64"]
 credential-vendor-gcp = ["dep:reqwest", "dep:serde", "dep:sha2", "dep:base64", "dep:ring", "dep:rustls-pki-types"]
@@ -50,6 +51,8 @@ object_store = { workspace = true }
 arrow = { workspace = true }
 arrow-ipc = { workspace = true }
 arrow-schema = { workspace = true }
+datafusion-common = { workspace = true }
+datafusion-physical-plan = { workspace = true }
 
 # REST adapter implementation dependencies (optional, enabled by "rest-adapter" feature)
 axum = { workspace = true, optional = true }
@@ -65,6 +68,8 @@ serde_json = { workspace = true }
 futures.workspace = true
 log.workspace = true
 rand.workspace = true
+roaring.workspace = true
+uuid.workspace = true
 
 # Shared credential vending dependencies
 sha2 = { version = "0.10", optional = true }
@@ -74,6 +79,11 @@ base64 = { version = "0.22", optional = true }
 aws-sdk-sts = { version = "1.38.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
 aws-config = { workspace = true, optional = true }
 
+# Pin: time 0.3.48 conflicts with aws-smithy-types (E0119: conflicting `From` impls), which this
+# crate pulls in via the AWS credential vendor. Capping time here forces the workspace resolver to
+# 0.3.47 even for no-lock builds. Not used directly; remove once the upstream conflict is resolved.
+time = "=0.3.47"
+
 # GCP credential vending dependencies (optional, enabled by "credential-vendor-gcp" feature)
 ring = { version = "0.17", optional = true }
 rustls-pki-types = { version = "1", optional = true }
@@ -81,16 +91,25 @@ rustls-pki-types = { version = "1", optional = true }
 # Azure credential vending dependencies (optional, enabled by "credential-vendor-azure" feature)
 chrono = { workspace = true, optional = true }
 hmac = { version = "0.12", optional = true }
-quick-xml = { version = "0.38", optional = true }
+quick-xml = { version = "0.40", optional = true }
 
 [dev-dependencies]
+opendal = { workspace = true, features = ["services-goosefs"] }
 tokio = { workspace = true, features = ["full"] }
 tempfile.workspace = true
 wiremock.workspace = true
 arrow = { workspace = true }
+arrow-array = { workspace = true }
 arrow-ipc = { workspace = true }
 rstest.workspace = true
 lance-table.workspace = true
+lance-arrow = { workspace = true }
+lance = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+
+[[example]]
+name = "manifest_bench"
+path = "examples/manifest_bench.rs"
 
 [lints]
 workspace = true
diff --git a/rust/lance-namespace-impls/benches/manifest_commit_sweep.sh b/rust/lance-namespace-impls/benches/manifest_commit_sweep.sh
new file mode 100755
index 00000000000..7384ced4152
--- /dev/null
+++ b/rust/lance-namespace-impls/benches/manifest_commit_sweep.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Copy-on-write __manifest commit benchmark sweep panel.
+#
+# Drives `cargo run --release --example manifest_bench` across a panel of:
+#   - bootstrap manifest sizes (rows already in __manifest)
+#   - inline scalar indices on vs off
+#   - continuous commit (single process, N commits) and
+#     concurrent commit (C processes, steady TPS over a fixed duration)
+#
+# Each run is isolated: a "golden" manifest is bootstrapped once per (size, index)
+# and server-side-copied to a fresh S3 prefix per run, so every run starts at exactly
+# the bootstrapped size. Results are written as JSONL (one BenchResult per line) and
+# summarised to CSV. The sweep is resumable: completed runs are skipped.
+#
+# Usage:
+#   S3_BASE=s3://jack-devland-build/manifest-cow-bench/$(date -u +%Y%m%dT%H%M%SZ) \
+#     ./manifest_commit_sweep.sh
+#
+# Env knobs (defaults match the requested panel):
+#   SIZES, CONCURRENCY, INLINE_VARIANTS, CONT_OPS, CONC_DURATION_SECS,
+#   AWS_REGION, OUT_DIR, BIN
+#
+# Resilient by design: a single failed run is logged and skipped rather than aborting
+# the sweep, and re-running fills the gaps (completed runs are detected and skipped).
+set -uo pipefail
+
+RUN_ID="${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}"
+S3_BASE="${S3_BASE:?set S3_BASE, e.g. s3://jack-devland-build/manifest-cow-bench/$RUN_ID}"
+AWS_REGION="${AWS_REGION:-us-east-1}"
+export AWS_REGION AWS_DEFAULT_REGION="$AWS_REGION"
+
+REPO_ROOT="${REPO_ROOT:-$HOME/oss/lance}"
+BIN="${BIN:-$REPO_ROOT/target/release/examples/manifest_bench}"
+OUT_DIR="${OUT_DIR:-$HOME/manifest_cow_bench_${RUN_ID}}"
+RESULTS="$OUT_DIR/results.jsonl"
+PROGRESS="$OUT_DIR/progress.log"
+mkdir -p "$OUT_DIR"
+
+SIZES=(${SIZES:-1000 2000 5000 10000 20000 50000 100000 200000 500000 1000000})
+CONCURRENCY=(${CONCURRENCY:-10 20 50 100 120 150 200})
+INLINE_VARIANTS=(${INLINE_VARIANTS:-true false})
+CONT_OPS="${CONT_OPS:-100}"
+CONC_DURATION_SECS="${CONC_DURATION_SECS:-30}"
+STORAGE_OPT=(--storage-option "aws_region=${AWS_REGION}")
+
+log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*" | tee -a "$PROGRESS"; }
+
+# Skip a run if its tag already appears in results.jsonl (resume support).
+done_already() { grep -q "\"bench_tag\":\"$1\"" "$RESULTS" 2>/dev/null; }
+
+# Append a result line, tagging it so reruns can resume and we can pivot later.
+record() {
+    local tag="$1"; shift
+    # shellcheck disable=SC2016
+    python3 -c 'import json,sys; d=json.load(sys.stdin); d["bench_tag"]=sys.argv[1]; print(json.dumps(d))' \
+        "$tag" >> "$RESULTS"
+}
+
+s3_copy() { aws s3 cp --recursive --quiet "$1" "$2" --region "$AWS_REGION"; }
+s3_rm()   { aws s3 rm --recursive --quiet "$1" --region "$AWS_REGION" || true; }
+
+# Backstops for unattended runs: cap any single run and clear leaked worker processes
+# (a killed coordinator can orphan its worker children) before the next run.
+RUN_TIMEOUT="${RUN_TIMEOUT:-1200}"
+clear_stragglers() { pkill -f 'examples/manifest_bench worker' 2>/dev/null || true; sleep 1; }
+
+for inline in "${INLINE_VARIANTS[@]}"; do
+  for rows in "${SIZES[@]}"; do
+    golden="${S3_BASE}/golden/inline_${inline}_rows_${rows}"
+    boot_tag="boot_inline_${inline}_rows_${rows}"
+
+    if ! done_already "$boot_tag"; then
+      log "BOOTSTRAP inline=$inline rows=$rows -> $golden"
+      s3_rm "$golden"
+      if "$BIN" seed-large --root "$golden" --count "$rows" \
+          --inline-optimization "$inline" "${STORAGE_OPT[@]}"; then
+        echo "{\"bench_tag\":\"$boot_tag\"}" >> "$RESULTS"
+      else
+        log "BOOTSTRAP FAILED inline=$inline rows=$rows (skipping this size)"
+        continue
+      fi
+    else
+      log "skip bootstrap $boot_tag (done)"
+    fi
+
+    # ---- Continuous: single process, CONT_OPS commits ----
+    cont_tag="cont_inline_${inline}_rows_${rows}"
+    if ! done_already "$cont_tag"; then
+      run_prefix="${S3_BASE}/run/${cont_tag}"
+      log "CONTINUOUS inline=$inline rows=$rows ops=$CONT_OPS"
+      clear_stragglers
+      s3_copy "$golden" "$run_prefix"
+      timeout "$RUN_TIMEOUT" "$BIN" run --root "$run_prefix" --operation write-create-namespace \
+        --concurrency 1 --operations "$CONT_OPS" --initial-entries "$rows" \
+        --inline-optimization "$inline" "${STORAGE_OPT[@]}" \
+        2>>"$PROGRESS" | while read -r line; do record "$cont_tag" <<<"$line"; done
+      s3_rm "$run_prefix"
+    else
+      log "skip continuous $cont_tag (done)"
+    fi
+
+    # ---- Concurrent: C processes, steady TPS over CONC_DURATION_SECS ----
+    for c in "${CONCURRENCY[@]}"; do
+      conc_tag="conc_inline_${inline}_rows_${rows}_c_${c}"
+      if done_already "$conc_tag"; then log "skip concurrent $conc_tag (done)"; continue; fi
+      run_prefix="${S3_BASE}/run/${conc_tag}"
+      log "CONCURRENT inline=$inline rows=$rows c=$c dur=${CONC_DURATION_SECS}s"
+      clear_stragglers
+      s3_copy "$golden" "$run_prefix"
+      timeout "$RUN_TIMEOUT" "$BIN" run --root "$run_prefix" --operation write-create-namespace \
+        --concurrency "$c" --duration-secs "$CONC_DURATION_SECS" --initial-entries "$rows" \
+        --inline-optimization "$inline" "${STORAGE_OPT[@]}" \
+        2>>"$PROGRESS" | while read -r line; do record "$conc_tag" <<<"$line"; done
+      s3_rm "$run_prefix"
+    done
+  done
+done
+
+# ---- Summarise to CSV ----
+CSV="$OUT_DIR/summary.csv"
+python3 - "$RESULTS" "$CSV" <<'PY'
+import json, sys, csv
+rows = []
+with open(sys.argv[1]) as f:
+    for line in f:
+        d = json.loads(line)
+        if "throughput_ops_per_sec" not in d:
+            continue  # bootstrap marker
+        mode = "continuous" if d["duration_secs"] == 0 else "concurrent"
+        rows.append({
+            "mode": mode, "variant": d["variant"], "initial_entries": d["initial_entries"],
+            "concurrency": d["concurrency"], "duration_secs": d["duration_secs"],
+            "ops": d["total_operations"], "errors": d["errors"],
+            "tps": round(d["throughput_ops_per_sec"], 3),
+            "avg_ms": round(d["avg_latency_ms"], 2), "p50_ms": round(d["p50_latency_ms"], 2),
+            "p90_ms": round(d["p90_latency_ms"], 2), "p99_ms": round(d["p99_latency_ms"], 2),
+        })
+rows.sort(key=lambda r: (r["mode"], r["variant"], r["initial_entries"], r["concurrency"]))
+with open(sys.argv[2], "w", newline="") as f:
+    w = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else [])
+    w.writeheader(); w.writerows(rows)
+print(f"wrote {len(rows)} rows to {sys.argv[2]}")
+PY
+
+log "SWEEP COMPLETE. Results: $RESULTS  Summary: $CSV"
+s3_rm "${S3_BASE}/golden" "${S3_BASE}/run" 2>/dev/null || true
diff --git a/rust/lance-namespace-impls/examples/manifest_bench.rs b/rust/lance-namespace-impls/examples/manifest_bench.rs
new file mode 100644
index 00000000000..4841f2471d7
--- /dev/null
+++ b/rust/lance-namespace-impls/examples/manifest_bench.rs
@@ -0,0 +1,714 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Copy-on-write `__manifest` directory-catalog commit benchmark (S3 capable).
+//!
+//! Measures how fast the directory catalog commits `__manifest` mutations as the
+//! manifest scales, with the inline scalar indices on or off.
+//!
+//! Modes:
+//!   seed-large — bootstrap a `__manifest` with N rows (direct dataset write + one
+//!                CoW rewrite to build indices)
+//!   run        — coordinator: spawn `--concurrency` worker processes committing for
+//!                either a fixed op count (continuous) or a fixed duration (steady TPS)
+//!   worker     — (internal) a single committing process spawned by `run`
+//!
+//! Examples:
+//!   # Bootstrap 100k rows with inline indices
+//!   manifest_bench seed-large --root s3://bucket/bench/p --count 100000 \
+//!     --inline-optimization true --storage-option aws_region=us-east-1
+//!
+//!   # Continuous: 100 commits, single process
+//!   manifest_bench run --root s3://bucket/bench/p --operation write-create-namespace \
+//!     --concurrency 1 --operations 100 --initial-entries 100000 --inline-optimization true
+//!
+//!   # Concurrent steady TPS: 50 processes committing for 30s
+//!   manifest_bench run --root s3://bucket/bench/p --operation write-create-namespace \
+//!     --concurrency 50 --duration-secs 30 --initial-entries 100000 --inline-optimization true
+
+// A CLI benchmark tool: workers emit JSON latency records on stdout and progress on
+// stderr, so stdout/stderr printing is intentional here.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::collections::HashMap;
+use std::io::{BufRead, BufReader};
+use std::process::{Command, Stdio};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use arrow::array::builder::{ListBuilder, StringBuilder};
+use arrow::array::{RecordBatch, RecordBatchIterator, StringArray};
+use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+use bytes::Bytes;
+use lance::dataset::{InsertBuilder, WriteMode, WriteParams};
+use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION;
+use lance_namespace::LanceNamespace;
+use lance_namespace::models::{
+    CreateNamespaceRequest, CreateTableRequest, DeclareTableRequest, DescribeTableRequest,
+    ListNamespacesRequest, ListTablesRequest,
+};
+use lance_namespace_impls::DirectoryNamespaceBuilder;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone)]
+struct LatencyRecord {
+    operation: String,
+    latency_ms: f64,
+    error: bool,
+}
+
+#[derive(Serialize)]
+struct BenchResult {
+    variant: String,
+    operation: String,
+    concurrency: usize,
+    initial_entries: usize,
+    duration_secs: u64,
+    total_operations: usize,
+    total_duration_ms: f64,
+    throughput_ops_per_sec: f64,
+    avg_latency_ms: f64,
+    p50_latency_ms: f64,
+    p90_latency_ms: f64,
+    p99_latency_ms: f64,
+    min_latency_ms: f64,
+    max_latency_ms: f64,
+    errors: usize,
+}
+
+fn percentile(sorted: &[f64], p: f64) -> f64 {
+    if sorted.is_empty() {
+        return 0.0;
+    }
+    let idx = ((sorted.len() as f64 - 1.0) * p).round() as usize;
+    sorted[idx.min(sorted.len() - 1)]
+}
+
+#[allow(clippy::too_many_arguments)]
+fn compute_result(
+    variant: &str,
+    operation: &str,
+    concurrency: usize,
+    initial_entries: usize,
+    duration_secs: u64,
+    wall_duration: Duration,
+    mut latencies: Vec<f64>,
+    errors: usize,
+) -> BenchResult {
+    latencies.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    let total = latencies.len();
+    let total_ms = wall_duration.as_secs_f64() * 1000.0;
+    let throughput = if total_ms > 0.0 {
+        total as f64 / (total_ms / 1000.0)
+    } else {
+        0.0
+    };
+    BenchResult {
+        variant: variant.to_string(),
+        operation: operation.to_string(),
+        concurrency,
+        initial_entries,
+        duration_secs,
+        total_operations: total,
+        total_duration_ms: total_ms,
+        throughput_ops_per_sec: throughput,
+        avg_latency_ms: if total > 0 {
+            latencies.iter().sum::<f64>() / total as f64
+        } else {
+            0.0
+        },
+        p50_latency_ms: percentile(&latencies, 0.50),
+        p90_latency_ms: percentile(&latencies, 0.90),
+        p99_latency_ms: percentile(&latencies, 0.99),
+        min_latency_ms: latencies.first().copied().unwrap_or(0.0),
+        max_latency_ms: latencies.last().copied().unwrap_or(0.0),
+        errors,
+    }
+}
+
+fn create_test_ipc_data() -> Vec<u8> {
+    use arrow::array::Int32Array;
+    use arrow_ipc::writer::StreamWriter;
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        Field::new("id", DataType::Int32, false),
+        Field::new("name", DataType::Utf8, false),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2, 3])),
+            Arc::new(StringArray::from(vec!["a", "b", "c"])),
+        ],
+    )
+    .unwrap();
+    let mut buffer = Vec::new();
+    {
+        let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap();
+        writer.write(&batch).unwrap();
+        writer.finish().unwrap();
+    }
+    buffer
+}
+
+/// The `__manifest` schema used by the copy-on-write directory catalog:
+/// `object_id`, `object_type`, `location`, `metadata` (Utf8), `base_objects` (List<Utf8>).
+fn manifest_schema() -> Arc<ArrowSchema> {
+    Arc::new(ArrowSchema::new(vec![
+        Field::new("object_id", DataType::Utf8, false).with_metadata(
+            [(
+                LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(),
+                "0".to_string(),
+            )]
+            .into_iter()
+            .collect(),
+        ),
+        Field::new("object_type", DataType::Utf8, false),
+        Field::new("location", DataType::Utf8, true),
+        Field::new("metadata", DataType::Utf8, true),
+        Field::new(
+            "base_objects",
+            DataType::List(Arc::new(Field::new("object_id", DataType::Utf8, true))),
+            true,
+        ),
+    ]))
+}
+
+async fn build_namespace(
+    root: &str,
+    inline_optimization: bool,
+    storage_options: &HashMap<String, String>,
+) -> Box<dyn LanceNamespace> {
+    let mut properties = HashMap::new();
+    properties.insert("root".to_string(), root.to_string());
+    properties.insert("dir_listing_enabled".to_string(), "false".to_string());
+    properties.insert(
+        "inline_optimization_enabled".to_string(),
+        inline_optimization.to_string(),
+    );
+    for (k, v) in storage_options {
+        properties.insert(format!("storage.{}", k), v.clone());
+    }
+    let builder = DirectoryNamespaceBuilder::from_properties(properties, None)
+        .expect("Failed to create namespace builder from properties");
+    Box::new(builder.build().await.expect("Failed to build namespace"))
+}
+
+// ──────────────────── seed-large mode ────────────────────
+// Bootstrap a `__manifest` with N rows by writing the Lance dataset directly (fast,
+// O(N) once), then trigger a single CoW rewrite via the namespace so the on-disk state
+// matches what the catalog produces (single fragment + inline indices when enabled).
+
+const SEED_LARGE_BATCH_SIZE: usize = 50_000;
+
+fn generate_manifest_batch(start_idx: usize, batch_size: usize, total_count: usize) -> RecordBatch {
+    let ns_count = total_count / 3;
+    let actual_size = batch_size.min(total_count - start_idx);
+
+    let mut object_ids = Vec::with_capacity(actual_size);
+    let mut object_types = Vec::with_capacity(actual_size);
+    let mut locations: Vec<Option<String>> = Vec::with_capacity(actual_size);
+    let mut metadatas: Vec<Option<String>> = Vec::with_capacity(actual_size);
+
+    for i in start_idx..start_idx + actual_size {
+        if i < ns_count {
+            object_ids.push(format!("ns_{}", i));
+            object_types.push("namespace".to_string());
+            locations.push(None);
+            metadatas.push(None);
+        } else {
+            let table_idx = i - ns_count;
+            object_ids.push(format!("table_{}", table_idx));
+            object_types.push("table".to_string());
+            locations.push(Some(format!("table_{}", table_idx)));
+            metadatas.push(Some(r#"{"bench":"true"}"#.to_string()));
+        }
+    }
+
+    // base_objects is null for every bootstrapped row.
+    let mut base_objects_builder = ListBuilder::new(StringBuilder::new())
+        .with_field(Arc::new(Field::new("object_id", DataType::Utf8, true)));
+    for _ in 0..actual_size {
+        base_objects_builder.append_null();
+    }
+
+    RecordBatch::try_new(
+        manifest_schema(),
+        vec![
+            Arc::new(StringArray::from(object_ids)),
+            Arc::new(StringArray::from(object_types)),
+            Arc::new(StringArray::from(
+                locations.iter().map(|l| l.as_deref()).collect::<Vec<_>>(),
+            )),
+            Arc::new(StringArray::from(
+                metadatas.iter().map(|m| m.as_deref()).collect::<Vec<_>>(),
+            )),
+            Arc::new(base_objects_builder.finish()),
+        ],
+    )
+    .expect("Failed to create manifest batch")
+}
+
+async fn seed_large(
+    root: &str,
+    count: usize,
+    inline_optimization: bool,
+    storage_options: &HashMap<String, String>,
+) {
+    let manifest_uri = format!("{}/{}", root, "__manifest");
+    eprintln!("Seed-large: writing {} rows to {}", count, manifest_uri);
+
+    let schema = manifest_schema();
+    let mut batches = Vec::new();
+    let mut offset = 0;
+    while offset < count {
+        let batch_size = SEED_LARGE_BATCH_SIZE.min(count - offset);
+        batches.push(generate_manifest_batch(offset, batch_size, count));
+        offset += batch_size;
+    }
+    eprintln!("  generated {} batches", batches.len());
+
+    let mut write_params = WriteParams {
+        mode: WriteMode::Create,
+        ..WriteParams::default()
+    };
+    if !storage_options.is_empty() {
+        let accessor = Arc::new(
+            lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                storage_options.clone(),
+            ),
+        );
+        write_params.store_params = Some(lance_io::object_store::ObjectStoreParams {
+            storage_options_accessor: Some(accessor),
+            ..Default::default()
+        });
+    }
+
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+    InsertBuilder::new(manifest_uri.as_str())
+        .with_params(&write_params)
+        .execute_stream(reader)
+        .await
+        .expect("Failed to write manifest dataset");
+    eprintln!("  wrote Lance dataset");
+
+    // Trigger one CoW rewrite so the manifest is in steady catalog form (single
+    // fragment; inline indices when enabled). For the no-index variant the first real
+    // commit performs this rewrite instead.
+    if inline_optimization {
+        eprintln!("  triggering initial CoW rewrite to build indices...");
+        let start = Instant::now();
+        let ns = build_namespace(root, true, storage_options).await;
+        let mut req = CreateNamespaceRequest::new();
+        req.id = Some(vec!["__seed_trigger__".to_string()]);
+        ns.create_namespace(req)
+            .await
+            .expect("Failed to trigger CoW rewrite");
+        eprintln!(
+            "  CoW rewrite with index build took {:.1}s",
+            start.elapsed().as_secs_f64()
+        );
+    }
+
+    let ns_count = count / 3;
+    eprintln!(
+        "Seed-large complete: {} rows ({} namespaces, {} tables)",
+        count,
+        ns_count,
+        count - ns_count
+    );
+}
+
+// ──────────────────── worker mode ────────────────────
+
+#[allow(clippy::too_many_arguments)]
+async fn worker(
+    root: &str,
+    operation: &str,
+    operations: usize,
+    duration_secs: u64,
+    warmup: usize,
+    worker_id: usize,
+    table_count: usize,
+    inline_optimization: bool,
+    storage_options: &HashMap<String, String>,
+) {
+    let ns = build_namespace(root, inline_optimization, storage_options).await;
+    let ipc_data = Bytes::from(create_test_ipc_data());
+
+    if operation.starts_with("warm-read") {
+        for _ in 0..warmup {
+            let _ =
+                run_operation(ns.as_ref(), operation, worker_id, 0, table_count, &ipc_data).await;
+        }
+    }
+
+    let emit = |op_idx: usize, start: Instant, err: bool| {
+        let record = LatencyRecord {
+            operation: operation.to_string(),
+            latency_ms: start.elapsed().as_secs_f64() * 1000.0,
+            error: err,
+        };
+        let _ = op_idx;
+        println!("{}", serde_json::to_string(&record).unwrap());
+    };
+
+    if duration_secs > 0 {
+        // Steady-TPS mode: commit continuously until the deadline.
+        let deadline = Instant::now() + Duration::from_secs(duration_secs);
+        let mut op_idx = 0;
+        while Instant::now() < deadline {
+            let start = Instant::now();
+            let err = run_operation(
+                ns.as_ref(),
+                operation,
+                worker_id,
+                op_idx,
+                table_count,
+                &ipc_data,
+            )
+            .await
+            .is_err();
+            emit(op_idx, start, err);
+            op_idx += 1;
+        }
+    } else {
+        for op_idx in 0..operations {
+            let start = Instant::now();
+            let err = run_operation(
+                ns.as_ref(),
+                operation,
+                worker_id,
+                op_idx,
+                table_count,
+                &ipc_data,
+            )
+            .await
+            .is_err();
+            emit(op_idx, start, err);
+        }
+    }
+}
+
+async fn run_operation(
+    ns: &dyn LanceNamespace,
+    operation: &str,
+    worker_id: usize,
+    op_idx: usize,
+    table_count: usize,
+    ipc_data: &Bytes,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    match operation {
+        "cold-read-list-namespaces" | "warm-read-list-namespaces" => {
+            let mut req = ListNamespacesRequest::new();
+            req.id = Some(vec![]);
+            ns.list_namespaces(req).await?;
+        }
+        "cold-read-list-tables" | "warm-read-list-tables" => {
+            let mut req = ListTablesRequest::new();
+            req.id = Some(vec![]);
+            ns.list_tables(req).await?;
+        }
+        "cold-read-describe-table" | "warm-read-describe-table" => {
+            let table_idx = (worker_id * 1_000_000 + op_idx) % table_count.max(1);
+            let req = DescribeTableRequest {
+                id: Some(vec![format!("table_{}", table_idx)]),
+                ..Default::default()
+            };
+            ns.describe_table(req).await?;
+        }
+        "write-create-namespace" => {
+            let mut req = CreateNamespaceRequest::new();
+            req.id = Some(vec![format!("bench_w{}_{}", worker_id, op_idx)]);
+            ns.create_namespace(req).await?;
+        }
+        "write-create-table" => {
+            let mut req = CreateTableRequest::new();
+            req.id = Some(vec![format!("bench_t{}_{}", worker_id, op_idx)]);
+            ns.create_table(req, ipc_data.clone()).await?;
+        }
+        "write-declare-table" => {
+            let req = DeclareTableRequest {
+                id: Some(vec![format!("bench_d{}_{}", worker_id, op_idx)]),
+                ..Default::default()
+            };
+            ns.declare_table(req).await?;
+        }
+        _ => {
+            return Err(format!("unknown operation: {}", operation).into());
+        }
+    }
+    Ok(())
+}
+
+// ──────────────────── run mode (coordinator) ────────────────────
+
+#[allow(clippy::too_many_arguments)]
+fn run_workers(
+    self_exe: &str,
+    root: &str,
+    operation: &str,
+    concurrency: usize,
+    operations: usize,
+    duration_secs: u64,
+    warmup: usize,
+    table_count: usize,
+    initial_entries: usize,
+    inline_optimization: bool,
+    variant: &str,
+    storage_options: &HashMap<String, String>,
+) -> BenchResult {
+    // Continuous mode splits a fixed op budget across workers; steady-TPS mode lets each
+    // worker run for the full duration.
+    let ops_per_worker = if duration_secs > 0 {
+        0
+    } else {
+        operations / concurrency.max(1)
+    };
+    if duration_secs == 0 && ops_per_worker == 0 {
+        return compute_result(
+            variant,
+            operation,
+            concurrency,
+            initial_entries,
+            duration_secs,
+            Duration::ZERO,
+            vec![],
+            0,
+        );
+    }
+
+    let wall_start = Instant::now();
+    let children: Vec<_> = (0..concurrency)
+        .map(|worker_id| {
+            let mut cmd = Command::new(self_exe);
+            cmd.arg("worker")
+                .arg("--root")
+                .arg(root)
+                .arg("--operation")
+                .arg(operation)
+                .arg("--operations")
+                .arg(ops_per_worker.to_string())
+                .arg("--duration-secs")
+                .arg(duration_secs.to_string())
+                .arg("--warmup")
+                .arg(warmup.to_string())
+                .arg("--worker-id")
+                .arg(worker_id.to_string())
+                .arg("--table-count")
+                .arg(table_count.to_string())
+                .arg("--inline-optimization")
+                .arg(inline_optimization.to_string());
+            for (k, v) in storage_options {
+                cmd.arg("--storage-option").arg(format!("{}={}", k, v));
+            }
+            cmd.stdout(Stdio::piped())
+                .stderr(Stdio::inherit())
+                .spawn()
+                .expect("Failed to spawn worker")
+        })
+        .collect();
+
+    let mut all_latencies = Vec::new();
+    let mut total_errors = 0;
+    for mut child in children {
+        let stdout = child.stdout.take().unwrap();
+        for line in BufReader::new(stdout).lines() {
+            let line = line.expect("failed to read worker output");
+            if let Ok(record) = serde_json::from_str::<LatencyRecord>(&line) {
+                if record.error {
+                    total_errors += 1;
+                } else {
+                    all_latencies.push(record.latency_ms);
+                }
+            }
+        }
+        let status = child.wait().expect("failed to wait for worker");
+        if !status.success() {
+            eprintln!("Worker exited with status: {}", status);
+        }
+    }
+
+    compute_result(
+        variant,
+        operation,
+        concurrency,
+        initial_entries,
+        duration_secs,
+        wall_start.elapsed(),
+        all_latencies,
+        total_errors,
+    )
+}
+
+fn parse_concurrency_list(s: &str) -> Vec<usize> {
+    s.split(',')
+        .filter_map(|v| v.trim().parse::<usize>().ok())
+        .filter(|v| *v > 0)
+        .collect()
+}
+
+#[tokio::main]
+async fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    if args.len() < 2 {
+        eprintln!("Usage: manifest_bench <seed-large|run|worker> [options]");
+        std::process::exit(1);
+    }
+
+    let mode = args[1].as_str();
+    let mut root = String::new();
+    let mut operation = String::new();
+    let mut operations: usize = 100;
+    let mut duration_secs: u64 = 0;
+    let mut warmup: usize = 0;
+    let mut concurrency_list = vec![1];
+    let mut count: usize = 1000;
+    let mut worker_id: usize = 0;
+    let mut table_count: usize = 667;
+    let mut initial_entries: usize = 0;
+    let mut inline_optimization = true;
+    let mut variant = String::new();
+    let mut storage_options: HashMap<String, String> = HashMap::new();
+
+    let mut i = 2;
+    while i < args.len() {
+        match args[i].as_str() {
+            "--root" => {
+                root = args[i + 1].clone();
+                i += 2;
+            }
+            "--operation" => {
+                operation = args[i + 1].clone();
+                i += 2;
+            }
+            "--operations" => {
+                operations = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--duration-secs" => {
+                duration_secs = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--warmup" => {
+                warmup = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--concurrency" => {
+                concurrency_list = parse_concurrency_list(&args[i + 1]);
+                i += 2;
+            }
+            "--count" => {
+                count = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--worker-id" => {
+                worker_id = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--table-count" => {
+                table_count = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--initial-entries" => {
+                initial_entries = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--inline-optimization" => {
+                inline_optimization = args[i + 1].parse().unwrap();
+                i += 2;
+            }
+            "--variant" => {
+                variant = args[i + 1].clone();
+                i += 2;
+            }
+            "--storage-option" => {
+                if let Some((k, v)) = args[i + 1].split_once('=') {
+                    storage_options.insert(k.to_string(), v.to_string());
+                }
+                i += 2;
+            }
+            other => {
+                eprintln!("Unknown argument: {}", other);
+                std::process::exit(1);
+            }
+        }
+    }
+
+    if variant.is_empty() {
+        variant = if inline_optimization {
+            "inline_index".to_string()
+        } else {
+            "no_index".to_string()
+        };
+    }
+
+    match mode {
+        "seed-large" => {
+            seed_large(&root, count, inline_optimization, &storage_options).await;
+        }
+        "worker" => {
+            worker(
+                &root,
+                &operation,
+                operations,
+                duration_secs,
+                warmup,
+                worker_id,
+                table_count,
+                inline_optimization,
+                &storage_options,
+            )
+            .await;
+        }
+        "run" => {
+            let self_exe = std::env::current_exe()
+                .expect("failed to get self exe path")
+                .to_string_lossy()
+                .to_string();
+            let op = if operation.is_empty() {
+                "write-create-namespace"
+            } else {
+                operation.as_str()
+            };
+
+            eprintln!("=== Manifest commit benchmark ===");
+            eprintln!(
+                "variant={} op={} root={} initial_entries={} concurrency={:?} operations={} duration_secs={}",
+                variant, op, root, initial_entries, concurrency_list, operations, duration_secs
+            );
+
+            for &concurrency in &concurrency_list {
+                let result = run_workers(
+                    &self_exe,
+                    &root,
+                    op,
+                    concurrency,
+                    operations,
+                    duration_secs,
+                    warmup,
+                    table_count,
+                    initial_entries,
+                    inline_optimization,
+                    &variant,
+                    &storage_options,
+                );
+                eprintln!(
+                    "  c={} -> {:.2} ops/s ({} ops, {} errors, p50={:.0}ms p99={:.0}ms)",
+                    concurrency,
+                    result.throughput_ops_per_sec,
+                    result.total_operations,
+                    result.errors,
+                    result.p50_latency_ms,
+                    result.p99_latency_ms
+                );
+                println!("{}", serde_json::to_string(&result).unwrap());
+            }
+            eprintln!("=== complete ===");
+        }
+        _ => {
+            eprintln!("Unknown mode: {}. Use seed-large, run, or worker.", mode);
+            std::process::exit(1);
+        }
+    }
+}
diff --git a/rust/lance-namespace-impls/src/credentials.rs b/rust/lance-namespace-impls/src/credentials.rs
index e841ac620f6..23f9346cfb3 100644
--- a/rust/lance-namespace-impls/src/credentials.rs
+++ b/rust/lance-namespace-impls/src/credentials.rs
@@ -223,6 +223,14 @@ pub mod aws_props {
     /// AWS credential duration in milliseconds.
     /// Default: 3600000 (1 hour). Range: 900000 (15 min) to 43200000 (12 hours).
     pub const DURATION_MILLIS: &str = "aws_duration_millis";
+
+    /// When "true", the scoped assume is performed via `AssumeRoleWithWebIdentity`
+    /// using the pod's projected service-account OIDC token
+    pub const ASSUME_VIA_POD_WEB_IDENTITY: &str = "aws_assume_via_pod_web_identity";
+
+    /// Explicit path to the pod's projected SA OIDC token file. Overrides
+    /// `AWS_WEB_IDENTITY_TOKEN_FILE` when set.
+    pub const POD_WEB_IDENTITY_TOKEN_FILE: &str = "aws_pod_web_identity_token_file";
 }
 
 /// GCP-specific property keys (short form, without prefix)
@@ -503,6 +511,43 @@ async fn create_aws_vendor(
         config = config.with_role_session_name(session_name);
     }
 
+    // Direct (non-chained) web-identity assume for the pod, when enabled. An
+    // explicit token-file path wins; otherwise, if opted in, resolve the
+    // EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`. Falling back to the chained
+    // AssumeRole path when neither is present keeps existing behavior.
+    let assume_via_pod = properties
+        .get(aws_props::ASSUME_VIA_POD_WEB_IDENTITY)
+        .map(|v| v.eq_ignore_ascii_case("true"))
+        .unwrap_or(false);
+    let pod_token_file = properties
+        .get(aws_props::POD_WEB_IDENTITY_TOKEN_FILE)
+        .cloned()
+        .or_else(|| {
+            assume_via_pod
+                .then(|| std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE").ok())
+                .flatten()
+        });
+    // Log the resolved assume path once at vendor init so a deployment can
+    // confirm at runtime which branch `assume_scoped` will take.
+    match &pod_token_file {
+        Some(path) => log::info!(
+            "AWS credential vendor (role {role_arn}): direct AssumeRoleWithWebIdentity \
+             via pod token file '{path}'"
+        ),
+        None if assume_via_pod => log::warn!(
+            "AWS credential vendor (role {role_arn}): aws_assume_via_pod_web_identity=true \
+             but no token file resolved (aws_pod_web_identity_token_file unset and \
+             AWS_WEB_IDENTITY_TOKEN_FILE not in env); falling back to chained AssumeRole"
+        ),
+        None => log::info!(
+            "AWS credential vendor (role {role_arn}): chained AssumeRole \
+             (pod web-identity not enabled)"
+        ),
+    }
+    if let Some(path) = pod_token_file {
+        config = config.with_pod_web_identity_token_file(path);
+    }
+
     let vendor = AwsCredentialVendor::new(config).await?;
     Ok(Some(Box::new(vendor)))
 }
diff --git a/rust/lance-namespace-impls/src/credentials/aws.rs b/rust/lance-namespace-impls/src/credentials/aws.rs
index 7dedaa6e108..56dc9a54c2c 100644
--- a/rust/lance-namespace-impls/src/credentials/aws.rs
+++ b/rust/lance-namespace-impls/src/credentials/aws.rs
@@ -24,6 +24,18 @@ use super::{
     redact_credential,
 };
 
+/// Render an error together with its full `source()` chain.
+fn full_error_chain(err: &dyn std::error::Error) -> String {
+    let mut out = err.to_string();
+    let mut source = err.source();
+    while let Some(cause) = source {
+        out.push_str(": ");
+        out.push_str(&cause.to_string());
+        source = cause.source();
+    }
+    out
+}
+
 /// Configuration for AWS credential vending.
 #[derive(Debug, Clone)]
 pub struct AwsCredentialVendorConfig {
@@ -60,6 +72,17 @@ pub struct AwsCredentialVendorConfig {
     /// When an API key is provided, its hash is looked up in this map.
     /// If found, the mapped permission is used instead of the default permission.
     pub api_key_hash_permissions: HashMap<String, VendedPermission>,
+
+    /// Optional path to the pod's projected service-account OIDC token file (typically the
+    /// EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`).
+    /// This is the recommended method when running in Kubernetes.
+    /// When set, the scoped assume is performed with `AssumeRoleWithWebIdentity`
+    /// using this token instead of a role-chained `AssumeRole` from the process's
+    /// ambient credentials. A web-identity assumption is *not* role chaining, so
+    /// STS honors the role's `MaxSessionDuration` (up to 12h) and the returned
+    /// expiration is accurate. When `None`, the `AssumeRole` path
+    /// is used and `duration_millis` is subject to the source role duration and may be invalid
+    pub pod_web_identity_token_file: Option<String>,
 }
 
 impl AwsCredentialVendorConfig {
@@ -74,6 +97,7 @@ impl AwsCredentialVendorConfig {
             permission: VendedPermission::default(),
             api_key_salt: None,
             api_key_hash_permissions: HashMap::new(),
+            pod_web_identity_token_file: None,
         }
     }
 
@@ -101,6 +125,14 @@ impl AwsCredentialVendorConfig {
         self
     }
 
+    /// Set the pod web-identity token file, enabling direct (non-chained)
+    /// `AssumeRoleWithWebIdentity` for the scoped assume. See
+    /// [`AwsCredentialVendorConfig::pod_web_identity_token_file`].
+    pub fn with_pod_web_identity_token_file(mut self, path: impl Into<String>) -> Self {
+        self.pod_web_identity_token_file = Some(path.into());
+        self
+    }
+
     /// Set the permission level for vended credentials.
     pub fn with_permission(mut self, permission: VendedPermission) -> Self {
         self.permission = permission;
@@ -407,7 +439,8 @@ impl AwsCredentialVendor {
                 lance_core::Error::from(NamespaceError::Internal {
                     message: format!(
                         "AssumeRoleWithWebIdentity failed for role '{}': {}",
-                        self.config.role_arn, e
+                        self.config.role_arn,
+                        full_error_chain(&e)
                     ),
                 })
             })?;
@@ -421,6 +454,95 @@ impl AwsCredentialVendor {
     }
 
     /// Vend credentials using AssumeRole with API key validation.
+    /// Perform the scoped assume for `(bucket, prefix, permission)`, attaching the
+    /// per-table session policy.
+    ///
+    /// When [`AwsCredentialVendorConfig::pod_web_identity_token_file`] is set this
+    /// uses `AssumeRoleWithWebIdentity` with the pod's projected SA OIDC token -- a
+    /// *direct*, non-chained assumption that honors the role's `MaxSessionDuration`
+    /// (so `expires_at_millis` is accurate). Otherwise it falls back to the legacy
+    /// role-chained `AssumeRole` from ambient credentials (STS-capped at 1h).
+    ///
+    /// `external_id` is only applied on the chained `AssumeRole` path;
+    /// `AssumeRoleWithWebIdentity` has no external-id parameter (the OIDC
+    /// `sub`/`aud` trust condition is the binding instead).
+    async fn assume_scoped(
+        &self,
+        bucket: &str,
+        prefix: &str,
+        permission: VendedPermission,
+        session_name: &str,
+        external_id: Option<&str>,
+    ) -> Result<VendedCredentials> {
+        let policy = Self::build_policy(bucket, prefix, permission);
+        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
+
+        let credentials = if let Some(token_file) = &self.config.pod_web_identity_token_file {
+            // DIRECT (non-chained): AssumeRoleWithWebIdentity with the pod's SA
+            // OIDC token. Re-read the file every vend -- kubelet rotates it.
+            let token = tokio::fs::read_to_string(token_file).await.map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "failed to read pod web identity token '{}': {}",
+                        token_file, e
+                    ),
+                })
+            })?;
+            debug!(
+                "AWS AssumeRoleWithWebIdentity (pod): role={}, session={}, permission={}",
+                self.config.role_arn, session_name, permission
+            );
+            let response = self
+                .sts_client
+                .assume_role_with_web_identity()
+                .role_arn(&self.config.role_arn)
+                .web_identity_token(token.trim())
+                .role_session_name(session_name)
+                .policy(&policy)
+                .duration_seconds(duration_secs)
+                .send()
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "AssumeRoleWithWebIdentity (pod) failed for role '{}': {}",
+                            self.config.role_arn,
+                            full_error_chain(&e)
+                        ),
+                    })
+                })?;
+            response.credentials().cloned()
+        } else {
+            // LEGACY chained path: AssumeRole from ambient credentials (1h cap).
+            debug!(
+                "AWS AssumeRole (chained): role={}, session={}, permission={}",
+                self.config.role_arn, session_name, permission
+            );
+            let mut request = self
+                .sts_client
+                .assume_role()
+                .role_arn(&self.config.role_arn)
+                .role_session_name(session_name)
+                .policy(&policy)
+                .duration_seconds(duration_secs);
+            if let Some(external_id) = external_id {
+                request = request.external_id(external_id);
+            }
+            let response = request.send().await.map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "AssumeRole failed for role '{}': {}",
+                        self.config.role_arn,
+                        full_error_chain(&e)
+                    ),
+                })
+            })?;
+            response.credentials().cloned()
+        };
+
+        self.extract_credentials(credentials.as_ref(), bucket, prefix, permission)
+    }
+
     async fn vend_with_api_key(
         &self,
         bucket: &str,
@@ -452,42 +574,19 @@ impl AwsCredentialVendor {
                 })
             })?;
 
-        let policy = Self::build_policy(bucket, prefix, permission);
+        // The api_key authorizes the client and picks the permission; the AWS
+        // assume itself goes through `assume_scoped` (pod web-identity when
+        // configured, else chained AssumeRole with the key hash as external_id).
         let session_name = Self::cap_session_name(&format!("lance-api-{}", &key_hash[..16]));
-        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
-
-        debug!(
-            "AWS AssumeRole with API key: role={}, session={}, permission={}",
-            self.config.role_arn, session_name, permission
-        );
-
-        let request = self
-            .sts_client
-            .assume_role()
-            .role_arn(&self.config.role_arn)
-            .role_session_name(&session_name)
-            .policy(&policy)
-            .duration_seconds(duration_secs)
-            .external_id(&key_hash); // Use hash as external_id
-
-        let response = request.send().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!(
-                    "AssumeRole with API key failed for role '{}': {}",
-                    self.config.role_arn, e
-                ),
-            })
-        })?;
-
-        self.extract_credentials(response.credentials(), bucket, prefix, permission)
+        self.assume_scoped(bucket, prefix, permission, &session_name, Some(&key_hash))
+            .await
     }
 
-    /// Vend credentials using AssumeRole with static configuration.
+    /// Vend credentials using the vendor's static (default) permission.
     async fn vend_with_static_config(
         &self,
         bucket: &str,
         prefix: &str,
-        policy: &str,
     ) -> Result<VendedCredentials> {
         let role_session_name = self
             .config
@@ -496,40 +595,14 @@ impl AwsCredentialVendor {
             .unwrap_or_else(|| "lance-credential-vending".to_string());
         let role_session_name = Self::cap_session_name(&role_session_name);
 
-        let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32;
-
-        debug!(
-            "AWS AssumeRole (static): role={}, session={}, permission={}",
-            self.config.role_arn, role_session_name, self.config.permission
-        );
-
-        let mut request = self
-            .sts_client
-            .assume_role()
-            .role_arn(&self.config.role_arn)
-            .role_session_name(&role_session_name)
-            .policy(policy)
-            .duration_seconds(duration_secs);
-
-        if let Some(ref external_id) = self.config.external_id {
-            request = request.external_id(external_id);
-        }
-
-        let response = request.send().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!(
-                    "AssumeRole failed for role '{}': {}",
-                    self.config.role_arn, e
-                ),
-            })
-        })?;
-
-        self.extract_credentials(
-            response.credentials(),
+        self.assume_scoped(
             bucket,
             prefix,
             self.config.permission,
+            &role_session_name,
+            self.config.external_id.as_deref(),
         )
+        .await
     }
 }
 
@@ -575,10 +648,8 @@ impl CredentialVendor for AwsCredentialVendor {
                 .into())
             }
             None => {
-                // Use AssumeRole with static configuration
-                let policy = Self::build_policy(&bucket, &prefix, self.config.permission);
-                self.vend_with_static_config(&bucket, &prefix, &policy)
-                    .await
+                // Use the vendor's static (default) permission
+                self.vend_with_static_config(&bucket, &prefix).await
             }
         }
     }
@@ -743,6 +814,41 @@ mod tests {
         assert_eq!(config.duration_millis, 7200000);
         assert_eq!(config.role_session_name, Some("my-session".to_string()));
         assert_eq!(config.region, Some("us-west-2".to_string()));
+        // Defaults to the legacy chained AssumeRole path.
+        assert_eq!(config.pod_web_identity_token_file, None);
+
+        let pod_config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole")
+            .with_pod_web_identity_token_file("/var/run/secrets/.../token");
+        assert_eq!(
+            pod_config.pod_web_identity_token_file,
+            Some("/var/run/secrets/.../token".to_string())
+        );
+    }
+
+    #[tokio::test]
+    async fn test_pod_web_identity_path_reads_token_file() {
+        // When pod_web_identity_token_file is set, the scoped assume takes the
+        // AssumeRoleWithWebIdentity branch and reads the token file first. Point
+        // it at a missing file so we deterministically hit the read error without
+        // needing a live STS -- this proves the branch selection + file read.
+        let sdk_config = aws_config::SdkConfig::builder()
+            .behavior_version(aws_config::BehaviorVersion::latest())
+            .region(aws_config::Region::new("us-east-2"))
+            .build();
+        let sts_client = StsClient::new(&sdk_config);
+        let config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole")
+            .with_pod_web_identity_token_file("/nonexistent/pod/web-identity-token");
+        let vendor = AwsCredentialVendor::with_sts_client(config, sts_client);
+
+        let err = vendor
+            .vend_credentials("s3://bucket/prefix", None)
+            .await
+            .expect_err("missing token file must fail before any STS call");
+        assert!(
+            err.to_string()
+                .contains("failed to read pod web identity token"),
+            "unexpected error: {err}"
+        );
     }
 
     // ============================================================================
diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs
index fb0b03ad239..814da6435c5 100644
--- a/rust/lance-namespace-impls/src/dir.rs
+++ b/rust/lance-namespace-impls/src/dir.rs
@@ -7,6 +7,7 @@
 //! that stores tables as Lance datasets in a filesystem directory structure.
 
 pub mod manifest;
+pub mod manifest_feature_flags;
 
 use arrow::array::Float32Array;
 use arrow::record_batch::RecordBatchIterator;
@@ -15,12 +16,13 @@ use async_trait::async_trait;
 use bytes::Bytes;
 use futures::{StreamExt, TryStreamExt};
 use lance::dataset::builder::DatasetBuilder;
+use lance::dataset::refs::check_valid_branch;
 use lance::dataset::scanner::Scanner;
 use lance::dataset::statistics::DatasetStatisticsExt;
 use lance::dataset::transaction::{Operation, Transaction};
 use lance::dataset::{
-    Dataset, MergeInsertBuilder, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, WriteMode,
-    WriteParams,
+    Dataset, MergeInsertBuilder, UpdateBuilder, WhenMatched, WhenNotMatched,
+    WhenNotMatchedBySource, WriteMode, WriteParams,
 };
 use lance::index::{DatasetIndexExt, IndexParams, vector::VectorIndexParams};
 use lance::session::Session;
@@ -41,34 +43,41 @@ use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, PutM
 use std::collections::HashMap;
 use std::io::Cursor;
 use std::sync::{Arc, Mutex};
+use tokio::sync::OnceCell;
 
 use crate::context::DynamicContextProvider;
 use lance_namespace::models::{
-    AnalyzeTableQueryPlanRequest, BatchDeleteTableVersionsRequest,
-    BatchDeleteTableVersionsResponse, CountTableRowsRequest, CreateNamespaceRequest,
-    CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest,
-    CreateTableResponse, CreateTableScalarIndexResponse, CreateTableTagRequest,
-    CreateTableTagResponse, CreateTableVersionRequest, CreateTableVersionResponse,
-    DeclareTableRequest, DeclareTableResponse, DeleteTableTagRequest, DeleteTableTagResponse,
-    DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableIndexStatsRequest,
-    DescribeTableIndexStatsResponse, DescribeTableRequest, DescribeTableResponse,
-    DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
-    DescribeTransactionResponse, DropNamespaceRequest, DropNamespaceResponse,
-    DropTableIndexRequest, DropTableIndexResponse, DropTableRequest, DropTableResponse,
-    ExplainTableQueryPlanRequest, FragmentStats, FragmentSummary, GetTableStatsRequest,
-    GetTableStatsResponse, GetTableTagVersionRequest, GetTableTagVersionResponse, Identity,
-    IndexContent, InsertIntoTableRequest, InsertIntoTableResponse, ListNamespacesRequest,
-    ListNamespacesResponse, ListTableIndicesRequest, ListTableIndicesResponse,
-    ListTableTagsRequest, ListTableTagsResponse, ListTableVersionsRequest,
-    ListTableVersionsResponse, ListTablesRequest, ListTablesResponse, MergeInsertIntoTableRequest,
-    MergeInsertIntoTableResponse, NamespaceExistsRequest, QueryTableRequest,
-    QueryTableRequestColumns, QueryTableRequestVector, RestoreTableRequest, RestoreTableResponse,
-    TableExistsRequest, TableVersion, TagContents as ModelTagContents,
-    UpdateTableSchemaMetadataRequest, UpdateTableSchemaMetadataResponse, UpdateTableTagRequest,
-    UpdateTableTagResponse,
+    AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest,
+    AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse,
+    AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest,
+    BatchDeleteTableVersionsRequest, BatchDeleteTableVersionsResponse,
+    BranchContents as ModelBranchContents, CountTableRowsRequest, CreateNamespaceRequest,
+    CreateNamespaceResponse, CreateTableBranchRequest, CreateTableBranchResponse,
+    CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, CreateTableResponse,
+    CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse,
+    CreateTableVersionRequest, CreateTableVersionResponse, DeclareTableRequest,
+    DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse,
+    DeleteTableBranchRequest, DeleteTableBranchResponse, DeleteTableTagRequest,
+    DeleteTableTagResponse, DescribeNamespaceRequest, DescribeNamespaceResponse,
+    DescribeTableIndexStatsRequest, DescribeTableIndexStatsResponse, DescribeTableRequest,
+    DescribeTableResponse, DescribeTableVersionRequest, DescribeTableVersionResponse,
+    DescribeTransactionRequest, DescribeTransactionResponse, DropNamespaceRequest,
+    DropNamespaceResponse, DropTableIndexRequest, DropTableIndexResponse, DropTableRequest,
+    DropTableResponse, ExplainTableQueryPlanRequest, FragmentStats, FragmentSummary,
+    GetTableStatsRequest, GetTableStatsResponse, GetTableTagVersionRequest,
+    GetTableTagVersionResponse, Identity, IndexContent, InsertIntoTableRequest,
+    InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse,
+    ListTableBranchesRequest, ListTableBranchesResponse, ListTableIndicesRequest,
+    ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
+    ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse,
+    MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest,
+    QueryTableRequest, QueryTableRequestColumns, QueryTableRequestVector, RestoreTableRequest,
+    RestoreTableResponse, TableExistsRequest, TableVersion, TagContents as ModelTagContents,
+    UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest,
+    UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse,
 };
 
-use lance_core::{Error, Result};
+use lance_core::{Error, Result, box_error};
 use lance_namespace::LanceNamespace;
 use lance_namespace::error::NamespaceError;
 use lance_namespace::schema::arrow_schema_to_json;
@@ -107,6 +116,70 @@ impl OpsMetrics {
     }
 }
 
+/// Build SQL expression list for the add_columns operation.
+/// Returns an explicit error when the expression is missing, instead of silently using an empty string.
+pub(crate) fn build_sql_expressions(
+    new_columns: &[lance_namespace::models::AddColumnsEntry],
+) -> Result<Vec<(String, String)>> {
+    new_columns
+        .iter()
+        .map(|col| {
+            // expression is Option<Option<String>>: outer Option means whether the
+            // field is present, inner Option means whether the value is JSON null.
+            let expression = col.expression.clone().and_then(|opt| opt).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Expression is required for new column '{}'",
+                    col.name
+                ))
+            })?;
+            Ok((col.name.clone(), expression))
+        })
+        .collect()
+}
+
+/// Build column alteration list for the alter_columns operation.
+/// Returns an explicit error when data_type conversion fails, instead of silently ignoring it.
+pub(crate) fn build_column_alterations(
+    alterations: &[lance_namespace::models::AlterColumnsEntry],
+) -> Result<Vec<lance::dataset::ColumnAlteration>> {
+    alterations
+        .iter()
+        .map(|entry| {
+            let mut alteration = lance::dataset::ColumnAlteration::new(entry.path.clone());
+            // rename is Option<Option<String>>: flatten to get the actual rename value.
+            if let Some(Some(rename)) = &entry.rename {
+                alteration = alteration.rename(rename.clone());
+            }
+            // nullable is Option<Option<bool>>: flatten to get the actual nullable value.
+            if let Some(Some(nullable)) = entry.nullable {
+                alteration = alteration.set_nullable(nullable);
+            }
+            // data_type is Option<serde_json::Value>: only process when present and not null.
+            if let Some(data_type) = &entry.data_type
+                && !data_type.is_null()
+            {
+                let type_str = data_type.as_str().ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "data_type for column '{}' must be a JSON string, got: {}",
+                        entry.path, data_type
+                    ))
+                })?;
+                let json_type =
+                    lance_namespace::models::JsonArrowDataType::new(type_str.to_string());
+                let dt =
+                    lance_namespace::schema::convert_json_arrow_type(&json_type).map_err(|e| {
+                        Error::invalid_input(format!(
+                            "Failed to parse data_type '{}' for column '{}': {}",
+                            type_str, entry.path, e
+                        ))
+                    })?;
+                alteration = alteration.cast_to(dt);
+            }
+            Ok(alteration)
+        })
+        .collect()
+}
+
 /// Result of checking table status atomically.
 ///
 /// This struct captures the state of a table directory in a single snapshot,
@@ -192,9 +265,6 @@ pub struct DirectoryNamespaceBuilder {
     dir_listing_enabled: bool,
     inline_optimization_enabled: bool,
     table_version_tracking_enabled: bool,
-    /// When true, table versions are stored in the `__manifest` table instead of
-    /// relying on Lance's native version management.
-    table_version_storage_enabled: bool,
     /// When true, enables migration mode where the namespace checks the manifest first
     /// before falling back to directory listing for root-level tables. When false (default),
     /// root-level tables use directory listing directly without checking the manifest,
@@ -230,10 +300,6 @@ impl std::fmt::Debug for DirectoryNamespaceBuilder {
                 "table_version_tracking_enabled",
                 &self.table_version_tracking_enabled,
             )
-            .field(
-                "table_version_storage_enabled",
-                &self.table_version_storage_enabled,
-            )
             .field(
                 "dir_listing_to_manifest_migration_enabled",
                 &self.dir_listing_to_manifest_migration_enabled,
@@ -270,7 +336,6 @@ impl DirectoryNamespaceBuilder {
             dir_listing_enabled: true, // Default to enabled for backwards compatibility
             inline_optimization_enabled: true,
             table_version_tracking_enabled: false, // Default to disabled
-            table_version_storage_enabled: false,  // Default to disabled
             dir_listing_to_manifest_migration_enabled: false, // Default to disabled
             credential_vendor_properties: HashMap::new(),
             context_provider: None,
@@ -310,11 +375,10 @@ impl DirectoryNamespaceBuilder {
         self
     }
 
-    /// Enable or disable inline optimization of the __manifest table.
+    /// Enable or disable replacement index maintenance for the __manifest table.
     ///
-    /// When enabled (default), performs compaction and indexing on the __manifest table
-    /// after every write operation to maintain optimal performance.
-    /// When disabled, manual optimization must be performed separately.
+    /// When enabled (default), copy-on-write manifest rewrites build replacement indices
+    /// for fast reads. When disabled, rewrites only replace data files.
     pub fn inline_optimization_enabled(mut self, enabled: bool) -> Self {
         self.inline_optimization_enabled = enabled;
         self
@@ -332,19 +396,6 @@ impl DirectoryNamespaceBuilder {
         self
     }
 
-    /// Enable or disable table version management through the `__manifest` table.
-    ///
-    /// When enabled, table versions are tracked as `table_version` entries in the
-    /// `__manifest` Lance table. This enables:
-    /// - Centralized version tracking instead of per-table `_versions/` directories
-    ///
-    /// Requires `manifest_enabled` to be true.
-    /// When disabled (default), version storage uses per-table storage operations.
-    pub fn table_version_storage_enabled(mut self, enabled: bool) -> Self {
-        self.table_version_storage_enabled = enabled;
-        self
-    }
-
     /// Create a DirectoryNamespaceBuilder from properties HashMap.
     ///
     /// This method parses a properties map into builder configuration.
@@ -352,7 +403,7 @@ impl DirectoryNamespaceBuilder {
     /// - `root`: The root directory path (required)
     /// - `manifest_enabled`: Enable manifest-based table tracking (optional, default: true)
     /// - `dir_listing_enabled`: Enable directory listing for table discovery (optional, default: true)
-    /// - `inline_optimization_enabled`: Enable inline optimization of __manifest table (optional, default: true)
+    /// - `inline_optimization_enabled`: Enable replacement indices on __manifest rewrites (optional, default: true)
     /// - `storage.*`: Storage options (optional, prefix will be stripped)
     ///
     /// Credential vendor properties (prefixed with `credential_vendor.`, prefix is stripped):
@@ -462,12 +513,6 @@ impl DirectoryNamespaceBuilder {
             .and_then(|v| v.parse::<bool>().ok())
             .unwrap_or(false);
 
-        // Extract table_version_storage_enabled (default: false)
-        let table_version_storage_enabled = properties
-            .get("table_version_storage_enabled")
-            .and_then(|v| v.parse::<bool>().ok())
-            .unwrap_or(false);
-
         // Extract dir_listing_to_manifest_migration_enabled (default: false)
         let dir_listing_to_manifest_migration_enabled = properties
             .get("dir_listing_to_manifest_migration_enabled")
@@ -514,7 +559,6 @@ impl DirectoryNamespaceBuilder {
             dir_listing_enabled,
             inline_optimization_enabled,
             table_version_tracking_enabled,
-            table_version_storage_enabled,
             dir_listing_to_manifest_migration_enabled,
             credential_vendor_properties,
             context_provider: None,
@@ -691,19 +735,11 @@ impl DirectoryNamespaceBuilder {
     /// - Connection to the storage backend fails
     /// - Storage options are invalid
     pub async fn build(self) -> Result<DirectoryNamespace> {
-        // Validate: table_version_storage_enabled requires manifest_enabled
-        if self.table_version_storage_enabled && !self.manifest_enabled {
-            return Err(NamespaceError::InvalidInput {
-                message: "table_version_storage_enabled requires manifest_enabled=true".to_string(),
-            }
-            .into());
-        }
-
         let (object_store, base_path) =
             Self::initialize_object_store(&self.root, &self.storage_options, &self.session).await?;
 
         let manifest_ns = if self.manifest_enabled {
-            match manifest::ManifestNamespace::from_directory(
+            match manifest::ManifestNamespace::open_from_directory(
                 self.root.clone(),
                 self.storage_options.clone(),
                 self.session.clone(),
@@ -712,23 +748,29 @@ impl DirectoryNamespaceBuilder {
                 self.dir_listing_enabled,
                 self.inline_optimization_enabled,
                 self.commit_retries,
-                self.table_version_storage_enabled,
             )
             .await
             {
                 Ok(ns) => Some(Arc::new(ns)),
-                Err(e) => {
-                    // Failed to initialize manifest namespace, fall back to directory listing only
-                    log::warn!(
-                        "Failed to initialize manifest namespace, falling back to directory listing only: {}",
-                        e
-                    );
+                Err(e) if manifest_feature_flags::is_incompatible_manifest_error(&e) => {
+                    // The manifest exists but was written with a feature flag this
+                    // build does not understand. Refuse rather than silently
+                    // degrading to a directory-listing view that ignores it.
+                    return Err(e);
+                }
+                Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => {
+                    log::debug!("Manifest namespace does not exist yet: {}", e);
                     None
                 }
+                Err(e) => return Err(e),
             }
         } else {
             None
         };
+        let manifest_cell = OnceCell::new();
+        if let Some(manifest_ns) = manifest_ns {
+            let _ = manifest_cell.set(manifest_ns);
+        }
 
         // Create credential vendor once during initialization if enabled
         let credential_vendor = if has_credential_vendor_config(&self.credential_vendor_properties)
@@ -752,12 +794,15 @@ impl DirectoryNamespaceBuilder {
             session: self.session,
             object_store,
             base_path,
-            manifest_ns,
+            manifest_ns: manifest_cell,
+            write_manifest_ns: OnceCell::new(),
+            manifest_enabled: self.manifest_enabled,
             dir_listing_enabled: self.dir_listing_enabled,
+            inline_optimization_enabled: self.inline_optimization_enabled,
+            commit_retries: self.commit_retries,
             dir_listing_to_manifest_migration_enabled: self
                 .dir_listing_to_manifest_migration_enabled,
             table_version_tracking_enabled: self.table_version_tracking_enabled,
-            table_version_storage_enabled: self.table_version_storage_enabled,
             credential_vendor,
             context_provider: self.context_provider,
             vend_input_storage_options: self.vend_input_storage_options,
@@ -831,8 +876,12 @@ pub struct DirectoryNamespace {
     session: Option<Arc<Session>>,
     object_store: Arc<ObjectStore>,
     base_path: Path,
-    manifest_ns: Option<Arc<manifest::ManifestNamespace>>,
+    manifest_ns: OnceCell<Arc<manifest::ManifestNamespace>>,
+    write_manifest_ns: OnceCell<Arc<manifest::ManifestNamespace>>,
+    manifest_enabled: bool,
     dir_listing_enabled: bool,
+    inline_optimization_enabled: bool,
+    commit_retries: Option<u32>,
     /// When true, root-level table operations check the manifest first before
     /// falling back to directory listing. When false, root-level tables skip
     /// the manifest check and use directory listing directly.
@@ -840,8 +889,6 @@ pub struct DirectoryNamespace {
     /// When true, `describe_table` returns `managed_versioning: true` to indicate
     /// commits should go through namespace table version APIs.
     table_version_tracking_enabled: bool,
-    /// When true, table versions are stored in the `__manifest` table.
-    table_version_storage_enabled: bool,
     /// Credential vendor created once during initialization.
     /// Used to vend temporary credentials for table access.
     credential_vendor: Option<Arc<dyn CredentialVendor>>,
@@ -877,7 +924,121 @@ struct TableDeleteEntry {
     ranges: Vec<(i64, i64)>,
 }
 
+/// Persistent record of `alter_transaction` outcomes for a single transaction.
+///
+/// Lance's transaction file is immutable once written, so we record any
+/// modifications (status transitions, extra properties, tombstoned properties)
+/// in a namespace-owned sidecar file. The sidecar is then merged into the
+/// response of subsequent `describe_transaction` / `alter_transaction` calls.
+///
+/// Serialization is implemented manually via `serde_json::Value` to avoid
+/// pulling in `serde`'s `derive` feature for this crate.
+#[derive(Debug, Clone, Default)]
+struct TransactionAlteration {
+    /// The most recently applied status, if any.
+    status: Option<String>,
+    /// User-defined properties layered on top of the immutable transaction
+    /// properties. Values here take precedence over the transaction's own
+    /// properties when both are present.
+    properties: HashMap<String, String>,
+    /// Names of transaction properties that have been tombstoned via
+    /// `unset_property_action`. A tombstoned key is hidden from the response
+    /// even when the immutable transaction still carries it.
+    removed_properties: std::collections::HashSet<String>,
+}
+
+impl TransactionAlteration {
+    /// JSON field names used for the sidecar on-disk representation.
+    const F_STATUS: &'static str = "status";
+    const F_PROPERTIES: &'static str = "properties";
+    const F_REMOVED_PROPERTIES: &'static str = "removed_properties";
+
+    /// Serialize this alteration to a JSON byte vector.
+    ///
+    /// Uses the same pattern as `dir/manifest.rs`: rely on the built-in
+    /// `Serialize` impls for `Option<String>`, `HashMap<String, String>` and
+    /// `HashSet<String>` provided by the `serde` crate (transitively pulled in
+    /// by `serde_json`), so no `serde` derive nor extra dependency is needed.
+    fn to_json_bytes(&self) -> serde_json::Result<Vec<u8>> {
+        serde_json::to_vec(&serde_json::json!({
+            Self::F_STATUS: self.status,
+            Self::F_PROPERTIES: self.properties,
+            Self::F_REMOVED_PROPERTIES: self.removed_properties,
+        }))
+    }
+
+    /// Deserialize an alteration from JSON bytes, mirroring the
+    /// `serde_json::from_slice::<HashMap<String, String>>(...)` idiom already
+    /// used in `dir/manifest.rs`. Missing / null fields fall back to defaults
+    /// so that the sidecar format stays forward-compatible.
+    fn from_json_slice(bytes: &[u8]) -> serde_json::Result<Self> {
+        let mut obj: serde_json::Map<String, serde_json::Value> = serde_json::from_slice(bytes)?;
+        Ok(Self {
+            status: serde_json::from_value(
+                obj.remove(Self::F_STATUS)
+                    .unwrap_or(serde_json::Value::Null),
+            )?,
+            properties: serde_json::from_value(
+                obj.remove(Self::F_PROPERTIES)
+                    .unwrap_or(serde_json::Value::Null),
+            )
+            .unwrap_or_default(),
+            removed_properties: serde_json::from_value(
+                obj.remove(Self::F_REMOVED_PROPERTIES)
+                    .unwrap_or(serde_json::Value::Null),
+            )
+            .unwrap_or_default(),
+        })
+    }
+}
+
 impl DirectoryNamespace {
+    fn manifest_ns_for_read(&self) -> Option<&Arc<manifest::ManifestNamespace>> {
+        self.write_manifest_ns
+            .get()
+            .or_else(|| self.manifest_ns.get())
+    }
+
+    async fn manifest_ns_for_write(&self) -> Result<Option<Arc<manifest::ManifestNamespace>>> {
+        if !self.manifest_enabled {
+            return Ok(None);
+        }
+
+        let manifest_ns = self
+            .write_manifest_ns
+            .get_or_try_init(|| async {
+                manifest::ManifestNamespace::from_directory(
+                    self.root.clone(),
+                    self.storage_options.clone(),
+                    self.session.clone(),
+                    self.object_store.clone(),
+                    self.base_path.clone(),
+                    self.dir_listing_enabled,
+                    self.inline_optimization_enabled,
+                    self.commit_retries,
+                )
+                .await
+                .map(Arc::new)
+            })
+            .await?;
+        Ok(Some(manifest_ns.clone()))
+    }
+
+    fn child_namespace_requires_manifest_error(&self) -> Error {
+        if self.manifest_enabled {
+            NamespaceError::NamespaceNotFound {
+                message: "Child namespace reads require an existing __manifest dataset".to_string(),
+            }
+            .into()
+        } else {
+            NamespaceError::Unsupported {
+                message: "Child namespaces are only supported when manifest mode is enabled"
+                    .to_string(),
+            }
+            .into()
+        }
+    }
+
     /// Apply pagination to a list of table names
     ///
     /// Sorts the list alphabetically and applies pagination using page_token (start_after) and limit.
@@ -1062,6 +1223,123 @@ impl DirectoryNamespace {
         }
     }
 
+    /// Map lance-core ref errors from branch operations to namespace errors.
+    ///
+    /// `RefConflict` is intentionally not handled here: create-time duplicates are rejected by
+    /// the existence pre-check before `create_branch` runs, and delete maps its own `RefConflict`
+    /// (branch still has dependents) inline.
+    fn map_branch_error(
+        err: lance_core::Error,
+        branch: &str,
+        table_uri: &str,
+    ) -> lance_core::Error {
+        match err {
+            lance_core::Error::RefNotFound { .. } => NamespaceError::TableBranchNotFound {
+                message: format!("branch '{}' for table at '{}'", branch, table_uri),
+            }
+            .into(),
+            lance_core::Error::InvalidRef { message } => NamespaceError::InvalidInput {
+                message: format!("invalid branch '{}': {}", branch, message),
+            }
+            .into(),
+            lance_core::Error::VersionNotFound { message } => {
+                NamespaceError::TableVersionNotFound {
+                    message: format!(
+                        "source version for branch '{}' not found for table at '{}': {}",
+                        branch, table_uri, message
+                    ),
+                }
+                .into()
+            }
+            other => NamespaceError::Internal {
+                message: format!(
+                    "branch operation failed for branch '{}' on table at '{}': {}",
+                    branch, table_uri, other
+                ),
+            }
+            .into(),
+        }
+    }
+
+    /// Map a Lance error from a table mutation (update / delete / merge-insert) into the most
+    /// specific `NamespaceError` we can determine from the underlying variant.
+    ///
+    /// Collapsing every failure into `InvalidInput`/`Internal` hides the real cause from callers;
+    /// mapping per variant lets them branch on a meaningful error code (e.g. retry on
+    /// `ConcurrentModification`, surface `TableNotFound` to the user).
+    ///
+    /// Commit-conflict variants are mapped consistently with `convert_lance_commit_error` in
+    /// `manifest.rs`: `CommitConflict` (retries exhausted, safe to retry) -> `Throttling`, while
+    /// semantic conflicts (`TooMuchWriteContention` / `RetryableCommitConflict` /
+    /// `IncompatibleTransaction` / `VersionConflict`) -> `ConcurrentModification`.
+    fn map_mutation_error(
+        err: lance_core::Error,
+        operation: &str,
+        table_uri: &str,
+    ) -> lance_core::Error {
+        let detail = err.to_string();
+        let ns_err = match &err {
+            lance_core::Error::InvalidInput { .. }
+            | lance_core::Error::Unprocessable { .. }
+            | lance_core::Error::InvalidRef { .. } => NamespaceError::InvalidInput {
+                message: format!(
+                    "Invalid input for {} on table at '{}': {}",
+                    operation, table_uri, detail
+                ),
+            },
+            lance_core::Error::NotFound { .. } | lance_core::Error::DatasetNotFound { .. } => {
+                NamespaceError::TableNotFound {
+                    message: format!(
+                        "Table at '{}' not found while running {}: {}",
+                        table_uri, operation, detail
+                    ),
+                }
+            }
+            lance_core::Error::SchemaMismatch { .. } | lance_core::Error::Schema { .. } => {
+                NamespaceError::TableSchemaValidationError {
+                    message: format!(
+                        "Schema validation failed for {} on table at '{}': {}",
+                        operation, table_uri, detail
+                    ),
+                }
+            }
+            // `CommitConflict` means the version-collision retries were exhausted; the operation
+            // is safe to retry as-is, so surface it as `Throttling` (kept aligned with
+            // `convert_lance_commit_error` in manifest.rs).
+            lance_core::Error::CommitConflict { .. } => NamespaceError::Throttling {
+                message: format!(
+                    "Too many concurrent writes for {} on table at '{}', please retry later: {}",
+                    operation, table_uri, detail
+                ),
+            },
+            // Semantic conflicts: a concurrent change is incompatible with this one and retrying
+            // as-is would not help, so surface them as `ConcurrentModification` (kept aligned with
+            // `convert_lance_commit_error` in manifest.rs).
+            lance_core::Error::TooMuchWriteContention { .. }
+            | lance_core::Error::RetryableCommitConflict { .. }
+            | lance_core::Error::IncompatibleTransaction { .. }
+            | lance_core::Error::VersionConflict { .. } => NamespaceError::ConcurrentModification {
+                message: format!(
+                    "Concurrent modification detected for {} on table at '{}': {}",
+                    operation, table_uri, detail
+                ),
+            },
+            lance_core::Error::NotSupported { .. } => NamespaceError::Unsupported {
+                message: format!(
+                    "{} is not supported on table at '{}': {}",
+                    operation, table_uri, detail
+                ),
+            },
+            _ => NamespaceError::Internal {
+                message: format!(
+                    "Failed to run {} on table at '{}': {}",
+                    operation, table_uri, detail
+                ),
+            },
+        };
+        ns_err.into()
+    }
+
     async fn table_has_actual_manifests(&self, table_name: &str) -> Result<bool> {
         manifest::ManifestNamespace::path_has_actual_manifests(
             &self.object_store,
@@ -1160,6 +1438,103 @@ impl DirectoryNamespace {
         ObjectStore::extract_path_from_uri(registry, uri)
     }
 
+    /// Normalize and validate a branch selector: `None`, empty, and `main` mean
+    /// the main branch; any other name is validated with lance's
+    /// `check_valid_branch` (lance skips this on the open path) so it cannot
+    /// escape the table root via `..`.
+    fn normalized_branch(branch: Option<&str>) -> Result<Option<&str>> {
+        match branch.filter(|b| !b.is_empty() && *b != "main") {
+            Some(branch) => {
+                check_valid_branch(branch).map_err(|e| {
+                    lance_core::Error::from(NamespaceError::InvalidInput {
+                        message: format!("invalid branch name '{}': {}", branch, e),
+                    })
+                })?;
+                Ok(Some(branch))
+            }
+            None => Ok(None),
+        }
+    }
+
+    async fn open_validated_branch(&self, table_uri: &str, branch: &str) -> Result<Dataset> {
+        let dataset = self
+            .configured_builder(table_uri)
+            .with_branch(branch, None)
+            .load()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableNotFound {
+                    message: format!(
+                        "branch '{}' not found for table at '{}': {}",
+                        branch, table_uri, e
+                    ),
+                })
+            })?;
+        dataset.branches().get(branch).await.map_err(|_| {
+            lance_core::Error::from(NamespaceError::TableNotFound {
+                message: format!("branch '{}' not found for table at '{}'", branch, table_uri),
+            })
+        })?;
+        Ok(dataset)
+    }
+
+    async fn resolve_branch_location(&self, table_uri: &str, branch: &str) -> Result<String> {
+        Ok(self
+            .open_validated_branch(table_uri, branch)
+            .await?
+            .branch_location()
+            .uri)
+    }
+
+    /// Resolves a branch to its `(uri, object-store path)` for `create_table_version`.
+    ///
+    /// `BranchContents` is the source of truth, so check the ref first: a
+    /// registered branch commits directly. With no ref, accept the commit only on
+    /// an empty chain (the `create_branch` bootstrap, whose first commit precedes
+    /// its ref); reject a chain that already holds committed versions as a zombie.
+    async fn resolve_branch_for_commit(
+        &self,
+        table_uri: &str,
+        branch: &str,
+    ) -> Result<(String, Path)> {
+        let main = self
+            .configured_builder(table_uri)
+            .load()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableNotFound {
+                    message: format!("table at '{}' not found: {}", table_uri, e),
+                })
+            })?;
+        let branch_location = main.branch_location().find_branch(Some(branch))?;
+        match main.branches().get(branch).await {
+            Ok(_) => Ok((branch_location.uri, branch_location.path)),
+            Err(lance_core::Error::RefNotFound { .. }) => {
+                if self
+                    .branch_has_committed_versions(&branch_location.path)
+                    .await?
+                {
+                    return Err(NamespaceError::TableNotFound {
+                        message: format!(
+                            "branch '{}' not found for table at '{}'",
+                            branch, table_uri
+                        ),
+                    }
+                    .into());
+                }
+                Ok((branch_location.uri, branch_location.path))
+            }
+            Err(e) => Err(e),
+        }
+    }
+
+    async fn branch_has_committed_versions(&self, branch_path: &Path) -> Result<bool> {
+        Ok(!self
+            .list_versions_under(branch_path, false, Some(1))
+            .await?
+            .is_empty())
+    }
+
     fn validate_dir_only_properties(
         properties: Option<&HashMap<String, String>>,
         operation: &str,
@@ -1217,6 +1592,13 @@ impl DirectoryNamespace {
         Ok(dataset)
     }
 
+    /// Logical table version parsed from a manifest filename, or `None` for
+    /// non-manifest / detached entries. Delegates to lance's scheme detection so
+    /// version listing and deletion stay consistent with the on-disk format.
+    fn manifest_version_from_filename(filename: &str) -> Option<u64> {
+        ManifestNamingScheme::detect_scheme(filename)?.parse_version(filename)
+    }
+
     async fn list_table_versions_from_storage(
         &self,
         table_uri: &str,
@@ -1224,6 +1606,19 @@ impl DirectoryNamespace {
         limit: Option<i32>,
     ) -> Result<Vec<TableVersion>> {
         let table_path = self.object_store_path_from_uri(table_uri)?;
+        self.list_versions_under(&table_path, descending, limit)
+            .await
+    }
+
+    /// List committed manifest versions under `table_path/_versions/`.
+    /// `table_path` must be an object-store `Path`; converting a URI to a path
+    /// can miss manifests on Windows.
+    async fn list_versions_under(
+        &self,
+        table_path: &Path,
+        descending: bool,
+        limit: Option<i32>,
+    ) -> Result<Vec<TableVersion>> {
         let versions_dir = table_path.clone().join(VERSIONS_DIR);
         let manifest_metas: Vec<_> = self
             .object_store
@@ -1233,8 +1628,8 @@ impl DirectoryNamespace {
             .map_err(|e| {
                 lance_core::Error::from(NamespaceError::Internal {
                     message: format!(
-                        "Failed to list manifest files for table at '{}': {}",
-                        table_uri, e
+                        "Failed to list manifest files under '{}': {}",
+                        versions_dir, e
                     ),
                 })
             })?;
@@ -1247,17 +1642,7 @@ impl DirectoryNamespace {
             .into_iter()
             .filter_map(|meta| {
                 let filename = meta.location.filename()?;
-                let version_str = filename.strip_suffix(".manifest")?;
-                if version_str.starts_with('d') {
-                    return None;
-                }
-                let file_version: u64 = version_str.parse().ok()?;
-
-                let actual_version = if file_version > u64::MAX / 2 {
-                    u64::MAX - file_version
-                } else {
-                    file_version
-                };
+                let actual_version = Self::manifest_version_from_filename(filename)?;
 
                 Some(TableVersion {
                     version: actual_version as i64,
@@ -1284,9 +1669,9 @@ impl DirectoryNamespace {
 
         if needs_sort {
             if descending {
-                table_versions.sort_by(|a, b| b.version.cmp(&a.version));
+                table_versions.sort_by_key(|v| std::cmp::Reverse(v.version));
             } else {
-                table_versions.sort_by(|a, b| a.version.cmp(&b.version));
+                table_versions.sort_by_key(|v| v.version);
             }
         }
 
@@ -1305,10 +1690,11 @@ impl DirectoryNamespace {
         request: DescribeTableRequest,
     ) -> Result<DescribeTableResponse> {
         let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1);
         let skip_manifest_for_root = self.dir_listing_enabled
             && is_root_level
             && !self.dir_listing_to_manifest_migration_enabled;
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !skip_manifest_for_root
         {
             match manifest_ns.describe_table(request.clone()).await {
@@ -1327,15 +1713,27 @@ impl DirectoryNamespace {
                     }
                     return Ok(response);
                 }
+                Err(e) if manifest_feature_flags::is_incompatible_manifest_error(&e) => {
+                    // An incompatible manifest must surface "please upgrade"
+                    // rather than degrading to a directory-listing view.
+                    return Err(e);
+                }
                 Err(_) if self.dir_listing_enabled && is_root_level => {
                     // Fall through to directory check only for single-level IDs
                 }
                 Err(e) => return Err(e),
             }
         }
+        if is_child_table {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
 
         let table_name = Self::table_name_from_id(&request.id)?;
         let table_id = Self::format_table_id_from_request(&request.id);
+        if !self.dir_listing_enabled {
+            return Err(NamespaceError::TableNotFound { message: table_id }.into());
+        }
+
         let table_uri = self.table_full_uri(&table_name);
 
         // Atomically check table existence and deregistration status
@@ -1525,6 +1923,19 @@ impl DirectoryNamespace {
         }
     }
 
+    /// Build a `DatasetBuilder` for `table_uri` with this namespace's storage
+    /// options and session applied. Callers add version/branch scoping.
+    fn configured_builder(&self, table_uri: &str) -> DatasetBuilder {
+        let mut builder = DatasetBuilder::from_uri(table_uri);
+        if let Some(opts) = &self.storage_options {
+            builder = builder.with_storage_options(opts.clone());
+        }
+        if let Some(sess) = &self.session {
+            builder = builder.with_session(sess.clone());
+        }
+        builder
+    }
+
     async fn load_dataset(
         &self,
         table_uri: &str,
@@ -1543,13 +1954,7 @@ impl DirectoryNamespace {
             .into());
         }
 
-        let mut builder = DatasetBuilder::from_uri(table_uri);
-        if let Some(opts) = &self.storage_options {
-            builder = builder.with_storage_options(opts.clone());
-        }
-        if let Some(sess) = &self.session {
-            builder = builder.with_session(sess.clone());
-        }
+        let builder = self.configured_builder(table_uri);
 
         let dataset = builder.load().await.map_err(|e| {
             lance_core::Error::from(NamespaceError::TableNotFound {
@@ -1792,12 +2197,29 @@ impl DirectoryNamespace {
     fn transaction_response(
         version: u64,
         transaction: &Transaction,
+        alteration: Option<TransactionAlteration>,
     ) -> DescribeTransactionResponse {
         let mut properties = transaction
             .transaction_properties
             .as_ref()
             .map(|properties| (**properties).clone())
             .unwrap_or_default();
+
+        // Apply persisted alterations on top of the immutable transaction
+        // properties so callers see the current effective state.
+        let mut effective_status = "SUCCEEDED".to_string();
+        if let Some(alteration) = alteration {
+            for key in &alteration.removed_properties {
+                properties.remove(key);
+            }
+            for (key, value) in alteration.properties {
+                properties.insert(key, value);
+            }
+            if let Some(status) = alteration.status {
+                effective_status = status;
+            }
+        }
+
         properties.insert("uuid".to_string(), transaction.uuid.clone());
         properties.insert("version".to_string(), version.to_string());
         properties.insert(
@@ -1813,7 +2235,7 @@ impl DirectoryNamespace {
         }
 
         DescribeTransactionResponse {
-            status: "SUCCEEDED".to_string(),
+            status: effective_status,
             properties: Some(properties),
         }
     }
@@ -1902,8 +2324,86 @@ impl DirectoryNamespace {
         .into())
     }
 
+    /// Relative directory (under a table's Lance root) used to persist
+    /// alter_transaction outcomes. The Lance transaction file itself is
+    /// immutable, so we keep alterations in a namespace-owned sidecar.
+    const TRANSACTION_ALTERATIONS_DIR: &'static str = "_alter_transactions";
+
+    fn transaction_alteration_path(&self, table_uri: &str, txn_uuid: &str) -> Result<Path> {
+        let table_path = self.object_store_path_from_uri(table_uri)?;
+        Ok(table_path
+            .join(Self::TRANSACTION_ALTERATIONS_DIR)
+            .join(format!("{}.json", txn_uuid).as_str()))
+    }
+
+    async fn load_transaction_alteration(
+        &self,
+        table_uri: &str,
+        txn_uuid: &str,
+    ) -> Result<Option<TransactionAlteration>> {
+        let path = self.transaction_alteration_path(table_uri, txn_uuid)?;
+        match self.object_store.inner.get(&path).await {
+            Ok(get_result) => {
+                let bytes = get_result.bytes().await.map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to read alter_transaction sidecar for '{}': {}",
+                            txn_uuid, e
+                        ),
+                    })
+                })?;
+                let alteration = TransactionAlteration::from_json_slice(&bytes).map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to parse alter_transaction sidecar for '{}': {}",
+                            txn_uuid, e
+                        ),
+                    })
+                })?;
+                Ok(Some(alteration))
+            }
+            Err(ObjectStoreError::NotFound { .. }) => Ok(None),
+            Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "Failed to load alter_transaction sidecar for '{}': {}",
+                    txn_uuid, e
+                ),
+            })),
+        }
+    }
+
+    async fn save_transaction_alteration(
+        &self,
+        table_uri: &str,
+        txn_uuid: &str,
+        alteration: &TransactionAlteration,
+    ) -> Result<()> {
+        let path = self.transaction_alteration_path(table_uri, txn_uuid)?;
+        let bytes = alteration.to_json_bytes().map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "Failed to serialize alter_transaction sidecar for '{}': {}",
+                    txn_uuid, e
+                ),
+            })
+        })?;
+        self.object_store
+            .inner
+            .put(&path, bytes.into())
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to persist alter_transaction sidecar for '{}': {}",
+                        txn_uuid, e
+                    ),
+                })
+            })?;
+        Ok(())
+    }
+
     fn table_full_uri(&self, table_name: &str) -> String {
-        format!("{}/{}.lance", &self.root, table_name)
+        format!("{}/{}.lance", self.root, table_name)
     }
 
     /// Get the object store path for a table (relative to base_path)
@@ -2050,6 +2550,7 @@ impl DirectoryNamespace {
     /// to the manifest to enable manifest-only mode:
     ///
     /// ```no_run
+    /// #![recursion_limit = "256"]
     /// # use lance_namespace_impls::DirectoryNamespaceBuilder;
     /// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
     /// // Create namespace with dual mode (manifest + directory listing)
@@ -2088,7 +2589,7 @@ impl DirectoryNamespace {
     /// - Manifest registration fails
     pub async fn migrate(&self) -> Result<usize> {
         // We only care about tables in the root namespace
-        let Some(ref manifest_ns) = self.manifest_ns else {
+        let Some(manifest_ns) = self.manifest_ns_for_write().await? else {
             return Ok(0); // No manifest, nothing to migrate
         };
 
@@ -2118,59 +2619,74 @@ impl DirectoryNamespace {
         Ok(migrated_count)
     }
 
-    /// Delete physical manifest files for the given table version ranges (best-effort).
+    /// Delete physical manifest files for the given table version ranges.
     ///
-    /// This helper is used by `batch_delete_table_versions` in both the manifest-enabled
-    /// and non-manifest paths. It resolves each table's storage location, computes the
-    /// version file paths, and attempts to delete them. Errors are logged (best-effort)
-    /// when `best_effort` is true, or returned immediately when false.
+    /// This helper backs `batch_delete_table_versions`. It resolves each table's storage
+    /// location, computes the version file paths, and deletes them, returning an error on
+    /// the first failure.
     ///
     /// Returns the number of files successfully deleted.
     async fn delete_physical_version_files(
         &self,
         table_entries: &[TableDeleteEntry],
-        best_effort: bool,
+        branch: Option<&str>,
     ) -> Result<i64> {
         let mut deleted_count = 0i64;
         for te in table_entries {
             let table_uri = self.resolve_table_location(&te.table_id).await?;
+            let table_uri = match branch {
+                Some(b) => self.resolve_branch_location(&table_uri, b).await?,
+                None => table_uri,
+            };
             let table_path = self.object_store_path_from_uri(&table_uri)?;
             let versions_dir_path = table_path.clone().join(VERSIONS_DIR);
 
-            for (start, end) in &te.ranges {
-                for version in *start..=*end {
-                    let version_path = versions_dir_path
-                        .clone()
-                        .join(format!("{}.manifest", version as u64));
-                    match self.object_store.inner.delete(&version_path).await {
-                        Ok(_) => {
-                            deleted_count += 1;
-                        }
-                        Err(object_store::Error::NotFound { .. }) => {}
-                        Err(e) => {
-                            if best_effort {
-                                log::warn!(
-                                    "Failed to delete manifest file for version {} of table {:?}: {:?}",
-                                    version,
-                                    te.table_id,
-                                    e
-                                );
-                            } else {
-                                return Err(NamespaceError::Internal {
-                                    message: format!(
-                                        "Failed to delete version {} for table at '{}': {}",
-                                        version, table_uri, e
-                                    ),
-                                }
-                                .into());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        Ok(deleted_count)
-    }
+            // Match listed files, not constructed names (`{version}.manifest` misses V2).
+            let manifest_metas: Vec<_> = self
+                .object_store
+                .read_dir_all(&versions_dir_path, None)
+                .try_collect()
+                .await
+                .map_err(|e| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to list manifest files for table at '{}': {}",
+                            table_uri, e
+                        ),
+                    })
+                })?;
+            let location_by_version: HashMap<u64, Path> = manifest_metas
+                .into_iter()
+                .filter_map(|meta| {
+                    let version = Self::manifest_version_from_filename(meta.location.filename()?)?;
+                    Some((version, meta.location))
+                })
+                .collect();
+
+            for (&v, version_path) in &location_by_version {
+                let vi = v as i64;
+                if !te.ranges.iter().any(|&(s, e)| vi >= s && (e < 0 || vi < e)) {
+                    continue;
+                }
+                match self.object_store.inner.delete(version_path).await {
+                    Ok(_) => {
+                        deleted_count += 1;
+                    }
+                    Err(object_store::Error::NotFound { .. }) => {}
+                    Err(e) => {
+                        return Err(NamespaceError::Internal {
+                            message: format!(
+                                "Failed to delete version {} for table at '{}': {}",
+                                v, table_uri, e
+                            ),
+                        }
+                        .into());
+                    }
+                }
+            }
+        }
+        Ok(deleted_count)
+    }
 
     /// Apply all query parameters from a `QueryTableRequest`-like source onto a `Scanner`.
     ///
@@ -2345,10 +2861,13 @@ impl LanceNamespace for DirectoryNamespace {
         request: ListNamespacesRequest,
     ) -> Result<ListNamespacesResponse> {
         self.record_op("list_namespaces");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.list_namespaces(request).await;
         }
 
+        if request.id.as_ref().is_some_and(|id| !id.is_empty()) {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
         Self::validate_root_namespace_id(&request.id)?;
         Ok(ListNamespacesResponse::new(vec![]))
     }
@@ -2358,10 +2877,13 @@ impl LanceNamespace for DirectoryNamespace {
         request: DescribeNamespaceRequest,
     ) -> Result<DescribeNamespaceResponse> {
         self.record_op("describe_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.describe_namespace(request).await;
         }
 
+        if request.id.as_ref().is_some_and(|id| !id.is_empty()) {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
         Self::validate_root_namespace_id(&request.id)?;
         #[allow(clippy::needless_update)]
         Ok(DescribeNamespaceResponse {
@@ -2375,7 +2897,7 @@ impl LanceNamespace for DirectoryNamespace {
         request: CreateNamespaceRequest,
     ) -> Result<CreateNamespaceResponse> {
         self.record_op("create_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.create_namespace(request).await;
         }
 
@@ -2395,7 +2917,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
         self.record_op("drop_namespace");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.drop_namespace(request).await;
         }
 
@@ -2415,7 +2937,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
         self.record_op("namespace_exists");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_read() {
             return manifest_ns.namespace_exists(request).await;
         }
 
@@ -2423,11 +2945,7 @@ impl LanceNamespace for DirectoryNamespace {
             return Ok(());
         }
 
-        Err(NamespaceError::NamespaceNotFound {
-            message: "Child namespaces are only supported when manifest mode is enabled"
-                .to_string(),
-        }
-        .into())
+        Err(self.child_namespace_requires_manifest_error())
     }
 
     async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
@@ -2441,31 +2959,30 @@ impl LanceNamespace for DirectoryNamespace {
 
         // For child namespaces, always delegate to manifest (if enabled)
         if !namespace_id.is_empty() {
-            if let Some(ref manifest_ns) = self.manifest_ns {
+            if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 return manifest_ns.list_tables(request).await;
             }
-            return Err(NamespaceError::Unsupported {
-                message: "Child namespaces are only supported when manifest mode is enabled"
-                    .to_string(),
-            }
-            .into());
+            return Err(self.child_namespace_requires_manifest_error());
         }
 
         // When only manifest is enabled (no directory listing), delegate directly to manifest
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !self.dir_listing_enabled
         {
             return manifest_ns.list_tables(request).await;
         }
+        if !self.dir_listing_enabled {
+            return Ok(ListTablesResponse::new(vec![]));
+        }
 
         // When both manifest and directory listing are enabled with migration mode,
         // we need to merge and deduplicate
-        let mut tables = if self.manifest_ns.is_some()
+        let mut tables = if self.manifest_ns_for_read().is_some()
             && self.dir_listing_enabled
             && self.dir_listing_to_manifest_migration_enabled
         {
             // Get all manifest table locations (for deduplication)
-            let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns {
+            let manifest_locations = if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 manifest_ns.list_manifest_table_locations().await?
             } else {
                 std::collections::HashSet::new()
@@ -2475,7 +2992,7 @@ impl LanceNamespace for DirectoryNamespace {
             let mut manifest_request = request.clone();
             manifest_request.limit = None;
             manifest_request.page_token = None;
-            let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns {
+            let manifest_tables = if let Some(manifest_ns) = self.manifest_ns_for_read() {
                 let manifest_response = manifest_ns.list_tables(manifest_request).await?;
                 manifest_response.tables
             } else {
@@ -2523,23 +3040,35 @@ impl LanceNamespace for DirectoryNamespace {
     async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
         self.record_op("table_exists");
         let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1);
+        let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1);
         let skip_manifest_for_root = self.dir_listing_enabled
             && is_root_level
             && !self.dir_listing_to_manifest_migration_enabled;
-        if let Some(ref manifest_ns) = self.manifest_ns
+        if let Some(manifest_ns) = self.manifest_ns_for_read()
             && !skip_manifest_for_root
         {
             match manifest_ns.table_exists(request.clone()).await {
                 Ok(()) => return Ok(()),
+                Err(e) if manifest_feature_flags::is_incompatible_manifest_error(&e) => {
+                    // An incompatible manifest must surface "please upgrade"
+                    // rather than degrading to a directory-listing view.
+                    return Err(e);
+                }
                 Err(_) if self.dir_listing_enabled && is_root_level => {
                     // Fall through to directory check only for single-level IDs
                 }
                 Err(e) => return Err(e),
             }
         }
+        if is_child_table {
+            return Err(self.child_namespace_requires_manifest_error());
+        }
 
         let table_name = Self::table_name_from_id(&request.id)?;
         let table_id = Self::format_table_id_from_request(&request.id);
+        if !self.dir_listing_enabled {
+            return Err(NamespaceError::TableNotFound { message: table_id }.into());
+        }
 
         // Atomically check table existence and deregistration status
         let status = self.check_table_status(&table_name).await;
@@ -2563,7 +3092,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
         self.record_op("drop_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.drop_table(request).await;
         }
 
@@ -2593,7 +3122,7 @@ impl LanceNamespace for DirectoryNamespace {
         request_data: Bytes,
     ) -> Result<CreateTableResponse> {
         self.record_op("create_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return manifest_ns.create_table(request, request_data).await;
         }
 
@@ -2640,7 +3169,7 @@ impl LanceNamespace for DirectoryNamespace {
 
     async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
         self.record_op("declare_table");
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             let mut response = manifest_ns.declare_table(request.clone()).await?;
             if let Some(ref location) = response.location {
                 // For backwards compatibility, only skip vending credentials when explicitly set to false
@@ -2731,7 +3260,7 @@ impl LanceNamespace for DirectoryNamespace {
     ) -> Result<lance_namespace::models::RegisterTableResponse> {
         self.record_op("register_table");
         // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return LanceNamespace::register_table(manifest_ns.as_ref(), request).await;
         }
 
@@ -2748,7 +3277,7 @@ impl LanceNamespace for DirectoryNamespace {
     ) -> Result<lance_namespace::models::DeregisterTableResponse> {
         self.record_op("deregister_table");
         // If manifest is enabled, delegate to manifest namespace
-        if let Some(ref manifest_ns) = self.manifest_ns {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
             return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await;
         }
 
@@ -2802,24 +3331,174 @@ impl LanceNamespace for DirectoryNamespace {
         })
     }
 
+    async fn alter_table_add_columns(
+        &self,
+        request: AlterTableAddColumnsRequest,
+    ) -> Result<AlterTableAddColumnsResponse> {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
+            return manifest_ns.alter_table_add_columns(request).await;
+        }
+
+        // Non-manifest mode: open Dataset directly via table URI and perform the operation
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+
+        // Check table existence and deregistration status before opening the dataset
+        let status = self.check_table_status(&table_name).await;
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_name,
+            }
+            .into());
+        }
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is deregistered: {}", table_name),
+            }
+            .into());
+        }
+
+        let mut dataset = self
+            .configured_builder(&table_uri)
+            .load()
+            .await
+            .map_err(|e| {
+                Error::io_source(box_error(std::io::Error::other(format!(
+                    "Failed to open dataset: {}",
+                    e
+                ))))
+            })?;
+
+        let sql_expressions = build_sql_expressions(&request.new_columns)?;
+
+        dataset
+            .add_columns(
+                lance::dataset::NewColumnTransform::SqlExpressions(sql_expressions),
+                None,
+                None,
+            )
+            .await
+            .map_err(|e| {
+                Error::io_source(box_error(std::io::Error::other(format!(
+                    "Failed to add columns: {}",
+                    e
+                ))))
+            })?;
+
+        let version = dataset.version().version as i64;
+        Ok(AlterTableAddColumnsResponse::new(version))
+    }
+
+    async fn alter_table_alter_columns(
+        &self,
+        request: AlterTableAlterColumnsRequest,
+    ) -> Result<AlterTableAlterColumnsResponse> {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
+            return manifest_ns.alter_table_alter_columns(request).await;
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+
+        // Check table existence and deregistration status before opening the dataset
+        let status = self.check_table_status(&table_name).await;
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_name,
+            }
+            .into());
+        }
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is deregistered: {}", table_name),
+            }
+            .into());
+        }
+
+        let mut dataset = self
+            .configured_builder(&table_uri)
+            .load()
+            .await
+            .map_err(|e| {
+                Error::io_source(box_error(std::io::Error::other(format!(
+                    "Failed to open dataset: {}",
+                    e
+                ))))
+            })?;
+
+        let alterations = build_column_alterations(&request.alterations)?;
+
+        dataset.alter_columns(&alterations).await.map_err(|e| {
+            Error::io_source(box_error(std::io::Error::other(format!(
+                "Failed to alter columns: {}",
+                e
+            ))))
+        })?;
+
+        let version = dataset.version().version as i64;
+        Ok(AlterTableAlterColumnsResponse::new(version))
+    }
+
+    async fn alter_table_drop_columns(
+        &self,
+        request: AlterTableDropColumnsRequest,
+    ) -> Result<AlterTableDropColumnsResponse> {
+        if let Some(manifest_ns) = self.manifest_ns_for_write().await? {
+            return manifest_ns.alter_table_drop_columns(request).await;
+        }
+
+        let table_name = Self::table_name_from_id(&request.id)?;
+        let table_uri = self.table_full_uri(&table_name);
+
+        // Check table existence and deregistration status before opening the dataset
+        let status = self.check_table_status(&table_name).await;
+        if !status.exists {
+            return Err(NamespaceError::TableNotFound {
+                message: table_name,
+            }
+            .into());
+        }
+        if status.is_deregistered {
+            return Err(NamespaceError::TableNotFound {
+                message: format!("Table is deregistered: {}", table_name),
+            }
+            .into());
+        }
+
+        let mut dataset = self
+            .configured_builder(&table_uri)
+            .load()
+            .await
+            .map_err(|e| {
+                Error::io_source(box_error(std::io::Error::other(format!(
+                    "Failed to open dataset: {}",
+                    e
+                ))))
+            })?;
+
+        let columns: Vec<&str> = request.columns.iter().map(|s| s.as_str()).collect();
+        dataset.drop_columns(&columns).await.map_err(|e| {
+            Error::io_source(box_error(std::io::Error::other(format!(
+                "Failed to drop columns: {}",
+                e
+            ))))
+        })?;
+
+        let version = dataset.version().version as i64;
+        Ok(AlterTableDropColumnsResponse::new(version))
+    }
+
     async fn list_table_versions(
         &self,
         request: ListTableVersionsRequest,
     ) -> Result<ListTableVersionsResponse> {
         self.record_op("list_table_versions");
-        // When table_version_storage_enabled, query from __manifest
-        if self.table_version_storage_enabled
-            && let Some(ref manifest_ns) = self.manifest_ns
-        {
-            let table_id = request.id.clone().unwrap_or_default();
-            let want_descending = request.descending == Some(true);
-            return manifest_ns
-                .list_table_versions(&table_id, want_descending, request.limit)
-                .await;
-        }
-
-        // Fallback when table_version_storage is not enabled: list from _versions/ directory
+        let branch = Self::normalized_branch(request.branch.as_deref())?;
         let table_uri = self.resolve_table_location(&request.id).await?;
+        let table_uri = match branch {
+            Some(b) => self.resolve_branch_location(&table_uri, b).await?,
+            None => table_uri,
+        };
         let want_descending = request.descending == Some(true);
         let table_versions = self
             .list_table_versions_from_storage(&table_uri, want_descending, request.limit)
@@ -2836,13 +3515,19 @@ impl LanceNamespace for DirectoryNamespace {
         request: CreateTableVersionRequest,
     ) -> Result<CreateTableVersionResponse> {
         self.record_op("create_table_version");
+        let branch = Self::normalized_branch(request.branch.as_deref())?;
         let table_uri = self.resolve_table_location(&request.id).await?;
+        let (table_uri, table_path) = match branch {
+            Some(b) => self.resolve_branch_for_commit(&table_uri, b).await?,
+            None => {
+                let table_path = self.object_store_path_from_uri(&table_uri)?;
+                (table_uri, table_path)
+            }
+        };
 
         let staging_manifest_path = &request.manifest_path;
         let version = request.version as u64;
 
-        let table_path = self.object_store_path_from_uri(&table_uri)?;
-
         // Determine naming scheme from request, default to V2
         let naming_scheme = match request.naming_scheme.as_deref() {
             Some("V1") => ManifestNamingScheme::V1,
@@ -2956,41 +3641,6 @@ impl LanceNamespace for DirectoryNamespace {
             );
         }
 
-        // If table_version_storage_enabled is enabled, also record in __manifest (best-effort)
-        if self.table_version_storage_enabled
-            && let Some(ref manifest_ns) = self.manifest_ns
-        {
-            let table_id_str =
-                manifest::ManifestNamespace::str_object_id(&request.id.clone().unwrap_or_default());
-            let object_id =
-                manifest::ManifestNamespace::build_version_object_id(&table_id_str, version as i64);
-            let metadata_json = serde_json::json!({
-                "manifest_path": final_path.to_string(),
-                "manifest_size": manifest_size,
-                "e_tag": final_meta.e_tag,
-                "naming_scheme": request.naming_scheme.as_deref().unwrap_or("V2"),
-            })
-            .to_string();
-
-            if let Err(e) = manifest_ns
-                .insert_into_manifest_with_metadata(
-                    vec![manifest::ManifestEntry {
-                        object_id,
-                        object_type: manifest::ObjectType::TableVersion,
-                        location: None,
-                        metadata: Some(metadata_json),
-                    }],
-                    None,
-                )
-                .await
-            {
-                log::warn!(
-                    "Failed to record table version in __manifest (best-effort): {:?}",
-                    e
-                );
-            }
-        }
-
         Ok(CreateTableVersionResponse {
             transaction_id: None,
             version: Some(Box::new(TableVersion {
@@ -3009,17 +3659,12 @@ impl LanceNamespace for DirectoryNamespace {
         request: DescribeTableVersionRequest,
     ) -> Result<DescribeTableVersionResponse> {
         self.record_op("describe_table_version");
-        // When table_version_storage_enabled and a specific version is requested,
-        // query from __manifest to avoid opening the entire dataset
-        if self.table_version_storage_enabled
-            && let (Some(manifest_ns), Some(version)) = (&self.manifest_ns, request.version)
-        {
-            let table_id = request.id.clone().unwrap_or_default();
-            return manifest_ns.describe_table_version(&table_id, version).await;
-        }
-
-        // Fallback when table_version_storage is not enabled: inspect physical manifests directly.
+        let branch = Self::normalized_branch(request.branch.as_deref())?;
         let table_uri = self.resolve_table_location(&request.id).await?;
+        let table_uri = match branch {
+            Some(b) => self.resolve_branch_location(&table_uri, b).await?,
+            None => table_uri,
+        };
         let versions = self
             .list_table_versions_from_storage(&table_uri, true, None)
             .await?;
@@ -3057,76 +3702,46 @@ impl LanceNamespace for DirectoryNamespace {
         request: BatchDeleteTableVersionsRequest,
     ) -> Result<BatchDeleteTableVersionsResponse> {
         self.record_op("batch_delete_table_versions");
+        let branch = Self::normalized_branch(request.branch.as_deref())?;
         // Single-table mode: use `id` (from path parameter) + `ranges` to delete
         // versions from one table.
         let ranges: Vec<(i64, i64)> = request
             .ranges
             .iter()
-            .map(|r| {
-                let start = r.start_version;
-                let end = if r.end_version > 0 {
-                    r.end_version
+            .map(|r| (r.start_version, r.end_version))
+            .collect();
+
+        // Reject pathological bounded ranges up front: an explicit huge bounded
+        // range like (0, i64::MAX) is almost certainly a mistake. A through-latest
+        // range (end < 0) is bounded by the manifests that actually exist on storage.
+        const MAX_VERSIONS_PER_REQUEST: i128 = 1_000_000;
+        let requested: i128 = ranges
+            .iter()
+            .map(|(s, e)| {
+                if *e < 0 {
+                    0
                 } else {
-                    start
-                };
-                (start, end)
+                    (*e as i128 - *s as i128).max(0)
+                }
             })
-            .collect();
+            .sum();
+        if requested > MAX_VERSIONS_PER_REQUEST {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "batch_delete requested {} versions; limit is {}",
+                    requested, MAX_VERSIONS_PER_REQUEST
+                ),
+            }
+            .into());
+        }
+
         let table_entries = vec![TableDeleteEntry {
             table_id: request.id.clone(),
             ranges,
         }];
 
-        let mut total_deleted_count = 0i64;
-
-        if self.table_version_storage_enabled
-            && let Some(ref manifest_ns) = self.manifest_ns
-        {
-            // Phase 1 (atomic commit point): Delete version records from __manifest
-            // for ALL tables in a single atomic operation. This is the authoritative
-            // source of truth — once __manifest entries are removed, the versions
-            // are logically deleted across all tables atomically.
-
-            // Collect all (table_id_str, ranges) for batch deletion
-            let mut all_object_ids: Vec<String> = Vec::new();
-            for te in &table_entries {
-                let table_id_str = manifest::ManifestNamespace::str_object_id(
-                    &te.table_id.clone().unwrap_or_default(),
-                );
-                for (start, end) in &te.ranges {
-                    for version in *start..=*end {
-                        let object_id = manifest::ManifestNamespace::build_version_object_id(
-                            &table_id_str,
-                            version,
-                        );
-                        all_object_ids.push(object_id);
-                    }
-                }
-            }
-
-            if !all_object_ids.is_empty() {
-                total_deleted_count = manifest_ns
-                    .batch_delete_table_versions_by_object_ids(&all_object_ids)
-                    .await?;
-            }
-
-            // Phase 2: Delete physical manifest files (best-effort).
-            // Even if some file deletions fail, the versions are already removed from
-            // __manifest, so they won't be visible to readers. Leftover files are
-            // orphaned but harmless and can be cleaned up later.
-            let _ = self
-                .delete_physical_version_files(&table_entries, true)
-                .await;
-
-            return Ok(BatchDeleteTableVersionsResponse {
-                deleted_count: Some(total_deleted_count),
-                transaction_id: None,
-            });
-        }
-
-        // Fallback when table_version_storage is not enabled: delete physical files directly (no __manifest)
-        total_deleted_count = self
-            .delete_physical_version_files(&table_entries, false)
+        let total_deleted_count = self
+            .delete_physical_version_files(&table_entries, branch)
             .await?;
 
         Ok(BatchDeleteTableVersionsResponse {
@@ -3213,6 +3828,11 @@ impl LanceNamespace for DirectoryNamespace {
         let dataset = self
             .load_dataset(&table_uri, request.version, "list_table_indices")
             .await?;
+        let total_rows = dataset.count_rows(None).await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to count rows for table '{}': {:?}", table_uri, e),
+            })
+        })? as u64;
         let mut indices = dataset
             .describe_indices(None)
             .await
@@ -3255,12 +3875,35 @@ impl LanceNamespace for DirectoryNamespace {
                     })
                     .collect::<Result<Vec<_>>>()?;
 
-                Ok(IndexContent {
+                let segments = description.segments();
+                let created_at = segments
+                    .iter()
+                    .filter_map(|segment| segment.created_at)
+                    .min()
+                    .map(|ts| ts.to_rfc3339());
+
+                // `..Default::default()` keeps this tolerant of additive reqwest
+                // client model changes (see #7212).
+                #[allow(clippy::needless_update)]
+                let content = IndexContent {
                     index_name: description.name().to_string(),
                     index_uuid: description.metadata()[0].uuid.to_string(),
                     columns,
                     status: "SUCCEEDED".to_string(),
-                })
+                    index_type: Some(description.index_type().to_string()),
+                    type_url: Some(description.type_url().to_string()),
+                    num_indexed_rows: Some(description.rows_indexed() as i64),
+                    num_unindexed_rows: Some(
+                        total_rows.saturating_sub(description.rows_indexed()) as i64,
+                    ),
+                    size_bytes: description.total_size_bytes().map(|size| size as i64),
+                    num_segments: Some(segments.len() as i32),
+                    created_at,
+                    index_version: segments.first().map(|segment| segment.index_version),
+                    index_details: description.details().ok(),
+                    ..Default::default()
+                };
+                Ok(content)
             })
             .collect::<Result<Vec<_>>>()?;
 
@@ -3354,70 +3997,275 @@ impl LanceNamespace for DirectoryNamespace {
             .await?;
         let (version, transaction) = self.find_transaction(&dataset, &id).await?;
 
-        Ok(Self::transaction_response(version, &transaction))
-    }
-
-    async fn create_table_scalar_index(
-        &self,
-        request: CreateTableIndexRequest,
-    ) -> Result<CreateTableScalarIndexResponse> {
-        self.record_op("create_table_scalar_index");
-        let index_type = Self::parse_index_type(&request.index_type)?;
-        if !index_type.is_scalar() {
-            return Err(NamespaceError::InvalidInput {
-                message: format!(
-                    "create_table_scalar_index only supports scalar index types, got {}",
-                    request.index_type
-                ),
-            }
-            .into());
-        }
+        // Merge any persisted alter_transaction changes stored in the sidecar
+        // so that describe_transaction reflects the latest altered state.
+        let sidecar = self
+            .load_transaction_alteration(&table_uri, &transaction.uuid)
+            .await?;
 
-        let response = self.create_table_index(request).await?;
-        Ok(CreateTableScalarIndexResponse {
-            transaction_id: response.transaction_id,
-        })
+        Ok(Self::transaction_response(version, &transaction, sidecar))
     }
 
-    async fn drop_table_index(
+    async fn alter_transaction(
         &self,
-        request: DropTableIndexRequest,
-    ) -> Result<DropTableIndexResponse> {
-        self.record_op("drop_table_index");
-        let table_uri = self.resolve_table_location(&request.id).await?;
-        let index_name = request.index_name.as_deref().ok_or_else(|| {
+        request: AlterTransactionRequest,
+    ) -> Result<AlterTransactionResponse> {
+        self.record_op("alter_transaction");
+
+        // Parse the request ID: must include table id and transaction identifier
+        let mut request_id = request.id.ok_or_else(|| {
             lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Index name is required for drop_table_index".to_string(),
+                message: "Transaction id must include table id and transaction identifier"
+                    .to_string(),
             })
         })?;
-        let mut dataset = self
-            .load_dataset(&table_uri, None, "drop_table_index")
-            .await?;
-        let metadatas = dataset
-            .load_indices_by_name(index_name)
-            .await
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::TableIndexNotFound {
-                    message: format!(
-                        "Failed to load index '{}' before dropping it from table '{}': {}",
-                        index_name, table_uri, e
-                    ),
-                })
-            })?;
-        if metadatas.first().is_some_and(is_system_index) {
-            return Err(NamespaceError::Unsupported {
+        if request_id.len() < 2 {
+            return Err(NamespaceError::InvalidInput {
                 message: format!(
-                    "System index '{}' cannot be dropped via this API",
-                    index_name
+                    "Transaction request id must include table id and transaction identifier, got {:?}",
+                    request_id
                 ),
             }
             .into());
         }
 
-        dataset.drop_index(index_name).await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::TableIndexNotFound {
-                message: format!(
-                    "Failed to drop index '{}' from table '{}': {}",
+        let txn_id = request_id.pop().expect("request_id len checked above");
+        let table_id = Some(request_id);
+        let table_uri = self.resolve_table_location(&table_id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, None, "alter_transaction")
+            .await?;
+        let (version, transaction) = self.find_transaction(&dataset, &txn_id).await?;
+
+        // Reserved keys are derived from the immutable Transaction metadata and
+        // must not be modified via alter_transaction. They are only surfaced in
+        // the response for the caller's convenience.
+        const RESERVED_KEYS: &[&str] = &["uuid", "version", "read_version", "operation", "tag"];
+        let is_reserved = |key: &str| RESERVED_KEYS.contains(&key);
+
+        // Load the existing sidecar (if any) so alterations accumulate across
+        // successive alter_transaction calls.
+        let mut sidecar = self
+            .load_transaction_alteration(&table_uri, &transaction.uuid)
+            .await?
+            .unwrap_or_default();
+
+        for action in &request.actions {
+            if let Some(ref set_status) = action.set_status_action
+                && let Some(ref status) = set_status.status
+            {
+                // Validate the status value (case-insensitive)
+                let normalized = status.to_lowercase().replace('_', "");
+                match normalized.as_str() {
+                    "queued" | "running" | "succeeded" | "failed" | "canceled" => {
+                        sidecar.status = Some(status.clone());
+                    }
+                    _ => {
+                        return Err(NamespaceError::InvalidInput {
+                            message: format!(
+                                "Invalid transaction status '{}'. Valid values are: Queued, Running, Succeeded, Failed, Canceled",
+                                status
+                            ),
+                        }
+                        .into());
+                    }
+                }
+            }
+
+            if let Some(ref set_property) = action.set_property_action
+                && let (Some(key), Some(value)) = (&set_property.key, &set_property.value)
+            {
+                if is_reserved(key) {
+                    return Err(NamespaceError::InvalidInput {
+                        message: format!("Property '{}' is reserved and cannot be modified", key),
+                    }
+                    .into());
+                }
+                let mode = set_property
+                    .mode
+                    .as_deref()
+                    .unwrap_or("Overwrite")
+                    .to_lowercase();
+                match mode.as_str() {
+                    "overwrite" => {
+                        sidecar.properties.insert(key.clone(), value.clone());
+                    }
+                    "fail" => {
+                        // Consider both the immutable transaction properties
+                        // and any values previously written to the sidecar.
+                        let exists = sidecar.properties.contains_key(key)
+                            || transaction
+                                .transaction_properties
+                                .as_ref()
+                                .is_some_and(|props| props.contains_key(key));
+                        if exists {
+                            return Err(NamespaceError::ConcurrentModification {
+                                message: format!(
+                                    "Property '{}' already exists and mode is 'Fail'",
+                                    key
+                                ),
+                            }
+                            .into());
+                        }
+                        sidecar.properties.insert(key.clone(), value.clone());
+                    }
+                    "skip" => {
+                        let exists = sidecar.properties.contains_key(key)
+                            || transaction
+                                .transaction_properties
+                                .as_ref()
+                                .is_some_and(|props| props.contains_key(key));
+                        if !exists {
+                            sidecar.properties.insert(key.clone(), value.clone());
+                        }
+                    }
+                    _ => {
+                        return Err(NamespaceError::InvalidInput {
+                            message: format!(
+                                "Invalid set_property mode '{}'. Valid values are: Overwrite, Fail, Skip",
+                                mode
+                            ),
+                        }
+                        .into());
+                    }
+                }
+            }
+
+            if let Some(ref unset_property) = action.unset_property_action
+                && let Some(ref key) = unset_property.key
+            {
+                if is_reserved(key) {
+                    return Err(NamespaceError::InvalidInput {
+                        message: format!("Property '{}' is reserved and cannot be modified", key),
+                    }
+                    .into());
+                }
+                let mode = unset_property
+                    .mode
+                    .as_deref()
+                    .unwrap_or("Skip")
+                    .to_lowercase();
+                let exists_in_transaction = transaction
+                    .transaction_properties
+                    .as_ref()
+                    .is_some_and(|props| props.contains_key(key));
+                match mode.as_str() {
+                    "skip" => {
+                        sidecar.properties.remove(key);
+                        if exists_in_transaction {
+                            // Track a tombstone so describe_transaction can
+                            // hide the immutable property from the response.
+                            sidecar.removed_properties.insert(key.clone());
+                        }
+                    }
+                    "fail" => {
+                        if !sidecar.properties.contains_key(key) && !exists_in_transaction {
+                            return Err(NamespaceError::InvalidInput {
+                                message: format!(
+                                    "Property '{}' does not exist and mode is 'Fail'",
+                                    key
+                                ),
+                            }
+                            .into());
+                        }
+                        sidecar.properties.remove(key);
+                        if exists_in_transaction {
+                            sidecar.removed_properties.insert(key.clone());
+                        }
+                    }
+                    _ => {
+                        return Err(NamespaceError::InvalidInput {
+                            message: format!(
+                                "Invalid unset_property mode '{}'. Valid values are: Skip, Fail",
+                                mode
+                            ),
+                        }
+                        .into());
+                    }
+                }
+            }
+        }
+
+        // Persist the accumulated alterations so subsequent calls observe
+        // them. The transaction file itself is immutable in Lance, so we
+        // record alter_transaction outcomes in a namespace-owned sidecar.
+        self.save_transaction_alteration(&table_uri, &transaction.uuid, &sidecar)
+            .await?;
+
+        // Assemble the response by merging the immutable transaction metadata
+        // with the persisted alterations.
+        let final_status = sidecar
+            .status
+            .clone()
+            .unwrap_or_else(|| "SUCCEEDED".to_string());
+        let response = Self::transaction_response(version, &transaction, Some(sidecar));
+        Ok(AlterTransactionResponse {
+            status: final_status,
+            properties: response.properties,
+        })
+    }
+
+    async fn create_table_scalar_index(
+        &self,
+        request: CreateTableIndexRequest,
+    ) -> Result<CreateTableScalarIndexResponse> {
+        self.record_op("create_table_scalar_index");
+        let index_type = Self::parse_index_type(&request.index_type)?;
+        if !index_type.is_scalar() {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "create_table_scalar_index only supports scalar index types, got {}",
+                    request.index_type
+                ),
+            }
+            .into());
+        }
+
+        let response = self.create_table_index(request).await?;
+        Ok(CreateTableScalarIndexResponse {
+            transaction_id: response.transaction_id,
+        })
+    }
+
+    async fn drop_table_index(
+        &self,
+        request: DropTableIndexRequest,
+    ) -> Result<DropTableIndexResponse> {
+        self.record_op("drop_table_index");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let index_name = request.index_name.as_deref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Index name is required for drop_table_index".to_string(),
+            })
+        })?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "drop_table_index")
+            .await?;
+        let metadatas = dataset
+            .load_indices_by_name(index_name)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                    message: format!(
+                        "Failed to load index '{}' before dropping it from table '{}': {}",
+                        index_name, table_uri, e
+                    ),
+                })
+            })?;
+        if metadatas.first().is_some_and(is_system_index) {
+            return Err(NamespaceError::Unsupported {
+                message: format!(
+                    "System index '{}' cannot be dropped via this API",
+                    index_name
+                ),
+            }
+            .into());
+        }
+
+        dataset.drop_index(index_name).await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::TableIndexNotFound {
+                message: format!(
+                    "Failed to drop index '{}' from table '{}': {}",
                     index_name, table_uri, e
                 ),
             })
@@ -3462,8 +4310,12 @@ impl LanceNamespace for DirectoryNamespace {
             ));
         }
 
+        let branch = Self::normalized_branch(request.branch.as_deref())?;
         let table_uri = self.resolve_table_location(&request.id).await?;
-        let mut dataset = self.load_dataset(&table_uri, None, "restore_table").await?;
+        let mut dataset = match branch {
+            Some(branch) => self.open_validated_branch(&table_uri, branch).await?,
+            None => self.load_dataset(&table_uri, None, "restore_table").await?,
+        };
 
         dataset = dataset
             .checkout_version(version as u64)
@@ -3863,12 +4715,7 @@ impl LanceNamespace for DirectoryNamespace {
             })?
             .execute_reader(reader)
             .await
-            .map_err(|e| NamespaceError::Internal {
-                message: format!(
-                    "Failed to merge_insert_into_table at '{}': {}",
-                    table_uri, e
-                ),
-            })?;
+            .map_err(|e| Self::map_mutation_error(e, "merge_insert_into_table", &table_uri))?;
 
         Ok(MergeInsertIntoTableResponse {
             transaction_id: None,
@@ -3879,6 +4726,119 @@ impl LanceNamespace for DirectoryNamespace {
         })
     }
 
+    async fn update_table(&self, request: UpdateTableRequest) -> Result<UpdateTableResponse> {
+        self.record_op("update_table");
+
+        if request.updates.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "update_table requires at least one [column, expression] pair".to_string(),
+            }
+            .into());
+        }
+
+        // Validate every update pair shape and detect duplicate columns up front so we
+        // surface a clean error instead of failing deep inside the planner.
+        let mut seen_columns: HashMap<String, ()> = HashMap::with_capacity(request.updates.len());
+        for (idx, pair) in request.updates.iter().enumerate() {
+            if pair.len() != 2 {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
+                        "update_table updates[{}] must be a [column, expression] pair, got {} elements",
+                        idx,
+                        pair.len()
+                    ),
+                }
+                .into());
+            }
+            let column = &pair[0];
+            if column.trim().is_empty() {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!("update_table updates[{}] has an empty column name", idx),
+                }
+                .into());
+            }
+            if seen_columns.insert(column.clone(), ()).is_some() {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
+                        "update_table cannot update column '{}' more than once",
+                        column
+                    ),
+                }
+                .into());
+            }
+        }
+
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = Arc::new(self.load_dataset(&table_uri, None, "update_table").await?);
+
+        let mut builder = UpdateBuilder::new(dataset);
+        for pair in &request.updates {
+            // Indexing by 0/1 is safe due to the length check above.
+            builder = builder.set(&pair[0], &pair[1]).map_err(|e| {
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: format!("Invalid update expression for column '{}': {}", pair[0], e),
+                })
+            })?;
+        }
+        if let Some(predicate) = request.predicate.as_deref()
+            && !predicate.trim().is_empty()
+        {
+            builder = builder.update_where(predicate).map_err(|e| {
+                lance_core::Error::from(NamespaceError::InvalidInput {
+                    message: format!("Invalid update_table predicate '{}': {}", predicate, e),
+                })
+            })?;
+        }
+
+        let job = builder.build().map_err(|e| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: format!("Failed to build update_table job: {}", e),
+            })
+        })?;
+
+        let result = job
+            .execute()
+            .await
+            .map_err(|e| Self::map_mutation_error(e, "update_table", &table_uri))?;
+
+        let version = result.new_dataset.version().version as i64;
+        Ok(UpdateTableResponse {
+            transaction_id: None,
+            updated_rows: result.rows_updated as i64,
+            version,
+            properties: None,
+        })
+    }
+
+    async fn delete_from_table(
+        &self,
+        request: DeleteFromTableRequest,
+    ) -> Result<DeleteFromTableResponse> {
+        self.record_op("delete_from_table");
+
+        if request.predicate.trim().is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "delete_from_table requires a non-empty predicate".to_string(),
+            }
+            .into());
+        }
+
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "delete_from_table")
+            .await?;
+
+        let result = dataset
+            .delete(&request.predicate)
+            .await
+            .map_err(|e| Self::map_mutation_error(e, "delete_from_table", &table_uri))?;
+
+        Ok(DeleteFromTableResponse {
+            transaction_id: None,
+            version: Some(result.new_dataset.version().version as i64),
+        })
+    }
+
     async fn query_table(&self, request: QueryTableRequest) -> Result<Bytes> {
         use arrow::ipc::writer::FileWriter;
 
@@ -4162,14 +5122,15 @@ impl LanceNamespace for DirectoryNamespace {
             .load_dataset(&table_uri, None, "get_table_tag_version")
             .await?;
 
-        let version = dataset
+        let contents = dataset
             .tags()
-            .get_version(&request.tag)
+            .get(&request.tag)
             .await
             .map_err(|e| Self::map_tag_error(e, &request.tag, &table_uri))?;
 
         Ok(GetTableTagVersionResponse {
-            version: version as i64,
+            version: contents.version as i64,
+            branch: contents.branch,
         })
     }
 
@@ -4275,56 +5236,254 @@ impl LanceNamespace for DirectoryNamespace {
         })
     }
 
-    fn namespace_id(&self) -> String {
-        format!("DirectoryNamespace {{ root: {:?} }}", self.root)
-    }
-}
+    async fn create_table_branch(
+        &self,
+        request: CreateTableBranchRequest,
+    ) -> Result<CreateTableBranchResponse> {
+        self.record_op("create_table_branch");
+        if request.name.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "branch name must not be empty for create_table_branch".to_string(),
+            }
+            .into());
+        }
+        let from_version = match request.from_version {
+            Some(v) if v <= 0 => {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
+                        "from_version must be a positive integer, got {} for create_table_branch",
+                        v
+                    ),
+                }
+                .into());
+            }
+            Some(v) => Some(v as u64),
+            None => None,
+        };
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow_ipc::reader::{FileReader, StreamReader};
-    use lance::dataset::Dataset;
-    use lance::index::DatasetIndexExt;
-    use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
-    use lance_core::utils::testing::CountingObjectStore;
-    use lance_io::object_store::{providers::local::FileStoreProvider, uri_to_url};
-    use lance_namespace::models::{
-        CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest,
-        QueryTableRequestColumns,
-    };
-    use lance_namespace::schema::convert_json_arrow_schema;
-    use std::io::Cursor;
-    use std::sync::{
-        Arc,
-        atomic::{AtomicUsize, Ordering},
-    };
-    use url::Url;
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "create_table_branch")
+            .await?;
 
-    fn assert_plan_contains_all(plan: &str, expected_fragments: &[&str], context: &str) {
-        for expected_fragment in expected_fragments {
-            assert!(
-                plan.contains(expected_fragment),
-                "{}. Missing fragment: '{}'. Plan:\n{}",
-                context,
-                expected_fragment,
-                plan
-            );
+        // Best-effort pre-check: a duplicate returns a clean TableBranchAlreadyExists conflict
+        // instead of the opaque Internal error create_branch raises on a pre-existing branch. A
+        // concurrent create can still race past this window. Remove once lance-core create_branch
+        // returns RefConflict up front.
+        if dataset.branches().get(&request.name).await.is_ok() {
+            return Err(NamespaceError::TableBranchAlreadyExists {
+                message: format!("branch '{}' for table at '{}'", request.name, table_uri),
+            }
+            .into());
         }
-    }
-
-    /// Helper to create a test DirectoryNamespace with a temporary directory
-    async fn create_test_namespace() -> (DirectoryNamespace, TempStdDir) {
-        let temp_dir = TempStdDir::default();
 
-        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
-            .build()
-            .await
+        dataset
+            .create_branch(
+                &request.name,
+                (request.from_branch.as_deref(), from_version),
+                None,
+            )
+            .await
+            .map_err(|e| {
+                // After load_dataset + the dup pre-check, a DatasetNotFound from create_branch
+                // means the requested fork source (from_branch/from_version) doesn't exist.
+                if matches!(e, lance_core::Error::DatasetNotFound { .. }) {
+                    NamespaceError::InvalidInput {
+                        message: format!(
+                            "from_branch/from_version for branch '{}' refers to a source that does not exist: {}",
+                            request.name, e
+                        ),
+                    }
+                    .into()
+                } else {
+                    Self::map_branch_error(e, &request.name, &table_uri)
+                }
+            })?;
+
+        Ok(CreateTableBranchResponse {
+            transaction_id: None,
+        })
+    }
+
+    async fn list_table_branches(
+        &self,
+        request: ListTableBranchesRequest,
+    ) -> Result<ListTableBranchesResponse> {
+        self.record_op("list_table_branches");
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let dataset = self
+            .load_dataset(&table_uri, None, "list_table_branches")
+            .await?;
+
+        let raw_branches = dataset.list_branches().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!(
+                    "Failed to list branches for table at '{}': {}",
+                    table_uri, e
+                ),
+            })
+        })?;
+
+        let branches = raw_branches
+            .into_iter()
+            .map(|(name, contents)| {
+                // The namespace `BranchContents` model has no `identifier` field, so the
+                // lance-core branch identifier is intentionally dropped here.
+                let mut branch_model = ModelBranchContents::new(
+                    contents.parent_version as i64,
+                    contents.create_at as i64,
+                    contents.manifest_size as i64,
+                );
+                branch_model.parent_branch = contents.parent_branch;
+                branch_model.metadata = if contents.metadata.is_empty() {
+                    None
+                } else {
+                    Some(contents.metadata)
+                };
+                (name, branch_model)
+            })
+            .collect();
+
+        Ok(ListTableBranchesResponse {
+            branches,
+            page_token: None,
+        })
+    }
+
+    async fn delete_table_branch(
+        &self,
+        request: DeleteTableBranchRequest,
+    ) -> Result<DeleteTableBranchResponse> {
+        self.record_op("delete_table_branch");
+        if request.name.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "branch name must not be empty for delete_table_branch".to_string(),
+            }
+            .into());
+        }
+
+        let table_uri = self.resolve_table_location(&request.id).await?;
+        let mut dataset = self
+            .load_dataset(&table_uri, None, "delete_table_branch")
+            .await?;
+
+        dataset
+            .delete_branch(&request.name)
+            .await
+            .map_err(|e| match e {
+                lance_core::Error::RefConflict { message } => NamespaceError::InvalidInput {
+                    message: format!(
+                        "branch '{}' for table at '{}': {}",
+                        request.name, table_uri, message
+                    ),
+                }
+                .into(),
+                other => Self::map_branch_error(other, &request.name, &table_uri),
+            })?;
+
+        Ok(DeleteTableBranchResponse {
+            transaction_id: None,
+        })
+    }
+
+    fn namespace_id(&self) -> String {
+        format!("DirectoryNamespace {{ root: {:?} }}", self.root)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_ipc::reader::{FileReader, StreamReader};
+    use lance::dataset::Dataset;
+    use lance::index::DatasetIndexExt;
+    use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
+    use lance_core::utils::testing::CountingObjectStore;
+    use lance_io::object_store::{providers::local::FileStoreProvider, uri_to_url};
+    use lance_namespace::error::ErrorCode;
+    use lance_namespace::models::{
+        CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest,
+        QueryTableRequestColumns,
+    };
+    use lance_namespace::schema::convert_json_arrow_schema;
+    use std::io::Cursor;
+    use std::sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering},
+    };
+    use url::Url;
+
+    fn assert_plan_contains_all(plan: &str, expected_fragments: &[&str], context: &str) {
+        for expected_fragment in expected_fragments {
+            assert!(
+                plan.contains(expected_fragment),
+                "{}. Missing fragment: '{}'. Plan:\n{}",
+                context,
+                expected_fragment,
+                plan
+            );
+        }
+    }
+
+    fn mutation_error_code(err: lance_core::Error) -> ErrorCode {
+        match err {
+            lance_core::Error::Namespace { source, .. } => source
+                .downcast_ref::<NamespaceError>()
+                .expect("mutation error should wrap a NamespaceError")
+                .code(),
+            other => panic!("expected Namespace error, got: {other:?}"),
+        }
+    }
+
+    /// `map_mutation_error` must classify commit-conflict variants the same way as
+    /// `convert_lance_commit_error` in `manifest.rs`: `CommitConflict` is a retries-exhausted
+    /// version collision that is safe to retry (`Throttling`), while the semantic-conflict variants
+    /// map to `ConcurrentModification`.
+    #[test]
+    fn test_map_mutation_error_commit_conflict_alignment() {
+        let boxed = || -> Box<dyn std::error::Error + Send + Sync + 'static> {
+            Box::<dyn std::error::Error + Send + Sync>::from("inner conflict")
+        };
+
+        let throttling_cases = vec![lance_core::Error::commit_conflict_source(1, boxed())];
+        for err in throttling_cases {
+            let code = mutation_error_code(DirectoryNamespace::map_mutation_error(
+                err,
+                "update",
+                "memory://t",
+            ));
+            assert_eq!(code, ErrorCode::Throttling);
+        }
+
+        let concurrent_cases = vec![
+            lance_core::Error::too_much_write_contention("contention"),
+            lance_core::Error::retryable_commit_conflict_source(1, boxed()),
+            lance_core::Error::incompatible_transaction_source(boxed()),
+            lance_core::Error::version_conflict("conflict", 0, 3),
+        ];
+        for err in concurrent_cases {
+            let code = mutation_error_code(DirectoryNamespace::map_mutation_error(
+                err,
+                "update",
+                "memory://t",
+            ));
+            assert_eq!(code, ErrorCode::ConcurrentModification);
+        }
+    }
+
+    /// Helper to create a test DirectoryNamespace with a temporary directory
+    async fn create_test_namespace() -> (DirectoryNamespace, TempStdDir) {
+        let temp_dir = TempStdDir::default();
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_dir.to_str().unwrap())
+            .build()
+            .await
             .unwrap();
         (namespace, temp_dir)
     }
 
     #[derive(Debug)]
+    #[allow(dead_code)]
     struct CountingFileStoreProvider {
         listing_count: Arc<AtomicUsize>,
     }
@@ -4360,6 +5519,7 @@ mod tests {
         }
     }
 
+    #[allow(dead_code)]
     fn file_object_store_uri(path: &str) -> String {
         let file_url = uri_to_url(path).unwrap();
         let mut url = Url::parse("file-object-store:///").unwrap();
@@ -4367,6 +5527,7 @@ mod tests {
         url.to_string()
     }
 
+    #[allow(dead_code)]
     fn build_listing_counting_session(listing_count: Arc<AtomicUsize>) -> Arc<Session> {
         let registry = Arc::new(ObjectStoreRegistry::default());
         registry.insert(
@@ -4485,6 +5646,43 @@ mod tests {
         create_ipc_data_from_batches(schema, vec![batch])
     }
 
+    async fn create_legacy_manifest_without_primary_key_metadata(root: &str) {
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use arrow::record_batch::{RecordBatch, RecordBatchIterator};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("object_id", DataType::Utf8, false),
+            Field::new("object_type", DataType::Utf8, false),
+            Field::new("location", DataType::Utf8, true),
+            Field::new("metadata", DataType::Utf8, true),
+            Field::new(
+                "base_objects",
+                DataType::List(Arc::new(Field::new("object_id", DataType::Utf8, true))),
+                true,
+            ),
+        ]));
+        let batch = RecordBatch::new_empty(schema.clone());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        Dataset::write(Box::new(reader), &format!("{}/__manifest", root), None)
+            .await
+            .unwrap();
+    }
+
+    async fn manifest_has_primary_key_metadata(root: &str) -> bool {
+        let dataset = Dataset::open(&format!("{}/__manifest", root))
+            .await
+            .unwrap();
+        dataset
+            .schema()
+            .field("object_id")
+            .map(|field| {
+                field
+                    .metadata
+                    .contains_key(lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+            })
+            .unwrap_or(false)
+    }
+
     fn create_vector_table_ipc_data() -> Vec<u8> {
         use arrow::array::{FixedSizeListArray, Float32Array, Int32Array};
         use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
@@ -4567,93 +5765,1496 @@ mod tests {
             .transaction_id
     }
 
-    #[tokio::test]
-    async fn test_create_table() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
-
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
-
-        let response = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
+    /// Fork `branch_name` from the table's current version and append
+    /// `extra_versions` commits to it (each a new version on the branch, written
+    /// with the default V2 naming). The main branch is left untouched. Returns
+    /// the branch's storage URI (`<root>/tree/<branch>`).
+    async fn create_branch_with_commits(
+        namespace: &DirectoryNamespace,
+        table_name: &str,
+        branch_name: &str,
+        extra_versions: usize,
+    ) -> String {
+        let mut main = open_dataset(namespace, table_name).await;
+        let fork_version = main.version().version;
+        let branch = main
+            .create_branch(branch_name, fork_version, None)
             .await
             .unwrap();
+        let branch_uri = branch.uri().to_string();
+        for i in 0..extra_versions {
+            append_scalar_version(&branch_uri, (i as i32 + 1) * 100).await;
+        }
+        branch_uri
+    }
 
-        assert!(response.location.is_some());
-        assert!(response.location.unwrap().ends_with("test_table.lance"));
-        assert_eq!(response.version, Some(1));
+    /// Append one scalar-schema batch to the dataset at `uri`, creating a new
+    /// version (default V2 naming). Shared by branch and main chain setup.
+    async fn append_scalar_version(uri: &str, seed: i32) {
+        use arrow::array::{Int32Array, StringArray};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![seed, seed + 1])),
+                Arc::new(StringArray::from(vec![Some("x"), Some("y")])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+    }
+
+    /// List a table's versions on `branch` (None == main) via the namespace.
+    async fn list_versions(
+        namespace: &DirectoryNamespace,
+        table_name: &str,
+        branch: Option<&str>,
+    ) -> Result<Vec<TableVersion>> {
+        let req = ListTableVersionsRequest {
+            id: Some(vec![table_name.to_string()]),
+            branch: branch.map(|b| b.to_string()),
+            ..Default::default()
+        };
+        namespace.list_table_versions(req).await.map(|r| r.versions)
     }
 
     #[tokio::test]
-    async fn test_create_table_without_data() {
+    async fn test_list_table_versions_on_branch() {
         let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        create_branch_with_commits(&namespace, "users", "exp", 2).await;
 
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_table".to_string()]);
+        // The branch lists its own chain, and every version resolves to a
+        // manifest under the branch's tree path.
+        let branch_versions = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        assert!(branch_versions.len() >= 2);
+        assert!(
+            branch_versions
+                .iter()
+                .all(|v| v.manifest_path.contains("tree/exp")),
+            "branch versions must resolve to branch manifests: {:?}",
+            branch_versions
+        );
 
-        let result = namespace.create_table(request, bytes::Bytes::new()).await;
-        assert!(result.is_err());
+        // Unset and "main" behave identically and never see the tree path.
+        let main_versions = list_versions(&namespace, "users", None).await.unwrap();
+        let main_explicit = list_versions(&namespace, "users", Some("main"))
+            .await
+            .unwrap();
+        assert_eq!(main_versions.len(), main_explicit.len());
         assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Arrow IPC stream) is required")
+            main_versions
+                .iter()
+                .all(|v| !v.manifest_path.contains("tree/"))
         );
+
+        // A non-existent branch is a clean not-found, not an empty list.
+        let missing = list_versions(&namespace, "users", Some("does-not-exist")).await;
+        assert!(missing.is_err());
+        assert!(missing.unwrap_err().to_string().contains("not found"));
     }
 
     #[tokio::test]
-    async fn test_create_table_with_invalid_id() {
+    async fn test_describe_table_version_on_branch() {
         let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        create_branch_with_commits(&namespace, "users", "exp", 2).await;
 
-        // Create test IPC data
-        let schema = create_test_schema();
-        let ipc_data = create_test_ipc_data(&schema);
-
-        // Test with empty ID
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec![]);
-
-        let result = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data.clone()))
-            .await;
-        assert!(result.is_err());
+        let branch_versions = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        let latest = branch_versions.iter().map(|v| v.version).max().unwrap();
 
-        // Test with multi-level ID - should now work with manifest enabled
-        // First create the parent namespace
-        let mut create_ns_req = CreateNamespaceRequest::new();
-        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
-        namespace.create_namespace(create_ns_req).await.unwrap();
+        // Describe latest on the branch returns the branch's manifest_path.
+        let req = DescribeTableVersionRequest {
+            id: Some(vec!["users".to_string()]),
+            branch: Some("exp".to_string()),
+            ..Default::default()
+        };
+        let resp = namespace.describe_table_version(req).await.unwrap();
+        assert_eq!(resp.version.version, latest);
+        assert!(resp.version.manifest_path.contains("tree/exp"));
+
+        // A specific existing branch version resolves.
+        let req = DescribeTableVersionRequest {
+            id: Some(vec!["users".to_string()]),
+            version: Some(latest),
+            branch: Some("exp".to_string()),
+            ..Default::default()
+        };
+        assert!(namespace.describe_table_version(req).await.is_ok());
 
-        // Now create table in the namespace
-        let mut request = CreateTableRequest::new();
-        request.id = Some(vec!["test_namespace".to_string(), "table".to_string()]);
+        // A version absent on the branch is not found.
+        let req = DescribeTableVersionRequest {
+            id: Some(vec!["users".to_string()]),
+            version: Some(999_999),
+            branch: Some("exp".to_string()),
+            ..Default::default()
+        };
+        assert!(namespace.describe_table_version(req).await.is_err());
 
-        let result = namespace
-            .create_table(request, bytes::Bytes::from(ipc_data))
-            .await;
-        // Should succeed with manifest enabled
-        assert!(
-            result.is_ok(),
-            "Multi-level table IDs should work with manifest enabled"
-        );
+        // A non-existent branch is not found.
+        let req = DescribeTableVersionRequest {
+            id: Some(vec!["users".to_string()]),
+            branch: Some("nope".to_string()),
+            ..Default::default()
+        };
+        let err = namespace.describe_table_version(req).await;
+        assert!(err.is_err() && err.unwrap_err().to_string().contains("not found"));
     }
 
     #[tokio::test]
-    async fn test_list_tables() {
-        let (namespace, _temp_dir) = create_test_namespace().await;
+    async fn test_restore_table_on_branch() {
+        use lance_namespace::models::RestoreTableRequest;
 
-        // Initially, no tables
-        let mut request = ListTablesRequest::new();
-        request.id = Some(vec![]);
-        let response = namespace.list_tables(request).await.unwrap();
-        assert_eq!(response.tables.len(), 0);
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        create_branch_with_commits(&namespace, "users", "exp", 2).await;
 
-        // Create test IPC data
-        let schema = create_test_schema();
+        let before = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        let branch_latest = before.iter().map(|v| v.version).max().unwrap();
+        let earliest = before.iter().map(|v| v.version).min().unwrap();
+        let main_before = list_versions(&namespace, "users", None)
+            .await
+            .unwrap()
+            .len();
+
+        // Restoring the branch to an earlier version commits a NEW version on
+        // the branch (restore is itself a commit), and must not touch main.
+        let req = RestoreTableRequest {
+            id: Some(vec!["users".to_string()]),
+            version: earliest,
+            branch: Some("exp".to_string()),
+            ..Default::default()
+        };
+        let resp = namespace.restore_table(req).await.unwrap();
+        assert!(resp.transaction_id.is_some());
+
+        let after = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        let new_latest = after.iter().map(|v| v.version).max().unwrap();
+        assert!(
+            new_latest > branch_latest,
+            "restore should add a branch version"
+        );
+
+        let main_after = list_versions(&namespace, "users", None)
+            .await
+            .unwrap()
+            .len();
+        assert_eq!(main_after, main_before, "main must be unaffected");
+    }
+
+    #[tokio::test]
+    async fn test_batch_delete_table_versions_on_branch() {
+        use lance_namespace::models::{BatchDeleteTableVersionsRequest, VersionRange};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        create_branch_with_commits(&namespace, "users", "exp", 2).await;
+
+        let before = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        let main_before = list_versions(&namespace, "users", None).await.unwrap();
+
+        // Delete the branch's whole history with a through-latest range (end = -1).
+        // The branch manifests use V2 naming (inverted, zero-padded), so a nonzero
+        // deleted_count proves the V2 fix: the old code constructed
+        // "{version}.manifest" and silently matched nothing.
+        let req = BatchDeleteTableVersionsRequest {
+            id: Some(vec!["users".to_string()]),
+            branch: Some("exp".to_string()),
+            ranges: vec![VersionRange::new(0, -1)],
+            ..Default::default()
+        };
+        let resp = namespace.batch_delete_table_versions(req).await.unwrap();
+        assert_eq!(
+            resp.deleted_count,
+            Some(before.len() as i64),
+            "every branch manifest should be physically deleted"
+        );
+
+        // The emptied branch now reads as not-found, and main is untouched.
+        assert!(
+            list_versions(&namespace, "users", Some("exp"))
+                .await
+                .is_err()
+        );
+        let main_after = list_versions(&namespace, "users", None).await.unwrap();
+        assert_eq!(
+            main_after.len(),
+            main_before.len(),
+            "main must be untouched"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_table_version_on_branch() {
+        use futures::TryStreamExt;
+        use lance_namespace::models::CreateTableVersionRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let branch_uri = create_branch_with_commits(&namespace, "users", "exp", 1).await;
+
+        // Stage a manifest by copying one of the branch's existing manifests.
+        let branch_ds = Dataset::open(&branch_uri).await.unwrap();
+        let versions_dir = branch_ds.versions_dir();
+        let store = branch_ds.object_store(None).await.unwrap();
+        let existing = store
+            .inner
+            .list(Some(&versions_dir))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+            .into_iter()
+            .find(|m| {
+                m.location
+                    .filename()
+                    .map(|f| f.ends_with(".manifest"))
+                    .unwrap_or(false)
+            })
+            .expect("a branch manifest");
+        let bytes = store
+            .inner
+            .get(&existing.location)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        let staging = versions_dir.join("staging_manifest");
+        store.inner.put(&staging, bytes.into()).await.unwrap();
+
+        let main_before = list_versions(&namespace, "users", None)
+            .await
+            .unwrap()
+            .len();
+        let new_version = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap()
+            .iter()
+            .map(|v| v.version)
+            .max()
+            .unwrap()
+            + 1;
+
+        let req = CreateTableVersionRequest {
+            id: Some(vec!["users".to_string()]),
+            version: new_version,
+            manifest_path: staging.to_string(),
+            naming_scheme: Some("V2".to_string()),
+            branch: Some("exp".to_string()),
+            ..Default::default()
+        };
+        let resp = namespace.create_table_version(req).await.unwrap();
+        let info = resp.version.expect("version info");
+        // The new manifest must land under the branch's tree path.
+        assert!(
+            info.manifest_path.contains("tree/exp"),
+            "got {}",
+            info.manifest_path
+        );
+
+        // It is visible on the branch, and main did not gain a version.
+        let after = list_versions(&namespace, "users", Some("exp"))
+            .await
+            .unwrap();
+        assert!(after.iter().any(|v| v.version == new_version));
+        let main_after = list_versions(&namespace, "users", None)
+            .await
+            .unwrap()
+            .len();
+        assert_eq!(main_after, main_before, "main must be unaffected");
+    }
+
+    /// The namespace-managed commit store derives the branch a request targets
+    /// from the base path it is handed, so a single store serves every branch of
+    /// the table: a branch-qualified base resolves and commits against the
+    /// branch chain while the table root targets main.
+    #[tokio::test]
+    async fn test_external_manifest_store_resolves_branch_from_base_path() {
+        use futures::TryStreamExt;
+        use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
+        use lance_table::io::commit::external_manifest::ExternalManifestStore;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await; // main: version 1
+        let branch_uri = create_branch_with_commits(&namespace, "users", "exp", 2).await;
+
+        let namespace = Arc::new(namespace);
+        let table_id = vec!["users".to_string()];
+        let branch_ds = Dataset::open(&branch_uri).await.unwrap();
+        let branch_base = branch_ds.branch_location().path;
+        let root_base = branch_ds.branch_location().find_main().unwrap().path;
+        let store = LanceNamespaceExternalManifestStore::new(
+            namespace.clone(),
+            table_id.clone(),
+            root_base.clone(),
+        );
+
+        // The branch-qualified base resolves the branch chain, the root base
+        // resolves main: proof the base path reaches list_table_versions.
+        let (branch_latest, branch_path) = store
+            .get_latest_version(branch_base.as_ref())
+            .await
+            .unwrap()
+            .expect("branch has versions");
+        let (_main_latest, main_path) = store
+            .get_latest_version(root_base.as_ref())
+            .await
+            .unwrap()
+            .expect("main has versions");
+        assert!(
+            branch_path.contains("tree/exp"),
+            "branch latest must resolve to the branch tree: {}",
+            branch_path
+        );
+        assert!(
+            !main_path.contains("tree/exp"),
+            "main latest must not resolve to a branch tree: {}",
+            main_path
+        );
+
+        // describe (get) with the branch base also resolves to the branch tree.
+        let described = store
+            .get(branch_base.as_ref(), branch_latest)
+            .await
+            .unwrap();
+        assert!(
+            described.contains("tree/exp"),
+            "describe on the branch must resolve to the branch tree: {}",
+            described
+        );
+
+        // A base that is neither the root nor a branch chain is rejected.
+        assert!(store.get_latest_version("somewhere/else").await.is_err());
+
+        // Commit (put) with the branch base: the new version must land on the
+        // branch chain. Stage a manifest by copying an existing branch manifest.
+        let versions_dir = branch_ds.versions_dir();
+        let obj = branch_ds.object_store(None).await.unwrap();
+        let existing = obj
+            .inner
+            .list(Some(&versions_dir))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+            .into_iter()
+            .find(|m| {
+                m.location
+                    .filename()
+                    .map(|f| f.ends_with(".manifest"))
+                    .unwrap_or(false)
+            })
+            .expect("a branch manifest");
+        let bytes = obj
+            .inner
+            .get(&existing.location)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        let size = bytes.len() as u64;
+        let staging = versions_dir.clone().join("staging_manifest");
+        obj.inner.put(&staging, bytes.into()).await.unwrap();
+
+        let committed = store
+            .put(
+                &branch_base,
+                branch_latest + 1,
+                &staging,
+                size,
+                None,
+                obj.inner.as_ref(),
+                ManifestNamingScheme::V2,
+            )
+            .await
+            .unwrap();
+        assert!(
+            committed.path.to_string().contains("tree/exp"),
+            "a commit through a branch-qualified base must land on the branch tree: {}",
+            committed.path
+        );
+    }
+
+    /// write_into_namespace_on_branch must append against the branch chain
+    /// THROUGH the managed commit handler: the version is registered with the
+    /// namespace (create_table_version), lands on the branch tree, and main's
+    /// catalog is untouched. The ops-metrics assertions exist because a
+    /// physical-only commit is invisible to DirectoryNamespace branch listing
+    /// (it lists storage), while a catalog-authoritative namespace would
+    /// silently lose the version.
+    #[tokio::test]
+    async fn test_write_into_namespace_on_branch_appends_to_branch() {
+        use lance::dataset::builder::DatasetBuilder;
+        use lance_namespace::models::CreateTableBranchRequest;
+
+        let temp = TempStdDir::default();
+        let namespace = Arc::new(
+            DirectoryNamespaceBuilder::new(temp.to_str().unwrap())
+                .manifest_enabled(true)
+                .table_version_tracking_enabled(true)
+                .ops_metrics_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
+        let ns: Arc<dyn LanceNamespace> = namespace.clone();
+        let table_id = vec!["t".to_string()];
+        create_managed_table(&ns, &table_id).await; // main: v1 (id=1), v2 (id=2)
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "exp".to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let main_chain_len = |ns: Arc<dyn LanceNamespace>, table_id: Vec<String>| async move {
+            ns.list_table_versions(ListTableVersionsRequest {
+                id: Some(table_id),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .versions
+            .len()
+        };
+        let main_before = main_chain_len(ns.clone(), table_id.clone()).await;
+        let commits_before = namespace
+            .retrieve_ops_metrics()
+            .get("create_table_version")
+            .copied()
+            .unwrap_or(0);
+
+        let branch_ds = Dataset::write_into_namespace_on_branch(
+            RecordBatchIterator::new(vec![Ok(single_int_batch(3))], single_int_schema()),
+            ns.clone(),
+            table_id.clone(),
+            "exp",
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(branch_ds.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&branch_ds).await, vec![1, 2, 3]);
+
+        // The append must commit through the namespace, not just write a
+        // physical manifest under the branch tree.
+        let commits_after = namespace
+            .retrieve_ops_metrics()
+            .get("create_table_version")
+            .copied()
+            .unwrap_or(0);
+        assert_eq!(
+            commits_after,
+            commits_before + 1,
+            "the branch append must register its version via create_table_version"
+        );
+        let exp_versions = ns
+            .list_table_versions(ListTableVersionsRequest {
+                id: Some(table_id.clone()),
+                branch: Some("exp".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .versions;
+        assert!(
+            exp_versions
+                .iter()
+                .all(|v| v.manifest_path.contains("tree/exp")),
+            "branch versions must resolve to the branch tree: {:?}",
+            exp_versions
+        );
+        assert_eq!(
+            main_chain_len(ns.clone(), table_id.clone()).await,
+            main_before,
+            "main's catalog must be untouched by the branch append"
+        );
+
+        // A managed main append through the same entry point must register in
+        // the catalog too, so a fresh managed open resolves the new latest.
+        Dataset::write_into_namespace(
+            RecordBatchIterator::new(vec![Ok(single_int_batch(100))], single_int_schema()),
+            ns.clone(),
+            table_id.clone(),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            main_chain_len(ns.clone(), table_id.clone()).await,
+            main_before + 1,
+            "a managed main append must register its version in the catalog"
+        );
+        let fresh = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(
+            scan_id_column(&fresh).await,
+            vec![1, 2, 100],
+            "a fresh managed open must resolve the appended version, not a stale latest"
+        );
+    }
+
+    /// CREATE on a branch is rejected: a branch forks from an existing version.
+    #[tokio::test]
+    async fn test_write_into_namespace_on_branch_rejects_create() {
+        use arrow::array::{Int32Array, StringArray};
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        let namespace = Arc::new(namespace);
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let batch = arrow::record_batch::RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1])),
+                Arc::new(StringArray::from(vec![Some("a")])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+
+        let result = Dataset::write_into_namespace_on_branch(
+            reader,
+            namespace.clone(),
+            vec!["new_table".to_string()],
+            "exp",
+            Some(WriteParams {
+                mode: WriteMode::Create,
+                ..Default::default()
+            }),
+        )
+        .await;
+        assert!(result.is_err(), "create on a branch must be rejected");
+        assert!(
+            result.unwrap_err().to_string().contains("branch"),
+            "error should mention the branch restriction"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_branch_name_validation_rejects_traversal() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+
+        // A traversal-style branch name is rejected as invalid input before any
+        // storage path is built from it.
+        let err = list_versions(&namespace, "users", Some("../evil")).await;
+        assert!(err.is_err());
+        assert!(err.unwrap_err().to_string().contains("invalid branch name"));
+    }
+
+    #[tokio::test]
+    async fn test_branch_ops_reject_zombie_branch() {
+        use futures::TryStreamExt;
+        use lance_namespace::models::{
+            BatchDeleteTableVersionsRequest, CreateTableVersionRequest, RestoreTableRequest,
+            VersionRange,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+
+        let dataset = open_dataset(&namespace, "users").await;
+        let store = dataset.object_store(None).await.unwrap();
+        let manifest = store
+            .inner
+            .list(Some(&dataset.versions_dir()))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap()
+            .into_iter()
+            .find(|m| {
+                m.location
+                    .filename()
+                    .map(|f| f.ends_with(".manifest"))
+                    .unwrap_or(false)
+            })
+            .expect("a manifest");
+        let bytes = store
+            .inner
+            .get(&manifest.location)
+            .await
+            .unwrap()
+            .bytes()
+            .await
+            .unwrap();
+        let zombie = dataset
+            .branch_location()
+            .find_branch(Some("ghost"))
+            .unwrap()
+            .path
+            .join(VERSIONS_DIR)
+            .join(manifest.location.filename().unwrap());
+        store.inner.put(&zombie, bytes.into()).await.unwrap();
+
+        assert!(dataset.branches().get("ghost").await.is_err());
+
+        fn rejected<T: std::fmt::Debug>(label: &str, r: Result<T>) {
+            match r {
+                Ok(v) => panic!("{label} must reject the zombie branch, got Ok({v:?})"),
+                Err(e) => assert!(e.to_string().contains("not found"), "{label}: {e}"),
+            }
+        }
+
+        rejected(
+            "list",
+            list_versions(&namespace, "users", Some("ghost")).await,
+        );
+        rejected(
+            "describe",
+            namespace
+                .describe_table_version(DescribeTableVersionRequest {
+                    id: Some(vec!["users".to_string()]),
+                    branch: Some("ghost".to_string()),
+                    ..Default::default()
+                })
+                .await,
+        );
+        rejected(
+            "create",
+            namespace
+                .create_table_version(CreateTableVersionRequest {
+                    id: Some(vec!["users".to_string()]),
+                    version: 2,
+                    manifest_path: zombie.to_string(),
+                    branch: Some("ghost".to_string()),
+                    ..Default::default()
+                })
+                .await,
+        );
+        rejected(
+            "restore",
+            namespace
+                .restore_table(RestoreTableRequest {
+                    id: Some(vec!["users".to_string()]),
+                    version: 1,
+                    branch: Some("ghost".to_string()),
+                    ..Default::default()
+                })
+                .await,
+        );
+        rejected(
+            "batch_delete",
+            namespace
+                .batch_delete_table_versions(BatchDeleteTableVersionsRequest {
+                    id: Some(vec!["users".to_string()]),
+                    branch: Some("ghost".to_string()),
+                    ranges: vec![VersionRange::new(1, 1)],
+                    ..Default::default()
+                })
+                .await,
+        );
+    }
+
+    /// V2 is the default naming scheme, and the pre-rewrite delete path
+    /// constructed `{version}.manifest` (a V1 name) and silently matched nothing
+    /// on a V2 table, returning deleted_count 0. This pins the fix on the main
+    /// chain (branch=None), which previously had no batch_delete coverage at all.
+    #[tokio::test]
+    async fn test_batch_delete_table_versions_main_v2() {
+        use lance_namespace::models::{BatchDeleteTableVersionsRequest, VersionRange};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await; // version 1
+        let main_uri = open_dataset(&namespace, "users").await.uri().to_string();
+        append_scalar_version(&main_uri, 100).await; // version 2
+        append_scalar_version(&main_uri, 200).await; // version 3
+
+        let before = list_versions(&namespace, "users", None).await.unwrap();
+        assert!(before.len() >= 3);
+        // Confirm these really are V2-named manifests (20-digit inverted version
+        // + ".manifest" == 29 chars), i.e. the case the old code skipped.
+        assert!(
+            before
+                .iter()
+                .all(|v| v.manifest_path.rsplit('/').next().unwrap().len() == 29),
+            "expected V2-named manifests: {:?}",
+            before
+        );
+        let min_v = before.iter().map(|v| v.version).min().unwrap();
+        let max_v = before.iter().map(|v| v.version).max().unwrap();
+
+        // Delete everything except the latest version. end is exclusive, so
+        // [min_v, max_v) keeps max_v.
+        let req = BatchDeleteTableVersionsRequest {
+            id: Some(vec!["users".to_string()]),
+            ranges: vec![VersionRange::new(min_v, max_v)],
+            ..Default::default()
+        };
+        let resp = namespace.batch_delete_table_versions(req).await.unwrap();
+        assert_eq!(
+            resp.deleted_count,
+            Some((before.len() - 1) as i64),
+            "V2 manifests must actually be deleted (was 0 before the fix)"
+        );
+
+        let after = list_versions(&namespace, "users", None).await.unwrap();
+        assert_eq!(after.len(), 1);
+        assert_eq!(after[0].version, max_v);
+    }
+
+    /// Pins the exclusive end of VersionRange: [v, v+1) must match only v.
+    #[tokio::test]
+    async fn test_batch_delete_end_is_exclusive() {
+        use lance_namespace::models::{BatchDeleteTableVersionsRequest, VersionRange};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await; // version 1
+        let main_uri = open_dataset(&namespace, "users").await.uri().to_string();
+        append_scalar_version(&main_uri, 100).await; // version 2
+        append_scalar_version(&main_uri, 200).await; // version 3
+
+        let before = list_versions(&namespace, "users", None).await.unwrap();
+        let min_v = before.iter().map(|v| v.version).min().unwrap();
+
+        let req = BatchDeleteTableVersionsRequest {
+            id: Some(vec!["users".to_string()]),
+            ranges: vec![VersionRange::new(min_v, min_v + 1)],
+            ..Default::default()
+        };
+        let resp = namespace.batch_delete_table_versions(req).await.unwrap();
+        assert_eq!(
+            resp.deleted_count,
+            Some(1),
+            "only min_v is in [min_v, min_v+1)"
+        );
+
+        let after = list_versions(&namespace, "users", None).await.unwrap();
+        assert!(
+            !after.iter().any(|v| v.version == min_v),
+            "min_v must be deleted"
+        );
+        assert_eq!(after.len(), before.len() - 1, "exactly one version removed");
+    }
+
+    #[tokio::test]
+    async fn test_batch_delete_rejects_unbounded_range() {
+        use lance_namespace::models::{BatchDeleteTableVersionsRequest, VersionRange};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+
+        // An unbounded range must be rejected up front, not turned into ~10^19
+        // iterations / an unbounded id list.
+        let req = BatchDeleteTableVersionsRequest {
+            id: Some(vec!["users".to_string()]),
+            ranges: vec![VersionRange::new(0, i64::MAX)],
+            ..Default::default()
+        };
+        let err = namespace.batch_delete_table_versions(req).await;
+        assert!(err.is_err());
+        assert!(
+            err.unwrap_err().to_string().contains("limit"),
+            "expected a range-too-large error"
+        );
+    }
+
+    /// Build a managed (manifest-tracked) namespace over `path`.
+    async fn create_managed_namespace(path: &str) -> Arc<dyn LanceNamespace> {
+        Arc::new(
+            DirectoryNamespaceBuilder::new(path)
+                .manifest_enabled(true)
+                .table_version_tracking_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        )
+    }
+
+    fn single_int_schema() -> Arc<arrow::datatypes::Schema> {
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]))
+    }
+
+    fn single_int_batch(seed: i32) -> arrow::record_batch::RecordBatch {
+        use arrow::array::Int32Array;
+        arrow::record_batch::RecordBatch::try_new(
+            single_int_schema(),
+            vec![Arc::new(Int32Array::from(vec![seed]))],
+        )
+        .unwrap()
+    }
+
+    /// Create a managed table with versions v1 (id=1) and v2 (id=2) on main and
+    /// return the main dataset handle.
+    async fn create_managed_table(ns: &Arc<dyn LanceNamespace>, table_id: &[String]) -> Dataset {
+        let mut ds = Dataset::write_into_namespace(
+            RecordBatchIterator::new(vec![Ok(single_int_batch(1))], single_int_schema()),
+            ns.clone(),
+            table_id.to_vec(),
+            Some(WriteParams {
+                mode: WriteMode::Create,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        ds.append(
+            RecordBatchIterator::new(vec![Ok(single_int_batch(2))], single_int_schema()),
+            None,
+        )
+        .await
+        .unwrap();
+        ds
+    }
+
+    /// Sorted values of the `id` column across a full scan.
+    async fn scan_id_column(ds: &Dataset) -> Vec<i32> {
+        use arrow::array::Int32Array;
+        use futures::TryStreamExt;
+        let batches: Vec<arrow::record_batch::RecordBatch> = ds
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let mut ids: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        ids.sort();
+        ids
+    }
+
+    /// E2e for the managed branch path through the builder: create a branch via the
+    /// namespace op, open it with `from_namespace(managed).with_branch`, commit on
+    /// it, and confirm the dataset is rooted at the branch chain (manifest, base
+    /// path and data placement) while main's catalog is untouched.
+    #[tokio::test]
+    async fn test_managed_branch_open_and_commit() {
+        use futures::TryStreamExt;
+        use lance::dataset::builder::DatasetBuilder;
+        use lance_namespace::models::CreateTableBranchRequest;
+
+        let temp = TempStdDir::default();
+        let ns = create_managed_namespace(temp.to_str().unwrap()).await;
+        let table_id = vec!["t".to_string()];
+        create_managed_table(&ns, &table_id).await;
+        let main_before = ns
+            .list_table_versions(ListTableVersionsRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .versions
+            .len();
+
+        // Create a branch via the namespace op (the FS-handler path, which succeeds
+        // on a managed table).
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "exp".to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        // Open the managed table on the branch: the base path is qualified up
+        // front and the manifest store derives the branch from it.
+        let mut branch_ds = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_branch("exp", None)
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(
+            branch_ds.manifest.branch.as_deref(),
+            Some("exp"),
+            "with_branch on a managed table must open the branch chain"
+        );
+        let branch_base = branch_ds.branch_location().path;
+        assert!(
+            branch_base.as_ref().ends_with("tree/exp"),
+            "the branch dataset must be rooted at the branch chain: {}",
+            branch_base
+        );
+        let branch_v_before = branch_ds.version().version;
+
+        // Commit on the branch.
+        branch_ds
+            .append(
+                RecordBatchIterator::new(vec![Ok(single_int_batch(3))], single_int_schema()),
+                None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(
+            branch_ds.manifest.branch.as_deref(),
+            Some("exp"),
+            "the commit must stay on the branch"
+        );
+        assert!(
+            branch_ds.version().version > branch_v_before,
+            "the branch version must advance after the commit"
+        );
+        assert_eq!(scan_id_column(&branch_ds).await, vec![1, 2, 3]);
+
+        // The committed data files live under the branch chain, not main's data
+        // dir, so unmanaged readers of the branch and main's cleanup see a
+        // consistent layout.
+        let store = branch_ds.object_store(None).await.unwrap();
+        let branch_data = branch_base.clone().join("data");
+        let branch_files = store
+            .inner
+            .list(Some(&branch_data))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert!(
+            !branch_files.is_empty(),
+            "the branch commit must place data files under the branch chain"
+        );
+
+        // The same branch is readable through the unmanaged (path-based) open.
+        let table_uri = ns
+            .describe_table(DescribeTableRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .location
+            .unwrap();
+        let fs_branch_ds = DatasetBuilder::from_uri(&table_uri)
+            .with_branch("exp", None)
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(fs_branch_ds.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&fs_branch_ds).await, vec![1, 2, 3]);
+
+        // Main's catalog is untouched (branches are not tracked in __manifest),
+        // and main still reads its own data.
+        let main_after = ns
+            .list_table_versions(ListTableVersionsRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .versions
+            .len();
+        assert_eq!(
+            main_after, main_before,
+            "committing on the branch must not change main's chain"
+        );
+        let main_ds = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(main_ds.manifest.branch, None);
+        assert_eq!(scan_id_column(&main_ds).await, vec![1, 2]);
+    }
+
+    /// Branch-pointing tags on a managed table: create them through the normal
+    /// API (from both the main and the branch handle), open the table at the
+    /// tag, and check the tag out from an already-open dataset. All of these
+    /// must resolve the branch chain, never main's chain.
+    #[tokio::test]
+    async fn test_managed_branch_tags() {
+        use lance::dataset::builder::DatasetBuilder;
+        use lance::dataset::refs::Ref;
+        use lance_namespace::models::CreateTableBranchRequest;
+
+        let temp = TempStdDir::default();
+        let ns = create_managed_namespace(temp.to_str().unwrap()).await;
+        let table_id = vec!["t".to_string()];
+        let main_ds = create_managed_table(&ns, &table_id).await;
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "exp".to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+        let mut branch_ds = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_branch("exp", None)
+            .load()
+            .await
+            .unwrap();
+        branch_ds
+            .append(
+                RecordBatchIterator::new(vec![Ok(single_int_batch(3))], single_int_schema()),
+                None,
+            )
+            .await
+            .unwrap();
+        let branch_version = branch_ds.version().version;
+
+        // A branch-pointing tag created from the main handle must validate
+        // against the branch chain (the version does not exist on main).
+        main_ds
+            .tags()
+            .create("exp-tag", ("exp", Some(branch_version)))
+            .await
+            .unwrap();
+        let tag = main_ds.tags().get("exp-tag").await.unwrap();
+        assert_eq!(tag.branch.as_deref(), Some("exp"));
+        assert_eq!(tag.version, branch_version);
+
+        // A tag created from the branch handle resolves the branch implicitly.
+        branch_ds
+            .tags()
+            .create("exp-tag2", branch_version)
+            .await
+            .unwrap();
+        let tag2 = branch_ds.tags().get("exp-tag2").await.unwrap();
+        assert_eq!(tag2.branch.as_deref(), Some("exp"));
+
+        // Opening the managed table at the branch-pointing tag checks out the
+        // branch chain.
+        let tag_open = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_tag("exp-tag")
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(tag_open.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(tag_open.version().version, branch_version);
+        assert_eq!(scan_id_column(&tag_open).await, vec![1, 2, 3]);
+
+        // So does checking the tag out from an already-open main dataset.
+        let tag_checkout = main_ds
+            .checkout_version(Ref::Tag("exp-tag".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(tag_checkout.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&tag_checkout).await, vec![1, 2, 3]);
+
+        // A missing tag on a managed table errors at open.
+        let err = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_tag("no-such-tag")
+            .load()
+            .await;
+        assert!(err.is_err(), "a missing tag must error");
+    }
+
+    /// Cross-branch checkout on a managed table, including version numbers that
+    /// exist on both chains (branch numbering continues from the fork point, so
+    /// overlap is the common case). Every checkout must land on the requested
+    /// chain and read that chain's data.
+    #[tokio::test]
+    async fn test_managed_cross_branch_checkout() {
+        use lance::dataset::builder::DatasetBuilder;
+        use lance::dataset::refs::Ref;
+        use lance_namespace::models::CreateTableBranchRequest;
+
+        let temp = TempStdDir::default();
+        let ns = create_managed_namespace(temp.to_str().unwrap()).await;
+        let table_id = vec!["t".to_string()];
+        let mut main_ds = create_managed_table(&ns, &table_id).await;
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "exp".to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        // exp gets id=3 at its tip; main gets id=100 at the same version number.
+        let mut branch_ds = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_branch("exp", None)
+            .load()
+            .await
+            .unwrap();
+        branch_ds
+            .append(
+                RecordBatchIterator::new(vec![Ok(single_int_batch(3))], single_int_schema()),
+                None,
+            )
+            .await
+            .unwrap();
+        let overlap_version = branch_ds.version().version;
+        while main_ds.version().version < overlap_version {
+            main_ds
+                .append(
+                    RecordBatchIterator::new(vec![Ok(single_int_batch(100))], single_int_schema()),
+                    None,
+                )
+                .await
+                .unwrap();
+        }
+
+        // main -> branch at the overlapping version number: must read the
+        // branch's data, not main's same-numbered version.
+        let on_branch = main_ds
+            .checkout_version(Ref::Version(Some("exp".to_string()), Some(overlap_version)))
+            .await
+            .unwrap();
+        assert_eq!(on_branch.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&on_branch).await, vec![1, 2, 3]);
+
+        // main -> branch latest.
+        let mut on_branch_latest = main_ds.checkout_branch("exp").await.unwrap();
+        assert_eq!(on_branch_latest.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(on_branch_latest.version().version, overlap_version);
+
+        // A commit through the checked-out handle (which shares main's commit
+        // handler) must land on the branch chain, not main's.
+        let main_chain_len = |ns: Arc<dyn LanceNamespace>, table_id: Vec<String>| async move {
+            ns.list_table_versions(ListTableVersionsRequest {
+                id: Some(table_id),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .versions
+            .len()
+        };
+        let main_before = main_chain_len(ns.clone(), table_id.clone()).await;
+        on_branch_latest
+            .append(
+                RecordBatchIterator::new(vec![Ok(single_int_batch(4))], single_int_schema()),
+                None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(on_branch_latest.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&on_branch_latest).await, vec![1, 2, 3, 4]);
+        assert_eq!(
+            main_chain_len(ns.clone(), table_id.clone()).await,
+            main_before,
+            "a commit on the checked-out branch must not advance main's chain"
+        );
+
+        // branch -> main at a specific version.
+        let on_main = branch_ds
+            .checkout_version(Ref::Version(None, Some(1)))
+            .await
+            .unwrap();
+        assert_eq!(on_main.manifest.branch, None);
+        assert_eq!(scan_id_column(&on_main).await, vec![1]);
+
+        // branch -> another branch.
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "exp2".to_string(),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+        let on_branch2 = branch_ds.checkout_branch("exp2").await.unwrap();
+        assert_eq!(on_branch2.manifest.branch.as_deref(), Some("exp2"));
+
+        // A version missing from the branch chain errors loudly.
+        let err = main_ds
+            .checkout_version(Ref::Version(Some("exp".to_string()), Some(999)))
+            .await;
+        assert!(err.is_err(), "a version missing from the branch must error");
+    }
+
+    /// CommitBuilder must honor an explicitly supplied commit handler for a
+    /// Dataset destination: a managed-versioning commit through a dataset that
+    /// was opened without the namespace handler (as the Java and Python commit
+    /// APIs allow) must still register with the catalog instead of silently
+    /// writing a physical manifest the catalog never sees.
+    #[tokio::test]
+    async fn test_commit_builder_honors_explicit_handler_for_dataset_dest() {
+        use lance::dataset::write::{CommitBuilder, InsertBuilder};
+        use lance::dataset::{WriteDestination, builder::DatasetBuilder};
+        use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
+        use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
+
+        let temp = TempStdDir::default();
+        let namespace = Arc::new(
+            DirectoryNamespaceBuilder::new(temp.to_str().unwrap())
+                .manifest_enabled(true)
+                .table_version_tracking_enabled(true)
+                .ops_metrics_enabled(true)
+                .build()
+                .await
+                .unwrap(),
+        );
+        let ns: Arc<dyn LanceNamespace> = namespace.clone();
+        let table_id = vec!["t".to_string()];
+        create_managed_table(&ns, &table_id).await; // main: v1 (id=1), v2 (id=2)
+
+        // Open WITHOUT the namespace handler, the way a binding caller can.
+        let table_uri = ns
+            .describe_table(DescribeTableRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap()
+            .location
+            .unwrap();
+        let plain_ds = Arc::new(Dataset::open(&table_uri).await.unwrap());
+
+        let transaction = InsertBuilder::new(WriteDestination::Dataset(plain_ds.clone()))
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute_uncommitted(vec![single_int_batch(3)])
+            .await
+            .unwrap();
+
+        let handler = Arc::new(ExternalManifestCommitHandler {
+            external_manifest_store: Arc::new(
+                LanceNamespaceExternalManifestStore::for_table_uri(
+                    ns.clone(),
+                    table_id.clone(),
+                    &table_uri,
+                )
+                .unwrap(),
+            ),
+        });
+        let commits_before = namespace
+            .retrieve_ops_metrics()
+            .get("create_table_version")
+            .copied()
+            .unwrap_or(0);
+        let committed = CommitBuilder::new(WriteDestination::Dataset(plain_ds))
+            .with_commit_handler(handler)
+            .execute(transaction)
+            .await
+            .unwrap();
+        assert_eq!(scan_id_column(&committed).await, vec![1, 2, 3]);
+
+        let commits_after = namespace
+            .retrieve_ops_metrics()
+            .get("create_table_version")
+            .copied()
+            .unwrap_or(0);
+        assert_eq!(
+            commits_after,
+            commits_before + 1,
+            "the explicit handler must route the commit through create_table_version"
+        );
+        let fresh = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(
+            scan_id_column(&fresh).await,
+            vec![1, 2, 3],
+            "a fresh managed open must resolve the committed version"
+        );
+    }
+
+    /// A branch forked from a non-latest version opens on its own chain.
+    #[tokio::test]
+    async fn test_managed_branch_from_non_latest_fork() {
+        use lance::dataset::builder::DatasetBuilder;
+        use lance_namespace::models::CreateTableBranchRequest;
+
+        let temp = TempStdDir::default();
+        let ns = create_managed_namespace(temp.to_str().unwrap()).await;
+        let table_id = vec!["t".to_string()];
+        create_managed_table(&ns, &table_id).await; // main: v1 (id=1), v2 (id=2)
+
+        ns.create_table_branch(CreateTableBranchRequest {
+            id: Some(table_id.clone()),
+            name: "old".to_string(),
+            from_version: Some(1),
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+        let old_ds = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_branch("old", None)
+            .load()
+            .await
+            .unwrap();
+        assert_eq!(old_ds.manifest.branch.as_deref(), Some("old"));
+        assert_eq!(
+            scan_id_column(&old_ds).await,
+            vec![1],
+            "the fork must contain only the fork-point data"
+        );
+    }
+
+    /// The shared parser must decode both naming schemes; this is the cheap
+    /// V1 no-regression guard (creating a real V1 table is not exposed here).
+    #[test]
+    fn test_manifest_version_from_filename() {
+        // V1: the plain version number.
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename("5.manifest"),
+            Some(5)
+        );
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename("0.manifest"),
+            Some(0)
+        );
+        // V2: version stored as u64::MAX - version, zero-padded to 20 digits.
+        let v2_five = format!("{:020}.manifest", u64::MAX - 5);
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename(&v2_five),
+            Some(5)
+        );
+        let v2_zero = format!("{:020}.manifest", u64::MAX);
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename(&v2_zero),
+            Some(0)
+        );
+        // Non-manifest and detached (`d`-prefixed) entries are ignored.
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename("data.lance"),
+            None
+        );
+        assert_eq!(
+            DirectoryNamespace::manifest_version_from_filename("d5.manifest"),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_table() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        assert!(response.location.is_some());
+        assert!(response.location.unwrap().ends_with("test_table.lance"));
+        assert_eq!(response.version, Some(1));
+    }
+
+    #[tokio::test]
+    async fn test_create_table_without_data() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_table".to_string()]);
+
+        let result = namespace.create_table(request, bytes::Bytes::new()).await;
+        assert!(result.is_err());
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Arrow IPC stream) is required")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_table_with_invalid_id() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create test IPC data
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+
+        // Test with empty ID
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec![]);
+
+        let result = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data.clone()))
+            .await;
+        assert!(result.is_err());
+
+        // Test with multi-level ID - should now work with manifest enabled
+        // First create the parent namespace
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["test_namespace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+
+        // Now create table in the namespace
+        let mut request = CreateTableRequest::new();
+        request.id = Some(vec!["test_namespace".to_string(), "table".to_string()]);
+
+        let result = namespace
+            .create_table(request, bytes::Bytes::from(ipc_data))
+            .await;
+        // Should succeed with manifest enabled
+        assert!(
+            result.is_ok(),
+            "Multi-level table IDs should work with manifest enabled"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_list_tables() {
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Initially, no tables
+        let mut request = ListTablesRequest::new();
+        request.id = Some(vec![]);
+        let response = namespace.list_tables(request).await.unwrap();
+        assert_eq!(response.tables.len(), 0);
+
+        // Create test IPC data
+        let schema = create_test_schema();
         let ipc_data = create_test_ipc_data(&schema);
 
         // Create a table
@@ -4828,7 +7429,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_list_table_indices() {
-        use lance_namespace::models::ListTableIndicesRequest;
+        use lance_namespace::models::{CreateTableIndexRequest, ListTableIndicesRequest};
 
         let (namespace, _temp_dir) = create_test_namespace().await;
         create_scalar_table(&namespace, "users").await;
@@ -4857,6 +7458,22 @@ mod tests {
         assert_eq!(users_id_idx.columns, vec!["id"]);
         assert_eq!(users_id_idx.status, "SUCCEEDED");
 
+        // Enriched fields populated from the index metadata for a scalar index.
+        assert_eq!(users_id_idx.index_type.as_deref(), Some("BTree"));
+        assert!(
+            users_id_idx
+                .type_url
+                .as_deref()
+                .is_some_and(|s| !s.is_empty())
+        );
+        assert_eq!(users_id_idx.num_indexed_rows, Some(3));
+        assert_eq!(users_id_idx.num_unindexed_rows, Some(0));
+        assert_eq!(users_id_idx.num_segments, Some(1));
+        assert!(users_id_idx.size_bytes.is_some_and(|size| size > 0));
+        assert!(users_id_idx.created_at.is_some());
+        assert!(users_id_idx.index_version.is_some());
+        assert!(users_id_idx.index_details.is_some());
+
         let dataset = open_dataset(&namespace, "users").await;
         let expected_transaction_id = dataset
             .read_transaction()
@@ -4900,6 +7517,44 @@ mod tests {
         assert_eq!(second_page.indexes.len(), 1);
         assert_eq!(second_page.indexes[0].index_name, "users_id_idx");
         assert!(second_page.page_token.is_none());
+
+        // A vector index exercises a different type_url, index_type, and details payload.
+        create_vector_table(&namespace, "vectors").await;
+        let mut create_index_request =
+            CreateTableIndexRequest::new("vector".to_string(), "IVF_FLAT".to_string());
+        create_index_request.id = Some(vec!["vectors".to_string()]);
+        create_index_request.name = Some("vector_idx".to_string());
+        create_index_request.distance_type = Some("l2".to_string());
+        namespace
+            .create_table_index(create_index_request)
+            .await
+            .unwrap();
+
+        let vector_response = namespace
+            .list_table_indices(ListTableIndicesRequest {
+                id: Some(vec!["vectors".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(vector_response.indexes.len(), 1);
+        let vector_idx = &vector_response.indexes[0];
+        assert_eq!(vector_idx.index_name, "vector_idx");
+        assert_eq!(vector_idx.columns, vec!["vector"]);
+        assert_eq!(vector_idx.index_type.as_deref(), Some("IVF_FLAT"));
+        assert!(
+            vector_idx
+                .type_url
+                .as_deref()
+                .is_some_and(|s| !s.is_empty())
+        );
+        assert!(vector_idx.num_indexed_rows.is_some());
+        assert!(vector_idx.num_unindexed_rows.is_some());
+        assert_eq!(vector_idx.num_segments, Some(1));
+        assert!(vector_idx.created_at.is_some());
+        assert!(vector_idx.index_version.is_some());
+        assert!(vector_idx.index_details.is_some());
     }
 
     #[tokio::test]
@@ -9275,154 +11930,203 @@ mod tests {
             assert!(total_rows > 0);
             assert!(total_rows < 3);
         }
-    }
 
-    /// Tests for multi-table transaction support via table_version_storage_enabled.
-    mod multi_table_transactions {
-        use super::*;
-        use futures::TryStreamExt;
-        use lance::dataset::builder::DatasetBuilder;
-        use lance_namespace::models::CreateTableVersionRequest;
+        // ---------------------- update_table / delete_from_table ----------------------
 
-        /// Helper to create a namespace with table_version_storage_enabled enabled
-        async fn create_managed_namespace(temp_path: &str) -> Arc<DirectoryNamespace> {
-            Arc::new(
-                DirectoryNamespaceBuilder::new(temp_path)
-                    .table_version_tracking_enabled(true)
-                    .table_version_storage_enabled(true)
-                    .manifest_enabled(true)
-                    .build()
-                    .await
-                    .unwrap(),
-            )
-        }
+        #[tokio::test]
+        async fn test_update_full_table() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-        /// Helper to create a table and get its staging manifest path
-        async fn create_table_and_get_staging(
-            namespace: Arc<dyn LanceNamespace>,
-            table_name: &str,
-        ) -> (Vec<String>, object_store::path::Path) {
-            let schema = create_test_schema();
-            let ipc_data = create_test_ipc_data(&schema);
-            let mut create_req = CreateTableRequest::new();
-            create_req.id = Some(vec![table_name.to_string()]);
-            namespace
-                .create_table(create_req, bytes::Bytes::from(ipc_data))
+            // Capture base version so we can assert the update bumped it.
+            let base_version = open_dataset(&namespace, &table_id[0])
                 .await
-                .unwrap();
+                .version()
+                .version;
 
-            let table_id = vec![table_name.to_string()];
-            let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
-                .await
-                .unwrap()
-                .load()
-                .await
-                .unwrap();
+            let request = UpdateTableRequest {
+                id: Some(table_id.clone()),
+                updates: vec![vec!["name".to_string(), "'updated'".to_string()]],
+                predicate: None,
+                ..Default::default()
+            };
+
+            let response = namespace.update_table(request).await.unwrap();
+            assert_eq!(response.updated_rows, 3);
+            assert!(response.version as u64 > base_version);
+
+            // Validate that all rows now carry the new value.
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: Some("name = 'updated'".to_string()),
+                ..Default::default()
+            };
+            assert_eq!(namespace.count_table_rows(count_req).await.unwrap(), 3);
+        }
+
+        #[tokio::test]
+        async fn test_update_with_predicate() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = UpdateTableRequest {
+                id: Some(table_id.clone()),
+                updates: vec![vec!["name".to_string(), "'matched'".to_string()]],
+                predicate: Some("id > 1".to_string()),
+                ..Default::default()
+            };
+
+            let response = namespace.update_table(request).await.unwrap();
+            assert_eq!(response.updated_rows, 2);
+
+            // Rows that did not match the predicate must remain unchanged.
+            let untouched = CountTableRowsRequest {
+                id: Some(table_id.clone()),
+                version: None,
+                predicate: Some("name = 'Alice'".to_string()),
+                ..Default::default()
+            };
+            assert_eq!(namespace.count_table_rows(untouched).await.unwrap(), 1);
+
+            let touched = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: Some("name = 'matched'".to_string()),
+                ..Default::default()
+            };
+            assert_eq!(namespace.count_table_rows(touched).await.unwrap(), 2);
+        }
+
+        #[tokio::test]
+        async fn test_update_invalid_expression_returns_invalid_input() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = UpdateTableRequest {
+                id: Some(table_id),
+                // Reference an unknown column on the right-hand side.
+                updates: vec![vec!["name".to_string(), "no_such_column + 1".to_string()]],
+                predicate: None,
+                ..Default::default()
+            };
+
+            let err = namespace.update_table(request).await.unwrap_err();
+            let msg = err.to_string();
+            assert!(
+                msg.contains("Invalid input"),
+                "expected Invalid input error, got: {}",
+                msg
+            );
+        }
+
+        #[tokio::test]
+        async fn test_update_rejects_duplicate_columns() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
+
+            let request = UpdateTableRequest {
+                id: Some(table_id),
+                updates: vec![
+                    vec!["name".to_string(), "'a'".to_string()],
+                    vec!["name".to_string(), "'b'".to_string()],
+                ],
+                predicate: None,
+                ..Default::default()
+            };
 
-            // Find existing manifest and create a staging copy
-            let versions_path = dataset.versions_dir();
-            let manifest_metas: Vec<_> = dataset
-                .object_store(None)
-                .await
-                .unwrap()
-                .inner
-                .list(Some(&versions_path))
-                .try_collect()
-                .await
-                .unwrap();
+            let err = namespace.update_table(request).await.unwrap_err();
+            let msg = err.to_string();
+            assert!(
+                msg.contains("Invalid input") && msg.contains("more than once"),
+                "expected duplicate column InvalidInput error, got: {}",
+                msg
+            );
+        }
 
-            let manifest_meta = manifest_metas
-                .iter()
-                .find(|m| {
-                    m.location
-                        .filename()
-                        .map(|f| f.ends_with(".manifest"))
-                        .unwrap_or(false)
-                })
-                .expect("No manifest file found");
+        #[tokio::test]
+        async fn test_delete_with_predicate() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-            let manifest_data = dataset
-                .object_store(None)
-                .await
-                .unwrap()
-                .inner
-                .get(&manifest_meta.location)
-                .await
-                .unwrap()
-                .bytes()
-                .await
-                .unwrap();
+            let request = DeleteFromTableRequest {
+                id: Some(table_id.clone()),
+                predicate: "id > 1".to_string(),
+                ..Default::default()
+            };
 
-            let staging_path = dataset
-                .versions_dir()
-                .join(format!("staging_{}", table_name));
-            dataset
-                .object_store(None)
-                .await
-                .unwrap()
-                .inner
-                .put(&staging_path, manifest_data.into())
-                .await
-                .unwrap();
+            let response = namespace.delete_from_table(request).await.unwrap();
+            assert!(response.version.is_some());
 
-            (table_id, staging_path)
+            let count_req = CountTableRowsRequest {
+                id: Some(table_id),
+                version: None,
+                predicate: None,
+                ..Default::default()
+            };
+            // Original rows = 3; after deleting `id > 1` only row id=1 remains.
+            assert_eq!(namespace.count_table_rows(count_req).await.unwrap(), 1);
         }
 
         #[tokio::test]
-        async fn test_table_version_storage_enabled_requires_manifest() {
-            // table_version_storage_enabled=true requires manifest_enabled=true
-            let temp_dir = TempStdDir::default();
-            let temp_path = temp_dir.to_str().unwrap();
+        async fn test_delete_empty_predicate_returns_invalid_input() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-            let result = DirectoryNamespaceBuilder::new(temp_path)
-                .table_version_storage_enabled(true)
-                .manifest_enabled(false)
-                .build()
-                .await;
+            let request = DeleteFromTableRequest {
+                id: Some(table_id),
+                predicate: "   ".to_string(),
+                ..Default::default()
+            };
 
+            let err = namespace.delete_from_table(request).await.unwrap_err();
+            let msg = err.to_string();
             assert!(
-                result.is_err(),
-                "Should fail when table_version_storage_enabled=true but manifest_enabled=false"
+                msg.contains("Invalid input") && msg.contains("non-empty predicate"),
+                "expected non-empty predicate InvalidInput error, got: {}",
+                msg
             );
         }
 
         #[tokio::test]
-        async fn test_create_table_version_records_in_manifest() {
-            // When table_version_storage_enabled is enabled, single create_table_version
-            // should also record the version in __manifest
-            let temp_dir = TempStrDir::default();
-            let temp_path: &str = &temp_dir;
+        async fn test_delete_table_not_found() {
+            let (namespace, _temp_dir) = create_test_namespace().await;
 
-            let namespace = create_managed_namespace(temp_path).await;
-            let ns: Arc<dyn LanceNamespace> = namespace.clone();
+            let request = DeleteFromTableRequest {
+                id: Some(vec!["does_not_exist".to_string()]),
+                predicate: "id = 1".to_string(),
+                ..Default::default()
+            };
 
-            let (table_id, staging_path) =
-                create_table_and_get_staging(ns.clone(), "table_managed").await;
+            let err = namespace.delete_from_table(request).await.unwrap_err();
+            let msg = err.to_string();
+            assert!(
+                msg.contains("Table not found"),
+                "expected TableNotFound for missing table, got: {}",
+                msg
+            );
+        }
 
-            // Create version 2
-            let mut create_req = CreateTableVersionRequest::new(2, staging_path.to_string());
-            create_req.id = Some(table_id.clone());
-            create_req.naming_scheme = Some("V2".to_string());
-            let response = namespace.create_table_version(create_req).await.unwrap();
+        #[tokio::test]
+        async fn test_delete_invalid_predicate_returns_invalid_input() {
+            let (namespace, _temp_dir, table_id) = create_ns_with_table().await;
 
-            assert!(response.version.is_some());
-            let version = response.version.unwrap();
-            assert_eq!(version.version, 2);
-
-            // Verify the version is recorded in __manifest by querying it
-            let manifest_ns = namespace.manifest_ns.as_ref().unwrap();
-            let table_id_str = manifest::ManifestNamespace::str_object_id(&table_id);
-            let versions = manifest_ns
-                .query_table_versions(&table_id_str, false, None)
-                .await
-                .unwrap();
+            // A predicate referencing a column that does not exist reaches `Dataset::delete`
+            // and surfaces as `Error::InvalidInput`, which must map to `InvalidInput` rather
+            // than a generic `Internal`.
+            let request = DeleteFromTableRequest {
+                id: Some(table_id),
+                predicate: "no_such_column = 1".to_string(),
+                ..Default::default()
+            };
 
-            assert!(
-                !versions.is_empty(),
-                "Version should be recorded in __manifest"
+            let err = namespace.delete_from_table(request).await.unwrap_err();
+            let lance_core::Error::Namespace { source, .. } = &err else {
+                panic!("expected a Namespace error, got: {}", err);
+            };
+            let ns_err = source
+                .downcast_ref::<NamespaceError>()
+                .expect("expected a NamespaceError source");
+            assert_eq!(
+                ns_err.code(),
+                lance_namespace::ErrorCode::InvalidInput,
+                "expected InvalidInput for an invalid delete predicate, got: {}",
+                err
             );
-            let (ver, _path) = &versions[0];
-            assert_eq!(*ver, 2, "Recorded version should be 2");
         }
     }
 
@@ -9479,6 +12183,55 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_alter_table_add_columns() {
+        use lance_namespace::models::{
+            AddColumnsEntry, AlterTableAddColumnsRequest, DescribeTableRequest,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Add a new column
+        let mut new_col = AddColumnsEntry::new("doubled_id".to_string());
+        new_col.expression = Some(Some("id * 2".to_string()));
+        let mut add_request = AlterTableAddColumnsRequest::new(vec![new_col]);
+        add_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .alter_table_add_columns(add_request)
+            .await
+            .unwrap();
+        assert!(
+            response.version > 1,
+            "Version should increment after adding columns"
+        );
+
+        // Verify via describe_table
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = namespace.describe_table(describe_request).await.unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let resp_schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = resp_schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            field_names.contains(&"doubled_id"),
+            "Column 'doubled_id' should exist, got: {:?}",
+            field_names
+        );
+    }
+
     #[tokio::test]
     async fn test_update_table_schema_metadata() {
         use lance_namespace::models::UpdateTableSchemaMetadataRequest;
@@ -9506,6 +12259,72 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_alter_table_add_columns_missing_id() {
+        use lance_namespace::models::{AddColumnsEntry, AlterTableAddColumnsRequest};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let new_col = AddColumnsEntry::new("col".to_string());
+        let request = AlterTableAddColumnsRequest::new(vec![new_col]);
+        let result = namespace.alter_table_add_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[tokio::test]
+    async fn test_alter_table_alter_columns_rename() {
+        use lance_namespace::models::{
+            AlterColumnsEntry, AlterTableAlterColumnsRequest, DescribeTableRequest,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Rename "name" to "full_name"
+        let mut entry = AlterColumnsEntry::new("name".to_string());
+        entry.rename = Some(Some("full_name".to_string()));
+        let mut alter_request = AlterTableAlterColumnsRequest::new(vec![entry]);
+        alter_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .alter_table_alter_columns(alter_request)
+            .await
+            .unwrap();
+        assert!(
+            response.version > 1,
+            "Version should increment after altering columns"
+        );
+
+        // Verify the rename
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = namespace.describe_table(describe_request).await.unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let resp_schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = resp_schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            field_names.contains(&"full_name"),
+            "Column should be renamed to 'full_name', got: {:?}",
+            field_names
+        );
+        assert!(
+            !field_names.contains(&"name"),
+            "Old column 'name' should not exist, got: {:?}",
+            field_names
+        );
+    }
+
     #[tokio::test]
     async fn test_get_table_stats() {
         use lance_namespace::models::GetTableStatsRequest;
@@ -9557,6 +12376,68 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_alter_table_alter_columns_missing_id() {
+        use lance_namespace::models::{AlterColumnsEntry, AlterTableAlterColumnsRequest};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let entry = AlterColumnsEntry::new("name".to_string());
+        let request = AlterTableAlterColumnsRequest::new(vec![entry]);
+        let result = namespace.alter_table_alter_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[tokio::test]
+    async fn test_alter_table_drop_columns() {
+        use lance_namespace::models::{AlterTableDropColumnsRequest, DescribeTableRequest};
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Create a table
+        let schema = create_test_schema();
+        let ipc_data = create_test_ipc_data(&schema);
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        namespace
+            .create_table(create_request, bytes::Bytes::from(ipc_data))
+            .await
+            .unwrap();
+
+        // Drop the "name" column
+        let mut drop_request = AlterTableDropColumnsRequest::new(vec!["name".to_string()]);
+        drop_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = namespace
+            .alter_table_drop_columns(drop_request)
+            .await
+            .unwrap();
+        assert!(
+            response.version > 1,
+            "Version should increment after dropping columns"
+        );
+
+        // Verify column was dropped
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = namespace.describe_table(describe_request).await.unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let resp_schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = resp_schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            !field_names.contains(&"name"),
+            "Column 'name' should be dropped, got: {:?}",
+            field_names
+        );
+        assert!(
+            field_names.contains(&"id"),
+            "Column 'id' should still exist, got: {:?}",
+            field_names
+        );
+    }
+
     #[tokio::test]
     async fn test_analyze_table_query_plan() {
         use lance_namespace::models::AnalyzeTableQueryPlanRequest;
@@ -9663,6 +12544,109 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_build_and_root_reads_do_not_create_manifest() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_path = std::path::Path::new(temp_path).join("__manifest");
+
+        let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(false)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        create_scalar_table(&dir_only_ns, "catalog").await;
+        assert!(!manifest_path.exists());
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        assert!(!manifest_path.exists());
+
+        let mut exists_req = TableExistsRequest::new();
+        exists_req.id = Some(vec!["catalog".to_string()]);
+        namespace.table_exists(exists_req).await.unwrap();
+        assert!(!manifest_path.exists());
+
+        let mut describe_req = DescribeTableRequest::new();
+        describe_req.id = Some(vec!["catalog".to_string()]);
+        namespace.describe_table(describe_req).await.unwrap();
+        assert!(!manifest_path.exists());
+
+        let list_response = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(list_response.tables, vec!["catalog".to_string()]);
+        assert!(!manifest_path.exists());
+
+        let mut list_namespaces_req = ListNamespacesRequest::new();
+        list_namespaces_req.id = Some(vec!["workspace".to_string()]);
+        let err = namespace
+            .list_namespaces(list_namespaces_req)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let err = namespace
+            .list_tables(ListTablesRequest {
+                id: Some(vec!["workspace".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut child_describe_req = DescribeTableRequest::new();
+        child_describe_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]);
+        let err = namespace
+            .describe_table(child_describe_req)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut child_exists_req = TableExistsRequest::new();
+        child_exists_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]);
+        let err = namespace.table_exists(child_exists_req).await.unwrap_err();
+        assert!(err.to_string().contains("__manifest"));
+        assert!(!manifest_path.exists());
+
+        let mut create_ns_req = CreateNamespaceRequest::new();
+        create_ns_req.id = Some(vec!["workspace".to_string()]);
+        namespace.create_namespace(create_ns_req).await.unwrap();
+        assert!(manifest_path.exists());
+    }
+
+    #[tokio::test]
+    async fn test_migrate_updates_read_opened_legacy_manifest() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        create_legacy_manifest_without_primary_key_metadata(temp_path).await;
+        assert!(!manifest_has_primary_key_metadata(temp_path).await);
+
+        let namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(true)
+            .build()
+            .await
+            .unwrap();
+        assert!(!manifest_has_primary_key_metadata(temp_path).await);
+
+        let migrated = namespace.migrate().await.unwrap();
+        assert_eq!(migrated, 0);
+        assert!(manifest_has_primary_key_metadata(temp_path).await);
+    }
+
     #[tokio::test]
     async fn test_describe_declared_table_checks_versions_only_when_requested() {
         let temp_dir = TempStdDir::default();
@@ -9770,13 +12754,47 @@ mod tests {
         describe_req.id = Some(vec!["test_table".to_string()]);
         hybrid_ns.describe_table(describe_req).await.unwrap();
 
-        let count = listing_count.load(Ordering::SeqCst);
-        assert_eq!(
-            count, 1,
-            "Expected exactly 1 listing call for describe_table with migration mode \
-             (table directory fallback; manifest reload uses the version hint), but got {}",
-            count
-        );
+        let count = listing_count.load(Ordering::SeqCst);
+        assert_eq!(
+            count, 1,
+            "Expected exactly 1 listing call for describe_table with migration mode \
+             (table directory fallback; manifest reload uses the version hint), but got {}",
+            count
+        );
+    }
+
+    #[tokio::test]
+    async fn test_manifest_reload_observes_new_version_from_other_namespace() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace_a = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(false)
+            .build()
+            .await
+            .unwrap();
+        create_scalar_table(&namespace_a, "alpha").await;
+
+        let namespace_b = DirectoryNamespaceBuilder::new(temp_path)
+            .manifest_enabled(true)
+            .dir_listing_enabled(false)
+            .build()
+            .await
+            .unwrap();
+        create_scalar_table(&namespace_b, "beta").await;
+
+        let response = namespace_a
+            .list_tables(ListTablesRequest {
+                id: Some(vec![]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let mut tables = response.tables;
+        tables.sort();
+        assert_eq!(tables, vec!["alpha", "beta"]);
     }
 
     #[tokio::test]
@@ -9858,56 +12876,450 @@ mod tests {
 
         assert!(versions >= 1, "versions must be at least 1");
 
-        let temp_dir = TempStdDir::default();
-        let temp_path = temp_dir.to_str().unwrap();
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let namespace = Arc::new(
+            DirectoryNamespaceBuilder::new(temp_path)
+                .build()
+                .await
+                .unwrap(),
+        );
+        let table_id = vec!["tag_table".to_string()];
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let initial_batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(initial_batch)], arrow_schema.clone());
+        let write_params = WriteParams {
+            mode: WriteMode::Create,
+            ..Default::default()
+        };
+
+        let mut dataset = Dataset::write_into_namespace(
+            batches,
+            namespace.clone() as Arc<dyn LanceNamespace>,
+            table_id.clone(),
+            Some(write_params),
+        )
+        .await
+        .unwrap();
+
+        for i in 1..versions {
+            let value_start = (i as i32) * 10;
+            let batch = RecordBatch::try_new(
+                arrow_schema.clone(),
+                vec![Arc::new(Int32Array::from(vec![
+                    value_start,
+                    value_start + 1,
+                ]))],
+            )
+            .unwrap();
+            let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
+            dataset.append(batches, None).await.unwrap();
+        }
+
+        (namespace, temp_dir, table_id)
+    }
+
+    /// Downcast a lance-core error to its NamespaceError code for precise assertions.
+    fn namespace_code(err: &Error) -> Option<ErrorCode> {
+        match err {
+            Error::Namespace { source, .. } => {
+                source.downcast_ref::<NamespaceError>().map(|e| e.code())
+            }
+            _ => None,
+        }
+    }
+
+    #[tokio::test]
+    async fn test_create_and_list_branches() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(3).await;
+
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "dev".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "staging".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let resp = namespace
+            .list_table_branches(ListTableBranchesRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(
+            resp.branches.len(),
+            2,
+            "expected 2 branches, got: {:?}",
+            resp.branches
+        );
+        assert!(resp.branches.contains_key("dev"));
+        assert!(resp.branches.contains_key("staging"));
+        assert!(resp.page_token.is_none());
+
+        // Deleting one branch is reflected in a subsequent list.
+        namespace
+            .delete_table_branch(DeleteTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "dev".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let resp = namespace
+            .list_table_branches(ListTableBranchesRequest {
+                id: Some(table_id),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(resp.branches.len(), 1, "expected 1 branch after delete");
+        assert!(!resp.branches.contains_key("dev"));
+        assert!(resp.branches.contains_key("staging"));
+    }
+
+    #[tokio::test]
+    async fn test_create_branch_from_version() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(3).await;
+
+        // Fork explicitly from version 1 of main.
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "from-v1".to_string(),
+                from_version: Some(1),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let resp = namespace
+            .list_table_branches(ListTableBranchesRequest {
+                id: Some(table_id),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        let branch = resp
+            .branches
+            .get("from-v1")
+            .expect("forked branch should be listed");
+        assert_eq!(
+            branch.parent_version, 1,
+            "branch should fork from version 1"
+        );
+        assert!(
+            branch.parent_branch.is_none(),
+            "a branch forked from main has no parent branch"
+        );
+    }
+
+    /// Forking from a NON-main source branch must clone that branch's chain.
+    /// Both chains are given a version 2 with diverged content, so a clone that
+    /// wrongly resolves the version under main succeeds silently with main's
+    /// data instead of erroring.
+    #[tokio::test]
+    async fn test_create_branch_from_other_branch() {
+        use lance::dataset::builder::DatasetBuilder;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await; // main v1: ids [1, 2, 3]
+        // dev: forked at v1, one append (ids 100, 101) -> dev v2
+        create_branch_with_commits(&namespace, "users", "dev", 1).await;
+        // Diverge main to the same version number with different content.
+        let main_ds = open_dataset(&namespace, "users").await;
+        append_scalar_version(main_ds.uri(), 500).await; // main v2: + ids [500, 501]
+
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(vec!["users".to_string()]),
+                name: "child".to_string(),
+                from_branch: Some("dev".to_string()),
+                from_version: Some(2),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let child_ds = DatasetBuilder::from_uri(main_ds.uri())
+            .with_branch("child", None)
+            .load()
+            .await
+            .unwrap();
+        let ids = scan_id_column(&child_ds).await;
+        assert!(
+            ids.contains(&100) && ids.contains(&101),
+            "child must contain dev's appended rows, got: {:?}",
+            ids
+        );
+        assert!(
+            !ids.contains(&500),
+            "child must not contain main's diverged rows, got: {:?}",
+            ids
+        );
+
+        // The recorded metadata and the cloned data must agree on the parent.
+        let listed = namespace
+            .list_table_branches(ListTableBranchesRequest {
+                id: Some(vec!["users".to_string()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(
+            listed
+                .branches
+                .get("child")
+                .unwrap()
+                .parent_branch
+                .as_deref(),
+            Some("dev")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_existing_branch_conflict() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
+
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "dev".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        let err = namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id),
+                name: "dev".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&err),
+            Some(ErrorCode::TableBranchAlreadyExists),
+            "expected TableBranchAlreadyExists, got: {}",
+            err
+        );
+        assert!(
+            err.to_string().to_lowercase().contains("already exists"),
+            "expected already-exists message, got: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_delete_unknown_branch() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
+
+        let err = namespace
+            .delete_table_branch(DeleteTableBranchRequest {
+                id: Some(table_id),
+                name: "does-not-exist".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&err),
+            Some(ErrorCode::TableBranchNotFound),
+            "expected TableBranchNotFound, got: {}",
+            err
+        );
+        assert!(
+            err.to_string().to_lowercase().contains("not found"),
+            "expected not-found message, got: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_delete_referenced_branch_conflict() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
+
+        // A child forked from `parent` (via from_branch) makes `parent` a referenced branch.
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "parent".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: "child".to_string(),
+                from_branch: Some("parent".to_string()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        // from_branch resolution: the child records its parent branch as its fork point.
+        let listed = namespace
+            .list_table_branches(ListTableBranchesRequest {
+                id: Some(table_id.clone()),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        let child = listed
+            .branches
+            .get("child")
+            .expect("child branch should be listed");
+        assert_eq!(
+            child.parent_branch.as_deref(),
+            Some("parent"),
+            "child should record parent branch as its fork point"
+        );
+        assert!(
+            child.parent_version >= 1,
+            "child should record the parent version it forked from, got {}",
+            child.parent_version
+        );
+
+        // Deleting a branch that still has dependents is refused. The delete spec has no 409,
+        // so it surfaces as a documented InvalidInput (400), not a conflict status.
+        let err = namespace
+            .delete_table_branch(DeleteTableBranchRequest {
+                id: Some(table_id),
+                name: "parent".to_string(),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&err),
+            Some(ErrorCode::InvalidInput),
+            "expected InvalidInput for deleting a referenced branch, got: {}",
+            err
+        );
+        assert!(
+            err.to_string().to_lowercase().contains("referenced"),
+            "error should explain the branch is still referenced, got: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_branch_name_required() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
+
+        let create_err = namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id.clone()),
+                name: String::new(),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&create_err),
+            Some(ErrorCode::InvalidInput),
+            "empty name on create should be InvalidInput, got: {}",
+            create_err
+        );
+        assert!(
+            create_err
+                .to_string()
+                .to_lowercase()
+                .contains("must not be empty")
+        );
+
+        let delete_err = namespace
+            .delete_table_branch(DeleteTableBranchRequest {
+                id: Some(table_id),
+                name: String::new(),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&delete_err),
+            Some(ErrorCode::InvalidInput),
+            "empty name on delete should be InvalidInput, got: {}",
+            delete_err
+        );
+        assert!(
+            delete_err
+                .to_string()
+                .to_lowercase()
+                .contains("must not be empty")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_branch_rejects_negative_from_version() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
 
-        let namespace = Arc::new(
-            DirectoryNamespaceBuilder::new(temp_path)
-                .build()
-                .await
-                .unwrap(),
+        let err = namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id),
+                name: "dev".to_string(),
+                from_version: Some(-1),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&err),
+            Some(ErrorCode::InvalidInput),
+            "negative from_version should be InvalidInput, got: {}",
+            err
         );
-        let table_id = vec!["tag_table".to_string()];
-        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
-            "id",
-            DataType::Int32,
-            false,
-        )]));
-        let initial_batch = RecordBatch::try_new(
-            arrow_schema.clone(),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-        let batches = RecordBatchIterator::new(vec![Ok(initial_batch)], arrow_schema.clone());
-        let write_params = WriteParams {
-            mode: WriteMode::Create,
-            ..Default::default()
-        };
-
-        let mut dataset = Dataset::write_into_namespace(
-            batches,
-            namespace.clone() as Arc<dyn LanceNamespace>,
-            table_id.clone(),
-            Some(write_params),
-        )
-        .await
-        .unwrap();
+        assert!(err.to_string().to_lowercase().contains("from_version"));
+    }
 
-        for i in 1..versions {
-            let value_start = (i as i32) * 10;
-            let batch = RecordBatch::try_new(
-                arrow_schema.clone(),
-                vec![Arc::new(Int32Array::from(vec![
-                    value_start,
-                    value_start + 1,
-                ]))],
-            )
-            .unwrap();
-            let batches = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema.clone());
-            dataset.append(batches, None).await.unwrap();
-        }
+    #[tokio::test]
+    async fn test_create_branch_nonexistent_from_version() {
+        let (namespace, _temp_dir, table_id) = create_tagged_test_table(2).await;
 
-        (namespace, temp_dir, table_id)
+        // Version 999 does not exist (the table has 2 versions). create_branch's clone phase
+        // raises DatasetNotFound, which we map to a documented InvalidInput (400).
+        let err = namespace
+            .create_table_branch(CreateTableBranchRequest {
+                id: Some(table_id),
+                name: "dev".to_string(),
+                from_version: Some(999),
+                ..Default::default()
+            })
+            .await
+            .unwrap_err();
+        assert_eq!(
+            namespace_code(&err),
+            Some(ErrorCode::InvalidInput),
+            "non-existent from_version should map to InvalidInput, got: {}",
+            err
+        );
+        assert!(
+            err.to_string().to_lowercase().contains("does not exist"),
+            "error should name the missing source, got: {}",
+            err
+        );
     }
 
     #[tokio::test]
@@ -9962,6 +13374,7 @@ mod tests {
         get_req.id = Some(table_id);
         let resp = namespace.get_table_tag_version(get_req).await.unwrap();
         assert_eq!(resp.version, 2);
+        assert_eq!(resp.branch, None);
     }
 
     #[tokio::test]
@@ -10087,4 +13500,390 @@ mod tests {
             err
         );
     }
+    #[tokio::test]
+    async fn test_alter_table_drop_columns_missing_id() {
+        use lance_namespace::models::AlterTableDropColumnsRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let request = AlterTableDropColumnsRequest::new(vec!["col".to_string()]);
+        let result = namespace.alter_table_drop_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[tokio::test]
+    async fn test_alter_table_drop_columns_nonexistent_table() {
+        use lance_namespace::models::AlterTableDropColumnsRequest;
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        let mut request = AlterTableDropColumnsRequest::new(vec!["col".to_string()]);
+        request.id = Some(vec!["nonexistent".to_string()]);
+        let result = namespace.alter_table_drop_columns(request).await;
+        assert!(result.is_err(), "Should fail when table does not exist");
+    }
+
+    #[tokio::test]
+    async fn test_create_branch_on_managed_dataset_succeeds() {
+        use lance::dataset::builder::DatasetBuilder;
+
+        let temp = TempStdDir::default();
+        let ns = create_managed_namespace(temp.to_str().unwrap()).await;
+        let table_id = vec!["t".to_string()];
+        let mut main = create_managed_table(&ns, &table_id).await;
+
+        let fork_version = main.version().version;
+        let branch = main
+            .create_branch("exp", fork_version, None)
+            .await
+            .expect("create_branch failed");
+        assert_eq!(branch.manifest.branch.as_deref(), Some("exp"));
+        assert_eq!(scan_id_column(&branch).await, vec![1, 2]);
+
+        let reopened = DatasetBuilder::from_namespace(ns.clone(), table_id.clone())
+            .await
+            .unwrap()
+            .with_branch("exp", None)
+            .load()
+            .await
+            .expect("reopen branch failed");
+        assert_eq!(scan_id_column(&reopened).await, vec![1, 2]);
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_set_status() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetStatus,
+            DescribeTransactionRequest,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let txn_id = create_scalar_index(&namespace, "users", "users_id_idx")
+            .await
+            .expect("create_scalar_index should return a transaction id");
+
+        // First verify the transaction exists
+        let describe_resp = namespace
+            .describe_transaction(DescribeTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(describe_resp.status, "SUCCEEDED");
+
+        // Alter the transaction status
+        let response = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                        status: Some("Canceled".to_string()),
+                    })),
+                    set_property_action: None,
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.status, "Canceled");
+        assert!(response.properties.is_some());
+        let props = response.properties.unwrap();
+        assert_eq!(props.get("uuid"), Some(&txn_id));
+        assert_eq!(props.get("operation"), Some(&"CreateIndex".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_set_property() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetProperty,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let txn_id = create_scalar_index(&namespace, "users", "users_id_idx")
+            .await
+            .expect("create_scalar_index should return a transaction id");
+
+        let response = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: None,
+                    set_property_action: Some(Box::new(AlterTransactionSetProperty {
+                        key: Some("custom_key".to_string()),
+                        value: Some("custom_value".to_string()),
+                        mode: None,
+                    })),
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.status, "SUCCEEDED");
+        let props = response.properties.unwrap();
+        assert_eq!(props.get("custom_key"), Some(&"custom_value".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_set_property_fail_mode() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetProperty,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let txn_id = create_scalar_index(&namespace, "users", "users_id_idx")
+            .await
+            .expect("create_scalar_index should return a transaction id");
+
+        // First, set a non-reserved property so it exists in the sidecar.
+        namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: None,
+                    set_property_action: Some(Box::new(AlterTransactionSetProperty {
+                        key: Some("custom_key".to_string()),
+                        value: Some("initial_value".to_string()),
+                        mode: None,
+                    })),
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        // Now try to set the same property again with Fail mode, which must
+        // exercise the mode='Fail' branch (not the reserved-key guard).
+        let result = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: None,
+                    set_property_action: Some(Box::new(AlterTransactionSetProperty {
+                        key: Some("custom_key".to_string()),
+                        value: Some("new_value".to_string()),
+                        mode: Some("Fail".to_string()),
+                    })),
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_unset_property() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetProperty,
+            AlterTransactionUnsetProperty,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let txn_id = create_scalar_index(&namespace, "users", "users_id_idx")
+            .await
+            .expect("create_scalar_index should return a transaction id");
+
+        // First set a custom property, then unset it
+        let response = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![
+                    AlterTransactionAction {
+                        set_status_action: None,
+                        set_property_action: Some(Box::new(AlterTransactionSetProperty {
+                            key: Some("temp_key".to_string()),
+                            value: Some("temp_value".to_string()),
+                            mode: None,
+                        })),
+                        unset_property_action: None,
+                    },
+                    AlterTransactionAction {
+                        set_status_action: None,
+                        set_property_action: None,
+                        unset_property_action: Some(Box::new(AlterTransactionUnsetProperty {
+                            key: Some("temp_key".to_string()),
+                            mode: None,
+                        })),
+                    },
+                ],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(response.status, "SUCCEEDED");
+        let props = response.properties.unwrap();
+        assert!(!props.contains_key("temp_key"));
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_invalid_status() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetStatus,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let txn_id = create_scalar_index(&namespace, "users", "users_id_idx")
+            .await
+            .expect("create_scalar_index should return a transaction id");
+
+        let result = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                        status: Some("InvalidStatus".to_string()),
+                    })),
+                    set_property_action: None,
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_not_found() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetStatus,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+
+        // Try to alter a non-existent transaction
+        let result = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), "non_existent_txn".to_string()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                        status: Some("Canceled".to_string()),
+                    })),
+                    set_property_action: None,
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_missing_id() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetStatus,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+
+        // Try with missing id
+        let result = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: None,
+                actions: vec![AlterTransactionAction {
+                    set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                        status: Some("Canceled".to_string()),
+                    })),
+                    set_property_action: None,
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await;
+        assert!(result.is_err());
+
+        // Try with insufficient id parts
+        let result = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string()]),
+                actions: vec![AlterTransactionAction {
+                    set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                        status: Some("Canceled".to_string()),
+                    })),
+                    set_property_action: None,
+                    unset_property_action: None,
+                }],
+                ..Default::default()
+            })
+            .await;
+        assert!(result.is_err());
+    }
+
+    #[tokio::test]
+    async fn test_alter_transaction_persists_changes() {
+        use lance_namespace::models::{
+            AlterTransactionAction, AlterTransactionRequest, AlterTransactionSetProperty,
+            AlterTransactionSetStatus, DescribeTransactionRequest,
+        };
+
+        let (namespace, _temp_dir) = create_test_namespace().await;
+        create_scalar_table(&namespace, "users").await;
+        let transaction_id = create_scalar_index(&namespace, "users", "users_id_idx").await;
+
+        let txn_id = transaction_id.expect("scalar index should produce a transaction id");
+
+        // Alter status and set a custom property.
+        namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![
+                    AlterTransactionAction {
+                        set_status_action: Some(Box::new(AlterTransactionSetStatus {
+                            status: Some("Canceled".to_string()),
+                        })),
+                        set_property_action: None,
+                        unset_property_action: None,
+                    },
+                    AlterTransactionAction {
+                        set_status_action: None,
+                        set_property_action: Some(Box::new(AlterTransactionSetProperty {
+                            key: Some("owner".to_string()),
+                            value: Some("alice".to_string()),
+                            mode: None,
+                        })),
+                        unset_property_action: None,
+                    },
+                ],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+
+        // The changes must survive across a fresh describe_transaction call,
+        // proving the alteration was persisted to the transaction file.
+        let describe_resp = namespace
+            .describe_transaction(DescribeTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        let props = describe_resp.properties.expect("properties should be set");
+        assert_eq!(props.get("owner"), Some(&"alice".to_string()));
+        // The internal `_status` marker should not leak into the response but
+        // must be present on disk so subsequent alter_transaction calls can
+        // observe the previously set status.
+        assert!(!props.contains_key("_status"));
+
+        let follow_up = namespace
+            .alter_transaction(AlterTransactionRequest {
+                id: Some(vec!["users".to_string(), txn_id.clone()]),
+                actions: vec![],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+        assert_eq!(follow_up.status, "Canceled");
+        let follow_up_props = follow_up.properties.unwrap();
+        assert_eq!(follow_up_props.get("owner"), Some(&"alice".to_string()));
+    }
 }
diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs
index 0e22f1e8b69..1e52fc85bfe 100644
--- a/rust/lance-namespace-impls/src/dir/manifest.rs
+++ b/rust/lance-namespace-impls/src/dir/manifest.rs
@@ -6,52 +6,76 @@
 //! This module provides a namespace implementation that uses a manifest table
 //! to track tables and nested namespaces.
 
+use super::manifest_feature_flags::{ensure_readable, ensure_writable};
 use arrow::array::builder::{ListBuilder, StringBuilder};
-use arrow::array::{Array, RecordBatch, RecordBatchIterator, StringArray};
-use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+use arrow::array::{Array, ListArray, RecordBatch, RecordBatchIterator, StringArray, UInt64Array};
+use arrow::datatypes::{DataType, Field, Schema as ArrowSchema, SchemaRef};
 use arrow_ipc::reader::StreamReader;
 use async_trait::async_trait;
 use bytes::Bytes;
-use futures::{FutureExt, TryStreamExt, stream::StreamExt};
-use lance::dataset::optimize::{CompactionOptions, compact_files};
+use datafusion_common::DataFusionError;
+use datafusion_physical_plan::{
+    SendableRecordBatchStream,
+    stream::RecordBatchStreamAdapter as DatafusionRecordBatchStreamAdapter,
+};
+use futures::{
+    FutureExt, TryStreamExt,
+    stream::{self, StreamExt},
+};
+use lance::dataset::index::LanceIndexStoreExt;
+use lance::dataset::transaction::{Operation, Transaction};
 use lance::dataset::{
-    DeleteBuilder, MergeInsertBuilder, ReadParams, WhenMatched, WhenNotMatched, WriteMode,
-    WriteParams, builder::DatasetBuilder,
+    InsertBuilder, ReadParams, WhenMatched, WriteMode, WriteParams, builder::DatasetBuilder,
 };
-use lance::index::DatasetIndexExt;
 use lance::session::Session;
 use lance::{Dataset, dataset::scanner::Scanner};
 use lance_core::Error as LanceError;
 use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION;
-use lance_core::{Error, Result};
-use lance_index::IndexType;
-use lance_index::optimize::OptimizeOptions;
-use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+use lance_core::{Error, ROW_ID, Result, box_error};
+use lance_index::progress::noop_progress;
+use lance_index::registry::IndexPluginRegistry;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_index::scalar::registry::VALUE_COLUMN_NAME;
+use lance_index::scalar::{
+    BuiltinIndexType, CreatedIndex, ScalarIndexParams, index_files_to_table,
+};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams};
+use lance_io::stream::RecordBatchStream as LanceRecordBatchStream;
 use lance_namespace::LanceNamespace;
 use lance_namespace::error::NamespaceError;
 use lance_namespace::models::{
+    AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest,
+    AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse,
     CreateNamespaceRequest, CreateNamespaceResponse, CreateTableRequest, CreateTableResponse,
     DeclareTableRequest, DeclareTableResponse, DeregisterTableRequest, DeregisterTableResponse,
     DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
-    DescribeTableResponse, DescribeTableVersionResponse, DropNamespaceRequest,
-    DropNamespaceResponse, DropTableRequest, DropTableResponse, ListNamespacesRequest,
-    ListNamespacesResponse, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse,
-    NamespaceExistsRequest, RegisterTableRequest, RegisterTableResponse, TableExistsRequest,
-    TableVersion,
+    DescribeTableResponse, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest,
+    DropTableResponse, ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest,
+    ListTablesResponse, NamespaceExistsRequest, RegisterTableRequest, RegisterTableResponse,
+    TableExistsRequest,
 };
 use lance_namespace::schema::arrow_schema_to_json;
+use lance_table::feature_flags::apply_feature_flags;
+use lance_table::format::{Fragment, IndexMetadata, Manifest};
+use lance_table::io::commit::{
+    CommitError, CommitHandler, commit_handler_from_url, write_manifest_file_to_path,
+};
 use object_store::{Error as ObjectStoreError, path::Path};
+use roaring::RoaringBitmap;
 use std::io::Cursor;
+use std::time::{SystemTime, UNIX_EPOCH};
 use std::{
-    collections::HashMap,
+    collections::{BTreeMap, HashMap, HashSet},
     hash::{DefaultHasher, Hash, Hasher},
     ops::{Deref, DerefMut},
-    sync::Arc,
+    sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard},
 };
 use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
+use uuid::Uuid;
 
 const MANIFEST_TABLE_NAME: &str = "__manifest";
+const LANCE_DATA_DIR: &str = "data";
+const LANCE_INDICES_DIR: &str = "_indices";
 const DELIMITER: &str = "$";
 /// Bounded concurrency for per-table `_versions/` probes when filtering declared tables.
 /// Higher values reduce latency but increase burst load against the object store.
@@ -64,24 +88,23 @@ const OBJECT_ID_INDEX_NAME: &str = "object_id_btree";
 const OBJECT_TYPE_INDEX_NAME: &str = "object_type_bitmap";
 /// LabelList index on the base_objects column for view dependencies
 const BASE_OBJECTS_INDEX_NAME: &str = "base_objects_label_list";
-/// Inline maintenance on the manifest table is expensive relative to a single-row mutation.
-/// Wait until enough fragments accumulate before compacting files or merging indices.
-const MANIFEST_INLINE_OPTIMIZATION_FRAGMENT_THRESHOLD: usize = 8;
+// Each retry reloads and rewrites the full manifest. Match the regular Lance
+// commit retry budget so multi-process namespace writes can make progress.
+const DEFAULT_MANIFEST_REWRITE_COMMIT_RETRIES: u32 = 20;
+const MANIFEST_INDEX_BATCH_SIZE: usize = 8192;
 
 /// Object types that can be stored in the manifest
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ObjectType {
     Namespace,
     Table,
-    TableVersion,
 }
 
 impl ObjectType {
-    pub fn as_str(&self) -> &str {
+    pub fn as_str(&self) -> &'static str {
         match self {
             Self::Namespace => "namespace",
             Self::Table => "table",
-            Self::TableVersion => "table_version",
         }
     }
 
@@ -89,7 +112,6 @@ impl ObjectType {
         match s {
             "namespace" => Ok(Self::Namespace),
             "table" => Ok(Self::Table),
-            "table_version" => Ok(Self::TableVersion),
             _ => Err(NamespaceError::Internal {
                 message: format!("Invalid object type: {}", s),
             }
@@ -152,7 +174,7 @@ pub struct TableInfo {
 pub struct ManifestEntry {
     /// The unique object identifier (e.g., table name or version object_id)
     pub object_id: String,
-    /// The type of the object (Namespace, Table, or TableVersion)
+    /// The type of the object (Namespace or Table)
     pub object_type: ObjectType,
     /// The storage location (e.g., directory name for tables)
     pub location: Option<String>,
@@ -160,6 +182,401 @@ pub struct ManifestEntry {
     pub metadata: Option<String>,
 }
 
+struct CopyOnWriteMutation<T> {
+    result: T,
+    has_changes: bool,
+}
+
+impl<T> CopyOnWriteMutation<T> {
+    fn updated(result: T) -> Self {
+        Self {
+            result,
+            has_changes: true,
+        }
+    }
+
+    fn unchanged(result: T) -> Self {
+        Self {
+            result,
+            has_changes: false,
+        }
+    }
+}
+
+struct ManifestIndexBuildInput {
+    index_name: &'static str,
+    column_name: &'static str,
+    params: ScalarIndexParams,
+    field: Field,
+    stream: SendableRecordBatchStream,
+}
+
+struct ManifestTrainedIndex {
+    index_name: &'static str,
+    column_name: &'static str,
+    uuid: Uuid,
+    created_index: CreatedIndex,
+}
+
+struct ManifestRowValue {
+    object_id: String,
+    object_type: ObjectType,
+    location: Option<String>,
+    metadata: Option<String>,
+    base_objects: Option<Vec<String>>,
+}
+
+struct ManifestOutputRow<'a> {
+    object_id: &'a str,
+    object_type: ObjectType,
+    location: Option<&'a str>,
+    metadata: Option<&'a str>,
+    base_objects: Option<&'a [String]>,
+}
+
+#[derive(Default)]
+struct ManifestIndexAccumulator {
+    object_ids: BTreeMap<Arc<str>, u64>,
+    object_types: BTreeMap<&'static str, RoaringBitmap>,
+    base_objects_values: Vec<Option<Vec<String>>>,
+    base_objects_row_ids: Vec<u64>,
+    row_count: u64,
+}
+
+impl ManifestIndexAccumulator {
+    fn next_row_id(&self) -> Result<u64> {
+        if self.row_count >= u64::from(u32::MAX) {
+            return Err(NamespaceError::Internal {
+                message: format!(
+                    "Manifest rewrite exceeded maximum single-fragment row count: {}",
+                    self.row_count
+                ),
+            }
+            .into());
+        }
+        Ok(self.row_count)
+    }
+
+    fn push(&mut self, row: &ManifestOutputRow<'_>) -> Result<u64> {
+        let row_id = self.next_row_id()?;
+        if self
+            .object_ids
+            .insert(Arc::<str>::from(row.object_id), row_id)
+            .is_some()
+        {
+            return Err(NamespaceError::Internal {
+                message: format!("Manifest contains duplicate object_id '{}'", row.object_id),
+            }
+            .into());
+        }
+        self.object_types
+            .entry(row.object_type.as_str())
+            .or_default()
+            .insert(row_id as u32);
+        self.base_objects_values
+            .push(row.base_objects.map(|objects| objects.to_vec()));
+        self.base_objects_row_ids.push(row_id);
+        self.row_count += 1;
+        Ok(row_id)
+    }
+}
+
+struct ManifestBatchBuilder {
+    object_ids: Vec<String>,
+    object_types: Vec<&'static str>,
+    locations: Vec<Option<String>>,
+    metadatas: Vec<Option<String>>,
+    base_objects: Vec<Option<Vec<String>>>,
+}
+
+impl ManifestBatchBuilder {
+    fn new() -> Self {
+        Self {
+            object_ids: Vec::new(),
+            object_types: Vec::new(),
+            locations: Vec::new(),
+            metadatas: Vec::new(),
+            base_objects: Vec::new(),
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.object_ids.is_empty()
+    }
+
+    fn append(
+        &mut self,
+        index_data: &mut ManifestIndexAccumulator,
+        row: ManifestOutputRow<'_>,
+    ) -> Result<()> {
+        index_data.push(&row)?;
+        self.object_ids.push(row.object_id.to_string());
+        self.object_types.push(row.object_type.as_str());
+        self.locations.push(row.location.map(ToString::to_string));
+        self.metadatas.push(row.metadata.map(ToString::to_string));
+        self.base_objects
+            .push(row.base_objects.map(|objects| objects.to_vec()));
+        Ok(())
+    }
+
+    fn finish(self) -> Result<RecordBatch> {
+        let base_objects_array = ManifestNamespace::base_objects_array(&self.base_objects);
+        RecordBatch::try_new(
+            ManifestNamespace::manifest_schema(),
+            vec![
+                Arc::new(StringArray::from(self.object_ids)),
+                Arc::new(StringArray::from(self.object_types)),
+                Arc::new(StringArray::from(self.locations)),
+                Arc::new(StringArray::from(self.metadatas)),
+                Arc::new(base_objects_array),
+            ],
+        )
+        .map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to create manifest snapshot batch: {:?}", e),
+            })
+        })
+    }
+}
+
+/// How to resolve a storage commit conflict (or an ambiguous commit error that did
+/// not land) against the latest catalog state, without re-staging the full rewrite.
+enum ConflictResolution<O> {
+    /// Re-read the latest manifest and re-apply the mutation (upserts, version-range
+    /// deletes). The staged data/index files are discarded and a new rewrite is attempted.
+    Retry,
+    /// Creating these object ids with fail-on-conflict semantics. If any of them now
+    /// exists in the latest manifest, the create lost the race and must fail with a
+    /// concurrent-modification error; otherwise retry the rewrite.
+    FailIfExists(Vec<String>),
+    /// Deleting `object_id`. If it is already absent from the latest manifest the delete
+    /// has effectively happened, so return `output` as success; otherwise retry.
+    SucceedIfAbsent { object_id: String, output: O },
+}
+
+trait ManifestStreamMutation: Send {
+    type Output: Clone + Send + 'static;
+
+    fn process_existing_row(
+        &mut self,
+        row: ManifestRowValue,
+        output: &mut ManifestBatchBuilder,
+        index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()>;
+
+    fn append_rows(
+        &mut self,
+        output: &mut ManifestBatchBuilder,
+        index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()>;
+
+    fn finish(&self) -> CopyOnWriteMutation<Self::Output>;
+
+    /// Declares how a storage commit conflict should be resolved against the latest
+    /// committed catalog state. Defaults to re-reading and re-applying.
+    fn conflict_resolution(&self) -> ConflictResolution<Self::Output> {
+        ConflictResolution::Retry
+    }
+}
+
+struct ManifestRewriteShared<M: ManifestStreamMutation> {
+    mutation: M,
+    index_data: Option<ManifestIndexAccumulator>,
+    result: Option<CopyOnWriteMutation<M::Output>>,
+    error: Option<LanceError>,
+}
+
+impl<M: ManifestStreamMutation> ManifestRewriteShared<M> {
+    fn new(mutation: M) -> Self {
+        Self {
+            mutation,
+            index_data: Some(ManifestIndexAccumulator::default()),
+            result: None,
+            error: None,
+        }
+    }
+}
+
+struct UpsertManifestMutation {
+    entries: Vec<ManifestEntry>,
+    base_objects: Vec<Option<Vec<String>>>,
+    entry_positions: HashMap<String, usize>,
+    matched: Vec<bool>,
+    when_matched: WhenMatched,
+}
+
+impl UpsertManifestMutation {
+    fn new(
+        entries: Vec<ManifestEntry>,
+        base_objects: Option<Vec<String>>,
+        when_matched: WhenMatched,
+    ) -> Self {
+        let entry_positions = entries
+            .iter()
+            .enumerate()
+            .map(|(index, entry)| (entry.object_id.clone(), index))
+            .collect();
+        let matched = vec![false; entries.len()];
+        let mut entry_base_objects = vec![None; entries.len()];
+        if !entry_base_objects.is_empty() {
+            entry_base_objects[0] = base_objects;
+        }
+        Self {
+            entries,
+            base_objects: entry_base_objects,
+            entry_positions,
+            matched,
+            when_matched,
+        }
+    }
+
+    fn entry_row(&self, index: usize) -> ManifestOutputRow<'_> {
+        let entry = &self.entries[index];
+        ManifestOutputRow {
+            object_id: &entry.object_id,
+            object_type: entry.object_type,
+            location: entry.location.as_deref(),
+            metadata: entry.metadata.as_deref(),
+            base_objects: self.base_objects[index].as_deref(),
+        }
+    }
+}
+
+impl ManifestStreamMutation for UpsertManifestMutation {
+    type Output = ();
+
+    fn process_existing_row(
+        &mut self,
+        row: ManifestRowValue,
+        output: &mut ManifestBatchBuilder,
+        index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()> {
+        if let Some(index) = self.entry_positions.get(&row.object_id).copied() {
+            match self.when_matched {
+                WhenMatched::Fail => {
+                    return Err(NamespaceError::ConcurrentModification {
+                        message: format!(
+                            "Object '{}' was concurrently created by another operation",
+                            row.object_id
+                        ),
+                    }
+                    .into());
+                }
+                WhenMatched::UpdateAll => {
+                    self.matched[index] = true;
+                    output.append(index_data, self.entry_row(index))?;
+                    return Ok(());
+                }
+                _ => {
+                    return Err(NamespaceError::Internal {
+                        message: format!(
+                            "Unsupported manifest rewrite matched action: {:?}",
+                            self.when_matched
+                        ),
+                    }
+                    .into());
+                }
+            }
+        }
+
+        output.append(
+            index_data,
+            ManifestOutputRow {
+                object_id: &row.object_id,
+                object_type: row.object_type,
+                location: row.location.as_deref(),
+                metadata: row.metadata.as_deref(),
+                base_objects: row.base_objects.as_deref(),
+            },
+        )
+    }
+
+    fn append_rows(
+        &mut self,
+        output: &mut ManifestBatchBuilder,
+        index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()> {
+        for index in 0..self.entries.len() {
+            if !self.matched[index] {
+                output.append(index_data, self.entry_row(index))?;
+            }
+        }
+        Ok(())
+    }
+
+    fn finish(&self) -> CopyOnWriteMutation<Self::Output> {
+        CopyOnWriteMutation::updated(())
+    }
+
+    fn conflict_resolution(&self) -> ConflictResolution<Self::Output> {
+        match self.when_matched {
+            // Fail-on-conflict create: a concurrent writer may have created one of these
+            // ids. Re-applying would still fail, so check directly instead of re-staging.
+            WhenMatched::Fail => ConflictResolution::FailIfExists(
+                self.entries.iter().map(|e| e.object_id.clone()).collect(),
+            ),
+            // Metadata upsert is last-writer-wins: re-read and re-apply.
+            _ => ConflictResolution::Retry,
+        }
+    }
+}
+
+struct DeleteObjectMutation {
+    object_id: String,
+    deleted: bool,
+}
+
+impl ManifestStreamMutation for DeleteObjectMutation {
+    type Output = ();
+
+    fn process_existing_row(
+        &mut self,
+        row: ManifestRowValue,
+        output: &mut ManifestBatchBuilder,
+        index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()> {
+        if row.object_id == self.object_id {
+            self.deleted = true;
+            return Ok(());
+        }
+
+        output.append(
+            index_data,
+            ManifestOutputRow {
+                object_id: &row.object_id,
+                object_type: row.object_type,
+                location: row.location.as_deref(),
+                metadata: row.metadata.as_deref(),
+                base_objects: row.base_objects.as_deref(),
+            },
+        )
+    }
+
+    fn append_rows(
+        &mut self,
+        _output: &mut ManifestBatchBuilder,
+        _index_data: &mut ManifestIndexAccumulator,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    fn finish(&self) -> CopyOnWriteMutation<Self::Output> {
+        if self.deleted {
+            CopyOnWriteMutation::updated(())
+        } else {
+            CopyOnWriteMutation::unchanged(())
+        }
+    }
+
+    fn conflict_resolution(&self) -> ConflictResolution<Self::Output> {
+        // If a concurrent writer already removed the object, the delete is satisfied.
+        ConflictResolution::SucceedIfAbsent {
+            object_id: self.object_id.clone(),
+            output: (),
+        }
+    }
+}
+
 /// Information about a namespace stored in the manifest
 #[derive(Debug, Clone)]
 pub struct NamespaceInfo {
@@ -171,13 +588,23 @@ pub struct NamespaceInfo {
 /// A wrapper around a Dataset that provides concurrent access.
 ///
 /// This can be cloned cheaply. It supports concurrent reads or exclusive writes.
-/// The manifest dataset is always kept strongly consistent by reloading on each read.
+/// The manifest dataset uses contiguous attached versions and this module never
+/// runs old-version cleanup on it, allowing reads to check only the immediate
+/// successor manifest before deciding whether a reload is needed.
 #[derive(Debug, Clone)]
 pub struct DatasetConsistencyWrapper(Arc<RwLock<Dataset>>);
 
 impl DatasetConsistencyWrapper {
     /// Create a new wrapper with the given dataset.
     pub fn new(dataset: Dataset) -> Self {
+        debug_assert!(
+            !dataset
+                .manifest()
+                .config
+                .keys()
+                .any(|key| key.starts_with("lance.auto_cleanup.")),
+            "the directory manifest dataset must not enable old-version cleanup"
+        );
         Self(Arc::new(RwLock::new(dataset)))
     }
 
@@ -185,18 +612,35 @@ impl DatasetConsistencyWrapper {
     /// Always reloads to ensure strong consistency.
     pub async fn get(&self) -> Result<DatasetReadGuard<'_>> {
         self.reload().await?;
-        Ok(DatasetReadGuard {
+        let guard = DatasetReadGuard {
             guard: self.0.read().await,
-        })
+        };
+        // Refuse manifests written with a reader feature flag this build does
+        // not understand instead of misreading them.
+        ensure_readable(guard.metadata())?;
+        Ok(guard)
+    }
+
+    /// Reload the dataset and return a reference.
+    pub async fn get_refreshed(&self) -> Result<DatasetReadGuard<'_>> {
+        self.reload().await?;
+        let guard = DatasetReadGuard {
+            guard: self.0.read().await,
+        };
+        ensure_readable(guard.metadata())?;
+        Ok(guard)
     }
 
     /// Get a mutable reference to the dataset.
     /// Always reloads to ensure strong consistency.
     pub async fn get_mut(&self) -> Result<DatasetWriteGuard<'_>> {
         self.reload().await?;
-        Ok(DatasetWriteGuard {
+        let guard = DatasetWriteGuard {
             guard: self.0.write().await,
-        })
+        };
+        ensure_readable(guard.metadata())?;
+        ensure_writable(guard.metadata())?;
+        Ok(guard)
     }
 
     /// Provide a known latest version of the dataset.
@@ -221,21 +665,25 @@ impl DatasetConsistencyWrapper {
             dataset_uri,
             current_version
         );
-        let latest_version = read_guard.latest_version_id().await.map_err(|e| {
+        // The directory manifest table uses contiguous attached versions and
+        // does not run old-version cleanup, so the immediate successor probe is
+        // enough to detect changes without resolving or loading the latest
+        // manifest on every namespace read.
+        let has_successor_version = read_guard.has_successor_version().await.map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to get latest version: {:?}", e),
+                message: format!("Failed to check dataset staleness: {:?}", e),
             })
         })?;
         log::debug!(
-            "Reload got latest_version={} for uri={}, current_version={}",
-            latest_version,
+            "Reload checked successor_version_exists={} for uri={}, current_version={}",
+            has_successor_version,
             dataset_uri,
             current_version
         );
         drop(read_guard);
 
         // If already up-to-date, return early
-        if latest_version == current_version {
+        if !has_successor_version {
             log::debug!("Already up-to-date for uri={}", dataset_uri);
             return Ok(());
         }
@@ -244,13 +692,13 @@ impl DatasetConsistencyWrapper {
         let mut write_guard = self.0.write().await;
 
         // Double-check after acquiring write lock (someone else might have reloaded)
-        let latest_version = write_guard.latest_version_id().await.map_err(|e| {
+        let has_successor_version = write_guard.has_successor_version().await.map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to get latest version: {:?}", e),
+                message: format!("Failed to check dataset staleness: {:?}", e),
             })
         })?;
 
-        if latest_version != write_guard.version().version {
+        if has_successor_version {
             write_guard.checkout_latest().await.map_err(|e| {
                 lance_core::Error::from(NamespaceError::Internal {
                     message: format!("Failed to checkout latest: {:?}", e),
@@ -306,8 +754,8 @@ pub struct ManifestNamespace {
     /// If true, root namespace tables use {table_name}.lance naming
     /// If false, they use namespace-prefixed names
     dir_listing_enabled: bool,
-    /// Whether to perform inline optimization (compaction and indexing) on the __manifest table
-    /// after every write. Defaults to true.
+    /// Whether copy-on-write manifest rewrites should build replacement indices.
+    /// Defaults to true.
     inline_optimization_enabled: bool,
     /// Number of retries for commit operations on the manifest table.
     /// If None, defaults to [`lance_table::io::commit::CommitConfig`] default (20).
@@ -401,17 +849,65 @@ impl ManifestNamespace {
         dir_listing_enabled: bool,
         inline_optimization_enabled: bool,
         commit_retries: Option<u32>,
-        table_version_storage_enabled: bool,
     ) -> Result<Self> {
-        let manifest_dataset = Self::ensure_manifest_table_up_to_date(
-            &root,
-            &storage_options,
-            session.clone(),
-            table_version_storage_enabled,
-        )
-        .await?;
+        let manifest_dataset =
+            Self::ensure_manifest_table_up_to_date(&root, &storage_options, session.clone())
+                .await?;
+
+        Ok(Self::new(
+            root,
+            storage_options,
+            session,
+            object_store,
+            base_path,
+            manifest_dataset,
+            dir_listing_enabled,
+            inline_optimization_enabled,
+            commit_retries,
+        ))
+    }
+
+    /// Open an existing manifest dataset without creating or migrating it.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn open_from_directory(
+        root: String,
+        storage_options: Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+        object_store: Arc<ObjectStore>,
+        base_path: Path,
+        dir_listing_enabled: bool,
+        inline_optimization_enabled: bool,
+        commit_retries: Option<u32>,
+    ) -> Result<Self> {
+        let manifest_dataset =
+            Self::open_manifest_table(&root, &storage_options, session.clone()).await?;
+
+        Ok(Self::new(
+            root,
+            storage_options,
+            session,
+            object_store,
+            base_path,
+            manifest_dataset,
+            dir_listing_enabled,
+            inline_optimization_enabled,
+            commit_retries,
+        ))
+    }
 
-        Ok(Self {
+    #[allow(clippy::too_many_arguments)]
+    fn new(
+        root: String,
+        storage_options: Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+        object_store: Arc<ObjectStore>,
+        base_path: Path,
+        manifest_dataset: DatasetConsistencyWrapper,
+        dir_listing_enabled: bool,
+        inline_optimization_enabled: bool,
+        commit_retries: Option<u32>,
+    ) -> Self {
+        Self {
             root,
             storage_options,
             session,
@@ -422,7 +918,7 @@ impl ManifestNamespace {
             inline_optimization_enabled,
             commit_retries,
             manifest_mutation_lock: Arc::new(Mutex::new(())),
-        })
+        }
     }
 
     /// Build object ID from namespace path and name
@@ -473,34 +969,6 @@ impl ManifestNamespace {
         format!("table id '{}'", Self::str_object_id(table_id))
     }
 
-    /// Format a version number as a zero-padded lexicographically sortable string.
-    ///
-    /// Versions are stored as 20-digit zero-padded integers (e.g., `00000000000000000001`
-    /// for version 1) so that string-based range queries and sorting work correctly.
-    pub fn format_table_version(version: i64) -> String {
-        format!("{:020}", version)
-    }
-
-    /// Build the object_id for a table version entry.
-    ///
-    /// Format: `{table_object_id}${zero_padded_version}`
-    pub fn build_version_object_id(table_object_id: &str, version: i64) -> String {
-        format!(
-            "{}{}{}",
-            table_object_id,
-            DELIMITER,
-            Self::format_table_version(version)
-        )
-    }
-
-    /// Parse a version number from the version suffix of a table version object_id.
-    ///
-    /// The object_id is formatted as `{table_id}${zero_padded_version}`.
-    pub fn parse_version_from_object_id(object_id: &str) -> Option<i64> {
-        let (_namespace, name) = Self::parse_object_id(object_id);
-        name.parse::<i64>().ok()
-    }
-
     /// Generate a new directory name in format: `<hash>_<object_id>`
     /// The hash is used to (1) optimize object store throughput,
     /// (2) have high enough entropy in a short period of time to prevent issues like
@@ -556,165 +1024,400 @@ impl ManifestNamespace {
         Ok(full_url.to_string())
     }
 
-    /// Perform inline optimization on the __manifest table.
-    ///
-    /// This method:
-    /// 1. Creates three indexes on the manifest table:
-    ///    - BTREE index on object_id for fast lookups
-    ///    - Bitmap index on object_type for filtering by type
-    ///    - LabelList index on base_objects for view dependencies
-    /// 2. Runs file compaction to merge small files
-    /// 3. Optimizes existing indices
-    ///
-    /// This is called automatically after writes when inline_optimization_enabled is true.
-    async fn run_inline_optimization(&self) -> Result<()> {
-        if !self.inline_optimization_enabled {
-            return Ok(());
+    fn string_list_array(values: &[Option<Vec<String>>], child_name: &str) -> ListArray {
+        let string_builder = StringBuilder::new();
+        let mut list_builder = ListBuilder::new(string_builder).with_field(Arc::new(Field::new(
+            child_name,
+            DataType::Utf8,
+            true,
+        )));
+        for value in values {
+            match value {
+                Some(objects) => {
+                    for object in objects {
+                        list_builder.values().append_value(object);
+                    }
+                    list_builder.append(true);
+                }
+                None => list_builder.append_null(),
+            }
         }
+        list_builder.finish()
+    }
 
-        // Get a mutable reference to the dataset to perform optimization
-        let mut dataset_guard = self.manifest_dataset.get_mut().await?;
-        let dataset: &mut Dataset = &mut dataset_guard;
+    fn base_objects_array(values: &[Option<Vec<String>>]) -> ListArray {
+        Self::string_list_array(values, "object_id")
+    }
 
-        // Step 1: Create indexes if they don't already exist
-        let indices = dataset.load_indices().await?;
+    fn value_row_id_schema(value_field: Field) -> SchemaRef {
+        Arc::new(ArrowSchema::new(vec![
+            value_field,
+            Field::new(ROW_ID, DataType::UInt64, false),
+        ]))
+    }
 
-        // Check which indexes already exist
-        let has_object_id_index = indices.iter().any(|idx| idx.name == OBJECT_ID_INDEX_NAME);
-        let has_object_type_index = indices.iter().any(|idx| idx.name == OBJECT_TYPE_INDEX_NAME);
-        let has_base_objects_index = indices
-            .iter()
-            .any(|idx| idx.name == BASE_OBJECTS_INDEX_NAME);
+    fn string_row_id_batch(
+        schema: SchemaRef,
+        values: Vec<String>,
+        row_ids: Vec<u64>,
+    ) -> Result<RecordBatch> {
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(StringArray::from(values)),
+                Arc::new(UInt64Array::from(row_ids)),
+            ],
+        )
+        .map_err(Into::into)
+    }
 
-        // Create BTREE index on object_id
-        if !has_object_id_index {
-            log::debug!(
-                "Creating BTREE index '{}' on object_id for __manifest table",
-                OBJECT_ID_INDEX_NAME
-            );
-            let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
-            if let Err(e) = dataset
-                .create_index(
-                    &["object_id"],
-                    IndexType::BTree,
-                    Some(OBJECT_ID_INDEX_NAME.to_string()),
-                    &params,
-                    true,
-                )
-                .await
-            {
-                log::warn!(
-                    "Failed to create BTREE index on object_id for __manifest table: {:?}. Query performance may be impacted.",
-                    e
-                );
-            } else {
-                log::info!(
-                    "Created BTREE index '{}' on object_id for __manifest table",
-                    OBJECT_ID_INDEX_NAME
-                );
-            }
-        }
+    fn list_row_id_batch(
+        schema: SchemaRef,
+        values: Vec<Option<Vec<String>>>,
+        row_ids: Vec<u64>,
+    ) -> Result<RecordBatch> {
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Self::string_list_array(&values, "item")),
+                Arc::new(UInt64Array::from(row_ids)),
+            ],
+        )
+        .map_err(Into::into)
+    }
 
-        // Create Bitmap index on object_type
-        if !has_object_type_index {
-            log::debug!(
-                "Creating Bitmap index '{}' on object_type for __manifest table",
-                OBJECT_TYPE_INDEX_NAME
-            );
-            let params = ScalarIndexParams::default();
-            if let Err(e) = dataset
-                .create_index(
-                    &["object_type"],
-                    IndexType::Bitmap,
-                    Some(OBJECT_TYPE_INDEX_NAME.to_string()),
-                    &params,
-                    true,
-                )
-                .await
-            {
-                log::warn!(
-                    "Failed to create Bitmap index on object_type for __manifest table: {:?}. Query performance may be impacted.",
-                    e
-                );
-            } else {
-                log::info!(
-                    "Created Bitmap index '{}' on object_type for __manifest table",
-                    OBJECT_TYPE_INDEX_NAME
-                );
-            }
-        }
+    fn object_id_index_stream(object_ids: BTreeMap<Arc<str>, u64>) -> SendableRecordBatchStream {
+        let schema =
+            Self::value_row_id_schema(Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false));
+        let stream_schema = schema.clone();
+        let stream = stream::unfold(
+            (object_ids.into_iter(), false, schema),
+            |(mut iter, emitted, schema)| async move {
+                let mut values = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                let mut row_ids = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                for _ in 0..MANIFEST_INDEX_BATCH_SIZE {
+                    let Some((value, row_id)) = iter.next() else {
+                        break;
+                    };
+                    values.push(value.to_string());
+                    row_ids.push(row_id);
+                }
+                if values.is_empty() {
+                    if emitted {
+                        None
+                    } else {
+                        let batch = Self::string_row_id_batch(schema.clone(), values, row_ids)
+                            .map_err(|err| DataFusionError::External(Box::new(err)));
+                        Some((batch, (iter, true, schema)))
+                    }
+                } else {
+                    let batch = Self::string_row_id_batch(schema.clone(), values, row_ids)
+                        .map_err(|err| DataFusionError::External(Box::new(err)));
+                    Some((batch, (iter, true, schema)))
+                }
+            },
+        );
+        Box::pin(DatafusionRecordBatchStreamAdapter::new(
+            stream_schema,
+            stream.fuse(),
+        ))
+    }
 
-        // Create LabelList index on base_objects
-        if !has_base_objects_index {
-            log::debug!(
-                "Creating LabelList index '{}' on base_objects for __manifest table",
-                BASE_OBJECTS_INDEX_NAME
-            );
-            let params = ScalarIndexParams::default();
-            if let Err(e) = dataset
-                .create_index(
-                    &["base_objects"],
-                    IndexType::LabelList,
-                    Some(BASE_OBJECTS_INDEX_NAME.to_string()),
-                    &params,
-                    true,
+    fn object_type_index_stream(
+        object_types: BTreeMap<&'static str, RoaringBitmap>,
+    ) -> SendableRecordBatchStream {
+        let schema =
+            Self::value_row_id_schema(Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false));
+        let stream_schema = schema.clone();
+        let entries = object_types
+            .into_iter()
+            .map(|(value, bitmap)| {
+                (
+                    value,
+                    Box::new(bitmap.into_iter()) as Box<dyn Iterator<Item = u32> + Send>,
                 )
-                .await
-            {
-                log::warn!(
-                    "Failed to create LabelList index on base_objects for __manifest table: {:?}. Query performance may be impacted.",
-                    e
-                );
-            } else {
-                log::info!(
-                    "Created LabelList index '{}' on base_objects for __manifest table",
-                    BASE_OBJECTS_INDEX_NAME
-                );
-            }
-        }
+            })
+            .collect::<Vec<_>>()
+            .into_iter();
+        let stream = stream::unfold(
+            (entries, None, false, schema),
+            |(mut entries, mut current, emitted, schema)| async move {
+                let mut values = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                let mut row_ids = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                while values.len() < MANIFEST_INDEX_BATCH_SIZE {
+                    if current.is_none() {
+                        current = entries.next();
+                    }
+                    let Some((value, iter)) = current.as_mut() else {
+                        break;
+                    };
+                    if let Some(row_id) = iter.next() {
+                        values.push((*value).to_string());
+                        row_ids.push(u64::from(row_id));
+                    } else {
+                        current = None;
+                    }
+                }
 
-        let should_compact_and_optimize =
-            dataset.count_fragments() >= MANIFEST_INLINE_OPTIMIZATION_FRAGMENT_THRESHOLD;
+                if values.is_empty() {
+                    if emitted {
+                        None
+                    } else {
+                        let batch = Self::string_row_id_batch(schema.clone(), values, row_ids)
+                            .map_err(|err| DataFusionError::External(Box::new(err)));
+                        Some((batch, (entries, current, true, schema)))
+                    }
+                } else {
+                    let batch = Self::string_row_id_batch(schema.clone(), values, row_ids)
+                        .map_err(|err| DataFusionError::External(Box::new(err)));
+                    Some((batch, (entries, current, true, schema)))
+                }
+            },
+        );
+        Box::pin(DatafusionRecordBatchStreamAdapter::new(
+            stream_schema,
+            stream.fuse(),
+        ))
+    }
 
-        if !should_compact_and_optimize {
-            return Ok(());
+    fn base_objects_index_stream(
+        base_objects_values: Vec<Option<Vec<String>>>,
+        base_objects_row_ids: Vec<u64>,
+    ) -> SendableRecordBatchStream {
+        let schema = Self::value_row_id_schema(Field::new(
+            VALUE_COLUMN_NAME,
+            DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
+            true,
+        ));
+        let stream_schema = schema.clone();
+        let stream = stream::unfold(
+            (
+                base_objects_values.into_iter().zip(base_objects_row_ids),
+                false,
+                schema,
+            ),
+            |(mut iter, emitted, schema)| async move {
+                let mut values = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                let mut row_ids = Vec::with_capacity(MANIFEST_INDEX_BATCH_SIZE);
+                for _ in 0..MANIFEST_INDEX_BATCH_SIZE {
+                    let Some((value, row_id)) = iter.next() else {
+                        break;
+                    };
+                    values.push(value);
+                    row_ids.push(row_id);
+                }
+                if values.is_empty() {
+                    if emitted {
+                        None
+                    } else {
+                        let batch = Self::list_row_id_batch(schema.clone(), values, row_ids)
+                            .map_err(|err| DataFusionError::External(Box::new(err)));
+                        Some((batch, (iter, true, schema)))
+                    }
+                } else {
+                    let batch = Self::list_row_id_batch(schema.clone(), values, row_ids)
+                        .map_err(|err| DataFusionError::External(Box::new(err)));
+                    Some((batch, (iter, true, schema)))
+                }
+            },
+        );
+        Box::pin(DatafusionRecordBatchStreamAdapter::new(
+            stream_schema,
+            stream.fuse(),
+        ))
+    }
+
+    async fn train_manifest_index(
+        dataset: &Dataset,
+        registry: Arc<IndexPluginRegistry>,
+        input: ManifestIndexBuildInput,
+        index_uuid: Uuid,
+    ) -> Result<ManifestTrainedIndex> {
+        let index_store = LanceIndexStore::from_dataset_for_new(dataset, &index_uuid)?;
+        let trainer = registry
+            .get_plugin_by_name(&input.params.index_type)?
+            .basic_trainer()
+            .ok_or_else(|| {
+                lance_core::Error::invalid_input_source(
+                    format!(
+                        "The '{}' index type does not support basic training, please refer to the index's documentation for more details on how to create this index.",
+                        input.params.index_type
+                    )
+                    .into(),
+                )
+            })?;
+        let training_request = trainer
+            .new_training_request(input.params.params.as_deref().unwrap_or("{}"), &input.field)?;
+        let created_index = trainer
+            .train_index(
+                input.stream,
+                &index_store,
+                training_request,
+                None,
+                noop_progress(),
+            )
+            .await?;
+        Ok(ManifestTrainedIndex {
+            index_name: input.index_name,
+            column_name: input.column_name,
+            uuid: index_uuid,
+            created_index,
+        })
+    }
+
+    fn manifest_index_metadata(
+        lance_schema: &lance_core::datatypes::Schema,
+        fragment_bitmap: &RoaringBitmap,
+        dataset_version: u64,
+        trained_index: ManifestTrainedIndex,
+    ) -> Result<IndexMetadata> {
+        Ok(IndexMetadata {
+            uuid: trained_index.uuid,
+            fields: vec![lance_schema.field_id(trained_index.column_name)?],
+            name: trained_index.index_name.to_string(),
+            dataset_version,
+            fragment_bitmap: Some(fragment_bitmap.clone()),
+            index_details: Some(Arc::new(trained_index.created_index.index_details)),
+            index_version: trained_index.created_index.index_version as i32,
+            created_at: None,
+            base_id: None,
+            files: Some(index_files_to_table(trained_index.created_index.files)),
+        })
+    }
+
+    fn manifest_fragment_bitmap(manifest: &Manifest) -> Result<RoaringBitmap> {
+        let mut bitmap = RoaringBitmap::new();
+        for fragment in manifest.fragments.iter() {
+            let fragment_id = u32::try_from(fragment.id).map_err(|_| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Manifest fragment id {} exceeds u32", fragment.id),
+                })
+            })?;
+            bitmap.insert(fragment_id);
         }
+        Ok(bitmap)
+    }
 
-        // Step 2: Run file compaction
-        log::debug!("Running file compaction on __manifest table");
-        match compact_files(dataset, CompactionOptions::default(), None).await {
-            Ok(compaction_metrics) => {
-                if compaction_metrics.fragments_removed > 0 {
-                    log::info!(
-                        "Compacted __manifest table: removed {} fragments, added {} fragments",
-                        compaction_metrics.fragments_removed,
-                        compaction_metrics.fragments_added
-                    );
+    fn manifest_from_overwrite_transaction(
+        previous: &Manifest,
+        schema: lance_core::datatypes::Schema,
+        fragments: &[Fragment],
+    ) -> Manifest {
+        let mut next_fragment_id = 0;
+        let mut fragments = fragments
+            .iter()
+            .cloned()
+            .map(|mut fragment| {
+                if fragment.id == 0 {
+                    fragment.id = next_fragment_id;
+                    next_fragment_id += 1;
                 }
-            }
-            Err(e) => {
-                log::warn!(
-                    "Failed to compact files for __manifest table: {:?}. Continuing with optimization.",
-                    e
-                );
-            }
-        }
+                fragment
+            })
+            .collect::<Vec<_>>();
+        fragments.sort_by_key(|fragment| fragment.id);
+        Manifest::new_from_previous(previous, schema, Arc::new(fragments))
+    }
 
-        // Step 3: Optimize indices
-        log::debug!("Optimizing indices on __manifest table");
-        match dataset.optimize_indices(&OptimizeOptions::default()).await {
-            Ok(_) => {
-                log::info!("Successfully optimized indices on __manifest table");
-            }
-            Err(e) => {
-                log::warn!(
-                    "Failed to optimize indices on __manifest table: {:?}. Continuing anyway.",
-                    e
-                );
-            }
-        }
+    async fn build_manifest_indices(
+        dataset: &Dataset,
+        manifest: &Manifest,
+        index_data: ManifestIndexAccumulator,
+        index_uuids: [Uuid; 3],
+    ) -> Result<Vec<IndexMetadata>> {
+        let fragment_bitmap = Self::manifest_fragment_bitmap(manifest)?;
+        let schema = &manifest.schema;
+        let ManifestIndexAccumulator {
+            object_ids,
+            object_types,
+            base_objects_values,
+            base_objects_row_ids,
+            ..
+        } = index_data;
+        let [object_id_uuid, object_type_uuid, base_objects_uuid] = index_uuids;
+        let registry = IndexPluginRegistry::with_default_plugins();
+
+        let dataset_version = manifest.version;
+        let object_id_index_fut = Self::build_manifest_index(
+            dataset,
+            registry.clone(),
+            schema,
+            ManifestIndexBuildInput {
+                index_name: OBJECT_ID_INDEX_NAME,
+                column_name: "object_id",
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::BTree),
+                field: Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false),
+                stream: Self::object_id_index_stream(object_ids),
+            },
+            &fragment_bitmap,
+            dataset_version,
+            object_id_uuid,
+        );
+        let object_type_index_fut = Self::build_manifest_index(
+            dataset,
+            registry.clone(),
+            schema,
+            ManifestIndexBuildInput {
+                index_name: OBJECT_TYPE_INDEX_NAME,
+                column_name: "object_type",
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap),
+                field: Field::new(VALUE_COLUMN_NAME, DataType::Utf8, false),
+                stream: Self::object_type_index_stream(object_types),
+            },
+            &fragment_bitmap,
+            dataset_version,
+            object_type_uuid,
+        );
+        let base_objects_index_fut = Self::build_manifest_index(
+            dataset,
+            registry,
+            schema,
+            ManifestIndexBuildInput {
+                index_name: BASE_OBJECTS_INDEX_NAME,
+                column_name: "base_objects",
+                params: ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList),
+                field: Field::new(
+                    VALUE_COLUMN_NAME,
+                    DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
+                    true,
+                ),
+                stream: Self::base_objects_index_stream(base_objects_values, base_objects_row_ids),
+            },
+            &fragment_bitmap,
+            dataset_version,
+            base_objects_uuid,
+        );
 
-        Ok(())
+        let (object_id_index, object_type_index, base_objects_index) = futures::join!(
+            object_id_index_fut,
+            object_type_index_fut,
+            base_objects_index_fut
+        );
+
+        Ok(vec![
+            object_id_index?,
+            object_type_index?,
+            base_objects_index?,
+        ])
+    }
+
+    async fn build_manifest_index(
+        dataset: &Dataset,
+        registry: Arc<IndexPluginRegistry>,
+        lance_schema: &lance_core::datatypes::Schema,
+        input: ManifestIndexBuildInput,
+        fragment_bitmap: &RoaringBitmap,
+        dataset_version: u64,
+        index_uuid: Uuid,
+    ) -> Result<IndexMetadata> {
+        let trained_index =
+            Self::train_manifest_index(dataset, registry, input, index_uuid).await?;
+        Self::manifest_index_metadata(
+            lance_schema,
+            fragment_bitmap,
+            dataset_version,
+            trained_index,
+        )
     }
 
     /// Get the manifest schema
@@ -783,130 +1486,751 @@ impl ManifestNamespace {
             })
     }
 
-    /// Check if the manifest contains an object with the given ID
-    async fn manifest_contains_object(&self, object_id: &str) -> Result<bool> {
-        let escaped_id = object_id.replace('\'', "''");
-        let filter = format!("object_id = '{}'", escaped_id);
+    fn required_string_value<'a>(
+        array: &'a StringArray,
+        row: usize,
+        column_name: &str,
+    ) -> Result<&'a str> {
+        if array.is_null(row) {
+            return Err(NamespaceError::Internal {
+                message: format!("Manifest column '{}' has null at row {}", column_name, row),
+            }
+            .into());
+        }
+        Ok(array.value(row))
+    }
 
-        let dataset_guard = self.manifest_dataset.get().await?;
-        let mut scanner = dataset_guard.scan();
+    fn optional_string_value(array: &StringArray, row: usize) -> Option<String> {
+        (!array.is_null(row)).then(|| array.value(row).to_string())
+    }
 
-        scanner.filter(&filter).map_err(|e| {
+    fn base_objects_column_values(batch: &RecordBatch) -> Result<Vec<Option<Vec<String>>>> {
+        let Some(column) = batch.column_by_name("base_objects") else {
+            return Ok(vec![None; batch.num_rows()]);
+        };
+        let array = column.as_any().downcast_ref::<ListArray>().ok_or_else(|| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
+                message: format!(
+                    "Column 'base_objects' is not a list array: {:?}",
+                    column.data_type()
+                ),
             })
         })?;
 
-        // Project no columns and enable row IDs for count_rows to work
-        scanner.project::<&str>(&[]).map_err(|e| {
+        let mut values = Vec::with_capacity(batch.num_rows());
+        for row in 0..batch.num_rows() {
+            if array.is_null(row) {
+                values.push(None);
+                continue;
+            }
+            let row_values = array.value(row);
+            let row_values = row_values
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .ok_or_else(|| {
+                    lance_core::Error::from(NamespaceError::Internal {
+                        message: "Column 'base_objects' values are not strings".to_string(),
+                    })
+                })?;
+            let mut objects = Vec::with_capacity(row_values.len());
+            for value_index in 0..row_values.len() {
+                if row_values.is_null(value_index) {
+                    return Err(NamespaceError::Internal {
+                        message: format!(
+                            "Manifest column 'base_objects' has null item at row {} item {}",
+                            row, value_index
+                        ),
+                    }
+                    .into());
+                }
+                objects.push(row_values.value(value_index).to_string());
+            }
+            values.push(Some(objects));
+        }
+        Ok(values)
+    }
+
+    async fn manifest_projected_stream(dataset: &Dataset) -> Result<SendableRecordBatchStream> {
+        let mut scanner = dataset.scan();
+        scanner
+            .project(&[
+                "object_id",
+                "object_type",
+                "location",
+                "metadata",
+                "base_objects",
+            ])
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to project manifest columns: {:?}", e),
+                })
+            })?;
+        let stream = scanner.try_into_stream().await.map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
+                message: format!("Failed to create manifest stream: {:?}", e),
             })
         })?;
+        let schema = stream.schema();
+        let stream = stream.map_err(|err| DataFusionError::External(Box::new(err)));
+        Ok(Box::pin(DatafusionRecordBatchStreamAdapter::new(
+            schema,
+            stream.fuse(),
+        )))
+    }
 
-        scanner.with_row_id();
+    fn manifest_rewrite_commit_retries(&self) -> u32 {
+        self.commit_retries
+            .unwrap_or(DEFAULT_MANIFEST_REWRITE_COMMIT_RETRIES)
+    }
 
-        let count = scanner.count_rows().await.map_err(|e| {
+    fn lock_manifest_rewrite_shared<M: ManifestStreamMutation>(
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> Result<StdMutexGuard<'_, ManifestRewriteShared<M>>> {
+        shared.lock().map_err(|_| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to count rows: {:?}", e),
+                message: "Manifest rewrite state mutex was poisoned".to_string(),
             })
-        })?;
+        })
+    }
 
-        Ok(count > 0)
+    fn set_manifest_rewrite_error<M: ManifestStreamMutation>(
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+        err: LanceError,
+    ) {
+        match shared.lock() {
+            Ok(mut guard) => {
+                guard.error = Some(err);
+            }
+            Err(poisoned) => {
+                let mut guard = poisoned.into_inner();
+                guard.error = Some(err);
+            }
+        }
     }
 
-    /// Query the manifest for a table with the given object ID
-    async fn query_manifest_for_table(&self, object_id: &str) -> Result<Option<TableInfo>> {
-        let escaped_id = object_id.replace('\'', "''");
-        let filter = format!("object_id = '{}' AND object_type = 'table'", escaped_id);
-        let mut scanner = self.manifest_scanner().await?;
-        scanner.filter(&filter).map_err(|e| {
+    fn take_manifest_rewrite_error<M: ManifestStreamMutation>(
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> Result<Option<LanceError>> {
+        let mut guard = Self::lock_manifest_rewrite_shared(shared)?;
+        Ok(guard.error.take())
+    }
+
+    fn process_manifest_rewrite_batch<M: ManifestStreamMutation>(
+        batch: RecordBatch,
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> Result<Option<RecordBatch>> {
+        let object_ids = Self::get_string_column(&batch, "object_id")?;
+        let object_types = Self::get_string_column(&batch, "object_type")?;
+        let locations = Self::get_string_column(&batch, "location")?;
+        let metadatas = Self::get_string_column(&batch, "metadata")?;
+        let base_objects = Self::base_objects_column_values(&batch)?;
+        let mut output = ManifestBatchBuilder::new();
+        let mut guard = Self::lock_manifest_rewrite_shared(shared)?;
+        let mut index_data = guard.index_data.take().ok_or_else(|| {
             lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
+                message: "Manifest rewrite index state is unavailable".to_string(),
             })
         })?;
-        scanner
-            .project(&["object_id", "location", "metadata"])
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!("Failed to project: {:?}", e),
-                })
-            })?;
-        let batches = Self::execute_scanner(scanner).await?;
-
-        let mut found_result: Option<TableInfo> = None;
-        let mut total_rows = 0;
+        for (row, base_objects) in base_objects.into_iter().enumerate().take(batch.num_rows()) {
+            let row_value = ManifestRowValue {
+                object_id: Self::required_string_value(object_ids, row, "object_id")?.to_string(),
+                object_type: ObjectType::parse(Self::required_string_value(
+                    object_types,
+                    row,
+                    "object_type",
+                )?)?,
+                location: Self::optional_string_value(locations, row),
+                metadata: Self::optional_string_value(metadatas, row),
+                base_objects,
+            };
+            guard
+                .mutation
+                .process_existing_row(row_value, &mut output, &mut index_data)?;
+        }
+        guard.index_data = Some(index_data);
+        if output.is_empty() {
+            return Ok(None);
+        }
+        Ok(Some(output.finish()?))
+    }
 
-        for batch in batches {
-            if batch.num_rows() == 0 {
-                continue;
-            }
+    fn finish_manifest_rewrite_stream<M: ManifestStreamMutation>(
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> Result<Option<RecordBatch>> {
+        let mut output = ManifestBatchBuilder::new();
+        let mut guard = Self::lock_manifest_rewrite_shared(shared)?;
+        let mut index_data = guard.index_data.take().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: "Manifest rewrite index state is unavailable".to_string(),
+            })
+        })?;
+        guard.mutation.append_rows(&mut output, &mut index_data)?;
+        let result = guard.mutation.finish();
+        let force_empty_batch = index_data.row_count == 0;
+        guard.result = Some(result);
+        guard.index_data = Some(index_data);
+        if output.is_empty() && !force_empty_batch {
+            Ok(None)
+        } else {
+            Ok(Some(output.finish()?))
+        }
+    }
 
-            total_rows += batch.num_rows();
-            if total_rows > 1 {
-                return Err(NamespaceError::Internal {
-                    message: format!(
-                        "Expected exactly 1 table with id '{}', found {}",
-                        object_id, total_rows
-                    ),
-                }
-                .into());
-            }
+    fn manifest_rewrite_output_stream<M: ManifestStreamMutation + 'static>(
+        source: SendableRecordBatchStream,
+        shared: Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> SendableRecordBatchStream {
+        enum Phase {
+            Source,
+            Finish,
+            Done,
+        }
 
-            let object_id_array = Self::get_string_column(&batch, "object_id")?;
-            let location_array = Self::get_string_column(&batch, "location")?;
-            let metadata_array = Self::get_string_column(&batch, "metadata")?;
-            let location = location_array.value(0).to_string();
-            let metadata = if !metadata_array.is_null(0) {
-                let metadata_str = metadata_array.value(0);
-                match serde_json::from_str::<HashMap<String, String>>(metadata_str) {
-                    Ok(map) => Some(map),
-                    Err(e) => {
-                        return Err(NamespaceError::Internal {
-                            message: format!(
-                                "Failed to deserialize metadata for table '{}': {}",
-                                object_id, e
-                            ),
+        let schema = Self::manifest_schema();
+        let stream = stream::unfold(
+            (source, shared, Phase::Source),
+            |(mut source, shared, mut phase)| async move {
+                loop {
+                    match phase {
+                        Phase::Source => match source.next().await {
+                            Some(Ok(batch)) => {
+                                match Self::process_manifest_rewrite_batch(batch, &shared) {
+                                    Ok(Some(batch)) => {
+                                        return Some((Ok(batch), (source, shared, phase)));
+                                    }
+                                    Ok(None) => continue,
+                                    Err(err) => {
+                                        let message = err.to_string();
+                                        Self::set_manifest_rewrite_error(&shared, err);
+                                        return Some((
+                                            Err(DataFusionError::External(Box::new(
+                                                std::io::Error::other(message),
+                                            ))),
+                                            (source, shared, Phase::Done),
+                                        ));
+                                    }
+                                }
+                            }
+                            Some(Err(err)) => {
+                                return Some((Err(err), (source, shared, Phase::Done)));
+                            }
+                            None => phase = Phase::Finish,
+                        },
+                        Phase::Finish => {
+                            phase = Phase::Done;
+                            match Self::finish_manifest_rewrite_stream(&shared) {
+                                Ok(Some(batch)) => {
+                                    return Some((Ok(batch), (source, shared, phase)));
+                                }
+                                Ok(None) => continue,
+                                Err(err) => {
+                                    let message = err.to_string();
+                                    Self::set_manifest_rewrite_error(&shared, err);
+                                    return Some((
+                                        Err(DataFusionError::External(Box::new(
+                                            std::io::Error::other(message),
+                                        ))),
+                                        (source, shared, Phase::Done),
+                                    ));
+                                }
+                            }
                         }
-                        .into());
+                        Phase::Done => return None,
                     }
                 }
-            } else {
-                None
-            };
-            let (namespace, name) = Self::parse_object_id(object_id_array.value(0));
-            found_result = Some(TableInfo {
-                namespace,
-                name,
-                location,
-                metadata,
-            });
-        }
+            },
+        );
+        Box::pin(DatafusionRecordBatchStreamAdapter::new(
+            schema,
+            stream.fuse(),
+        ))
+    }
 
-        Ok(found_result)
+    fn take_manifest_rewrite_result<M: ManifestStreamMutation>(
+        shared: &Arc<StdMutex<ManifestRewriteShared<M>>>,
+    ) -> Result<(CopyOnWriteMutation<M::Output>, ManifestIndexAccumulator)> {
+        let mut guard = Self::lock_manifest_rewrite_shared(shared)?;
+        let result = guard.result.take().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: "Manifest rewrite stream did not finish".to_string(),
+            })
+        })?;
+        let index_data = guard.index_data.take().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: "Manifest rewrite index state is unavailable".to_string(),
+            })
+        })?;
+        Ok((result, index_data))
     }
 
-    fn serialize_metadata(
-        properties: Option<&HashMap<String, String>>,
-        object_type: &str,
-        object_id: &str,
-    ) -> Result<Option<String>> {
-        match properties {
-            Some(properties) if !properties.is_empty() => {
-                serde_json::to_string(properties).map(Some).map_err(|e| {
-                    LanceError::from(NamespaceError::Internal {
-                        message: format!(
-                            "Failed to serialize {} metadata for '{}': {}",
-                            object_type, object_id, e
-                        ),
-                    })
-                })
+    /// Delete the staged (uncommitted) data files and index directories for a rewrite.
+    /// Only call this once the rewrite is known *not* to have landed (a put-if-not-exists
+    /// conflict, or an ambiguous error whose target version does not reference our data
+    /// file) — otherwise it would orphan files a committed manifest still references.
+    async fn cleanup_staged_manifest_files(
+        &self,
+        object_store: &ObjectStore,
+        data_files: &HashSet<String>,
+        index_uuids: &[Uuid],
+    ) {
+        let data_dir = self
+            .base_path
+            .clone()
+            .join(MANIFEST_TABLE_NAME)
+            .join(LANCE_DATA_DIR);
+        for path in data_files {
+            let data_path = data_dir.clone().join(path.as_str());
+            if let Err(err) = object_store.delete(&data_path).await {
+                log::warn!(
+                    "Failed to clean up uncommitted manifest rewrite data file '{}': {}",
+                    data_path,
+                    err
+                );
             }
-            _ => Ok(None),
         }
+        self.cleanup_uncommitted_manifest_index_dirs(object_store, index_uuids.iter().copied())
+            .await;
     }
 
-    pub(crate) async fn path_has_actual_manifests(
+    async fn cleanup_uncommitted_manifest_index_dirs(
+        &self,
+        object_store: &ObjectStore,
+        index_uuids: impl IntoIterator<Item = Uuid>,
+    ) {
+        for index_uuid in index_uuids {
+            let index_dir = self
+                .base_path
+                .clone()
+                .join(MANIFEST_TABLE_NAME)
+                .join(LANCE_INDICES_DIR)
+                .join(index_uuid.to_string());
+            if let Err(err) = object_store.remove_dir_all(index_dir.clone()).await
+                && !matches!(err, LanceError::NotFound { .. })
+            {
+                log::warn!(
+                    "Failed to clean up uncommitted manifest rewrite index directory '{}': {}",
+                    index_dir,
+                    err
+                );
+            }
+        }
+    }
+
+    /// Resolve the commit handler for the `__manifest` dataset's storage backend.
+    async fn manifest_commit_handler(&self) -> Result<Arc<dyn CommitHandler>> {
+        commit_handler_from_url(&self.root, &None)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to resolve manifest commit handler: {:?}", e),
+                })
+            })
+    }
+
+    /// Directly write the rewritten `__manifest` as a new version using the storage
+    /// backend's atomic put-if-not-exists. The overwrite transaction is embedded inline
+    /// (no separate transaction file) and the commit handler writes the version hint.
+    async fn commit_manifest_overwrite(
+        &self,
+        dataset: &Dataset,
+        commit_handler: &dyn CommitHandler,
+        manifest: &mut Manifest,
+        indices: Option<Vec<IndexMetadata>>,
+        transaction: Transaction,
+    ) -> std::result::Result<(), CommitError> {
+        apply_feature_flags(manifest, false, false).map_err(CommitError::from)?;
+        let timestamp_nanos = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_nanos())
+            .unwrap_or(0);
+        manifest.set_timestamp(timestamp_nanos);
+        manifest.update_max_fragment_id();
+
+        // Commit through the dataset's own object store, not `self.object_store`: for
+        // stores like `memory://` the namespace and the dataset can hold different
+        // instances, and a commit written to the wrong one is invisible to reads.
+        let object_store = dataset
+            .object_store(None)
+            .await
+            .map_err(CommitError::from)?;
+        let base_path = self.base_path.clone().join(MANIFEST_TABLE_NAME);
+        let naming_scheme = dataset.manifest_location().naming_scheme;
+        commit_handler
+            .commit(
+                manifest,
+                indices,
+                &base_path,
+                &object_store,
+                write_manifest_file_to_path,
+                naming_scheme,
+                Some((&transaction).into()),
+            )
+            .await
+            .map(|_location| ())
+    }
+
+    /// After an ambiguous commit error, determine whether our overwrite actually landed at
+    /// `target_version`. A network failure can leave the manifest committed even though the
+    /// client observed an error; in that case the committed version references one of our
+    /// staged data files, and deleting them would corrupt the catalog.
+    async fn manifest_commit_landed(
+        &self,
+        dataset: &Dataset,
+        target_version: u64,
+        data_files: &HashSet<String>,
+    ) -> bool {
+        let Ok(committed) = dataset.checkout_version(target_version).await else {
+            return false;
+        };
+        committed.manifest().fragments.iter().any(|fragment| {
+            fragment
+                .files
+                .iter()
+                .any(|file| data_files.contains(file.path.as_str()))
+        })
+    }
+
+    /// Resolve a storage commit conflict against the latest committed catalog state.
+    /// Returns `Some(output)` when the mutation's intent is already satisfied (no retry
+    /// needed), `Ok(None)` to retry the rewrite, or an error for a terminal conflict.
+    async fn resolve_manifest_conflict<O: Clone>(
+        &self,
+        resolution: &ConflictResolution<O>,
+    ) -> Result<Option<O>> {
+        match resolution {
+            ConflictResolution::Retry => Ok(None),
+            ConflictResolution::FailIfExists(object_ids) => {
+                for object_id in object_ids {
+                    if self.manifest_contains_object(object_id).await? {
+                        return Err(NamespaceError::ConcurrentModification {
+                            message: format!(
+                                "Object '{}' was concurrently created by another operation",
+                                object_id
+                            ),
+                        }
+                        .into());
+                    }
+                }
+                Ok(None)
+            }
+            ConflictResolution::SucceedIfAbsent { object_id, output } => {
+                if self.manifest_contains_object(object_id).await? {
+                    Ok(None)
+                } else {
+                    Ok(Some(output.clone()))
+                }
+            }
+        }
+    }
+
+    /// Validate that this build can write the current `__manifest` before a
+    /// mutating operation performs any side effect (e.g. writing table data), so
+    /// a refused write leaves nothing orphaned behind. The eventual
+    /// `rewrite_manifest` commit re-checks `ensure_writable` on each retry, so a
+    /// concurrent upgrade in between is still caught.
+    async fn ensure_manifest_writable(&self) -> Result<()> {
+        let dataset_guard = self.manifest_dataset.get().await?;
+        ensure_writable(dataset_guard.metadata())
+    }
+
+    async fn rewrite_manifest<M, F>(
+        &self,
+        operation: &str,
+        mut make_mutation: F,
+    ) -> Result<M::Output>
+    where
+        M: ManifestStreamMutation + 'static,
+        F: FnMut() -> M,
+    {
+        let _mutation_guard = self.manifest_mutation_lock.lock().await;
+        let max_retries = self.manifest_rewrite_commit_retries();
+        let mut retries = 0;
+        let build_indices = self.inline_optimization_enabled;
+        let commit_handler = self.manifest_commit_handler().await?;
+
+        loop {
+            let dataset_guard = self.manifest_dataset.get_refreshed().await?;
+            let dataset = Arc::new(dataset_guard.clone());
+            drop(dataset_guard);
+            // Refuse to mutate a manifest written with a writer feature flag this
+            // build does not understand.
+            ensure_writable(dataset.metadata())?;
+            // Staged files, indices, the commit, and cleanup must all use the dataset's
+            // own object store (see `commit_manifest_overwrite`).
+            let object_store = dataset.object_store(None).await?;
+
+            let source = Self::manifest_projected_stream(&dataset).await?;
+            let resolution = make_mutation().conflict_resolution();
+            let shared = Arc::new(StdMutex::new(ManifestRewriteShared::new(make_mutation())));
+            let output_stream = Self::manifest_rewrite_output_stream(source, shared.clone());
+            // Pin both limits so the overwrite never splits into multiple fragments: the
+            // replacement indices map each row to address `(0 << 32) | offset`, valid only
+            // for a single fragment with id 0. The row count is bounded below u32::MAX by
+            // `ManifestIndexAccumulator::next_row_id`.
+            let write_params = WriteParams {
+                mode: WriteMode::Overwrite,
+                session: self.session.clone(),
+                max_rows_per_file: u32::MAX as usize,
+                max_bytes_per_file: usize::MAX,
+                skip_auto_cleanup: true,
+                ..WriteParams::default()
+            };
+
+            let transaction = match InsertBuilder::new(dataset.clone())
+                .with_params(&write_params)
+                .execute_uncommitted_stream(output_stream)
+                .await
+            {
+                Ok(transaction) => transaction,
+                Err(err) => {
+                    if let Some(stream_err) = Self::take_manifest_rewrite_error(&shared)? {
+                        return Err(stream_err);
+                    }
+                    return Err(convert_lance_commit_error(&err, operation, None));
+                }
+            };
+
+            let (mutation, index_data) = Self::take_manifest_rewrite_result(&shared)?;
+
+            let Operation::Overwrite {
+                fragments, schema, ..
+            } = &transaction.operation
+            else {
+                return Err(NamespaceError::Internal {
+                    message: "Manifest rewrite transaction is not an overwrite".to_string(),
+                }
+                .into());
+            };
+            // Unique data files this attempt staged. Used to clean up orphans and to
+            // attribute an ambiguous commit error back to us.
+            let staged_data_files = fragments
+                .iter()
+                .flat_map(|fragment| fragment.files.iter())
+                .filter(|file| file.base_id.is_none())
+                .map(|file| file.path.clone())
+                .collect::<HashSet<_>>();
+
+            if !mutation.has_changes {
+                self.cleanup_staged_manifest_files(&object_store, &staged_data_files, &[])
+                    .await;
+                return Ok(mutation.result);
+            }
+
+            let mut manifest = Self::manifest_from_overwrite_transaction(
+                dataset.manifest(),
+                schema.clone(),
+                fragments,
+            );
+            let target_version = manifest.version;
+
+            let index_uuids = [Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()];
+            let indices = if build_indices {
+                match Self::build_manifest_indices(&dataset, &manifest, index_data, index_uuids)
+                    .await
+                {
+                    Ok(indices) => Some(indices),
+                    Err(err) => {
+                        self.cleanup_staged_manifest_files(
+                            &object_store,
+                            &staged_data_files,
+                            &index_uuids,
+                        )
+                        .await;
+                        return Err(err);
+                    }
+                }
+            } else {
+                None
+            };
+            let staged_index_uuids: &[Uuid] = if build_indices { &index_uuids } else { &[] };
+
+            let commit_result = self
+                .commit_manifest_overwrite(
+                    &dataset,
+                    commit_handler.as_ref(),
+                    &mut manifest,
+                    indices,
+                    transaction,
+                )
+                .await;
+
+            match commit_result {
+                Ok(()) => {
+                    let _ = self.manifest_dataset.get_refreshed().await;
+                    return Ok(mutation.result);
+                }
+                Err(err) => {
+                    // The put may have landed even though the client saw an error (lost
+                    // ack). Verify before deleting anything so we never orphan files that a
+                    // committed manifest still references.
+                    if self
+                        .manifest_commit_landed(&dataset, target_version, &staged_data_files)
+                        .await
+                    {
+                        let _ = self.manifest_dataset.get_refreshed().await;
+                        return Ok(mutation.result);
+                    }
+                    self.cleanup_staged_manifest_files(
+                        &object_store,
+                        &staged_data_files,
+                        staged_index_uuids,
+                    )
+                    .await;
+                    match err {
+                        CommitError::CommitConflict => {
+                            if let Some(output) =
+                                self.resolve_manifest_conflict(&resolution).await?
+                            {
+                                return Ok(output);
+                            }
+                            if retries >= max_retries {
+                                return Err(NamespaceError::ConcurrentModification {
+                                    message: format!(
+                                        "{}: still conflicting after {} retries",
+                                        operation, max_retries
+                                    ),
+                                }
+                                .into());
+                            }
+                            retries += 1;
+                            tokio::time::sleep(std::time::Duration::from_millis(
+                                10 * u64::from(retries),
+                            ))
+                            .await;
+                        }
+                        CommitError::OtherError(err) => {
+                            return Err(convert_lance_commit_error(&err, operation, None));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// Check if the manifest contains an object with the given ID
+    async fn manifest_contains_object(&self, object_id: &str) -> Result<bool> {
+        let escaped_id = object_id.replace('\'', "''");
+        let filter = format!("object_id = '{}'", escaped_id);
+
+        let dataset_guard = self.manifest_dataset.get().await?;
+        let mut scanner = dataset_guard.scan();
+
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+
+        // Project no columns and enable row IDs for count_rows to work
+        scanner.project::<&str>(&[]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+
+        scanner.with_row_id();
+
+        let count = scanner.count_rows().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to count rows: {:?}", e),
+            })
+        })?;
+
+        Ok(count > 0)
+    }
+
+    /// Query the manifest for a table with the given object ID
+    async fn query_manifest_for_table(&self, object_id: &str) -> Result<Option<TableInfo>> {
+        let escaped_id = object_id.replace('\'', "''");
+        let filter = format!("object_id = '{}' AND object_type = 'table'", escaped_id);
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner
+            .project(&["object_id", "location", "metadata"])
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to project: {:?}", e),
+                })
+            })?;
+        let batches = Self::execute_scanner(scanner).await?;
+
+        let mut found_result: Option<TableInfo> = None;
+        let mut total_rows = 0;
+
+        for batch in batches {
+            if batch.num_rows() == 0 {
+                continue;
+            }
+
+            total_rows += batch.num_rows();
+            if total_rows > 1 {
+                return Err(NamespaceError::Internal {
+                    message: format!(
+                        "Expected exactly 1 table with id '{}', found {}",
+                        object_id, total_rows
+                    ),
+                }
+                .into());
+            }
+
+            let object_id_array = Self::get_string_column(&batch, "object_id")?;
+            let location_array = Self::get_string_column(&batch, "location")?;
+            let metadata_array = Self::get_string_column(&batch, "metadata")?;
+            let location = location_array.value(0).to_string();
+            let metadata = if !metadata_array.is_null(0) {
+                let metadata_str = metadata_array.value(0);
+                match serde_json::from_str::<HashMap<String, String>>(metadata_str) {
+                    Ok(map) => Some(map),
+                    Err(e) => {
+                        return Err(NamespaceError::Internal {
+                            message: format!(
+                                "Failed to deserialize metadata for table '{}': {}",
+                                object_id, e
+                            ),
+                        }
+                        .into());
+                    }
+                }
+            } else {
+                None
+            };
+            let (namespace, name) = Self::parse_object_id(object_id_array.value(0));
+            found_result = Some(TableInfo {
+                namespace,
+                name,
+                location,
+                metadata,
+            });
+        }
+
+        Ok(found_result)
+    }
+
+    fn serialize_metadata(
+        properties: Option<&HashMap<String, String>>,
+        object_type: &str,
+        object_id: &str,
+    ) -> Result<Option<String>> {
+        match properties {
+            Some(properties) if !properties.is_empty() => {
+                serde_json::to_string(properties).map(Some).map_err(|e| {
+                    LanceError::from(NamespaceError::Internal {
+                        message: format!(
+                            "Failed to serialize {} metadata for '{}': {}",
+                            object_type, object_id, e
+                        ),
+                    })
+                })
+            }
+            _ => Ok(None),
+        }
+    }
+
+    pub(crate) async fn path_has_actual_manifests(
         object_store: &ObjectStore,
         table_path: &Path,
     ) -> Result<bool> {
@@ -999,7 +2323,6 @@ impl ManifestNamespace {
     /// Insert one or more entries into the manifest table with metadata and base_objects.
     ///
     /// This is the unified entry point for both single and batch inserts.
-    /// Uses a single MergeInsert operation to insert all entries at once.
     /// If any entry already exists (matching object_id), the entire batch fails.
     pub async fn insert_into_manifest_with_metadata(
         &self,
@@ -1029,181 +2352,55 @@ impl ManifestNamespace {
             return Ok(());
         }
 
-        let schema = Self::manifest_schema();
-
-        let mut object_ids = Vec::with_capacity(entries.len());
-        let mut object_types = Vec::with_capacity(entries.len());
-        let mut locations: Vec<Option<String>> = Vec::with_capacity(entries.len());
-        let mut metadatas: Vec<Option<String>> = Vec::with_capacity(entries.len());
+        self.rewrite_manifest("Failed to overwrite manifest", || {
+            UpsertManifestMutation::new(entries.clone(), base_objects.clone(), when_matched.clone())
+        })
+        .await
+    }
 
-        let string_builder = StringBuilder::new();
-        let mut list_builder = ListBuilder::new(string_builder).with_field(Arc::new(Field::new(
-            "object_id",
-            DataType::Utf8,
-            true,
-        )));
+    /// Delete an entry from the manifest table
+    pub async fn delete_from_manifest(&self, object_id: &str) -> Result<()> {
+        let object_id = object_id.to_string();
+        self.rewrite_manifest("Failed to delete from manifest", || DeleteObjectMutation {
+            object_id: object_id.clone(),
+            deleted: false,
+        })
+        .await
+    }
 
-        for (i, entry) in entries.iter().enumerate() {
-            object_ids.push(entry.object_id.as_str());
-            object_types.push(entry.object_type.as_str());
-            locations.push(entry.location.clone());
-            metadatas.push(entry.metadata.clone());
-
-            // Only the first entry gets the base_objects (for single-entry inserts
-            // with base_objects like view creation); batch entries use null.
-            if i == 0 {
-                match &base_objects {
-                    Some(objects) => {
-                        for obj in objects {
-                            list_builder.values().append_value(obj);
-                        }
-                        list_builder.append(true);
-                    }
-                    None => {
-                        list_builder.append_null();
-                    }
-                }
-            } else {
-                list_builder.append_null();
+    /// Register a table in the manifest without creating the physical table (internal helper for migration)
+    pub async fn register_table(&self, name: &str, location: String) -> Result<()> {
+        let object_id = Self::build_object_id(&[], name);
+        if self.manifest_contains_object(&object_id).await? {
+            return Err(NamespaceError::Internal {
+                message: format!("Table '{}' already exists", name),
             }
+            .into());
         }
 
-        let base_objects_array = list_builder.finish();
+        self.insert_into_manifest(object_id, ObjectType::Table, Some(location))
+            .await
+    }
 
-        let location_array: Arc<dyn Array> = Arc::new(StringArray::from(
-            locations.iter().map(|l| l.as_deref()).collect::<Vec<_>>(),
-        ));
+    /// Validate that all levels of a namespace path exist
+    async fn validate_namespace_levels_exist(&self, namespace_path: &[String]) -> Result<()> {
+        for i in 1..=namespace_path.len() {
+            let partial_path = &namespace_path[..i];
+            let object_id = partial_path.join(DELIMITER);
+            if !self.manifest_contains_object(&object_id).await? {
+                return Err(NamespaceError::NamespaceNotFound {
+                    message: format!("parent namespace '{}'", object_id),
+                }
+                .into());
+            }
+        }
+        Ok(())
+    }
 
-        let metadata_array: Arc<dyn Array> = Arc::new(StringArray::from(
-            metadatas.iter().map(|m| m.as_deref()).collect::<Vec<_>>(),
-        ));
-
-        let batch = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(StringArray::from(object_ids)),
-                Arc::new(StringArray::from(object_types.to_vec())),
-                location_array,
-                metadata_array,
-                Arc::new(base_objects_array),
-            ],
-        )
-        .map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to create manifest entries: {:?}", e),
-            })
-        })?;
-
-        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
-
-        // Use MergeInsert so callers can choose fail-on-existing inserts or metadata upserts.
-        let _mutation_guard = self.manifest_mutation_lock.lock().await;
-        let dataset_guard = self.manifest_dataset.get().await?;
-        let dataset_arc = Arc::new(dataset_guard.clone());
-        drop(dataset_guard); // Drop read guard before merge insert
-
-        let mut merge_builder =
-            MergeInsertBuilder::try_new(dataset_arc, vec!["object_id".to_string()]).map_err(
-                |e| {
-                    lance_core::Error::from(NamespaceError::Internal {
-                        message: format!("Failed to create merge builder: {:?}", e),
-                    })
-                },
-            )?;
-        merge_builder.when_matched(when_matched);
-        merge_builder.when_not_matched(WhenNotMatched::InsertAll);
-        // Use conflict_retries to handle cross-process races on manifest mutations.
-        merge_builder.conflict_retries(5);
-        // TODO: after BTREE index creation on object_id, has_scalar_index=true causes
-        // MergeInsert to use V1 path which lacks bloom filters for conflict detection. This
-        // results in (Some, None) filter mismatch when rebasing against V2 operations.
-        // Setting use_index=false ensures all operations consistently use V2 path.
-        merge_builder.use_index(false);
-        if let Some(retries) = self.commit_retries {
-            merge_builder.commit_retries(retries);
-        }
-
-        let (new_dataset_arc, _merge_stats) = merge_builder
-            .try_build()
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!("Failed to build merge: {:?}", e),
-                })
-            })?
-            .execute_reader(Box::new(reader))
-            .await
-            .map_err(|e| {
-                convert_lance_commit_error(&e, "Failed to execute merge insert into manifest", None)
-            })?;
-
-        let new_dataset = Arc::try_unwrap(new_dataset_arc).unwrap_or_else(|arc| (*arc).clone());
-        self.manifest_dataset.set_latest(new_dataset).await;
-
-        // Run inline optimization after write
-        if let Err(e) = self.run_inline_optimization().await {
-            log::warn!(
-                "Unexpected failure when running inline optimization: {:?}",
-                e
-            );
-        }
-
-        Ok(())
-    }
-
-    /// Delete an entry from the manifest table
-    pub async fn delete_from_manifest(&self, object_id: &str) -> Result<()> {
-        let predicate = format!("object_id = '{}'", object_id);
-
-        // Get dataset and use DeleteBuilder with configured retries
-        let _mutation_guard = self.manifest_mutation_lock.lock().await;
-        let dataset_guard = self.manifest_dataset.get().await?;
-        let dataset = Arc::new(dataset_guard.clone());
-        drop(dataset_guard); // Drop read guard before delete
-
-        let new_dataset = DeleteBuilder::new(dataset, &predicate)
-            .execute()
-            .await
-            .map_err(|e| convert_lance_commit_error(&e, "Failed to delete", None))?;
-
-        // Update the wrapper with the new dataset
-        self.manifest_dataset
-            .set_latest(
-                Arc::try_unwrap(new_dataset.new_dataset).unwrap_or_else(|arc| (*arc).clone()),
-            )
-            .await;
-
-        // Run inline optimization after delete
-        if let Err(e) = self.run_inline_optimization().await {
-            log::warn!(
-                "Unexpected failure when running inline optimization: {:?}",
-                e
-            );
-        }
-
-        Ok(())
-    }
-
-    /// Query the manifest for all versions of a table, sorted by version.
-    ///
-    /// Returns a list of (version, metadata_json_string) tuples where metadata_json_string
-    /// contains the full metadata JSON stored in the manifest (manifest_path, manifest_size,
-    /// e_tag, naming_scheme).
-    ///
-    /// **Known limitation**: All matching rows are loaded into memory, sorted in Rust,
-    /// and then truncated. For tables with a very large number of versions this may be
-    /// expensive. Pushing sort/limit into the scan is not yet supported by Lance.
-    pub async fn query_table_versions(
-        &self,
-        object_id: &str,
-        descending: bool,
-        limit: Option<i32>,
-    ) -> Result<Vec<(i64, String)>> {
+    /// Query the manifest for a namespace with the given object ID
+    async fn query_manifest_for_namespace(&self, object_id: &str) -> Result<Option<NamespaceInfo>> {
         let escaped_id = object_id.replace('\'', "''");
-        // table_version object_ids are formatted as "{object_id}${zero_padded_version}"
-        let filter = format!(
-            "object_type = 'table_version' AND starts_with(object_id, '{}{}')",
-            escaped_id, DELIMITER
-        );
+        let filter = format!("object_id = '{}' AND object_type = 'namespace'", escaped_id);
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
@@ -1217,200 +2414,322 @@ impl ManifestNamespace {
         })?;
         let batches = Self::execute_scanner(scanner).await?;
 
-        let mut versions: Vec<(i64, String)> = Vec::new();
+        let mut found_result: Option<NamespaceInfo> = None;
+        let mut total_rows = 0;
+
         for batch in batches {
             if batch.num_rows() == 0 {
                 continue;
             }
+
+            total_rows += batch.num_rows();
+            if total_rows > 1 {
+                return Err(NamespaceError::Internal {
+                    message: format!(
+                        "Expected exactly 1 namespace with id '{}', found {}",
+                        object_id, total_rows
+                    ),
+                }
+                .into());
+            }
+
             let object_id_array = Self::get_string_column(&batch, "object_id")?;
             let metadata_array = Self::get_string_column(&batch, "metadata")?;
-            for i in 0..batch.num_rows() {
-                let oid = object_id_array.value(i);
-                // Parse version from object_id
-                if let Some(version) = Self::parse_version_from_object_id(oid) {
-                    let metadata_str = metadata_array.value(i).to_string();
-                    versions.push((version, metadata_str));
+
+            let object_id_str = object_id_array.value(0);
+            let metadata = if !metadata_array.is_null(0) {
+                let metadata_str = metadata_array.value(0);
+                match serde_json::from_str::<HashMap<String, String>>(metadata_str) {
+                    Ok(map) => Some(map),
+                    Err(e) => {
+                        return Err(NamespaceError::Internal {
+                            message: format!(
+                                "Failed to deserialize metadata for namespace '{}': {}",
+                                object_id, e
+                            ),
+                        }
+                        .into());
+                    }
                 }
-            }
-        }
+            } else {
+                None
+            };
 
-        if descending {
-            versions.sort_by(|a, b| b.0.cmp(&a.0));
-        } else {
-            versions.sort_by(|a, b| a.0.cmp(&b.0));
+            let (namespace, name) = Self::parse_object_id(object_id_str);
+            found_result = Some(NamespaceInfo {
+                namespace,
+                name,
+                metadata,
+            });
         }
 
-        if let Some(limit) = limit {
-            versions.truncate(limit as usize);
-        }
+        Ok(found_result)
+    }
 
-        Ok(versions)
+    /// Load an existing manifest dataset without creating or migrating it.
+    async fn open_manifest_table(
+        root: &str,
+        storage_options: &Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+    ) -> Result<DatasetConsistencyWrapper> {
+        let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME);
+        log::debug!("Attempting to load manifest from {}", manifest_path);
+        let store_options = ObjectStoreParams {
+            storage_options_accessor: storage_options.as_ref().map(|opts| {
+                Arc::new(
+                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                        opts.clone(),
+                    ),
+                )
+            }),
+            ..Default::default()
+        };
+        let read_params = ReadParams {
+            session,
+            store_options: Some(store_options),
+            ..Default::default()
+        };
+        let dataset = DatasetBuilder::from_uri(&manifest_path)
+            .with_read_params(read_params)
+            .load()
+            .await?;
+        ensure_readable(dataset.metadata())?;
+        Ok(DatasetConsistencyWrapper::new(dataset))
     }
 
-    /// Query the manifest for a specific version of a table.
-    ///
-    /// Returns the full metadata JSON string if found, which contains
-    /// manifest_path, manifest_size, e_tag, and naming_scheme.
+    /// Create or load the manifest dataset, ensuring it has the latest schema setup.
     ///
-    pub async fn query_table_version(
-        &self,
-        object_id: &str,
-        version: i64,
-    ) -> Result<Option<String>> {
-        let version_object_id = Self::build_version_object_id(object_id, version);
-        self.query_table_version_by_object_id(&version_object_id)
-            .await
-    }
+    /// This function will:
+    /// 1. Try to load an existing manifest table
+    /// 2. If it exists, check and migrate the schema if needed (e.g., add primary key metadata)
+    /// 3. If it doesn't exist, create a new manifest table with the current schema
+    async fn ensure_manifest_table_up_to_date(
+        root: &str,
+        storage_options: &Option<HashMap<String, String>>,
+        session: Option<Arc<Session>>,
+    ) -> Result<DatasetConsistencyWrapper> {
+        let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME);
+        log::debug!("Attempting to load manifest from {}", manifest_path);
+        let store_options = ObjectStoreParams {
+            storage_options_accessor: storage_options.as_ref().map(|opts| {
+                Arc::new(
+                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                        opts.clone(),
+                    ),
+                )
+            }),
+            ..Default::default()
+        };
+        let read_params = ReadParams {
+            session: session.clone(),
+            store_options: Some(store_options.clone()),
+            ..Default::default()
+        };
+        let dataset_result = DatasetBuilder::from_uri(&manifest_path)
+            .with_read_params(read_params)
+            .load()
+            .await;
+        match dataset_result {
+            Ok(mut dataset) => {
+                // Reject a manifest written with a reader feature flag this build
+                // does not understand before touching it.
+                ensure_readable(dataset.metadata())?;
+
+                // Check if the object_id field has primary key metadata, migrate if not
+                let needs_pk_migration = dataset
+                    .schema()
+                    .field("object_id")
+                    .map(|f| {
+                        !f.metadata
+                            .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+                    })
+                    .unwrap_or(false);
 
-    /// Query a specific table version by its exact object_id.
-    async fn query_table_version_by_object_id(
-        &self,
-        version_object_id: &str,
-    ) -> Result<Option<String>> {
-        let escaped_id = version_object_id.replace('\'', "''");
-        let filter = format!(
-            "object_id = '{}' AND object_type = 'table_version'",
-            escaped_id
-        );
-        let mut scanner = self.manifest_scanner().await?;
-        scanner.filter(&filter).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
-            })
-        })?;
-        scanner.project(&["metadata"]).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
-            })
-        })?;
-        let batches = Self::execute_scanner(scanner).await?;
+                if needs_pk_migration {
+                    // This legacy migration writes to the manifest, so confirm this
+                    // build is allowed to write the current format first.
+                    ensure_writable(dataset.metadata())?;
+                    log::info!(
+                        "Migrating __manifest table to add primary key metadata on object_id"
+                    );
+                    dataset
+                        .update_field_metadata()
+                        .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")])
+                        .map_err(|e| {
+                            lance_core::Error::from(NamespaceError::Internal {
+                                message: format!(
+                                    "Failed to find object_id field for migration: {:?}",
+                                    e
+                                ),
+                            })
+                        })?
+                        .await
+                        .map_err(|e| {
+                            lance_core::Error::from(NamespaceError::Internal {
+                                message: format!("Failed to migrate primary key metadata: {:?}", e),
+                            })
+                        })?;
+                }
 
-        for batch in batches {
-            if batch.num_rows() == 0 {
-                continue;
+                Ok(DatasetConsistencyWrapper::new(dataset))
             }
-            let metadata_array = Self::get_string_column(&batch, "metadata")?;
-            return Ok(Some(metadata_array.value(0).to_string()));
-        }
+            Err(err) if Self::is_not_found_load_error(&err) => {
+                log::info!("Creating new manifest table at {}", manifest_path);
+                let schema = Self::manifest_schema();
+                let empty_batch = RecordBatch::new_empty(schema.clone());
+                let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone());
+
+                let store_params = ObjectStoreParams {
+                    storage_options_accessor: storage_options.as_ref().map(|opts| {
+                        Arc::new(
+                            lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                                opts.clone(),
+                            ),
+                        )
+                    }),
+                    ..Default::default()
+                };
+                let write_params = WriteParams {
+                    session: session.clone(),
+                    store_params: Some(store_params),
+                    ..Default::default()
+                };
 
-        Ok(None)
+                let dataset =
+                    Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await;
+
+                // Handle race condition where another process created the manifest concurrently
+                match dataset {
+                    Ok(dataset) => {
+                        log::info!(
+                            "Successfully created manifest table at {}, version={}, uri={}",
+                            manifest_path,
+                            dataset.version().version,
+                            dataset.uri()
+                        );
+                        Ok(DatasetConsistencyWrapper::new(dataset))
+                    }
+                    Err(ref e)
+                        if matches!(
+                            e,
+                            LanceError::DatasetAlreadyExists { .. }
+                                | LanceError::CommitConflict { .. }
+                                | LanceError::IncompatibleTransaction { .. }
+                                | LanceError::RetryableCommitConflict { .. }
+                        ) =>
+                    {
+                        // Another process created the manifest concurrently, try to load it
+                        log::info!(
+                            "Manifest table was created by another process, loading it: {}",
+                            manifest_path
+                        );
+                        let recovery_store_options = ObjectStoreParams {
+                            storage_options_accessor: storage_options.as_ref().map(|opts| {
+                                Arc::new(
+                                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
+                                        opts.clone(),
+                                    ),
+                                )
+                            }),
+                            ..Default::default()
+                        };
+                        let recovery_read_params = ReadParams {
+                            session,
+                            store_options: Some(recovery_store_options),
+                            ..Default::default()
+                        };
+                        let dataset = DatasetBuilder::from_uri(&manifest_path)
+                            .with_read_params(recovery_read_params)
+                            .load()
+                            .await
+                            .map_err(|e| {
+                                lance_core::Error::from(NamespaceError::Internal {
+                                    message: format!(
+                                        "Failed to load manifest dataset after creation conflict: {}",
+                                        e
+                                    ),
+                                })
+                            })?;
+                        Ok(DatasetConsistencyWrapper::new(dataset))
+                    }
+                    Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
+                        message: format!("Failed to create manifest dataset: {:?}", e),
+                    })),
+                }
+            }
+            Err(err) => Err(err),
+        }
     }
 
-    /// Delete table version entries from the manifest for a given table and version ranges.
-    ///
-    /// Each range is (start_version, end_version) inclusive. Deletes all matching
-    /// `object_type = 'table_version'` entries whose object_id matches
-    /// `{object_id}${zero_padded_version}`.
+    /// Sorts names alphabetically and applies pagination using page_token (start_after) and limit.
     ///
-    /// Builds a single filter expression covering all version ranges and executes
-    /// one bulk delete operation instead of deleting versions one at a time.
-    pub async fn delete_table_versions(
-        &self,
-        object_id: &str,
-        ranges: &[(i64, i64)],
-    ) -> Result<i64> {
-        if ranges.is_empty() {
-            return Ok(0);
-        }
+    /// Returns the next page token (last item in this page) if more results exist beyond the limit,
+    /// or `None` if this is the last page.
+    fn apply_pagination(
+        names: &mut Vec<String>,
+        page_token: Option<String>,
+        limit: Option<i32>,
+    ) -> Option<String> {
+        names.sort();
 
-        // Collect all object_ids to delete (both new zero-padded and legacy formats)
-        let mut object_id_conditions: Vec<String> = Vec::new();
-        for (start, end) in ranges {
-            for version in *start..=*end {
-                let oid = Self::build_version_object_id(object_id, version);
-                let escaped = oid.replace('\'', "''");
-                object_id_conditions.push(format!("'{}'", escaped));
+        if let Some(start_after) = page_token {
+            if let Some(index) = names
+                .iter()
+                .position(|name| name.as_str() > start_after.as_str())
+            {
+                names.drain(0..index);
+            } else {
+                names.clear();
             }
         }
 
-        if object_id_conditions.is_empty() {
-            return Ok(0);
+        if let Some(limit) = limit
+            && limit >= 0
+        {
+            let limit = limit as usize;
+            if names.len() > limit {
+                let next_page_token = if limit > 0 {
+                    Some(names[limit - 1].clone())
+                } else {
+                    None
+                };
+                names.truncate(limit);
+                return next_page_token;
+            }
         }
 
-        // First, count how many entries exist so we can report the deleted count
-        let in_list = object_id_conditions.join(", ");
-        let filter = format!(
-            "object_type = 'table_version' AND object_id IN ({})",
-            in_list
-        );
+        None
+    }
+}
 
-        let mut scanner = self.manifest_scanner().await?;
-        scanner.filter(&filter).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
-            })
-        })?;
-        scanner.project(&["object_id", "location"]).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
-            })
-        })?;
-        let batches = Self::execute_scanner(scanner).await?;
-        let deleted_count: i64 = batches.iter().map(|b| b.num_rows() as i64).sum();
-
-        if deleted_count == 0 {
-            return Ok(0);
-        }
-
-        // Execute a single bulk delete with the combined filter
-        let _mutation_guard = self.manifest_mutation_lock.lock().await;
-        let dataset_guard = self.manifest_dataset.get().await?;
-        let dataset = Arc::new(dataset_guard.clone());
-        drop(dataset_guard);
-
-        let new_dataset = DeleteBuilder::new(dataset, &filter)
-            .execute()
-            .await
-            .map_err(|e| {
-                convert_lance_commit_error(&e, "Failed to batch delete table versions", None)
-            })?;
-
-        self.manifest_dataset
-            .set_latest(
-                Arc::try_unwrap(new_dataset.new_dataset).unwrap_or_else(|arc| (*arc).clone()),
-            )
-            .await;
-
-        if let Err(e) = self.run_inline_optimization().await {
-            log::warn!(
-                "Unexpected failure when running inline optimization: {:?}",
-                e
-            );
-        }
-
-        Ok(deleted_count)
+#[async_trait]
+impl LanceNamespace for ManifestNamespace {
+    fn namespace_id(&self) -> String {
+        self.root.clone()
     }
 
-    /// Atomically delete table version entries from the manifest by their object_ids.
-    ///
-    /// This method supports multi-table transactional deletion: all specified
-    /// object_ids (which may span multiple tables) are deleted in a single atomic
-    /// `DeleteBuilder` operation. Either all entries are removed or none are.
-    ///
-    /// Object IDs are formatted as `{table_id}${version}`.
-    pub async fn batch_delete_table_versions_by_object_ids(
-        &self,
-        object_ids: &[String],
-    ) -> Result<i64> {
-        if object_ids.is_empty() {
-            return Ok(0);
-        }
-
-        let in_list: String = object_ids
-            .iter()
-            .map(|oid| {
-                let escaped = oid.replace('\'', "''");
-                format!("'{}'", escaped)
+    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
             })
-            .collect::<Vec<_>>()
-            .join(", ");
+        })?;
 
-        let filter = format!(
-            "object_type = 'table_version' AND object_id IN ({})",
-            in_list
-        );
+        // Build filter to find tables in this namespace
+        let filter = if namespace_id.is_empty() {
+            // Root namespace: find tables without a namespace prefix
+            "object_type = 'table' AND NOT contains(object_id, '$')".to_string()
+        } else {
+            // Namespaced: find tables that start with namespace$ but have no additional $
+            let prefix = namespace_id.join(DELIMITER);
+            format!(
+                "object_type = 'table' AND starts_with(object_id, '{}{}') AND NOT contains(substring(object_id, {}), '$')",
+                prefix,
+                DELIMITER,
+                prefix.len() + 2
+            )
+        };
 
-        // Count how many entries exist so we can report the deleted count
         let mut scanner = self.manifest_scanner().await?;
         scanner.filter(&filter).map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
@@ -1422,576 +2741,703 @@ impl ManifestNamespace {
                 message: format!("Failed to project: {:?}", e),
             })
         })?;
-        let batches = Self::execute_scanner(scanner).await?;
-        let deleted_count: i64 = batches.iter().map(|b| b.num_rows() as i64).sum();
-
-        if deleted_count == 0 {
-            return Ok(0);
-        }
-
-        // Execute a single atomic bulk delete covering all tables
-        let _mutation_guard = self.manifest_mutation_lock.lock().await;
-        let dataset_guard = self.manifest_dataset.get().await?;
-        let dataset = Arc::new(dataset_guard.clone());
-        drop(dataset_guard);
-
-        let new_dataset = DeleteBuilder::new(dataset, &filter)
-            .execute()
-            .await
-            .map_err(|e| {
-                convert_lance_commit_error(
-                    &e,
-                    "Failed to batch delete table versions across multiple tables",
-                    None,
-                )
-            })?;
-
-        self.manifest_dataset
-            .set_latest(
-                Arc::try_unwrap(new_dataset.new_dataset).unwrap_or_else(|arc| (*arc).clone()),
-            )
-            .await;
 
-        if let Err(e) = self.run_inline_optimization().await {
-            log::warn!(
-                "Unexpected failure when running inline optimization: {:?}",
-                e
-            );
-        }
+        let batches = Self::execute_scanner(scanner).await?;
 
-        Ok(deleted_count)
-    }
+        let mut table_entries = Vec::new();
+        for batch in batches {
+            if batch.num_rows() == 0 {
+                continue;
+            }
 
-    /// Set a property flag in the __manifest table's metadata key-value map.
-    ///
-    /// This uses `dataset.update_metadata()` to persist the flag in the
-    /// __manifest dataset's table metadata, rather than inserting a row.
-    /// If the property already exists with the same value, this is a no-op.
-    pub async fn set_property(&self, name: &str, value: &str) -> Result<()> {
-        let _mutation_guard = self.manifest_mutation_lock.lock().await;
-        let dataset_guard = self.manifest_dataset.get().await?;
-        if dataset_guard.metadata().get(name) == Some(&value.to_string()) {
-            return Ok(());
+            let object_id_array = Self::get_string_column(&batch, "object_id")?;
+            let location_array = Self::get_string_column(&batch, "location")?;
+            for i in 0..batch.num_rows() {
+                let object_id = object_id_array.value(i);
+                let location = location_array.value(i);
+                let (_namespace, name) = Self::parse_object_id(object_id);
+                table_entries.push((name, location.to_string()));
+            }
         }
-        drop(dataset_guard);
-
-        let mut dataset_guard = self.manifest_dataset.get_mut().await?;
-        dataset_guard
-            .update_metadata([(name, value)])
-            .await
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!(
-                        "Failed to set property '{}' in __manifest metadata: {}",
-                        name, e
-                    ),
-                })
-            })?;
-        Ok(())
-    }
 
-    /// Check if a property flag exists in the __manifest table's metadata key-value map.
-    pub async fn has_property(&self, name: &str) -> Result<bool> {
-        let dataset_guard = self.manifest_dataset.get().await?;
-        Ok(dataset_guard.metadata().contains_key(name))
-    }
+        let mut tables: Vec<String> = if request.include_declared.unwrap_or(true) {
+            table_entries.into_iter().map(|(name, _)| name).collect()
+        } else {
+            let mut stream = futures::stream::iter(table_entries.into_iter().map(
+                |(name, location)| async move {
+                    // `include_declared=false` is an explicit opt-in. We still pay one
+                    // `_versions/` probe per table so declared-state is derived from actual
+                    // manifests. This is linear in the total number of listed tables, and we do
+                    // the probes with bounded concurrency before pagination.
+                    if self.location_has_actual_manifests(&location).await? {
+                        Ok::<Option<String>, Error>(Some(name))
+                    } else {
+                        Ok::<Option<String>, Error>(None)
+                    }
+                },
+            ))
+            .buffered(DECLARED_FILTER_CONCURRENCY);
 
-    /// Parse metadata JSON into a `TableVersion`.
-    ///
-    /// Returns `None` if metadata is invalid or missing required fields.
-    fn parse_table_version(version: i64, metadata_str: &str) -> Option<TableVersion> {
-        let meta: serde_json::Value = match serde_json::from_str(metadata_str) {
-            Ok(v) => v,
-            Err(e) => {
-                log::warn!(
-                    "Skipping version {} due to invalid metadata JSON: {}",
-                    version,
-                    e
-                );
-                return None;
-            }
-        };
-        let manifest_path = match meta.get("manifest_path").and_then(|v| v.as_str()) {
-            Some(p) => p.to_string(),
-            None => {
-                log::warn!(
-                    "Skipping version {} due to missing 'manifest_path' in metadata — \
-                     this may indicate data corruption",
-                    version
-                );
-                return None;
+            let mut filtered = Vec::new();
+            while let Some(result) = stream.next().await {
+                if let Some(name) = result? {
+                    filtered.push(name);
+                }
             }
+            filtered
         };
-        let manifest_size = meta.get("manifest_size").and_then(|v| v.as_i64());
-        let e_tag = meta
-            .get("e_tag")
-            .and_then(|v| v.as_str())
-            .map(|s| s.to_string());
-        Some(TableVersion {
-            version,
-            manifest_path,
-            manifest_size,
-            e_tag,
-            timestamp_millis: None,
-            metadata: None,
-        })
-    }
-
-    /// List table versions from the __manifest table.
-    ///
-    /// Queries the manifest for all versions of the given table and returns
-    /// them as a `ListTableVersionsResponse`.
-    pub async fn list_table_versions(
-        &self,
-        table_id: &[String],
-        descending: bool,
-        limit: Option<i32>,
-    ) -> Result<ListTableVersionsResponse> {
-        let object_id = Self::str_object_id(table_id);
-        let manifest_versions = self
-            .query_table_versions(&object_id, descending, limit)
-            .await?;
 
-        let table_versions: Vec<TableVersion> = manifest_versions
-            .into_iter()
-            .filter_map(|(version, metadata_str)| Self::parse_table_version(version, &metadata_str))
-            .collect();
-
-        Ok(ListTableVersionsResponse {
-            versions: table_versions,
-            page_token: None,
-        })
+        let next_page_token =
+            Self::apply_pagination(&mut tables, request.page_token, request.limit);
+        let mut response = ListTablesResponse::new(tables);
+        response.page_token = next_page_token;
+        Ok(response)
     }
 
-    /// Describe a specific table version from the __manifest table.
-    ///
-    /// Queries the manifest for a specific version and returns it as a
-    /// `DescribeTableVersionResponse`. Returns an error if the version is not found.
-    pub async fn describe_table_version(
-        &self,
-        table_id: &[String],
-        version: i64,
-    ) -> Result<DescribeTableVersionResponse> {
-        let object_id = Self::str_object_id(table_id);
-        if let Some(metadata_str) = self.query_table_version(&object_id, version).await?
-            && let Some(tv) = Self::parse_table_version(version, &metadata_str)
-        {
-            return Ok(DescribeTableVersionResponse {
-                version: Box::new(tv),
-            });
-        }
-        Err(NamespaceError::TableVersionNotFound {
-            message: format!("version {} for table {:?}", version, table_id),
-        }
-        .into())
-    }
+    async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
 
-    /// Register a table in the manifest without creating the physical table (internal helper for migration)
-    pub async fn register_table(&self, name: &str, location: String) -> Result<()> {
-        let object_id = Self::build_object_id(&[], name);
-        if self.manifest_contains_object(&object_id).await? {
-            return Err(NamespaceError::Internal {
-                message: format!("Table '{}' already exists", name),
+        if table_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
             }
             .into());
         }
 
-        self.insert_into_manifest(object_id, ObjectType::Table, Some(location))
-            .await
-    }
-
-    /// Validate that all levels of a namespace path exist
-    async fn validate_namespace_levels_exist(&self, namespace_path: &[String]) -> Result<()> {
-        for i in 1..=namespace_path.len() {
-            let partial_path = &namespace_path[..i];
-            let object_id = partial_path.join(DELIMITER);
-            if !self.manifest_contains_object(&object_id).await? {
-                return Err(NamespaceError::NamespaceNotFound {
-                    message: format!("parent namespace '{}'", object_id),
-                }
-                .into());
-            }
-        }
-        Ok(())
-    }
+        let object_id = Self::str_object_id(table_id);
+        let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
 
-    /// Query the manifest for a namespace with the given object ID
-    async fn query_manifest_for_namespace(&self, object_id: &str) -> Result<Option<NamespaceInfo>> {
-        let escaped_id = object_id.replace('\'', "''");
-        let filter = format!("object_id = '{}' AND object_type = 'namespace'", escaped_id);
-        let mut scanner = self.manifest_scanner().await?;
-        scanner.filter(&filter).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
-            })
-        })?;
-        scanner.project(&["object_id", "metadata"]).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
-            })
-        })?;
-        let batches = Self::execute_scanner(scanner).await?;
+        // Extract table name and namespace from table_id
+        let table_name = table_id.last().cloned().unwrap_or_default();
+        let namespace_id: Vec<String> = if table_id.len() > 1 {
+            table_id[..table_id.len() - 1].to_vec()
+        } else {
+            vec![]
+        };
 
-        let mut found_result: Option<NamespaceInfo> = None;
-        let mut total_rows = 0;
+        let load_detailed_metadata = request.load_detailed_metadata.unwrap_or(false);
+        let should_check_declared =
+            load_detailed_metadata || request.check_declared.unwrap_or(false);
+        // For backwards compatibility, only skip vending credentials when explicitly set to false
+        let vend_credentials = request.vend_credentials.unwrap_or(true);
 
-        for batch in batches {
-            if batch.num_rows() == 0 {
-                continue;
-            }
+        match table_info {
+            Some(info) => {
+                // Construct full URI from relative location
+                let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
 
-            total_rows += batch.num_rows();
-            if total_rows > 1 {
-                return Err(NamespaceError::Internal {
-                    message: format!(
-                        "Expected exactly 1 namespace with id '{}', found {}",
-                        object_id, total_rows
-                    ),
-                }
-                .into());
-            }
+                let storage_options = if vend_credentials {
+                    self.storage_options.clone()
+                } else {
+                    None
+                };
+                let is_only_declared = if should_check_declared {
+                    Some(!self.location_has_actual_manifests(&info.location).await?)
+                } else {
+                    None
+                };
 
-            let object_id_array = Self::get_string_column(&batch, "object_id")?;
-            let metadata_array = Self::get_string_column(&batch, "metadata")?;
+                if !load_detailed_metadata {
+                    return Ok(DescribeTableResponse {
+                        table: Some(table_name),
+                        namespace: Some(namespace_id),
+                        location: Some(table_uri.clone()),
+                        table_uri: Some(table_uri),
+                        storage_options,
+                        properties: info.metadata,
+                        is_only_declared,
+                        ..Default::default()
+                    });
+                }
 
-            let object_id_str = object_id_array.value(0);
-            let metadata = if !metadata_array.is_null(0) {
-                let metadata_str = metadata_array.value(0);
-                match serde_json::from_str::<HashMap<String, String>>(metadata_str) {
-                    Ok(map) => Some(map),
-                    Err(e) => {
-                        return Err(NamespaceError::Internal {
-                            message: format!(
-                                "Failed to deserialize metadata for namespace '{}': {}",
-                                object_id, e
-                            ),
+                if is_only_declared == Some(true) {
+                    return Ok(DescribeTableResponse {
+                        table: Some(table_name),
+                        namespace: Some(namespace_id),
+                        location: Some(table_uri.clone()),
+                        table_uri: Some(table_uri),
+                        storage_options,
+                        properties: info.metadata,
+                        is_only_declared,
+                        ..Default::default()
+                    });
+                }
+
+                let mut builder = DatasetBuilder::from_uri(&table_uri);
+                if let Some(opts) = &self.storage_options {
+                    builder = builder.with_storage_options(opts.clone());
+                }
+                if let Some(session) = &self.session {
+                    builder = builder.with_session(session.clone());
+                }
+
+                match builder.load().await {
+                    Ok(mut dataset) => {
+                        // If a specific version is requested, checkout that version
+                        if let Some(requested_version) = request.version {
+                            dataset = dataset.checkout_version(requested_version as u64).await?;
                         }
-                        .into());
+
+                        let version = dataset.version().version;
+                        let lance_schema = dataset.schema();
+                        let arrow_schema: arrow_schema::Schema = lance_schema.into();
+                        let json_schema = arrow_schema_to_json(&arrow_schema)?;
+
+                        Ok(DescribeTableResponse {
+                            table: Some(table_name.clone()),
+                            namespace: Some(namespace_id.clone()),
+                            version: Some(version as i64),
+                            location: Some(table_uri.clone()),
+                            table_uri: Some(table_uri),
+                            schema: Some(Box::new(json_schema)),
+                            storage_options,
+                            properties: info.metadata.clone(),
+                            is_only_declared,
+                            ..Default::default()
+                        })
                     }
+                    Err(err) => Err(NamespaceError::Internal {
+                        message: format!(
+                            "Table exists in manifest but failed to load dataset '{}': {}",
+                            object_id, err
+                        ),
+                    }
+                    .into()),
                 }
-            } else {
-                None
-            };
+            }
+            None => Err(NamespaceError::TableNotFound {
+                message: Self::format_table_id(table_id),
+            }
+            .into()),
+        }
+    }
 
-            let (namespace, name) = Self::parse_object_id(object_id_str);
-            found_result = Some(NamespaceInfo {
-                namespace,
-                name,
-                metadata,
-            });
+    async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
+
+        if table_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
         }
 
-        Ok(found_result)
+        let object_id = Self::str_object_id(table_id);
+        let exists = self.manifest_contains_object(&object_id).await?;
+        if exists {
+            Ok(())
+        } else {
+            Err(NamespaceError::TableNotFound {
+                message: Self::format_table_id(table_id),
+            }
+            .into())
+        }
     }
 
-    /// Create or load the manifest dataset, ensuring it has the latest schema setup.
-    ///
-    /// This function will:
-    /// 1. Try to load an existing manifest table
-    /// 2. If it exists, check and migrate the schema if needed (e.g., add primary key metadata)
-    /// 3. If it doesn't exist, create a new manifest table with the current schema
-    /// 4. Persist feature flags (e.g., table_version_storage_enabled) if requested
-    async fn ensure_manifest_table_up_to_date(
-        root: &str,
-        storage_options: &Option<HashMap<String, String>>,
-        session: Option<Arc<Session>>,
-        table_version_storage_enabled: bool,
-    ) -> Result<DatasetConsistencyWrapper> {
-        let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME);
-        log::debug!("Attempting to load manifest from {}", manifest_path);
-        let store_options = ObjectStoreParams {
-            storage_options_accessor: storage_options.as_ref().map(|opts| {
+    async fn create_table(
+        &self,
+        request: CreateTableRequest,
+        data: Bytes,
+    ) -> Result<CreateTableResponse> {
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
+
+        if table_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
+        }
+
+        let (namespace, table_name) = Self::split_object_id(table_id);
+        let object_id = Self::build_object_id(&namespace, &table_name);
+
+        // Refuse before writing any table data if this build cannot write the
+        // manifest, so a refused create leaves no orphaned dataset behind.
+        self.ensure_manifest_writable().await?;
+
+        let existing_table = self.query_manifest_for_table(&object_id).await?;
+        let existing_has_manifests = if let Some(existing_table) = &existing_table {
+            Some(
+                self.location_has_actual_manifests(&existing_table.location)
+                    .await?,
+            )
+        } else {
+            None
+        };
+
+        if existing_has_manifests == Some(false)
+            && request
+                .properties
+                .as_ref()
+                .is_some_and(|properties| !properties.is_empty())
+        {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "create_table cannot set properties for already declared table '{}'",
+                    object_id
+                ),
+            }
+            .into());
+        }
+
+        let create_mode = if existing_has_manifests == Some(false) {
+            CreateTableMode::Create
+        } else {
+            CreateTableMode::parse(request.mode.as_deref())?
+        };
+        let dir_name = if let Some(existing_table) = &existing_table {
+            existing_table.location.clone()
+        } else if namespace.is_empty() && self.dir_listing_enabled {
+            format!("{}.lance", table_name)
+        } else {
+            Self::generate_dir_name(&object_id)
+        };
+        let table_uri = Self::construct_full_uri(&self.root, &dir_name)?;
+        let overwriting_existing_table =
+            existing_has_manifests == Some(true) && create_mode == CreateTableMode::Overwrite;
+
+        if existing_has_manifests == Some(true) {
+            match create_mode {
+                CreateTableMode::Create => {
+                    return Err(NamespaceError::TableAlreadyExists {
+                        message: table_name.clone(),
+                    }
+                    .into());
+                }
+                CreateTableMode::ExistOk => {
+                    let properties = existing_table
+                        .as_ref()
+                        .and_then(|table| table.metadata.clone());
+                    return Ok(CreateTableResponse {
+                        location: Some(table_uri),
+                        storage_options: self.storage_options.clone(),
+                        properties,
+                        ..Default::default()
+                    });
+                }
+                CreateTableMode::Overwrite => {}
+            }
+        }
+
+        // Validate that request_data is provided
+        if data.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Request data (Arrow IPC stream) is required for create_table".to_string(),
+            }
+            .into());
+        }
+
+        // Write the data using Lance Dataset
+        let cursor = Cursor::new(data.to_vec());
+        let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to read IPC stream: {:?}", e),
+            })
+        })?;
+
+        let batches: Vec<RecordBatch> = stream_reader
+            .collect::<std::result::Result<Vec<_>, _>>()
+            .map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to collect batches: {:?}", e),
+            })
+        })?;
+
+        if batches.is_empty() {
+            return Err(NamespaceError::Internal {
+                message: "No data provided for table creation".to_string(),
+            }
+            .into());
+        }
+
+        let schema = batches[0].schema();
+        let batch_results: Vec<std::result::Result<RecordBatch, arrow_schema::ArrowError>> =
+            batches.into_iter().map(Ok).collect();
+        let reader = RecordBatchIterator::new(batch_results, schema);
+
+        let mut write_storage_options = self.storage_options.clone().unwrap_or_default();
+        if let Some(request_storage_options) = request.storage_options.as_ref() {
+            write_storage_options.extend(request_storage_options.clone());
+        }
+
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: (!write_storage_options.is_empty()).then(|| {
                 Arc::new(
                     lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                        opts.clone(),
+                        write_storage_options,
                     ),
                 )
             }),
             ..Default::default()
         };
-        let read_params = ReadParams {
-            session: session.clone(),
-            store_options: Some(store_options.clone()),
+        let write_params = WriteParams {
+            mode: create_mode.write_mode(),
+            session: self.session.clone(),
+            store_params: Some(store_params),
             ..Default::default()
         };
-        let dataset_result = DatasetBuilder::from_uri(&manifest_path)
-            .with_read_params(read_params)
-            .load()
-            .await;
-        if let Ok(mut dataset) = dataset_result {
-            // Check if the object_id field has primary key metadata, migrate if not
-            let needs_pk_migration = dataset
-                .schema()
-                .field("object_id")
-                .map(|f| {
-                    !f.metadata
-                        .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+        let dataset = Dataset::write(Box::new(reader), &table_uri, Some(write_params))
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!("Failed to write dataset: {:?}", e),
                 })
-                .unwrap_or(false);
+            })?;
+        let version = dataset.version().version as i64;
+
+        if overwriting_existing_table {
+            let metadata =
+                Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
+            self.upsert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id,
+                    object_type: ObjectType::Table,
+                    location: Some(dir_name),
+                    metadata,
+                }],
+                None,
+            )
+            .await?;
+
+            Ok(CreateTableResponse {
+                version: Some(version),
+                location: Some(table_uri),
+                storage_options: self.storage_options.clone(),
+                properties: request.properties,
+                ..Default::default()
+            })
+        } else {
+            match existing_table {
+                Some(existing_table) => Ok(CreateTableResponse {
+                    version: Some(version),
+                    location: Some(table_uri),
+                    storage_options: self.storage_options.clone(),
+                    properties: existing_table.metadata,
+                    ..Default::default()
+                }),
+                None => {
+                    let metadata =
+                        Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
+                    // Register in manifest (store dir_name, not full URI)
+                    self.insert_into_manifest_with_metadata(
+                        vec![ManifestEntry {
+                            object_id,
+                            object_type: ObjectType::Table,
+                            location: Some(dir_name.clone()),
+                            metadata,
+                        }],
+                        None,
+                    )
+                    .await?;
+
+                    Ok(CreateTableResponse {
+                        version: Some(version),
+                        location: Some(table_uri),
+                        storage_options: self.storage_options.clone(),
+                        properties: request.properties,
+                        ..Default::default()
+                    })
+                }
+            }
+        }
+    }
+
+    async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
+            })
+        })?;
+
+        if table_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
+            }
+            .into());
+        }
+
+        let (namespace, table_name) = Self::split_object_id(table_id);
+        let object_id = Self::build_object_id(&namespace, &table_name);
+
+        // Query manifest for table location
+        let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
+
+        match table_info {
+            Some(info) => {
+                // Delete from manifest first
+                self.delete_from_manifest(&object_id).boxed().await?;
+
+                // Delete physical data directory using the dir_name from manifest
+                let table_path = self.base_path.clone().join(info.location.as_str());
+                let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
 
-            if needs_pk_migration {
-                log::info!("Migrating __manifest table to add primary key metadata on object_id");
-                dataset
-                    .update_field_metadata()
-                    .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")])
-                    .map_err(|e| {
-                        lance_core::Error::from(NamespaceError::Internal {
-                            message: format!(
-                                "Failed to find object_id field for migration: {:?}",
-                                e
-                            ),
-                        })
-                    })?
+                // Remove the table directory
+                self.object_store
+                    .remove_dir_all(table_path)
+                    .boxed()
                     .await
                     .map_err(|e| {
                         lance_core::Error::from(NamespaceError::Internal {
-                            message: format!("Failed to migrate primary key metadata: {:?}", e),
+                            message: format!("Failed to delete table directory: {:?}", e),
                         })
                     })?;
+
+                Ok(DropTableResponse {
+                    id: request.id.clone(),
+                    location: Some(table_uri),
+                    ..Default::default()
+                })
+            }
+            None => Err(NamespaceError::TableNotFound {
+                message: table_name.to_string(),
             }
+            .into()),
+        }
+    }
+
+    async fn list_namespaces(
+        &self,
+        request: ListNamespacesRequest,
+    ) -> Result<ListNamespacesResponse> {
+        let parent_namespace = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
-            // Persist table_version_storage_enabled flag in __manifest so that once
-            // enabled, it becomes a permanent property of this namespace.
-            if table_version_storage_enabled {
-                let needs_flag = dataset
-                    .metadata()
-                    .get("table_version_storage_enabled")
-                    .map(|v| v != "true")
-                    .unwrap_or(true);
+        // Build filter to find direct child namespaces
+        let filter = if parent_namespace.is_empty() {
+            // Root namespace: find all namespaces without a parent
+            "object_type = 'namespace' AND NOT contains(object_id, '$')".to_string()
+        } else {
+            // Non-root: find namespaces that start with parent$ but have no additional $
+            let prefix = parent_namespace.join(DELIMITER);
+            format!(
+                "object_type = 'namespace' AND starts_with(object_id, '{}{}') AND NOT contains(substring(object_id, {}), '$')",
+                prefix,
+                DELIMITER,
+                prefix.len() + 2
+            )
+        };
 
-                if needs_flag
-                    && let Err(e) = dataset
-                        .update_metadata([("table_version_storage_enabled", "true")])
-                        .await
-                {
-                    log::warn!(
-                        "Failed to persist table_version_storage_enabled flag in __manifest: {:?}",
-                        e
-                    );
-                }
+        let mut scanner = self.manifest_scanner().await?;
+        scanner.filter(&filter).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to filter: {:?}", e),
+            })
+        })?;
+        scanner.project(&["object_id"]).map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to project: {:?}", e),
+            })
+        })?;
+
+        let batches = Self::execute_scanner(scanner).await?;
+        let mut namespaces = Vec::new();
+
+        for batch in batches {
+            if batch.num_rows() == 0 {
+                continue;
             }
 
-            Ok(DatasetConsistencyWrapper::new(dataset))
-        } else {
-            log::info!("Creating new manifest table at {}", manifest_path);
-            let schema = Self::manifest_schema();
-            let empty_batch = RecordBatch::new_empty(schema.clone());
-            let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone());
-
-            let store_params = ObjectStoreParams {
-                storage_options_accessor: storage_options.as_ref().map(|opts| {
-                    Arc::new(
-                        lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                            opts.clone(),
-                        ),
-                    )
-                }),
-                ..Default::default()
-            };
-            let write_params = WriteParams {
-                session: session.clone(),
-                store_params: Some(store_params),
+            let object_id_array = Self::get_string_column(&batch, "object_id")?;
+            for i in 0..batch.num_rows() {
+                let object_id = object_id_array.value(i);
+                let (_namespace, name) = Self::parse_object_id(object_id);
+                namespaces.push(name);
+            }
+        }
+
+        let next_page_token =
+            Self::apply_pagination(&mut namespaces, request.page_token, request.limit);
+        let mut response = ListNamespacesResponse::new(namespaces);
+        response.page_token = next_page_token;
+        Ok(response)
+    }
+
+    async fn describe_namespace(
+        &self,
+        request: DescribeNamespaceRequest,
+    ) -> Result<DescribeNamespaceResponse> {
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
+
+        // Root namespace always exists
+        if namespace_id.is_empty() {
+            #[allow(clippy::needless_update)]
+            return Ok(DescribeNamespaceResponse {
+                properties: Some(HashMap::new()),
                 ..Default::default()
-            };
+            });
+        }
 
-            let dataset =
-                Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await;
+        // Check if namespace exists in manifest
+        let object_id = namespace_id.join(DELIMITER);
+        let namespace_info = self.query_manifest_for_namespace(&object_id).await?;
 
-            // Handle race condition where another process created the manifest concurrently
-            match dataset {
-                Ok(dataset) => {
-                    log::info!(
-                        "Successfully created manifest table at {}, version={}, uri={}",
-                        manifest_path,
-                        dataset.version().version,
-                        dataset.uri()
-                    );
-                    Ok(DatasetConsistencyWrapper::new(dataset))
-                }
-                Err(ref e)
-                    if matches!(
-                        e,
-                        LanceError::DatasetAlreadyExists { .. }
-                            | LanceError::CommitConflict { .. }
-                            | LanceError::IncompatibleTransaction { .. }
-                            | LanceError::RetryableCommitConflict { .. }
-                    ) =>
-                {
-                    // Another process created the manifest concurrently, try to load it
-                    log::info!(
-                        "Manifest table was created by another process, loading it: {}",
-                        manifest_path
-                    );
-                    let recovery_store_options = ObjectStoreParams {
-                        storage_options_accessor: storage_options.as_ref().map(|opts| {
-                            Arc::new(
-                                lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                                    opts.clone(),
-                                ),
-                            )
-                        }),
-                        ..Default::default()
-                    };
-                    let recovery_read_params = ReadParams {
-                        session,
-                        store_options: Some(recovery_store_options),
-                        ..Default::default()
-                    };
-                    let dataset = DatasetBuilder::from_uri(&manifest_path)
-                        .with_read_params(recovery_read_params)
-                        .load()
-                        .await
-                        .map_err(|e| {
-                            lance_core::Error::from(NamespaceError::Internal {
-                                message: format!(
-                                    "Failed to load manifest dataset after creation conflict: {}",
-                                    e
-                                ),
-                            })
-                        })?;
-                    Ok(DatasetConsistencyWrapper::new(dataset))
-                }
-                Err(e) => Err(lance_core::Error::from(NamespaceError::Internal {
-                    message: format!("Failed to create manifest dataset: {:?}", e),
-                })),
+        match namespace_info {
+            #[allow(clippy::needless_update)]
+            Some(info) => Ok(DescribeNamespaceResponse {
+                properties: info.metadata,
+                ..Default::default()
+            }),
+            None => Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
             }
+            .into()),
         }
     }
 
-    /// Sorts names alphabetically and applies pagination using page_token (start_after) and limit.
-    ///
-    /// Returns the next page token (last item in this page) if more results exist beyond the limit,
-    /// or `None` if this is the last page.
-    fn apply_pagination(
-        names: &mut Vec<String>,
-        page_token: Option<String>,
-        limit: Option<i32>,
-    ) -> Option<String> {
-        names.sort();
+    async fn create_namespace(
+        &self,
+        request: CreateNamespaceRequest,
+    ) -> Result<CreateNamespaceResponse> {
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
 
-        if let Some(start_after) = page_token {
-            if let Some(index) = names
-                .iter()
-                .position(|name| name.as_str() > start_after.as_str())
-            {
-                names.drain(0..index);
-            } else {
-                names.clear();
+        // Root namespace always exists and cannot be created
+        if namespace_id.is_empty() {
+            return Err(NamespaceError::NamespaceAlreadyExists {
+                message: "root namespace".to_string(),
             }
+            .into());
         }
 
-        if let Some(limit) = limit
-            && limit >= 0
-        {
-            let limit = limit as usize;
-            if names.len() > limit {
-                let next_page_token = if limit > 0 {
-                    Some(names[limit - 1].clone())
-                } else {
-                    None
-                };
-                names.truncate(limit);
-                return next_page_token;
+        // Validate parent namespaces exist (but not the namespace being created)
+        if namespace_id.len() > 1 {
+            self.validate_namespace_levels_exist(&namespace_id[..namespace_id.len() - 1])
+                .await?;
+        }
+
+        let object_id = namespace_id.join(DELIMITER);
+        if self.manifest_contains_object(&object_id).await? {
+            return Err(NamespaceError::NamespaceAlreadyExists {
+                message: object_id.to_string(),
             }
+            .into());
         }
 
-        None
-    }
-}
+        let metadata =
+            Self::serialize_metadata(request.properties.as_ref(), "namespace", &object_id)?;
 
-#[async_trait]
-impl LanceNamespace for ManifestNamespace {
-    fn namespace_id(&self) -> String {
-        self.root.clone()
+        self.insert_into_manifest_with_metadata(
+            vec![ManifestEntry {
+                object_id,
+                object_type: ObjectType::Namespace,
+                location: None,
+                metadata,
+            }],
+            None,
+        )
+        .await?;
+
+        Ok(CreateNamespaceResponse {
+            properties: request.properties,
+            ..Default::default()
+        })
     }
 
-    async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
+    async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
         let namespace_id = request.id.as_ref().ok_or_else(|| {
             lance_core::Error::from(NamespaceError::InvalidInput {
                 message: "Namespace ID is required".to_string(),
             })
         })?;
 
-        // Build filter to find tables in this namespace
-        let filter = if namespace_id.is_empty() {
-            // Root namespace: find tables without a namespace prefix
-            "object_type = 'table' AND NOT contains(object_id, '$')".to_string()
-        } else {
-            // Namespaced: find tables that start with namespace$ but have no additional $
-            let prefix = namespace_id.join(DELIMITER);
-            format!(
-                "object_type = 'table' AND starts_with(object_id, '{}{}') AND NOT contains(substring(object_id, {}), '$')",
-                prefix,
-                DELIMITER,
-                prefix.len() + 2
-            )
-        };
+        // Root namespace always exists and cannot be dropped
+        if namespace_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Root namespace cannot be dropped".to_string(),
+            }
+            .into());
+        }
+
+        let object_id = namespace_id.join(DELIMITER);
+
+        // Check if namespace exists
+        if !self.manifest_contains_object(&object_id).boxed().await? {
+            return Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
+            }
+            .into());
+        }
 
-        let mut scanner = self.manifest_scanner().await?;
+        // Check for child namespaces
+        let escaped_id = object_id.replace('\'', "''");
+        let prefix = format!("{}{}", escaped_id, DELIMITER);
+        let filter = format!("starts_with(object_id, '{}')", prefix);
+        let mut scanner = self.manifest_scanner().boxed().await?;
         scanner.filter(&filter).map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
                 message: format!("Failed to filter: {:?}", e),
             })
         })?;
-        scanner.project(&["object_id", "location"]).map_err(|e| {
+        scanner.project::<&str>(&[]).map_err(|e| {
             lance_core::Error::from(NamespaceError::Internal {
                 message: format!("Failed to project: {:?}", e),
             })
         })?;
+        scanner.with_row_id();
+        let count = scanner.count_rows().boxed().await.map_err(|e| {
+            lance_core::Error::from(NamespaceError::Internal {
+                message: format!("Failed to count rows: {:?}", e),
+            })
+        })?;
 
-        let batches = Self::execute_scanner(scanner).await?;
-
-        let mut table_entries = Vec::new();
-        for batch in batches {
-            if batch.num_rows() == 0 {
-                continue;
+        if count > 0 {
+            return Err(NamespaceError::NamespaceNotEmpty {
+                message: format!("'{}' (contains {} child objects)", object_id, count),
             }
+            .into());
+        }
 
-            let object_id_array = Self::get_string_column(&batch, "object_id")?;
-            let location_array = Self::get_string_column(&batch, "location")?;
-            for i in 0..batch.num_rows() {
-                let object_id = object_id_array.value(i);
-                let location = location_array.value(i);
-                let (_namespace, name) = Self::parse_object_id(object_id);
-                table_entries.push((name, location.to_string()));
-            }
+        self.delete_from_manifest(&object_id).boxed().await?;
+
+        Ok(DropNamespaceResponse::default())
+    }
+
+    async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
+        let namespace_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Namespace ID is required".to_string(),
+            })
+        })?;
+
+        // Root namespace always exists
+        if namespace_id.is_empty() {
+            return Ok(());
         }
 
-        let mut tables: Vec<String> = if request.include_declared.unwrap_or(true) {
-            table_entries.into_iter().map(|(name, _)| name).collect()
+        let object_id = namespace_id.join(DELIMITER);
+        if self.manifest_contains_object(&object_id).await? {
+            Ok(())
         } else {
-            let mut stream = futures::stream::iter(table_entries.into_iter().map(
-                |(name, location)| async move {
-                    // `include_declared=false` is an explicit opt-in. We still pay one
-                    // `_versions/` probe per table so declared-state is derived from actual
-                    // manifests. This is linear in the total number of listed tables, and we do
-                    // the probes with bounded concurrency before pagination.
-                    if self.location_has_actual_manifests(&location).await? {
-                        Ok::<Option<String>, Error>(Some(name))
-                    } else {
-                        Ok::<Option<String>, Error>(None)
-                    }
-                },
-            ))
-            .buffered(DECLARED_FILTER_CONCURRENCY);
-
-            let mut filtered = Vec::new();
-            while let Some(result) = stream.next().await {
-                if let Some(name) = result? {
-                    filtered.push(name);
-                }
+            Err(NamespaceError::NamespaceNotFound {
+                message: object_id.to_string(),
             }
-            filtered
-        };
-
-        let next_page_token =
-            Self::apply_pagination(&mut tables, request.page_token, request.limit);
-        let mut response = ListTablesResponse::new(tables);
-        response.page_token = next_page_token;
-        Ok(response)
+            .into())
+        }
     }
 
-    async fn describe_table(&self, request: DescribeTableRequest) -> Result<DescribeTableResponse> {
+    async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
         let table_id = request.id.as_ref().ok_or_else(|| {
             lance_core::Error::from(NamespaceError::InvalidInput {
                 message: "Table ID is required".to_string(),
@@ -2005,115 +3451,107 @@ impl LanceNamespace for ManifestNamespace {
             .into());
         }
 
-        let object_id = Self::str_object_id(table_id);
-        let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
+        let (namespace, table_name) = Self::split_object_id(table_id);
+        let object_id = Self::build_object_id(&namespace, &table_name);
 
-        // Extract table name and namespace from table_id
-        let table_name = table_id.last().cloned().unwrap_or_default();
-        let namespace_id: Vec<String> = if table_id.len() > 1 {
-            table_id[..table_id.len() - 1].to_vec()
+        // Check if table already exists in manifest
+        let existing = self.query_manifest_for_table(&object_id).await?;
+        if existing.is_some() {
+            return Err(NamespaceError::TableAlreadyExists {
+                message: table_name.to_string(),
+            }
+            .into());
+        }
+
+        // Create table location path with hash-based naming
+        // When dir_listing_enabled is true and it's a root table, use directory-style naming: {table_name}.lance
+        // Otherwise, use hash-based naming: {hash}_{object_id}
+        let dir_name = if namespace.is_empty() && self.dir_listing_enabled {
+            // Root table with directory listing enabled: use {table_name}.lance
+            format!("{}.lance", table_name)
         } else {
-            vec![]
+            // Child namespace table or dir listing disabled: use hash-based naming
+            Self::generate_dir_name(&object_id)
         };
+        let table_path = self.base_path.clone().join(dir_name.as_str());
+        let table_uri = Self::construct_full_uri(&self.root, &dir_name)?;
 
-        let load_detailed_metadata = request.load_detailed_metadata.unwrap_or(false);
-        let should_check_declared =
-            load_detailed_metadata || request.check_declared.unwrap_or(false);
-        // For backwards compatibility, only skip vending credentials when explicitly set to false
-        let vend_credentials = request.vend_credentials.unwrap_or(true);
-
-        match table_info {
-            Some(info) => {
-                // Construct full URI from relative location
-                let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
-
-                let storage_options = if vend_credentials {
-                    self.storage_options.clone()
-                } else {
-                    None
-                };
-                let is_only_declared = if should_check_declared {
-                    Some(!self.location_has_actual_manifests(&info.location).await?)
-                } else {
-                    None
-                };
-
-                if !load_detailed_metadata {
-                    return Ok(DescribeTableResponse {
-                        table: Some(table_name),
-                        namespace: Some(namespace_id),
-                        location: Some(table_uri.clone()),
-                        table_uri: Some(table_uri),
-                        storage_options,
-                        properties: info.metadata,
-                        is_only_declared,
-                        ..Default::default()
-                    });
+        // Validate location if provided
+        if let Some(req_location) = &request.location {
+            let req_location = req_location.trim_end_matches('/');
+            if req_location != table_uri {
+                return Err(NamespaceError::InvalidInput {
+                    message: format!(
+                        "Cannot declare table {} at location {}, must be at location {}",
+                        table_name, req_location, table_uri
+                    ),
                 }
+                .into());
+            }
+        }
 
-                if is_only_declared == Some(true) {
-                    return Ok(DescribeTableResponse {
-                        table: Some(table_name),
-                        namespace: Some(namespace_id),
-                        location: Some(table_uri.clone()),
-                        table_uri: Some(table_uri),
-                        storage_options,
-                        properties: info.metadata,
-                        is_only_declared,
-                        ..Default::default()
-                    });
-                }
+        // Create the .lance-reserved file to mark the table as existing
+        let reserved_file_path = table_path.clone().join(".lance-reserved");
 
-                let mut builder = DatasetBuilder::from_uri(&table_uri);
-                if let Some(opts) = &self.storage_options {
-                    builder = builder.with_storage_options(opts.clone());
-                }
-                if let Some(session) = &self.session {
-                    builder = builder.with_session(session.clone());
-                }
+        self.object_store
+            .create(&reserved_file_path)
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to create .lance-reserved file for table {}: {}",
+                        table_name, e
+                    ),
+                })
+            })?
+            .shutdown()
+            .await
+            .map_err(|e| {
+                lance_core::Error::from(NamespaceError::Internal {
+                    message: format!(
+                        "Failed to finalize .lance-reserved file for table {}: {}",
+                        table_name, e
+                    ),
+                })
+            })?;
 
-                match builder.load().await {
-                    Ok(mut dataset) => {
-                        // If a specific version is requested, checkout that version
-                        if let Some(requested_version) = request.version {
-                            dataset = dataset.checkout_version(requested_version as u64).await?;
-                        }
+        let metadata = Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
 
-                        let version = dataset.version().version;
-                        let lance_schema = dataset.schema();
-                        let arrow_schema: arrow_schema::Schema = lance_schema.into();
-                        let json_schema = arrow_schema_to_json(&arrow_schema)?;
+        // Add entry to manifest marking this as a declared table (store dir_name, not full path)
+        self.insert_into_manifest_with_metadata(
+            vec![ManifestEntry {
+                object_id,
+                object_type: ObjectType::Table,
+                location: Some(dir_name),
+                metadata,
+            }],
+            None,
+        )
+        .await?;
 
-                        Ok(DescribeTableResponse {
-                            table: Some(table_name.clone()),
-                            namespace: Some(namespace_id.clone()),
-                            version: Some(version as i64),
-                            location: Some(table_uri.clone()),
-                            table_uri: Some(table_uri),
-                            schema: Some(Box::new(json_schema)),
-                            storage_options,
-                            properties: info.metadata.clone(),
-                            is_only_declared,
-                            ..Default::default()
-                        })
-                    }
-                    Err(err) => Err(NamespaceError::Internal {
-                        message: format!(
-                            "Table exists in manifest but failed to load dataset '{}': {}",
-                            object_id, err
-                        ),
-                    }
-                    .into()),
-                }
-            }
-            None => Err(NamespaceError::TableNotFound {
-                message: Self::format_table_id(table_id),
-            }
-            .into()),
-        }
+        log::info!(
+            "Declared table '{}' in manifest at {}",
+            table_name,
+            table_uri
+        );
+
+        // For backwards compatibility, only skip vending credentials when explicitly set to false
+        let vend_credentials = request.vend_credentials.unwrap_or(true);
+        let storage_options = if vend_credentials {
+            self.storage_options.clone()
+        } else {
+            None
+        };
+
+        Ok(DeclareTableResponse {
+            location: Some(table_uri),
+            storage_options,
+            properties: request.properties,
+            ..Default::default()
+        })
     }
 
-    async fn table_exists(&self, request: TableExistsRequest) -> Result<()> {
+    async fn register_table(&self, request: RegisterTableRequest) -> Result<RegisterTableResponse> {
         let table_id = request.id.as_ref().ok_or_else(|| {
             lance_core::Error::from(NamespaceError::InvalidInput {
                 message: "Table ID is required".to_string(),
@@ -2127,783 +3565,1103 @@ impl LanceNamespace for ManifestNamespace {
             .into());
         }
 
-        let object_id = Self::str_object_id(table_id);
-        let exists = self.manifest_contains_object(&object_id).await?;
-        if exists {
-            Ok(())
-        } else {
-            Err(NamespaceError::TableNotFound {
-                message: Self::format_table_id(table_id),
+        let location = request.location.clone();
+
+        // Validate that location is a relative path within the root directory
+        // We don't allow absolute URIs or paths that escape the root
+        if location.contains("://") {
+            return Err(NamespaceError::InvalidInput {
+                message: format!(
+                    "Absolute URIs are not allowed for register_table. Location must be a relative path within the root directory: {}",
+                    location
+                ),
             }
-            .into())
+            .into());
         }
-    }
-
-    async fn create_table(
-        &self,
-        request: CreateTableRequest,
-        data: Bytes,
-    ) -> Result<CreateTableResponse> {
-        let table_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Table ID is required".to_string(),
-            })
-        })?;
 
-        if table_id.is_empty() {
+        if location.starts_with('/') {
             return Err(NamespaceError::InvalidInput {
-                message: "Table ID cannot be empty".to_string(),
+                message: format!(
+                    "Absolute paths are not allowed for register_table. Location must be a relative path within the root directory: {}",
+                    location
+                ),
             }
             .into());
         }
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
-
-        let existing_table = self.query_manifest_for_table(&object_id).await?;
-        let existing_has_manifests = if let Some(existing_table) = &existing_table {
-            Some(
-                self.location_has_actual_manifests(&existing_table.location)
-                    .await?,
-            )
-        } else {
-            None
-        };
-
-        if existing_has_manifests == Some(false)
-            && request
-                .properties
-                .as_ref()
-                .is_some_and(|properties| !properties.is_empty())
-        {
+        // Check for path traversal attempts
+        if location.contains("..") {
             return Err(NamespaceError::InvalidInput {
                 message: format!(
-                    "create_table cannot set properties for already declared table '{}'",
-                    object_id
+                    "Path traversal is not allowed. Location must be a relative path within the root directory: {}",
+                    location
                 ),
             }
             .into());
         }
 
-        let create_mode = if existing_has_manifests == Some(false) {
-            CreateTableMode::Create
-        } else {
-            CreateTableMode::parse(request.mode.as_deref())?
-        };
-        let dir_name = if let Some(existing_table) = &existing_table {
-            existing_table.location.clone()
-        } else if namespace.is_empty() && self.dir_listing_enabled {
-            format!("{}.lance", table_name)
-        } else {
-            Self::generate_dir_name(&object_id)
-        };
-        let table_uri = Self::construct_full_uri(&self.root, &dir_name)?;
-        let overwriting_existing_table =
-            existing_has_manifests == Some(true) && create_mode == CreateTableMode::Overwrite;
+        let (namespace, table_name) = Self::split_object_id(table_id);
+        let object_id = Self::build_object_id(&namespace, &table_name);
 
-        if existing_has_manifests == Some(true) {
-            match create_mode {
-                CreateTableMode::Create => {
-                    return Err(NamespaceError::TableAlreadyExists {
-                        message: table_name.clone(),
-                    }
-                    .into());
-                }
-                CreateTableMode::ExistOk => {
-                    let properties = existing_table
-                        .as_ref()
-                        .and_then(|table| table.metadata.clone());
-                    return Ok(CreateTableResponse {
-                        location: Some(table_uri),
-                        storage_options: self.storage_options.clone(),
-                        properties,
-                        ..Default::default()
-                    });
-                }
-                CreateTableMode::Overwrite => {}
-            }
+        // Validate that parent namespaces exist (if not root)
+        if !namespace.is_empty() {
+            self.validate_namespace_levels_exist(&namespace).await?;
         }
 
-        // Validate that request_data is provided
-        if data.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Request data (Arrow IPC stream) is required for create_table".to_string(),
+        // Check if table already exists
+        if self.manifest_contains_object(&object_id).await? {
+            return Err(NamespaceError::TableAlreadyExists {
+                message: object_id.to_string(),
             }
             .into());
         }
 
-        // Write the data using Lance Dataset
-        let cursor = Cursor::new(data.to_vec());
-        let stream_reader = StreamReader::try_new(cursor, None).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to read IPC stream: {:?}", e),
-            })
-        })?;
+        // Register the table with its location in the manifest
+        self.insert_into_manifest(object_id, ObjectType::Table, Some(location.clone()))
+            .await?;
 
-        let batches: Vec<RecordBatch> = stream_reader
-            .collect::<std::result::Result<Vec<_>, _>>()
-            .map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to collect batches: {:?}", e),
+        Ok(RegisterTableResponse {
+            location: Some(location),
+            ..Default::default()
+        })
+    }
+
+    async fn deregister_table(
+        &self,
+        request: DeregisterTableRequest,
+    ) -> Result<DeregisterTableResponse> {
+        let table_id = request.id.as_ref().ok_or_else(|| {
+            lance_core::Error::from(NamespaceError::InvalidInput {
+                message: "Table ID is required".to_string(),
             })
         })?;
 
-        if batches.is_empty() {
-            return Err(NamespaceError::Internal {
-                message: "No data provided for table creation".to_string(),
+        if table_id.is_empty() {
+            return Err(NamespaceError::InvalidInput {
+                message: "Table ID cannot be empty".to_string(),
             }
             .into());
         }
 
-        let schema = batches[0].schema();
-        let batch_results: Vec<std::result::Result<RecordBatch, arrow_schema::ArrowError>> =
-            batches.into_iter().map(Ok).collect();
-        let reader = RecordBatchIterator::new(batch_results, schema);
-
-        let mut write_storage_options = self.storage_options.clone().unwrap_or_default();
-        if let Some(request_storage_options) = request.storage_options.as_ref() {
-            write_storage_options.extend(request_storage_options.clone());
-        }
-
-        let store_params = ObjectStoreParams {
-            storage_options_accessor: (!write_storage_options.is_empty()).then(|| {
-                Arc::new(
-                    lance_io::object_store::StorageOptionsAccessor::with_static_options(
-                        write_storage_options,
-                    ),
-                )
-            }),
-            ..Default::default()
-        };
-        let write_params = WriteParams {
-            mode: create_mode.write_mode(),
-            session: self.session.clone(),
-            store_params: Some(store_params),
-            ..Default::default()
-        };
-        let dataset = Dataset::write(Box::new(reader), &table_uri, Some(write_params))
-            .await
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!("Failed to write dataset: {:?}", e),
-                })
-            })?;
-        let version = dataset.version().version as i64;
-
-        if overwriting_existing_table {
-            let metadata =
-                Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
-            self.upsert_into_manifest_with_metadata(
-                vec![ManifestEntry {
-                    object_id,
-                    object_type: ObjectType::Table,
-                    location: Some(dir_name),
-                    metadata,
-                }],
-                None,
-            )
-            .await?;
+        let (namespace, table_name) = Self::split_object_id(table_id);
+        let object_id = Self::build_object_id(&namespace, &table_name);
 
-            Ok(CreateTableResponse {
-                version: Some(version),
-                location: Some(table_uri),
-                storage_options: self.storage_options.clone(),
-                properties: request.properties,
-                ..Default::default()
-            })
-        } else {
-            match existing_table {
-                Some(existing_table) => Ok(CreateTableResponse {
-                    version: Some(version),
-                    location: Some(table_uri),
-                    storage_options: self.storage_options.clone(),
-                    properties: existing_table.metadata,
-                    ..Default::default()
-                }),
-                None => {
-                    let metadata =
-                        Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
-                    // Register in manifest (store dir_name, not full URI)
-                    self.insert_into_manifest_with_metadata(
-                        vec![ManifestEntry {
-                            object_id,
-                            object_type: ObjectType::Table,
-                            location: Some(dir_name.clone()),
-                            metadata,
-                        }],
-                        None,
-                    )
-                    .await?;
+        // Get table info before deleting
+        let table_info = self.query_manifest_for_table(&object_id).await?;
 
-                    Ok(CreateTableResponse {
-                        version: Some(version),
-                        location: Some(table_uri),
-                        storage_options: self.storage_options.clone(),
-                        properties: request.properties,
-                        ..Default::default()
-                    })
+        let table_uri = match table_info {
+            Some(info) => {
+                // Delete from manifest only (leave physical data intact)
+                self.delete_from_manifest(&object_id).boxed().await?;
+                Self::construct_full_uri(&self.root, &info.location)?
+            }
+            None => {
+                return Err(NamespaceError::TableNotFound {
+                    message: object_id.to_string(),
                 }
+                .into());
             }
-        }
+        };
+
+        Ok(DeregisterTableResponse {
+            id: request.id.clone(),
+            location: Some(table_uri),
+            ..Default::default()
+        })
     }
 
-    async fn drop_table(&self, request: DropTableRequest) -> Result<DropTableResponse> {
-        let table_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Table ID is required".to_string(),
-            })
-        })?;
+    /// Add columns to a table.
+    ///
+    /// Converts the API `AddColumnsEntry` (SQL expressions) into Lance's
+    /// `NewColumnTransform::SqlExpressions` and delegates to `Dataset::add_columns`.
+    async fn alter_table_add_columns(
+        &self,
+        request: AlterTableAddColumnsRequest,
+    ) -> Result<AlterTableAddColumnsResponse> {
+        let table_id = request
+            .id
+            .as_ref()
+            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
 
         if table_id.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Table ID cannot be empty".to_string(),
-            }
-            .into());
+            return Err(Error::invalid_input_source(
+                "Table ID cannot be empty".into(),
+            ));
         }
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
-
-        // Query manifest for table location
+        let object_id = Self::str_object_id(table_id);
         let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
 
         match table_info {
             Some(info) => {
-                // Delete from manifest first
-                self.delete_from_manifest(&object_id).boxed().await?;
-
-                // Delete physical data directory using the dir_name from manifest
-                let table_path = self.base_path.clone().join(info.location.as_str());
                 let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
+                // Use DatasetBuilder with storage options to align with describe_table
+                // and to support custom storage backends (e.g. S3 with custom endpoints).
+                let mut builder = DatasetBuilder::from_uri(&table_uri);
+                if let Some(opts) = &self.storage_options {
+                    builder = builder.with_storage_options(opts.clone());
+                }
+                if let Some(session) = &self.session {
+                    builder = builder.with_session(session.clone());
+                }
+                let mut dataset = builder.load().await.map_err(|e| {
+                    Error::io_source(box_error(std::io::Error::other(format!(
+                        "Failed to open dataset: {}",
+                        e
+                    ))))
+                })?;
 
-                // Remove the table directory
-                self.object_store
-                    .remove_dir_all(table_path)
-                    .boxed()
+                // Use shared helper to build SQL expressions, ensuring a clear error when expression is missing
+                let sql_expressions = super::build_sql_expressions(&request.new_columns)?;
+
+                dataset
+                    .add_columns(
+                        lance::dataset::NewColumnTransform::SqlExpressions(sql_expressions),
+                        None,
+                        None,
+                    )
                     .await
                     .map_err(|e| {
-                        lance_core::Error::from(NamespaceError::Internal {
-                            message: format!("Failed to delete table directory: {:?}", e),
-                        })
+                        // Surface specific commit/conflict errors (CommitConflict,
+                        // RetryableCommitConflict, IncompatibleTransaction, ...) rather than
+                        // collapsing every failure into a generic IO error.
+                        convert_lance_commit_error(&e, "add_columns", Some(&object_id))
                     })?;
 
-                Ok(DropTableResponse {
-                    id: request.id.clone(),
-                    location: Some(table_uri),
-                    ..Default::default()
-                })
-            }
-            None => Err(NamespaceError::TableNotFound {
-                message: table_name.to_string(),
+                let version = dataset.version().version as i64;
+                Ok(AlterTableAddColumnsResponse::new(version))
             }
-            .into()),
+            None => Err(NamespaceError::TableNotFound { message: object_id }.into()),
         }
     }
 
-    async fn list_namespaces(
+    /// Alter columns in a table (rename, change type, change nullability).
+    ///
+    /// Converts the API `AlterColumnsEntry` into Lance's `ColumnAlteration`
+    /// and delegates to `Dataset::alter_columns`.
+    async fn alter_table_alter_columns(
         &self,
-        request: ListNamespacesRequest,
-    ) -> Result<ListNamespacesResponse> {
-        let parent_namespace = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Namespace ID is required".to_string(),
-            })
-        })?;
+        request: AlterTableAlterColumnsRequest,
+    ) -> Result<AlterTableAlterColumnsResponse> {
+        let table_id = request
+            .id
+            .as_ref()
+            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
 
-        // Build filter to find direct child namespaces
-        let filter = if parent_namespace.is_empty() {
-            // Root namespace: find all namespaces without a parent
-            "object_type = 'namespace' AND NOT contains(object_id, '$')".to_string()
-        } else {
-            // Non-root: find namespaces that start with parent$ but have no additional $
-            let prefix = parent_namespace.join(DELIMITER);
-            format!(
-                "object_type = 'namespace' AND starts_with(object_id, '{}{}') AND NOT contains(substring(object_id, {}), '$')",
-                prefix,
-                DELIMITER,
-                prefix.len() + 2
-            )
-        };
+        if table_id.is_empty() {
+            return Err(Error::invalid_input_source(
+                "Table ID cannot be empty".into(),
+            ));
+        }
 
-        let mut scanner = self.manifest_scanner().await?;
-        scanner.filter(&filter).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
-            })
-        })?;
-        scanner.project(&["object_id"]).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
-            })
-        })?;
+        let object_id = Self::str_object_id(table_id);
+        let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
 
-        let batches = Self::execute_scanner(scanner).await?;
-        let mut namespaces = Vec::new();
+        match table_info {
+            Some(info) => {
+                let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
+                let mut builder = DatasetBuilder::from_uri(&table_uri);
+                if let Some(opts) = &self.storage_options {
+                    builder = builder.with_storage_options(opts.clone());
+                }
+                if let Some(session) = &self.session {
+                    builder = builder.with_session(session.clone());
+                }
+                let mut dataset = builder.load().await.map_err(|e| {
+                    Error::io_source(box_error(std::io::Error::other(format!(
+                        "Failed to open dataset: {}",
+                        e
+                    ))))
+                })?;
 
-        for batch in batches {
-            if batch.num_rows() == 0 {
-                continue;
-            }
+                // Use shared helper to build column alterations, ensuring a clear error when data_type conversion fails
+                let alterations = super::build_column_alterations(&request.alterations)?;
 
-            let object_id_array = Self::get_string_column(&batch, "object_id")?;
-            for i in 0..batch.num_rows() {
-                let object_id = object_id_array.value(i);
-                let (_namespace, name) = Self::parse_object_id(object_id);
-                namespaces.push(name);
+                dataset.alter_columns(&alterations).await.map_err(|e| {
+                    convert_lance_commit_error(&e, "alter_columns", Some(&object_id))
+                })?;
+
+                let version = dataset.version().version as i64;
+                Ok(AlterTableAlterColumnsResponse::new(version))
             }
+            None => Err(NamespaceError::TableNotFound { message: object_id }.into()),
         }
-
-        let next_page_token =
-            Self::apply_pagination(&mut namespaces, request.page_token, request.limit);
-        let mut response = ListNamespacesResponse::new(namespaces);
-        response.page_token = next_page_token;
-        Ok(response)
     }
 
-    async fn describe_namespace(
+    /// Drop columns from a table.
+    ///
+    /// Delegates to `Dataset::drop_columns` with the column names from the request.
+    async fn alter_table_drop_columns(
         &self,
-        request: DescribeNamespaceRequest,
-    ) -> Result<DescribeNamespaceResponse> {
-        let namespace_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Namespace ID is required".to_string(),
-            })
-        })?;
+        request: AlterTableDropColumnsRequest,
+    ) -> Result<AlterTableDropColumnsResponse> {
+        let table_id = request
+            .id
+            .as_ref()
+            .ok_or_else(|| Error::invalid_input_source("Table ID is required".into()))?;
 
-        // Root namespace always exists
-        if namespace_id.is_empty() {
-            #[allow(clippy::needless_update)]
-            return Ok(DescribeNamespaceResponse {
-                properties: Some(HashMap::new()),
-                ..Default::default()
-            });
+        if table_id.is_empty() {
+            return Err(Error::invalid_input_source(
+                "Table ID cannot be empty".into(),
+            ));
         }
 
-        // Check if namespace exists in manifest
-        let object_id = namespace_id.join(DELIMITER);
-        let namespace_info = self.query_manifest_for_namespace(&object_id).await?;
+        let object_id = Self::str_object_id(table_id);
+        let table_info = self.query_manifest_for_table(&object_id).boxed().await?;
 
-        match namespace_info {
-            #[allow(clippy::needless_update)]
-            Some(info) => Ok(DescribeNamespaceResponse {
-                properties: info.metadata,
-                ..Default::default()
-            }),
-            None => Err(NamespaceError::NamespaceNotFound {
-                message: object_id.to_string(),
-            }
-            .into()),
-        }
-    }
+        match table_info {
+            Some(info) => {
+                let table_uri = Self::construct_full_uri(&self.root, &info.location)?;
+                let mut builder = DatasetBuilder::from_uri(&table_uri);
+                if let Some(opts) = &self.storage_options {
+                    builder = builder.with_storage_options(opts.clone());
+                }
+                if let Some(session) = &self.session {
+                    builder = builder.with_session(session.clone());
+                }
+                let mut dataset = builder.load().await.map_err(|e| {
+                    Error::io_source(box_error(std::io::Error::other(format!(
+                        "Failed to open dataset: {}",
+                        e
+                    ))))
+                })?;
 
-    async fn create_namespace(
-        &self,
-        request: CreateNamespaceRequest,
-    ) -> Result<CreateNamespaceResponse> {
-        let namespace_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Namespace ID is required".to_string(),
-            })
-        })?;
+                let columns: Vec<&str> = request.columns.iter().map(|s| s.as_str()).collect();
+                dataset.drop_columns(&columns).await.map_err(|e| {
+                    convert_lance_commit_error(&e, "drop_columns", Some(&object_id))
+                })?;
 
-        // Root namespace always exists and cannot be created
-        if namespace_id.is_empty() {
-            return Err(NamespaceError::NamespaceAlreadyExists {
-                message: "root namespace".to_string(),
+                let version = dataset.version().version as i64;
+                Ok(AlterTableDropColumnsResponse::new(version))
             }
-            .into());
-        }
-
-        // Validate parent namespaces exist (but not the namespace being created)
-        if namespace_id.len() > 1 {
-            self.validate_namespace_levels_exist(&namespace_id[..namespace_id.len() - 1])
-                .await?;
+            None => Err(NamespaceError::TableNotFound { message: object_id }.into()),
         }
+    }
+}
 
-        let object_id = namespace_id.join(DELIMITER);
-        if self.manifest_contains_object(&object_id).await? {
-            return Err(NamespaceError::NamespaceAlreadyExists {
-                message: object_id.to_string(),
-            }
-            .into());
-        }
+#[cfg(test)]
+mod tests {
+    use super::{
+        BASE_OBJECTS_INDEX_NAME, ConflictResolution, CopyOnWriteMutation, DeleteObjectMutation,
+        LANCE_DATA_DIR, LANCE_INDICES_DIR, MANIFEST_TABLE_NAME, ManifestBatchBuilder,
+        ManifestEntry, ManifestIndexAccumulator, ManifestNamespace, ManifestOutputRow,
+        ManifestRowValue, ManifestStreamMutation, OBJECT_ID_INDEX_NAME, OBJECT_TYPE_INDEX_NAME,
+        ObjectType,
+    };
+    use crate::DirectoryNamespaceBuilder;
+    use arrow::datatypes::DataType;
+    use bytes::Bytes;
+    use futures::StreamExt;
+    use lance::index::DatasetIndexExt;
+    use lance_core::utils::tempfile::TempStdDir;
+    use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
+    use lance_namespace::LanceNamespace;
+    use lance_namespace::models::{
+        CreateNamespaceRequest, CreateTableRequest, DescribeTableRequest, DropTableRequest,
+        ListTablesRequest, TableExistsRequest,
+    };
+    use lance_table::format::Fragment;
+    use rstest::rstest;
+    use std::collections::{HashMap, HashSet};
+    use std::sync::Arc;
 
-        let metadata =
-            Self::serialize_metadata(request.properties.as_ref(), "namespace", &object_id)?;
+    async fn create_manifest_namespace(
+        root: &str,
+        inline_optimization_enabled: bool,
+    ) -> ManifestNamespace {
+        create_manifest_namespace_with_retries(root, inline_optimization_enabled, None).await
+    }
 
-        self.insert_into_manifest_with_metadata(
-            vec![ManifestEntry {
-                object_id,
-                object_type: ObjectType::Namespace,
-                location: None,
-                metadata,
-            }],
+    async fn create_manifest_namespace_with_retries(
+        root: &str,
+        inline_optimization_enabled: bool,
+        commit_retries: Option<u32>,
+    ) -> ManifestNamespace {
+        let (object_store, base_path) = ObjectStore::from_uri_and_params(
+            Arc::new(ObjectStoreRegistry::default()),
+            root,
+            &ObjectStoreParams::default(),
+        )
+        .await
+        .unwrap();
+        ManifestNamespace::from_directory(
+            root.to_string(),
             None,
+            None,
+            object_store,
+            base_path,
+            true,
+            inline_optimization_enabled,
+            commit_retries,
         )
-        .await?;
-
-        Ok(CreateNamespaceResponse {
-            properties: request.properties,
-            ..Default::default()
-        })
-    }
-
-    async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
-        let namespace_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Namespace ID is required".to_string(),
-            })
-        })?;
+        .await
+        .unwrap()
+    }
 
-        // Root namespace always exists and cannot be dropped
-        if namespace_id.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Root namespace cannot be dropped".to_string(),
-            }
-            .into());
-        }
+    struct CommitConflictAfterRewriteMutation {
+        root: String,
+        conflict_object_id: String,
+    }
 
-        let object_id = namespace_id.join(DELIMITER);
+    impl ManifestStreamMutation for CommitConflictAfterRewriteMutation {
+        type Output = ();
+
+        fn process_existing_row(
+            &mut self,
+            row: ManifestRowValue,
+            output: &mut ManifestBatchBuilder,
+            index_data: &mut ManifestIndexAccumulator,
+        ) -> lance_core::Result<()> {
+            output.append(
+                index_data,
+                ManifestOutputRow {
+                    object_id: &row.object_id,
+                    object_type: row.object_type,
+                    location: row.location.as_deref(),
+                    metadata: row.metadata.as_deref(),
+                    base_objects: row.base_objects.as_deref(),
+                },
+            )
+        }
 
-        // Check if namespace exists
-        if !self.manifest_contains_object(&object_id).boxed().await? {
-            return Err(NamespaceError::NamespaceNotFound {
-                message: object_id.to_string(),
-            }
-            .into());
+        fn append_rows(
+            &mut self,
+            output: &mut ManifestBatchBuilder,
+            index_data: &mut ManifestIndexAccumulator,
+        ) -> lance_core::Result<()> {
+            output.append(
+                index_data,
+                ManifestOutputRow {
+                    object_id: "attempted_table",
+                    object_type: ObjectType::Table,
+                    location: Some("attempted_table.lance"),
+                    metadata: None,
+                    base_objects: None,
+                },
+            )
         }
 
-        // Check for child namespaces
-        let escaped_id = object_id.replace('\'', "''");
-        let prefix = format!("{}{}", escaped_id, DELIMITER);
-        let filter = format!("starts_with(object_id, '{}')", prefix);
-        let mut scanner = self.manifest_scanner().boxed().await?;
-        scanner.filter(&filter).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to filter: {:?}", e),
-            })
-        })?;
-        scanner.project::<&str>(&[]).map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to project: {:?}", e),
-            })
-        })?;
-        scanner.with_row_id();
-        let count = scanner.count_rows().boxed().await.map_err(|e| {
-            lance_core::Error::from(NamespaceError::Internal {
-                message: format!("Failed to count rows: {:?}", e),
+        fn finish(&self) -> CopyOnWriteMutation<Self::Output> {
+            let root = self.root.clone();
+            let object_id = self.conflict_object_id.clone();
+            std::thread::spawn(move || {
+                let runtime = tokio::runtime::Runtime::new().unwrap();
+                runtime.block_on(async move {
+                    let writer = create_manifest_namespace(&root, false).await;
+                    writer
+                        .insert_into_manifest_with_metadata(
+                            vec![ManifestEntry {
+                                object_id,
+                                object_type: ObjectType::Table,
+                                location: Some("conflicting_table.lance".to_string()),
+                                metadata: None,
+                            }],
+                            None,
+                        )
+                        .await
+                        .unwrap();
+                });
             })
-        })?;
-
-        if count > 0 {
-            return Err(NamespaceError::NamespaceNotEmpty {
-                message: format!("'{}' (contains {} child objects)", object_id, count),
-            }
-            .into());
+            .join()
+            .unwrap();
+            CopyOnWriteMutation::updated(())
         }
+    }
 
-        self.delete_from_manifest(&object_id).boxed().await?;
-
-        Ok(DropNamespaceResponse::default())
+    /// A delete mutation that, during staging, has a concurrent writer delete the same
+    /// object and commit first, so our own commit hits a conflict while the object is
+    /// already gone — exercising `ConflictResolution::SucceedIfAbsent`.
+    struct ConcurrentDeleteBeforeCommitMutation {
+        inner: DeleteObjectMutation,
+        root: String,
+        target: String,
     }
 
-    async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> {
-        let namespace_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Namespace ID is required".to_string(),
-            })
-        })?;
+    impl ManifestStreamMutation for ConcurrentDeleteBeforeCommitMutation {
+        type Output = ();
 
-        // Root namespace always exists
-        if namespace_id.is_empty() {
-            return Ok(());
+        fn process_existing_row(
+            &mut self,
+            row: ManifestRowValue,
+            output: &mut ManifestBatchBuilder,
+            index_data: &mut ManifestIndexAccumulator,
+        ) -> lance_core::Result<()> {
+            self.inner.process_existing_row(row, output, index_data)
         }
 
-        let object_id = namespace_id.join(DELIMITER);
-        if self.manifest_contains_object(&object_id).await? {
-            Ok(())
-        } else {
-            Err(NamespaceError::NamespaceNotFound {
-                message: object_id.to_string(),
-            }
-            .into())
+        fn append_rows(
+            &mut self,
+            output: &mut ManifestBatchBuilder,
+            index_data: &mut ManifestIndexAccumulator,
+        ) -> lance_core::Result<()> {
+            self.inner.append_rows(output, index_data)
         }
-    }
 
-    async fn declare_table(&self, request: DeclareTableRequest) -> Result<DeclareTableResponse> {
-        let table_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Table ID is required".to_string(),
+        fn finish(&self) -> CopyOnWriteMutation<Self::Output> {
+            let root = self.root.clone();
+            let target = self.target.clone();
+            std::thread::spawn(move || {
+                let runtime = tokio::runtime::Runtime::new().unwrap();
+                runtime.block_on(async move {
+                    let writer = create_manifest_namespace(&root, false).await;
+                    writer.delete_from_manifest(&target).await.unwrap();
+                });
             })
-        })?;
+            .join()
+            .unwrap();
+            self.inner.finish()
+        }
 
-        if table_id.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Table ID cannot be empty".to_string(),
+        fn conflict_resolution(&self) -> ConflictResolution<Self::Output> {
+            ConflictResolution::SucceedIfAbsent {
+                object_id: self.target.clone(),
+                output: (),
             }
-            .into());
         }
+    }
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
-
-        // Check if table already exists in manifest
-        let existing = self.query_manifest_for_table(&object_id).await?;
-        if existing.is_some() {
-            return Err(NamespaceError::TableAlreadyExists {
-                message: table_name.to_string(),
+    async fn manifest_base_objects(
+        manifest_ns: &ManifestNamespace,
+    ) -> HashMap<String, Option<Vec<String>>> {
+        let mut scanner = manifest_ns.manifest_scanner().await.unwrap();
+        scanner.project(&["object_id", "base_objects"]).unwrap();
+        let batches = ManifestNamespace::execute_scanner(scanner).await.unwrap();
+        let mut rows = HashMap::new();
+        for batch in batches {
+            let object_ids = ManifestNamespace::get_string_column(&batch, "object_id").unwrap();
+            let base_objects = ManifestNamespace::base_objects_column_values(&batch).unwrap();
+            for (row, value) in base_objects.into_iter().enumerate() {
+                rows.insert(object_ids.value(row).to_string(), value);
             }
-            .into());
         }
+        rows
+    }
 
-        // Create table location path with hash-based naming
-        // When dir_listing_enabled is true and it's a root table, use directory-style naming: {table_name}.lance
-        // Otherwise, use hash-based naming: {hash}_{object_id}
-        let dir_name = if namespace.is_empty() && self.dir_listing_enabled {
-            // Root table with directory listing enabled: use {table_name}.lance
-            format!("{}.lance", table_name)
-        } else {
-            // Child namespace table or dir listing disabled: use hash-based naming
-            Self::generate_dir_name(&object_id)
-        };
-        let table_path = self.base_path.clone().join(dir_name.as_str());
-        let table_uri = Self::construct_full_uri(&self.root, &dir_name)?;
+    async fn manifest_data_paths(manifest_ns: &ManifestNamespace) -> HashSet<String> {
+        let data_dir = manifest_ns
+            .base_path
+            .clone()
+            .join(MANIFEST_TABLE_NAME)
+            .join(LANCE_DATA_DIR);
+        let mut stream = manifest_ns.object_store.read_dir_all(&data_dir, None);
+        let mut paths = HashSet::new();
+        while let Some(meta) = stream.next().await.transpose().unwrap() {
+            paths.insert(meta.location.to_string());
+        }
+        paths
+    }
 
-        // Validate location if provided
-        if let Some(req_location) = &request.location {
-            let req_location = req_location.trim_end_matches('/');
-            if req_location != table_uri {
-                return Err(NamespaceError::InvalidInput {
-                    message: format!(
-                        "Cannot declare table {} at location {}, must be at location {}",
-                        table_name, req_location, table_uri
-                    ),
-                }
-                .into());
-            }
+    async fn manifest_index_paths(manifest_ns: &ManifestNamespace) -> HashSet<String> {
+        let index_dir = manifest_ns
+            .base_path
+            .clone()
+            .join(MANIFEST_TABLE_NAME)
+            .join(LANCE_INDICES_DIR);
+        let mut stream = manifest_ns.object_store.read_dir_all(&index_dir, None);
+        let mut paths = HashSet::new();
+        while let Some(meta) = stream.next().await.transpose().unwrap() {
+            paths.insert(meta.location.to_string());
         }
+        paths
+    }
 
-        // Create the .lance-reserved file to mark the table as existing
-        let reserved_file_path = table_path.clone().join(".lance-reserved");
+    fn create_test_ipc_data() -> Vec<u8> {
+        use arrow::array::{Int32Array, StringArray};
+        use arrow::datatypes::{DataType, Field, Schema};
+        use arrow::ipc::writer::StreamWriter;
+        use arrow::record_batch::RecordBatch;
+        use std::sync::Arc;
 
-        self.object_store
-            .create(&reserved_file_path)
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+        ]));
+
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["a", "b", "c"])),
+            ],
+        )
+        .unwrap();
+
+        let mut buffer = Vec::new();
+        {
+            let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap();
+            writer.write(&batch).unwrap();
+            writer.finish().unwrap();
+        }
+        buffer
+    }
+
+    /// Open the `__manifest` dataset directly and set a table-metadata key,
+    /// simulating a future Lance client that persisted a feature flag.
+    async fn set_manifest_table_metadata(temp_path: &str, key: &str, value: &str) {
+        use lance::dataset::builder::DatasetBuilder;
+        let mut ds = DatasetBuilder::from_uri(format!("{}/{}", temp_path, MANIFEST_TABLE_NAME))
+            .load()
             .await
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!(
-                        "Failed to create .lance-reserved file for table {}: {}",
-                        table_name, e
-                    ),
-                })
-            })?
-            .shutdown()
+            .unwrap();
+        ds.update_metadata([(key, value)]).await.unwrap();
+    }
+
+    async fn create_namespace_with_one_table(temp_path: &str) {
+        let ns = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
             .await
-            .map_err(|e| {
-                lance_core::Error::from(NamespaceError::Internal {
-                    message: format!(
-                        "Failed to finalize .lance-reserved file for table {}: {}",
-                        table_name, e
-                    ),
-                })
-            })?;
+            .unwrap();
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["t1".to_string()]);
+        ns.create_table(create_request, Bytes::from(create_test_ipc_data()))
+            .await
+            .unwrap();
+    }
 
-        let metadata = Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?;
+    /// This is a forward-compatibility checker only: it must not set any feature
+    /// flag, so existing clients keep treating the manifest as compatible.
+    #[tokio::test]
+    async fn test_manifest_has_no_feature_flags_by_default() {
+        use lance::dataset::builder::DatasetBuilder;
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        create_namespace_with_one_table(temp_path).await;
 
-        // Add entry to manifest marking this as a declared table (store dir_name, not full path)
-        self.insert_into_manifest_with_metadata(
-            vec![ManifestEntry {
-                object_id,
-                object_type: ObjectType::Table,
-                location: Some(dir_name),
-                metadata,
-            }],
-            None,
+        let ds = DatasetBuilder::from_uri(format!("{}/{}", temp_path, MANIFEST_TABLE_NAME))
+            .load()
+            .await
+            .unwrap();
+        assert!(
+            !ds.metadata()
+                .contains_key(crate::dir::manifest_feature_flags::READER_FEATURE_FLAGS_KEY)
+        );
+        assert!(
+            !ds.metadata()
+                .contains_key(crate::dir::manifest_feature_flags::WRITER_FEATURE_FLAGS_KEY)
+        );
+    }
+
+    /// An unknown reader feature flag must block opening the catalog with a clear
+    /// "please upgrade" error rather than silently degrading to directory listing.
+    #[tokio::test]
+    async fn test_unknown_reader_flag_blocks_access() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        create_namespace_with_one_table(temp_path).await;
+        set_manifest_table_metadata(
+            temp_path,
+            crate::dir::manifest_feature_flags::READER_FEATURE_FLAGS_KEY,
+            "1",
         )
-        .await?;
+        .await;
+
+        let err = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
+            .await
+            .expect_err("opening a manifest with an unknown reader flag should fail");
+        assert!(
+            err.to_string().to_lowercase().contains("upgrade"),
+            "expected an upgrade error, got: {err}"
+        );
+    }
+
+    /// An unknown writer feature flag must still allow reads but block writes.
+    #[tokio::test]
+    async fn test_unknown_writer_flag_blocks_writes_but_allows_reads() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        create_namespace_with_one_table(temp_path).await;
+        set_manifest_table_metadata(
+            temp_path,
+            crate::dir::manifest_feature_flags::WRITER_FEATURE_FLAGS_KEY,
+            "1",
+        )
+        .await;
+
+        let ns = DirectoryNamespaceBuilder::new(temp_path)
+            .build()
+            .await
+            .expect("reads should still be allowed with only a writer flag set");
+        let mut list_request = ListTablesRequest::new();
+        list_request.id = Some(vec![]);
+        assert_eq!(ns.list_tables(list_request).await.unwrap().tables.len(), 1);
+
+        // A refused write must not leave an orphaned table dataset behind.
+        let entries_before = dir_entry_names(temp_path);
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["t2".to_string()]);
+        let err = ns
+            .create_table(create_request, Bytes::from(create_test_ipc_data()))
+            .await
+            .expect_err("writing through an unknown writer flag should fail");
+        assert!(
+            err.to_string().to_lowercase().contains("upgrade"),
+            "expected an upgrade error, got: {err}"
+        );
+        assert_eq!(
+            entries_before,
+            dir_entry_names(temp_path),
+            "a refused create_table must not create an orphaned table directory"
+        );
+
+        // Mutations that go straight through rewrite_manifest (no early
+        // create_table check) must also be refused: an insert (create_namespace)
+        // and a delete (drop_table). This proves the writer check is enforced at
+        // the single copy-on-write chokepoint, not just on the create_table path.
+        let mut create_ns = CreateNamespaceRequest::new();
+        create_ns.id = Some(vec!["ns1".to_string()]);
+        let err = ns
+            .create_namespace(create_ns)
+            .await
+            .expect_err("create_namespace through an unknown writer flag should fail");
+        assert!(
+            err.to_string().to_lowercase().contains("upgrade"),
+            "expected an upgrade error, got: {err}"
+        );
+
+        let mut drop_request = DropTableRequest::new();
+        drop_request.id = Some(vec!["t1".to_string()]);
+        let err = ns
+            .drop_table(drop_request)
+            .await
+            .expect_err("drop_table through an unknown writer flag should fail");
+        assert!(
+            err.to_string().to_lowercase().contains("upgrade"),
+            "expected an upgrade error, got: {err}"
+        );
+    }
+
+    fn dir_entry_names(path: &str) -> std::collections::BTreeSet<String> {
+        std::fs::read_dir(path)
+            .unwrap()
+            .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn test_manifest_rewrite_preserves_utf8_metadata_and_base_objects() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, true).await;
+
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "view".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("view.lance".to_string()),
+                    metadata: Some(r#"{"kind":"view"}"#.to_string()),
+                }],
+                Some(vec!["base_a".to_string(), "base_b".to_string()]),
+            )
+            .await
+            .unwrap();
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "other".to_string(),
+                    object_type: ObjectType::Namespace,
+                    location: None,
+                    metadata: Some(r#"{"kind":"namespace"}"#.to_string()),
+                }],
+                None,
+            )
+            .await
+            .unwrap();
 
-        log::info!(
-            "Declared table '{}' in manifest at {}",
-            table_name,
-            table_uri
+        let dataset_guard = manifest_ns.manifest_dataset.get().await.unwrap();
+        let metadata_field = dataset_guard.schema().field("metadata").unwrap();
+        assert_eq!(metadata_field.data_type(), DataType::Utf8);
+        drop(dataset_guard);
+
+        let base_objects = manifest_base_objects(&manifest_ns).await;
+        assert_eq!(
+            base_objects.get("view").cloned().unwrap(),
+            Some(vec!["base_a".to_string(), "base_b".to_string()])
         );
+        assert_eq!(base_objects.get("other").cloned().unwrap(), None);
+    }
 
-        // For backwards compatibility, only skip vending credentials when explicitly set to false
-        let vend_credentials = request.vend_credentials.unwrap_or(true);
-        let storage_options = if vend_credentials {
-            self.storage_options.clone()
-        } else {
-            None
-        };
+    #[tokio::test]
+    async fn test_manifest_rewrite_replacement_indices_are_versioned() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, true).await;
 
-        Ok(DeclareTableResponse {
-            location: Some(table_uri),
-            storage_options,
-            properties: request.properties,
-            ..Default::default()
-        })
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                Some(vec!["base".to_string()]),
+            )
+            .await
+            .unwrap();
+
+        let dataset_guard = manifest_ns.manifest_dataset.get().await.unwrap();
+        let dataset_version = dataset_guard.version().version;
+        let indices = dataset_guard.load_indices().await.unwrap();
+        let names = indices
+            .iter()
+            .map(|index| index.name.as_str())
+            .collect::<HashSet<_>>();
+        assert!(names.contains(OBJECT_ID_INDEX_NAME));
+        assert!(names.contains(OBJECT_TYPE_INDEX_NAME));
+        assert!(names.contains(BASE_OBJECTS_INDEX_NAME));
+        for index in indices.iter() {
+            assert_eq!(index.dataset_version, dataset_version);
+            assert!(!index.fragment_bitmap.as_ref().unwrap().is_empty());
+        }
     }
 
-    async fn register_table(&self, request: RegisterTableRequest) -> Result<RegisterTableResponse> {
-        let table_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Table ID is required".to_string(),
-            })
-        })?;
+    #[tokio::test]
+    async fn test_manifest_rewrite_empty_manifest_keeps_replacement_indices_valid() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, true).await;
 
-        if table_id.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Table ID cannot be empty".to_string(),
-            }
-            .into());
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+        manifest_ns.delete_from_manifest("table").await.unwrap();
+
+        assert!(!manifest_ns.manifest_contains_object("table").await.unwrap());
+        let mut scanner = manifest_ns.manifest_scanner().await.unwrap();
+        scanner.project(&["object_id"]).unwrap();
+        let rows = ManifestNamespace::execute_scanner(scanner)
+            .await
+            .unwrap()
+            .into_iter()
+            .map(|batch| batch.num_rows())
+            .sum::<usize>();
+        assert_eq!(rows, 0);
+
+        let dataset_guard = manifest_ns.manifest_dataset.get().await.unwrap();
+        let dataset_version = dataset_guard.version().version;
+        let indices = dataset_guard.load_indices().await.unwrap();
+        let names = indices
+            .iter()
+            .map(|index| index.name.as_str())
+            .collect::<HashSet<_>>();
+        assert!(names.contains(OBJECT_ID_INDEX_NAME));
+        assert!(names.contains(OBJECT_TYPE_INDEX_NAME));
+        assert!(names.contains(BASE_OBJECTS_INDEX_NAME));
+        for index in indices.iter() {
+            assert_eq!(index.dataset_version, dataset_version);
         }
+    }
 
-        let location = request.location.clone();
+    #[tokio::test]
+    async fn test_manifest_rewrite_fragment_bitmap_uses_overwrite_fragment_ids() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, false).await;
+        let dataset_guard = manifest_ns.manifest_dataset.get().await.unwrap();
+        let fragments = vec![Fragment::new(0), Fragment::new(0), Fragment::new(7)];
+
+        let manifest = ManifestNamespace::manifest_from_overwrite_transaction(
+            dataset_guard.manifest(),
+            dataset_guard.manifest().schema.clone(),
+            &fragments,
+        );
 
-        // Validate that location is a relative path within the root directory
-        // We don't allow absolute URIs or paths that escape the root
-        if location.contains("://") {
-            return Err(NamespaceError::InvalidInput {
-                message: format!(
-                    "Absolute URIs are not allowed for register_table. Location must be a relative path within the root directory: {}",
-                    location
-                ),
-            }
-            .into());
-        }
+        let fragment_ids = manifest
+            .fragments
+            .iter()
+            .map(|fragment| fragment.id)
+            .collect::<Vec<_>>();
+        assert_eq!(fragment_ids, vec![0, 1, 7]);
+        assert_eq!(
+            ManifestNamespace::manifest_fragment_bitmap(&manifest)
+                .unwrap()
+                .into_iter()
+                .collect::<Vec<_>>(),
+            vec![0, 1, 7]
+        );
+    }
 
-        if location.starts_with('/') {
-            return Err(NamespaceError::InvalidInput {
-                message: format!(
-                    "Absolute paths are not allowed for register_table. Location must be a relative path within the root directory: {}",
-                    location
-                ),
-            }
-            .into());
-        }
+    #[tokio::test]
+    async fn test_manifest_noop_delete_uses_latest_snapshot() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let stale_ns = create_manifest_namespace(temp_path, false).await;
+        let writer_ns = create_manifest_namespace(temp_path, false).await;
 
-        // Check for path traversal attempts
-        if location.contains("..") {
-            return Err(NamespaceError::InvalidInput {
-                message: format!(
-                    "Path traversal is not allowed. Location must be a relative path within the root directory: {}",
-                    location
-                ),
-            }
-            .into());
-        }
+        writer_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "late_table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("late_table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
+        stale_ns.delete_from_manifest("late_table").await.unwrap();
 
-        // Validate that parent namespaces exist (if not root)
-        if !namespace.is_empty() {
-            self.validate_namespace_levels_exist(&namespace).await?;
-        }
+        let check_ns = create_manifest_namespace(temp_path, false).await;
+        assert!(
+            !check_ns
+                .manifest_contains_object("late_table")
+                .await
+                .unwrap()
+        );
+    }
 
-        // Check if table already exists
-        if self.manifest_contains_object(&object_id).await? {
-            return Err(NamespaceError::TableAlreadyExists {
-                message: object_id.to_string(),
-            }
-            .into());
-        }
+    #[tokio::test]
+    async fn test_manifest_noop_delete_cleans_uncommitted_data_file() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, false).await;
 
-        // Register the table with its location in the manifest
-        self.insert_into_manifest(object_id, ObjectType::Table, Some(location.clone()))
-            .await?;
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
 
-        Ok(RegisterTableResponse {
-            location: Some(location),
-            ..Default::default()
-        })
+        let before = manifest_data_paths(&manifest_ns).await;
+        assert!(!before.is_empty());
+
+        manifest_ns
+            .delete_from_manifest("missing_table")
+            .await
+            .unwrap();
+
+        let after = manifest_data_paths(&manifest_ns).await;
+        assert_eq!(after, before);
     }
 
-    async fn deregister_table(
-        &self,
-        request: DeregisterTableRequest,
-    ) -> Result<DeregisterTableResponse> {
-        let table_id = request.id.as_ref().ok_or_else(|| {
-            lance_core::Error::from(NamespaceError::InvalidInput {
-                message: "Table ID is required".to_string(),
+    #[tokio::test]
+    async fn test_manifest_final_commit_failure_cleans_uncommitted_rewrite_files() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace_with_retries(temp_path, true, Some(0)).await;
+
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+
+        let before_data_paths = manifest_data_paths(&manifest_ns).await;
+        let before_index_paths = manifest_index_paths(&manifest_ns).await;
+
+        let result = manifest_ns
+            .rewrite_manifest("Failed to test manifest cleanup", || {
+                CommitConflictAfterRewriteMutation {
+                    root: temp_path.to_string(),
+                    conflict_object_id: "conflicting_table".to_string(),
+                }
             })
-        })?;
+            .await;
+        assert!(result.is_err());
 
-        if table_id.is_empty() {
-            return Err(NamespaceError::InvalidInput {
-                message: "Table ID cannot be empty".to_string(),
-            }
-            .into());
-        }
+        let after_data_paths = manifest_data_paths(&manifest_ns).await;
+        assert!(before_data_paths.is_subset(&after_data_paths));
+        assert_eq!(after_data_paths.len(), before_data_paths.len() + 1);
+        assert_eq!(manifest_index_paths(&manifest_ns).await, before_index_paths);
+        assert!(
+            manifest_ns
+                .manifest_contains_object("conflicting_table")
+                .await
+                .unwrap()
+        );
+        assert!(
+            !manifest_ns
+                .manifest_contains_object("attempted_table")
+                .await
+                .unwrap()
+        );
+    }
 
-        let (namespace, table_name) = Self::split_object_id(table_id);
-        let object_id = Self::build_object_id(&namespace, &table_name);
+    #[tokio::test]
+    async fn test_manifest_commit_visible_on_memory_store() {
+        // Regression: the commit must use the same object store the manifest dataset reads
+        // from. On `memory://` the namespace store and the dataset store can be different
+        // in-memory instances, so a commit written to the wrong one is invisible to reads
+        // (manifests as stale version -> endless conflict / "not found").
+        let manifest_ns = create_manifest_namespace("memory://test_commit_visible", false).await;
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+        assert!(manifest_ns.manifest_contains_object("table").await.unwrap());
+        // A second sequential commit must not falsely conflict.
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table2".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table2.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+        assert!(
+            manifest_ns
+                .manifest_contains_object("table2")
+                .await
+                .unwrap()
+        );
+    }
 
-        // Get table info before deleting
-        let table_info = self.query_manifest_for_table(&object_id).await?;
+    #[tokio::test]
+    async fn test_manifest_commit_uses_inline_transaction() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, false).await;
 
-        let table_uri = match table_info {
-            Some(info) => {
-                // Delete from manifest only (leave physical data intact)
-                self.delete_from_manifest(&object_id).boxed().await?;
-                Self::construct_full_uri(&self.root, &info.location)?
-            }
-            None => {
-                return Err(NamespaceError::TableNotFound {
-                    message: object_id.to_string(),
-                }
-                .into());
-            }
-        };
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
 
-        Ok(DeregisterTableResponse {
-            id: request.id.clone(),
-            location: Some(table_uri),
-            ..Default::default()
-        })
+        let dataset_guard = manifest_ns.manifest_dataset.get().await.unwrap();
+        let manifest = dataset_guard.manifest();
+        // The overwrite transaction is embedded inline in the manifest, never written as a
+        // separate _transactions/*.txn file.
+        assert!(manifest.transaction_section.is_some());
+        assert!(manifest.transaction_file.is_none());
     }
-}
 
-#[cfg(test)]
-mod tests {
-    use crate::{DirectoryNamespaceBuilder, ManifestNamespace};
-    use bytes::Bytes;
-    use lance_core::utils::tempfile::TempStdDir;
-    use lance_namespace::LanceNamespace;
-    use lance_namespace::models::{
-        CreateNamespaceRequest, CreateTableRequest, DescribeTableRequest, DropTableRequest,
-        ListTablesRequest, TableExistsRequest,
-    };
-    use rstest::rstest;
+    #[tokio::test]
+    async fn test_manifest_commit_landed_attributes_data_file() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace(temp_path, false).await;
+
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+
+        let dataset = Arc::new(manifest_ns.manifest_dataset.get().await.unwrap().clone());
+        let version = dataset.manifest().version;
+        let our_files = dataset
+            .manifest()
+            .fragments
+            .iter()
+            .flat_map(|fragment| fragment.files.iter())
+            .map(|file| file.path.clone())
+            .collect::<HashSet<_>>();
+        assert!(!our_files.is_empty());
 
-    fn create_test_ipc_data() -> Vec<u8> {
-        use arrow::array::{Int32Array, StringArray};
-        use arrow::datatypes::{DataType, Field, Schema};
-        use arrow::ipc::writer::StreamWriter;
-        use arrow::record_batch::RecordBatch;
-        use std::sync::Arc;
+        // The committed version references our data file => attributed to us (a lost-ack
+        // commit must be treated as success, not cleaned up).
+        assert!(
+            manifest_ns
+                .manifest_commit_landed(&dataset, version, &our_files)
+                .await
+        );
+        // A different file set is not attributed to us.
+        let other = HashSet::from(["missing.lance".to_string()]);
+        assert!(
+            !manifest_ns
+                .manifest_commit_landed(&dataset, version, &other)
+                .await
+        );
+        // A version that does not exist did not land.
+        assert!(
+            !manifest_ns
+                .manifest_commit_landed(&dataset, version + 100, &our_files)
+                .await
+        );
+    }
 
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::Int32, false),
-            Field::new("name", DataType::Utf8, false),
-        ]));
+    #[tokio::test]
+    async fn test_manifest_delete_conflict_with_concurrent_delete_succeeds() {
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+        let manifest_ns = create_manifest_namespace_with_retries(temp_path, false, Some(0)).await;
 
-        let batch = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 2, 3])),
-                Arc::new(StringArray::from(vec!["a", "b", "c"])),
-            ],
-        )
-        .unwrap();
+        manifest_ns
+            .insert_into_manifest_with_metadata(
+                vec![ManifestEntry {
+                    object_id: "table".to_string(),
+                    object_type: ObjectType::Table,
+                    location: Some("table.lance".to_string()),
+                    metadata: None,
+                }],
+                None,
+            )
+            .await
+            .unwrap();
+        assert!(manifest_ns.manifest_contains_object("table").await.unwrap());
+
+        // A concurrent writer deletes "table" and commits first, so our own delete commit
+        // conflicts while "table" is already gone. Native resolution treats the goal as
+        // achieved and succeeds instead of erroring or retrying forever.
+        let result = manifest_ns
+            .rewrite_manifest("Failed to delete from manifest", || {
+                ConcurrentDeleteBeforeCommitMutation {
+                    inner: DeleteObjectMutation {
+                        object_id: "table".to_string(),
+                        deleted: false,
+                    },
+                    root: temp_path.to_string(),
+                    target: "table".to_string(),
+                }
+            })
+            .await;
 
-        let mut buffer = Vec::new();
-        {
-            let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap();
-            writer.write(&batch).unwrap();
-            writer.finish().unwrap();
-        }
-        buffer
+        assert!(result.is_ok(), "delete should succeed: {result:?}");
+        assert!(!manifest_ns.manifest_contains_object("table").await.unwrap());
     }
 
     #[rstest]
@@ -3939,9 +5697,9 @@ mod tests {
     /// Test that concurrent create_table calls for the same table name don't
     /// create duplicate entries in the manifest. Uses two independent
     /// ManifestNamespace instances pointing at the same directory to simulate
-    /// two separate OS processes racing on table creation. The conflict_retries
-    /// setting on the MergeInsert ensures the second operation properly detects
-    /// the duplicate via WhenMatched::Fail after retrying against the latest data.
+    /// two separate OS processes racing on table creation. Copy-on-write rewrite
+    /// retries ensure the second operation detects the duplicate after retrying
+    /// against the latest data.
     #[tokio::test]
     async fn test_concurrent_create_table_no_duplicates() {
         let temp_dir = TempStdDir::default();
@@ -4069,4 +5827,297 @@ mod tests {
         assert_eq!(n, names(&["c", "d"]));
         assert_eq!(next, Some("d".to_string()));
     }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_add_columns(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{
+            AddColumnsEntry, AlterTableAddColumnsRequest, DescribeTableRequest,
+        };
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a table with id and name columns
+        let buffer = create_test_ipc_data();
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        dir_namespace
+            .create_table(create_request, Bytes::from(buffer))
+            .await
+            .unwrap();
+
+        // Add a new column using SQL expression
+        let mut new_col = AddColumnsEntry::new("doubled_id".to_string());
+        new_col.expression = Some(Some("id * 2".to_string()));
+        let mut add_request = AlterTableAddColumnsRequest::new(vec![new_col]);
+        add_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = dir_namespace
+            .alter_table_add_columns(add_request)
+            .await
+            .unwrap();
+        // Version should have incremented
+        assert!(response.version > 1);
+
+        // Verify the column was added by describing the table with detailed metadata
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = dir_namespace
+            .describe_table(describe_request)
+            .await
+            .unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            field_names.contains(&"doubled_id"),
+            "Column 'doubled_id' should exist after add_columns, got: {:?}",
+            field_names
+        );
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_add_columns_missing_id(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{AddColumnsEntry, AlterTableAddColumnsRequest};
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        // Request without ID should fail
+        let new_col = AddColumnsEntry::new("col".to_string());
+        let request = AlterTableAddColumnsRequest::new(vec![new_col]);
+        let result = dir_namespace.alter_table_add_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_add_columns_nonexistent_table(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{AddColumnsEntry, AlterTableAddColumnsRequest};
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        // Request with non-existent table should fail
+        let new_col = AddColumnsEntry::new("col".to_string());
+        let mut request = AlterTableAddColumnsRequest::new(vec![new_col]);
+        request.id = Some(vec!["nonexistent".to_string()]);
+        let result = dir_namespace.alter_table_add_columns(request).await;
+        assert!(result.is_err(), "Should fail when table does not exist");
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_alter_columns_rename(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{
+            AlterColumnsEntry, AlterTableAlterColumnsRequest, DescribeTableRequest,
+        };
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a table
+        let buffer = create_test_ipc_data();
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        dir_namespace
+            .create_table(create_request, Bytes::from(buffer))
+            .await
+            .unwrap();
+
+        // Rename the "name" column to "full_name"
+        let mut entry = AlterColumnsEntry::new("name".to_string());
+        entry.rename = Some(Some("full_name".to_string()));
+        let mut alter_request = AlterTableAlterColumnsRequest::new(vec![entry]);
+        alter_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = dir_namespace
+            .alter_table_alter_columns(alter_request)
+            .await
+            .unwrap();
+        assert!(response.version > 1);
+
+        // Verify the column was renamed
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = dir_namespace
+            .describe_table(describe_request)
+            .await
+            .unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            field_names.contains(&"full_name"),
+            "Column should be renamed to 'full_name', got: {:?}",
+            field_names
+        );
+        assert!(
+            !field_names.contains(&"name"),
+            "Old column name 'name' should no longer exist, got: {:?}",
+            field_names
+        );
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_alter_columns_missing_id(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{AlterColumnsEntry, AlterTableAlterColumnsRequest};
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        let entry = AlterColumnsEntry::new("name".to_string());
+        let request = AlterTableAlterColumnsRequest::new(vec![entry]);
+        let result = dir_namespace.alter_table_alter_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_drop_columns(#[case] inline_optimization: bool) {
+        use lance_namespace::models::{AlterTableDropColumnsRequest, DescribeTableRequest};
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        // Create a table with id and name columns
+        let buffer = create_test_ipc_data();
+        let mut create_request = CreateTableRequest::new();
+        create_request.id = Some(vec!["test_table".to_string()]);
+        dir_namespace
+            .create_table(create_request, Bytes::from(buffer))
+            .await
+            .unwrap();
+
+        // Drop the "name" column
+        let mut drop_request = AlterTableDropColumnsRequest::new(vec!["name".to_string()]);
+        drop_request.id = Some(vec!["test_table".to_string()]);
+
+        let response = dir_namespace
+            .alter_table_drop_columns(drop_request)
+            .await
+            .unwrap();
+        assert!(response.version > 1);
+
+        // Verify the column was dropped
+        let mut describe_request = DescribeTableRequest::new();
+        describe_request.id = Some(vec!["test_table".to_string()]);
+        describe_request.load_detailed_metadata = Some(true);
+        let describe_response = dir_namespace
+            .describe_table(describe_request)
+            .await
+            .unwrap();
+        assert!(describe_response.schema.is_some());
+
+        let schema = describe_response.schema.unwrap();
+        let field_names: Vec<&str> = schema.fields.iter().map(|f| f.name.as_str()).collect();
+        assert!(
+            !field_names.contains(&"name"),
+            "Column 'name' should have been dropped, got: {:?}",
+            field_names
+        );
+        assert!(
+            field_names.contains(&"id"),
+            "Column 'id' should still exist, got: {:?}",
+            field_names
+        );
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_drop_columns_missing_id(#[case] inline_optimization: bool) {
+        use lance_namespace::models::AlterTableDropColumnsRequest;
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        let request = AlterTableDropColumnsRequest::new(vec!["col".to_string()]);
+        let result = dir_namespace.alter_table_drop_columns(request).await;
+        assert!(result.is_err(), "Should fail when table ID is missing");
+    }
+
+    #[rstest]
+    #[case::with_optimization(true)]
+    #[case::without_optimization(false)]
+    #[tokio::test]
+    async fn test_alter_table_drop_columns_nonexistent_table(#[case] inline_optimization: bool) {
+        use lance_namespace::models::AlterTableDropColumnsRequest;
+
+        let temp_dir = TempStdDir::default();
+        let temp_path = temp_dir.to_str().unwrap();
+
+        let dir_namespace = DirectoryNamespaceBuilder::new(temp_path)
+            .inline_optimization_enabled(inline_optimization)
+            .build()
+            .await
+            .unwrap();
+
+        let mut request = AlterTableDropColumnsRequest::new(vec!["col".to_string()]);
+        request.id = Some(vec!["nonexistent".to_string()]);
+        let result = dir_namespace.alter_table_drop_columns(request).await;
+        assert!(result.is_err(), "Should fail when table does not exist");
+    }
 }
diff --git a/rust/lance-namespace-impls/src/dir/manifest_feature_flags.rs b/rust/lance-namespace-impls/src/dir/manifest_feature_flags.rs
new file mode 100644
index 00000000000..d0849ceda4f
--- /dev/null
+++ b/rust/lance-namespace-impls/src/dir/manifest_feature_flags.rs
@@ -0,0 +1,194 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Reader/writer feature flags for the directory-catalog `__manifest` dataset.
+//!
+//! Forward-compatibility infrastructure for the `__manifest` Lance dataset,
+//! analogous to the Lance table format's `reader_feature_flags` /
+//! `writer_feature_flags` but describing the *catalog manifest* format (schema
+//! and semantics) rather than the underlying Lance file format. The flags are
+//! persisted in the `__manifest` dataset's `table_metadata` map.
+//!
+//! Each manifest feature owns one bit in a `u64` bitmask. A build may read a
+//! `__manifest` only if it understands every set reader-flag bit, and may write
+//! it only if it understands every set writer-flag bit; otherwise it fails fast
+//! with a clear "please upgrade" error instead of silently misreading data. The
+//! set of bits a build understands is `READER_KNOWN_FLAGS` / `WRITER_KNOWN_FLAGS`.
+//!
+//! This is the mechanism only: no manifest feature is defined yet, so the known
+//! masks are `0` and nothing is ever set — every current manifest reads and
+//! writes unchanged. The first format change that needs forward-compatibility
+//! protection adds its bit to the known masks and stamps it on write; from then
+//! on, builds without that bit refuse the new format rather than misreading it.
+//! Manifests written before this mechanism carry no flag keys, which parse as
+//! `0` and stay compatible with every build.
+
+use std::collections::HashMap;
+
+use lance_core::{Error, Result};
+use lance_namespace::error::NamespaceError;
+
+/// `table_metadata` key holding the reader feature-flag bitmask (decimal `u64`).
+pub const READER_FEATURE_FLAGS_KEY: &str = "lance.namespace.manifest.reader_feature_flags";
+/// `table_metadata` key holding the writer feature-flag bitmask (decimal `u64`).
+pub const WRITER_FEATURE_FLAGS_KEY: &str = "lance.namespace.manifest.writer_feature_flags";
+
+/// Reader feature-flag bits this build understands. No manifest feature is
+/// defined yet, so this build understands none and refuses any non-zero reader
+/// flag. A future format change adds its bit here.
+const READER_KNOWN_FLAGS: u64 = 0;
+/// Writer feature-flag bits this build understands.
+const WRITER_KNOWN_FLAGS: u64 = 0;
+
+/// Whether this build can read a `__manifest` whose persisted reader feature
+/// flags are `reader_flags` — i.e. it understands every set bit.
+pub fn can_read_manifest(reader_flags: u64) -> bool {
+    (reader_flags & !READER_KNOWN_FLAGS) == 0
+}
+
+/// Whether this build can write a `__manifest` whose persisted writer feature
+/// flags are `writer_flags` — i.e. it understands every set bit.
+pub fn can_write_manifest(writer_flags: u64) -> bool {
+    (writer_flags & !WRITER_KNOWN_FLAGS) == 0
+}
+
+fn parse_flags(table_metadata: &HashMap<String, String>, key: &str) -> Result<u64> {
+    match table_metadata.get(key) {
+        None => Ok(0),
+        Some(raw) => raw.parse::<u64>().map_err(|e| {
+            Error::from(NamespaceError::Unsupported {
+                message: format!(
+                    "The __manifest dataset has an unparsable feature-flag value '{raw}' for \
+                     '{key}': {e}. This likely means it was written by a newer, incompatible \
+                     version of Lance; please upgrade Lance to use this catalog."
+                ),
+            })
+        }),
+    }
+}
+
+/// Reader feature flags persisted in the `__manifest` `table_metadata` (`0` if absent).
+pub fn reader_flags(table_metadata: &HashMap<String, String>) -> Result<u64> {
+    parse_flags(table_metadata, READER_FEATURE_FLAGS_KEY)
+}
+
+/// Writer feature flags persisted in the `__manifest` `table_metadata` (`0` if absent).
+pub fn writer_flags(table_metadata: &HashMap<String, String>) -> Result<u64> {
+    parse_flags(table_metadata, WRITER_FEATURE_FLAGS_KEY)
+}
+
+/// Validate that this build can READ the `__manifest` described by `table_metadata`,
+/// returning a clear "please upgrade" error otherwise.
+pub fn ensure_readable(table_metadata: &HashMap<String, String>) -> Result<()> {
+    let flags = reader_flags(table_metadata)?;
+    if !can_read_manifest(flags) {
+        return Err(Error::from(NamespaceError::Unsupported {
+            message: format!(
+                "The __manifest dataset was written with reader feature flags {flags}, which this \
+                 version of Lance does not understand (known reader flags: {READER_KNOWN_FLAGS}). \
+                 Please upgrade Lance to read this catalog."
+            ),
+        }));
+    }
+    Ok(())
+}
+
+/// Validate that this build can WRITE the `__manifest` described by `table_metadata`,
+/// returning a clear "please upgrade" error otherwise.
+pub fn ensure_writable(table_metadata: &HashMap<String, String>) -> Result<()> {
+    let flags = writer_flags(table_metadata)?;
+    if !can_write_manifest(flags) {
+        return Err(Error::from(NamespaceError::Unsupported {
+            message: format!(
+                "The __manifest dataset was written with writer feature flags {flags}, which this \
+                 version of Lance does not understand (known writer flags: {WRITER_KNOWN_FLAGS}). \
+                 Please upgrade Lance to modify this catalog."
+            ),
+        }));
+    }
+    Ok(())
+}
+
+/// Whether `err` indicates the `__manifest` is in a format this build cannot
+/// handle — i.e. it carries an unknown reader/writer feature flag, surfaced by
+/// [`ensure_readable`] / [`ensure_writable`] as a [`NamespaceError::Unsupported`].
+///
+/// Catalog initialization uses this to refuse opening such a manifest rather
+/// than silently degrading to a directory-listing view that ignores it. The
+/// `__manifest` open path raises no other `Unsupported` error, so matching the
+/// code is sufficient and avoids brittle message matching.
+pub fn is_incompatible_manifest_error(err: &Error) -> bool {
+    matches!(
+        err,
+        Error::Namespace { source, .. }
+            if source
+                .downcast_ref::<NamespaceError>()
+                .is_some_and(|e| matches!(e, NamespaceError::Unsupported { .. }))
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn meta(pairs: &[(&str, &str)]) -> HashMap<String, String> {
+        pairs
+            .iter()
+            .map(|(k, v)| (k.to_string(), v.to_string()))
+            .collect()
+    }
+
+    #[test]
+    fn unflagged_is_compatible() {
+        assert!(can_read_manifest(0));
+        assert!(can_write_manifest(0));
+        let empty = HashMap::new();
+        assert!(ensure_readable(&empty).is_ok());
+        assert!(ensure_writable(&empty).is_ok());
+        assert_eq!(reader_flags(&empty).unwrap(), 0);
+        assert_eq!(writer_flags(&empty).unwrap(), 0);
+        // Explicit zeroes are also compatible.
+        let zeroed = meta(&[
+            (READER_FEATURE_FLAGS_KEY, "0"),
+            (WRITER_FEATURE_FLAGS_KEY, "0"),
+        ]);
+        assert!(ensure_readable(&zeroed).is_ok());
+        assert!(ensure_writable(&zeroed).is_ok());
+    }
+
+    #[test]
+    fn any_unknown_flag_is_refused() {
+        // This build understands no feature flags, so any non-zero bit is refused.
+        assert!(!can_read_manifest(1));
+        assert!(!can_write_manifest(1));
+        assert!(!can_read_manifest(1 << 30));
+        assert!(!can_write_manifest(1 << 63));
+
+        let reader = meta(&[(READER_FEATURE_FLAGS_KEY, "1")]);
+        let err = ensure_readable(&reader).unwrap_err();
+        assert!(err.to_string().to_lowercase().contains("upgrade"));
+        assert!(is_incompatible_manifest_error(&err));
+        // A reader flag does not block writers that the writer mask allows.
+        assert!(ensure_writable(&reader).is_ok());
+
+        let writer = meta(&[(WRITER_FEATURE_FLAGS_KEY, "2")]);
+        let err = ensure_writable(&writer).unwrap_err();
+        assert!(err.to_string().to_lowercase().contains("upgrade"));
+        assert!(is_incompatible_manifest_error(&err));
+    }
+
+    #[test]
+    fn unparsable_value_is_refused() {
+        let m = meta(&[(READER_FEATURE_FLAGS_KEY, "not-a-number")]);
+        assert!(reader_flags(&m).is_err());
+        assert!(ensure_readable(&m).is_err());
+    }
+
+    #[test]
+    fn unrelated_error_is_not_an_incompatibility() {
+        let other = Error::from(NamespaceError::TableNotFound {
+            message: "x".to_string(),
+        });
+        assert!(!is_incompatible_manifest_error(&other));
+    }
+}
diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs
index 27a563d2807..c245a1e6dc1 100644
--- a/rust/lance-namespace-impls/src/rest.rs
+++ b/rust/lance-namespace-impls/src/rest.rs
@@ -23,11 +23,13 @@ use lance_namespace::models::{
     AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest,
     BatchDeleteTableVersionsRequest, BatchDeleteTableVersionsResponse, CountTableRowsRequest,
     CreateMaterializedViewRequest, CreateMaterializedViewResponse, CreateNamespaceRequest,
-    CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest,
-    CreateTableResponse, CreateTableScalarIndexResponse, CreateTableTagRequest,
-    CreateTableTagResponse, CreateTableVersionRequest, CreateTableVersionResponse,
-    DeclareTableRequest, DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse,
-    DeleteTableTagRequest, DeleteTableTagResponse, DeregisterTableRequest, DeregisterTableResponse,
+    CreateNamespaceResponse, CreateTableBranchRequest, CreateTableBranchResponse,
+    CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, CreateTableResponse,
+    CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse,
+    CreateTableVersionRequest, CreateTableVersionResponse, DeclareTableRequest,
+    DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse,
+    DeleteTableBranchRequest, DeleteTableBranchResponse, DeleteTableTagRequest,
+    DeleteTableTagResponse, DeregisterTableRequest, DeregisterTableResponse,
     DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableIndexStatsRequest,
     DescribeTableIndexStatsResponse, DescribeTableRequest, DescribeTableResponse,
     DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
@@ -36,7 +38,8 @@ use lance_namespace::models::{
     ErrorResponse, ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse,
     GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest,
     InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse,
-    ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
+    ListTableBranchesRequest, ListTableBranchesResponse, ListTableIndicesRequest,
+    ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
     ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse,
     MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest,
     QueryTableRequest, RefreshMaterializedViewRequest, RefreshMaterializedViewResponse,
@@ -1294,6 +1297,13 @@ impl LanceNamespace for RestNamespace {
             descending_str = descending.to_string();
             query.push(("descending", descending_str.as_str()));
         }
+        // Forward branch as a query param (this op sends no body).
+        // describe_table_version differs: branch rides its body, already serialized.
+        let branch_str;
+        if let Some(ref branch) = request.branch {
+            branch_str = branch.clone();
+            query.push(("branch", branch_str.as_str()));
+        }
         self.post_json(&path, &query, &(), "list_table_versions", &id)
             .await
     }
@@ -1553,6 +1563,55 @@ impl LanceNamespace for RestNamespace {
             .await
     }
 
+    async fn create_table_branch(
+        &self,
+        request: CreateTableBranchRequest,
+    ) -> Result<CreateTableBranchResponse> {
+        self.record_op("create_table_branch");
+        let id = object_id_str(&request.id, &self.delimiter)?;
+        let encoded_id = urlencode(&id);
+        let path = format!("/v1/table/{}/branches/create", encoded_id);
+        let query = [("delimiter", self.delimiter.as_str())];
+        self.post_json(&path, &query, &request, "create_table_branch", &id)
+            .await
+    }
+
+    async fn list_table_branches(
+        &self,
+        request: ListTableBranchesRequest,
+    ) -> Result<ListTableBranchesResponse> {
+        self.record_op("list_table_branches");
+        let id = object_id_str(&request.id, &self.delimiter)?;
+        let encoded_id = urlencode(&id);
+        let path = format!("/v1/table/{}/branches/list", encoded_id);
+        let mut query = vec![("delimiter", self.delimiter.as_str())];
+        let page_token_str;
+        if let Some(ref pt) = request.page_token {
+            page_token_str = pt.clone();
+            query.push(("page_token", page_token_str.as_str()));
+        }
+        let limit_str;
+        if let Some(limit) = request.limit {
+            limit_str = limit.to_string();
+            query.push(("limit", limit_str.as_str()));
+        }
+        self.post_json(&path, &query, &request, "list_table_branches", &id)
+            .await
+    }
+
+    async fn delete_table_branch(
+        &self,
+        request: DeleteTableBranchRequest,
+    ) -> Result<DeleteTableBranchResponse> {
+        self.record_op("delete_table_branch");
+        let id = object_id_str(&request.id, &self.delimiter)?;
+        let encoded_id = urlencode(&id);
+        let path = format!("/v1/table/{}/branches/delete", encoded_id);
+        let query = [("delimiter", self.delimiter.as_str())];
+        self.post_json(&path, &query, &request, "delete_table_branch", &id)
+            .await
+    }
+
     fn namespace_id(&self) -> String {
         format!(
             "RestNamespace {{ endpoint: {:?}, delimiter: {:?} }}",
diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs
index f7de5dc2240..44ebd866810 100644
--- a/rust/lance-namespace-impls/src/rest_adapter.rs
+++ b/rust/lance-namespace-impls/src/rest_adapter.rs
@@ -141,6 +141,10 @@ impl RestAdapter {
             .route("/v1/table/:id/tags/create", post(create_table_tag))
             .route("/v1/table/:id/tags/delete", post(delete_table_tag))
             .route("/v1/table/:id/tags/update", post(update_table_tag))
+            // Branch operations
+            .route("/v1/table/:id/branches/create", post(create_table_branch))
+            .route("/v1/table/:id/branches/list", post(list_table_branches))
+            .route("/v1/table/:id/branches/delete", post(delete_table_branch))
             // Query plan operations
             .route("/v1/table/:id/explain_plan", post(explain_table_query_plan))
             .route("/v1/table/:id/analyze_plan", post(analyze_table_query_plan))
@@ -302,6 +306,7 @@ struct PaginationQuery {
     limit: Option<i32>,
     include_declared: Option<bool>,
     descending: Option<bool>,
+    branch: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -325,11 +330,13 @@ fn error_code_to_status(code: u32) -> StatusCode {
         | Some(lance_namespace::error::ErrorCode::TableTagNotFound)
         | Some(lance_namespace::error::ErrorCode::TransactionNotFound)
         | Some(lance_namespace::error::ErrorCode::TableVersionNotFound)
-        | Some(lance_namespace::error::ErrorCode::TableColumnNotFound) => StatusCode::NOT_FOUND,
+        | Some(lance_namespace::error::ErrorCode::TableColumnNotFound)
+        | Some(lance_namespace::error::ErrorCode::TableBranchNotFound) => StatusCode::NOT_FOUND,
         Some(lance_namespace::error::ErrorCode::NamespaceAlreadyExists)
         | Some(lance_namespace::error::ErrorCode::TableAlreadyExists)
         | Some(lance_namespace::error::ErrorCode::TableIndexAlreadyExists)
         | Some(lance_namespace::error::ErrorCode::TableTagAlreadyExists)
+        | Some(lance_namespace::error::ErrorCode::TableBranchAlreadyExists)
         | Some(lance_namespace::error::ErrorCode::ConcurrentModification) => StatusCode::CONFLICT,
         Some(lance_namespace::error::ErrorCode::NamespaceNotEmpty)
         | Some(lance_namespace::error::ErrorCode::InvalidTableState) => StatusCode::CONFLICT,
@@ -847,6 +854,7 @@ async fn list_table_versions(
         page_token: params.page_token,
         limit: params.limit,
         descending: params.descending,
+        branch: params.branch,
         identity: extract_identity(&headers),
         ..Default::default()
     };
@@ -872,6 +880,7 @@ async fn create_table_version(
         manifest_size: body.manifest_size,
         e_tag: body.e_tag,
         metadata: body.metadata,
+        branch: body.branch,
         ..Default::default()
     };
 
@@ -891,6 +900,7 @@ async fn describe_table_version(
     let request = DescribeTableVersionRequest {
         id: Some(parse_id(&id, query.delimiter.as_deref())),
         version: body.version,
+        branch: body.branch,
         identity: extract_identity(&headers),
         ..Default::default()
     };
@@ -912,6 +922,7 @@ async fn batch_delete_table_versions(
         id: Some(parse_id(&id, params.delimiter.as_deref())),
         identity: extract_identity(&headers),
         ranges: body.ranges,
+        branch: body.branch,
         ..Default::default()
     };
 
@@ -1261,6 +1272,62 @@ async fn update_table_tag(
     }
 }
 
+// ============================================================================
+// Branch Operation Handlers
+// ============================================================================
+
+async fn create_table_branch(
+    State(backend): State<Arc<dyn LanceNamespace>>,
+    headers: HeaderMap,
+    Path(id): Path<String>,
+    Query(params): Query<DelimiterQuery>,
+    Json(mut request): Json<CreateTableBranchRequest>,
+) -> Response {
+    request.id = Some(parse_id(&id, params.delimiter.as_deref()));
+    request.identity = extract_identity(&headers);
+
+    match backend.create_table_branch(request).await {
+        Ok(response) => (StatusCode::CREATED, Json(response)).into_response(),
+        Err(e) => error_to_response(e),
+    }
+}
+
+async fn list_table_branches(
+    State(backend): State<Arc<dyn LanceNamespace>>,
+    headers: HeaderMap,
+    Path(id): Path<String>,
+    Query(params): Query<PaginationQuery>,
+) -> Response {
+    let request = ListTableBranchesRequest {
+        id: Some(parse_id(&id, params.delimiter.as_deref())),
+        page_token: params.page_token,
+        limit: params.limit,
+        identity: extract_identity(&headers),
+        ..Default::default()
+    };
+
+    match backend.list_table_branches(request).await {
+        Ok(response) => (StatusCode::OK, Json(response)).into_response(),
+        Err(e) => error_to_response(e),
+    }
+}
+
+async fn delete_table_branch(
+    State(backend): State<Arc<dyn LanceNamespace>>,
+    headers: HeaderMap,
+    Path(id): Path<String>,
+    Query(params): Query<DelimiterQuery>,
+    Json(mut request): Json<DeleteTableBranchRequest>,
+) -> Response {
+    request.id = Some(parse_id(&id, params.delimiter.as_deref()));
+    request.identity = extract_identity(&headers);
+
+    match backend.delete_table_branch(request).await {
+        Ok(response) => (StatusCode::OK, Json(response)).into_response(),
+        Err(e) => error_to_response(e),
+    }
+}
+
 // ============================================================================
 // Query Plan Operation Handlers
 // ============================================================================
@@ -1456,15 +1523,25 @@ mod tests {
 
         impl RestServerFixture {
             async fn new() -> Self {
+                Self::build(false).await
+            }
+
+            /// Like [`Self::new`], with managed versioning (table version
+            /// tracking) enabled on the backend.
+            async fn new_managed() -> Self {
+                Self::build(true).await
+            }
+
+            async fn build(managed_versioning: bool) -> Self {
                 let temp_dir = TempDir::new().unwrap();
                 let temp_path = temp_dir.path().to_str().unwrap().to_string();
 
                 // Create DirectoryNamespace backend with manifest enabled
-                let backend = DirectoryNamespaceBuilder::new(&temp_path)
-                    .manifest_enabled(true)
-                    .build()
-                    .await
-                    .unwrap();
+                let mut builder = DirectoryNamespaceBuilder::new(&temp_path).manifest_enabled(true);
+                if managed_versioning {
+                    builder = builder.table_version_tracking_enabled(true);
+                }
+                let backend = builder.build().await.unwrap();
                 let backend = Arc::new(backend);
 
                 // Start REST server with port 0 (OS assigns available port)
@@ -3095,13 +3172,7 @@ mod tests {
                     "context_test_ns".to_string(),
                     "test_table".to_string(),
                 ]),
-                with_table_uri: None,
-                load_detailed_metadata: None,
-                check_declared: None,
-                vend_credentials: None,
-                version: None,
-                identity: None,
-                context: None,
+                ..Default::default()
             };
             let result = namespace.describe_table(describe_req).await;
             assert!(result.is_ok(), "Failed to describe table: {:?}", result);
@@ -3195,6 +3266,455 @@ mod tests {
             );
         }
 
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_branch_param_forwarded_end_to_end() {
+            let fixture = RestServerFixture::new().await;
+
+            fixture
+                .namespace
+                .create_namespace(CreateNamespaceRequest {
+                    id: Some(vec!["branch_fwd_ns".to_string()]),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            fixture
+                .namespace
+                .create_table(
+                    CreateTableRequest {
+                        id: Some(vec![
+                            "branch_fwd_ns".to_string(),
+                            "branch_fwd_table".to_string(),
+                        ]),
+                        mode: Some("create".to_string()),
+                        ..Default::default()
+                    },
+                    create_test_arrow_data(),
+                )
+                .await
+                .unwrap();
+
+            let id = vec!["branch_fwd_ns".to_string(), "branch_fwd_table".to_string()];
+
+            // Control: no branch succeeds (resolves the main chain).
+            assert!(
+                fixture
+                    .namespace
+                    .list_table_versions(ListTableVersionsRequest {
+                        id: Some(id.clone()),
+                        ..Default::default()
+                    })
+                    .await
+                    .is_ok()
+            );
+
+            // list forwards branch as a query param; a bogus branch 404s at the backend.
+            assert!(
+                fixture
+                    .namespace
+                    .list_table_versions(ListTableVersionsRequest {
+                        id: Some(id.clone()),
+                        branch: Some("ghost".to_string()),
+                        ..Default::default()
+                    })
+                    .await
+                    .is_err(),
+                "branch must be forwarded as a query param and honored by the backend"
+            );
+
+            // describe carries branch in the request body; a bogus branch likewise 404s.
+            assert!(
+                fixture
+                    .namespace
+                    .describe_table_version(DescribeTableVersionRequest {
+                        id: Some(id.clone()),
+                        branch: Some("ghost".to_string()),
+                        ..Default::default()
+                    })
+                    .await
+                    .is_err(),
+                "branch must be forwarded in the request body and honored by the backend"
+            );
+
+            fixture.server_handle.shutdown();
+        }
+
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_branch_crud_end_to_end() {
+            let fixture = RestServerFixture::new().await;
+
+            fixture
+                .namespace
+                .create_namespace(CreateNamespaceRequest {
+                    id: Some(vec!["branch_crud_ns".to_string()]),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            fixture
+                .namespace
+                .create_table(
+                    CreateTableRequest {
+                        id: Some(vec![
+                            "branch_crud_ns".to_string(),
+                            "branch_crud_table".to_string(),
+                        ]),
+                        mode: Some("create".to_string()),
+                        ..Default::default()
+                    },
+                    create_test_arrow_data(),
+                )
+                .await
+                .unwrap();
+
+            let id = vec![
+                "branch_crud_ns".to_string(),
+                "branch_crud_table".to_string(),
+            ];
+
+            // create -> list shows it (client -> server -> directory backend)
+            fixture
+                .namespace
+                .create_table_branch(CreateTableBranchRequest {
+                    id: Some(id.clone()),
+                    name: "dev".to_string(),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+
+            let listed = fixture
+                .namespace
+                .list_table_branches(ListTableBranchesRequest {
+                    id: Some(id.clone()),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            assert!(
+                listed.branches.contains_key("dev"),
+                "created branch should appear in list: {:?}",
+                listed.branches
+            );
+
+            // duplicate create -> 409 Conflict
+            let port = fixture.server_handle.port();
+            let client = reqwest::Client::new();
+            let table_path = "branch_crud_ns%24branch_crud_table";
+            let resp = client
+                .post(format!(
+                    "http://127.0.0.1:{}/v1/table/{}/branches/create",
+                    port, table_path
+                ))
+                .query(&[("delimiter", "$")])
+                .json(&serde_json::json!({ "name": "dev" }))
+                .send()
+                .await
+                .unwrap();
+            assert_eq!(
+                resp.status(),
+                409,
+                "duplicate branch create should map to 409, got {}",
+                resp.status()
+            );
+
+            // delete -> list no longer shows it
+            fixture
+                .namespace
+                .delete_table_branch(DeleteTableBranchRequest {
+                    id: Some(id.clone()),
+                    name: "dev".to_string(),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+
+            let listed = fixture
+                .namespace
+                .list_table_branches(ListTableBranchesRequest {
+                    id: Some(id.clone()),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            assert!(
+                !listed.branches.contains_key("dev"),
+                "deleted branch must not appear in list: {:?}",
+                listed.branches
+            );
+
+            // delete missing -> 404 Not Found (raw HTTP validates TableBranchNotFound -> 404).
+            let resp = client
+                .post(format!(
+                    "http://127.0.0.1:{}/v1/table/{}/branches/delete",
+                    port, table_path
+                ))
+                .query(&[("delimiter", "$")])
+                .json(&serde_json::json!({ "name": "dev" }))
+                .send()
+                .await
+                .unwrap();
+            assert_eq!(
+                resp.status(),
+                404,
+                "deleting a missing branch should map to 404, got {}",
+                resp.status()
+            );
+
+            fixture.server_handle.shutdown();
+        }
+
+        /// The managed (manifest-tracked) branch flow over REST: create a
+        /// managed table and a branch through the RestNamespace client, open
+        /// the branch via `from_namespace(...).with_branch`, commit on it,
+        /// check out across branches at an overlapping version number, and
+        /// round-trip a branch-pointing tag. Mirrors the DirectoryNamespace
+        /// e2e to prove the REST layer forwards everything the managed commit
+        /// store needs.
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_namespace_managed_branch_e2e() {
+            use arrow::array::Int32Array;
+            use arrow::datatypes::{DataType, Field as ArrowField, Schema as ArrowSchema};
+            use arrow::record_batch::{RecordBatch, RecordBatchIterator};
+            use futures::TryStreamExt;
+            use lance::dataset::builder::DatasetBuilder;
+            use lance::dataset::refs::Ref;
+            use lance::dataset::{Dataset, WriteMode, WriteParams};
+            use lance_namespace::LanceNamespace;
+
+            async fn scan_ids(ds: &Dataset) -> Vec<i32> {
+                let batches: Vec<RecordBatch> = ds
+                    .scan()
+                    .try_into_stream()
+                    .await
+                    .unwrap()
+                    .try_collect()
+                    .await
+                    .unwrap();
+                let mut ids: Vec<i32> = batches
+                    .iter()
+                    .flat_map(|b| {
+                        b.column(0)
+                            .as_any()
+                            .downcast_ref::<Int32Array>()
+                            .unwrap()
+                            .values()
+                            .to_vec()
+                    })
+                    .collect();
+                ids.sort();
+                ids
+            }
+
+            let fixture = RestServerFixture::new_managed().await;
+            let namespace = Arc::new(fixture.namespace.clone()) as Arc<dyn LanceNamespace>;
+            let table_id = vec!["mb_table".to_string()];
+
+            let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                "id",
+                DataType::Int32,
+                false,
+            )]));
+            let batch = |seed: i32| {
+                RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![seed]))])
+                    .unwrap()
+            };
+
+            // Managed main: v1 (id=1), v2 (id=2).
+            let mut main_ds = Dataset::write_into_namespace(
+                RecordBatchIterator::new(vec![Ok(batch(1))], schema.clone()),
+                namespace.clone(),
+                table_id.clone(),
+                Some(WriteParams {
+                    mode: WriteMode::Create,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap();
+            main_ds
+                .append(
+                    RecordBatchIterator::new(vec![Ok(batch(2))], schema.clone()),
+                    None,
+                )
+                .await
+                .unwrap();
+
+            // The REST layer must surface managed versioning for the deferred
+            // commit handler to engage.
+            let described = namespace
+                .describe_table(DescribeTableRequest {
+                    id: Some(table_id.clone()),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            assert_eq!(
+                described.managed_versioning,
+                Some(true),
+                "managed_versioning must survive the REST round trip"
+            );
+            let main_chain_len = |ns: Arc<dyn LanceNamespace>, table_id: Vec<String>| async move {
+                ns.list_table_versions(ListTableVersionsRequest {
+                    id: Some(table_id),
+                    ..Default::default()
+                })
+                .await
+                .unwrap()
+                .versions
+                .len()
+            };
+            assert_eq!(main_chain_len(namespace.clone(), table_id.clone()).await, 2);
+
+            // Branch via the REST client, then open and commit on it.
+            namespace
+                .create_table_branch(CreateTableBranchRequest {
+                    id: Some(table_id.clone()),
+                    name: "exp".to_string(),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            let mut branch_ds = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
+                .await
+                .unwrap()
+                .with_branch("exp", None)
+                .load()
+                .await
+                .unwrap();
+            assert_eq!(branch_ds.manifest().branch.as_deref(), Some("exp"));
+            assert!(
+                branch_ds
+                    .branch_location()
+                    .path
+                    .as_ref()
+                    .ends_with("tree/exp"),
+                "the branch dataset must be rooted at the branch chain"
+            );
+            branch_ds
+                .append(
+                    RecordBatchIterator::new(vec![Ok(batch(3))], schema.clone()),
+                    None,
+                )
+                .await
+                .unwrap();
+            assert_eq!(branch_ds.manifest().branch.as_deref(), Some("exp"));
+            assert_eq!(scan_ids(&branch_ds).await, vec![1, 2, 3]);
+            assert_eq!(
+                main_chain_len(namespace.clone(), table_id.clone()).await,
+                2,
+                "a branch commit must not advance main's chain"
+            );
+
+            // Cross-branch checkout at an overlapping version number must land
+            // on the branch chain (branch numbering continues from the fork
+            // point, so both chains have this version).
+            let overlap_version = branch_ds.version().version;
+            while main_ds.version().version < overlap_version {
+                main_ds
+                    .append(
+                        RecordBatchIterator::new(vec![Ok(batch(100))], schema.clone()),
+                        None,
+                    )
+                    .await
+                    .unwrap();
+            }
+            let on_branch = main_ds
+                .checkout_version(Ref::Version(Some("exp".to_string()), Some(overlap_version)))
+                .await
+                .unwrap();
+            assert_eq!(on_branch.manifest().branch.as_deref(), Some("exp"));
+            assert_eq!(scan_ids(&on_branch).await, vec![1, 2, 3]);
+            let on_branch_latest = main_ds.checkout_branch("exp").await.unwrap();
+            assert_eq!(on_branch_latest.manifest().branch.as_deref(), Some("exp"));
+
+            // Branch-pointing tag round trip through the builder.
+            main_ds
+                .tags()
+                .create("exp-tag", ("exp", Some(overlap_version)))
+                .await
+                .unwrap();
+            let tag_open = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone())
+                .await
+                .unwrap()
+                .with_tag("exp-tag")
+                .load()
+                .await
+                .unwrap();
+            assert_eq!(tag_open.manifest().branch.as_deref(), Some("exp"));
+            assert_eq!(tag_open.version().version, overlap_version);
+            assert_eq!(scan_ids(&tag_open).await, vec![1, 2, 3]);
+
+            fixture.server_handle.shutdown();
+        }
+
+        #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+        async fn test_list_branches_bodyless_post() {
+            let fixture = RestServerFixture::new().await;
+
+            fixture
+                .namespace
+                .create_namespace(CreateNamespaceRequest {
+                    id: Some(vec!["list_post_ns".to_string()]),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+            fixture
+                .namespace
+                .create_table(
+                    CreateTableRequest {
+                        id: Some(vec![
+                            "list_post_ns".to_string(),
+                            "list_post_table".to_string(),
+                        ]),
+                        mode: Some("create".to_string()),
+                        ..Default::default()
+                    },
+                    create_test_arrow_data(),
+                )
+                .await
+                .unwrap();
+            fixture
+                .namespace
+                .create_table_branch(CreateTableBranchRequest {
+                    id: Some(vec![
+                        "list_post_ns".to_string(),
+                        "list_post_table".to_string(),
+                    ]),
+                    name: "dev".to_string(),
+                    ..Default::default()
+                })
+                .await
+                .unwrap();
+
+            let port = fixture.server_handle.port();
+            let client = reqwest::Client::new();
+            let resp = client
+                .post(format!(
+                    "http://127.0.0.1:{}/v1/table/list_post_ns%24list_post_table/branches/list",
+                    port
+                ))
+                .query(&[("delimiter", "$")])
+                .send()
+                .await
+                .unwrap();
+            assert_eq!(
+                resp.status(),
+                200,
+                "bodyless list POST should succeed, got {}",
+                resp.status()
+            );
+            let body: ListTableBranchesResponse = resp.json().await.unwrap();
+            assert!(
+                body.branches.contains_key("dev"),
+                "bodyless list should return the branch, got: {:?}",
+                body.branches
+            );
+
+            fixture.server_handle.shutdown();
+        }
+
         #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
         async fn test_describe_table_version() {
             let fixture = RestServerFixture::new().await;
diff --git a/rust/lance-namespace/src/error.rs b/rust/lance-namespace/src/error.rs
index 8a73d4db8e8..5ed05541be3 100644
--- a/rust/lance-namespace/src/error.rs
+++ b/rust/lance-namespace/src/error.rs
@@ -78,6 +78,10 @@ pub enum ErrorCode {
     TableSchemaValidationError = 20,
     /// Request was throttled due to rate limiting or too many concurrent operations
     Throttling = 21,
+    /// The specified table branch does not exist
+    TableBranchNotFound = 22,
+    /// A table branch with this name already exists
+    TableBranchAlreadyExists = 23,
 }
 
 impl ErrorCode {
@@ -113,6 +117,8 @@ impl ErrorCode {
             19 => Some(Self::InvalidTableState),
             20 => Some(Self::TableSchemaValidationError),
             21 => Some(Self::Throttling),
+            22 => Some(Self::TableBranchNotFound),
+            23 => Some(Self::TableBranchAlreadyExists),
             _ => None,
         }
     }
@@ -143,6 +149,8 @@ impl std::fmt::Display for ErrorCode {
             Self::InvalidTableState => "InvalidTableState",
             Self::TableSchemaValidationError => "TableSchemaValidationError",
             Self::Throttling => "Throttling",
+            Self::TableBranchNotFound => "TableBranchNotFound",
+            Self::TableBranchAlreadyExists => "TableBranchAlreadyExists",
         };
         write!(f, "{}", name)
     }
@@ -260,6 +268,14 @@ pub enum NamespaceError {
     /// Request was throttled due to rate limiting or too many concurrent operations.
     #[snafu(display("Throttling: {message}"))]
     Throttling { message: String },
+
+    /// The specified table branch does not exist.
+    #[snafu(display("Table branch not found: {message}"))]
+    TableBranchNotFound { message: String },
+
+    /// A table branch with this name already exists.
+    #[snafu(display("Table branch already exists: {message}"))]
+    TableBranchAlreadyExists { message: String },
 }
 
 impl NamespaceError {
@@ -291,7 +307,9 @@ impl NamespaceError {
             | Self::Internal { message }
             | Self::InvalidTableState { message }
             | Self::TableSchemaValidationError { message }
-            | Self::Throttling { message } => message,
+            | Self::Throttling { message }
+            | Self::TableBranchNotFound { message }
+            | Self::TableBranchAlreadyExists { message } => message,
         }
     }
 
@@ -322,6 +340,8 @@ impl NamespaceError {
             Self::InvalidTableState { .. } => ErrorCode::InvalidTableState,
             Self::TableSchemaValidationError { .. } => ErrorCode::TableSchemaValidationError,
             Self::Throttling { .. } => ErrorCode::Throttling,
+            Self::TableBranchNotFound { .. } => ErrorCode::TableBranchNotFound,
+            Self::TableBranchAlreadyExists { .. } => ErrorCode::TableBranchAlreadyExists,
         }
     }
 
@@ -355,6 +375,8 @@ impl NamespaceError {
                 Self::TableSchemaValidationError { message }
             }
             Some(ErrorCode::Throttling) => Self::Throttling { message },
+            Some(ErrorCode::TableBranchNotFound) => Self::TableBranchNotFound { message },
+            Some(ErrorCode::TableBranchAlreadyExists) => Self::TableBranchAlreadyExists { message },
             None => Self::Internal { message },
         }
     }
@@ -380,7 +402,7 @@ mod tests {
 
     #[test]
     fn test_error_code_roundtrip() {
-        for code in 0..=21 {
+        for code in 0..=23 {
             let error_code = ErrorCode::from_u32(code).unwrap();
             assert_eq!(error_code.as_u32(), code);
         }
diff --git a/rust/lance-namespace/src/namespace.rs b/rust/lance-namespace/src/namespace.rs
index 0ee93b80b2e..faa7ec57367 100644
--- a/rust/lance-namespace/src/namespace.rs
+++ b/rust/lance-namespace/src/namespace.rs
@@ -14,11 +14,13 @@ use lance_namespace_reqwest_client::models::{
     AlterTransactionRequest, AlterTransactionResponse, AnalyzeTableQueryPlanRequest,
     BatchDeleteTableVersionsRequest, BatchDeleteTableVersionsResponse, CountTableRowsRequest,
     CreateMaterializedViewRequest, CreateMaterializedViewResponse, CreateNamespaceRequest,
-    CreateNamespaceResponse, CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest,
-    CreateTableResponse, CreateTableScalarIndexResponse, CreateTableTagRequest,
-    CreateTableTagResponse, CreateTableVersionRequest, CreateTableVersionResponse,
-    DeclareTableRequest, DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse,
-    DeleteTableTagRequest, DeleteTableTagResponse, DeregisterTableRequest, DeregisterTableResponse,
+    CreateNamespaceResponse, CreateTableBranchRequest, CreateTableBranchResponse,
+    CreateTableIndexRequest, CreateTableIndexResponse, CreateTableRequest, CreateTableResponse,
+    CreateTableScalarIndexResponse, CreateTableTagRequest, CreateTableTagResponse,
+    CreateTableVersionRequest, CreateTableVersionResponse, DeclareTableRequest,
+    DeclareTableResponse, DeleteFromTableRequest, DeleteFromTableResponse,
+    DeleteTableBranchRequest, DeleteTableBranchResponse, DeleteTableTagRequest,
+    DeleteTableTagResponse, DeregisterTableRequest, DeregisterTableResponse,
     DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableIndexStatsRequest,
     DescribeTableIndexStatsResponse, DescribeTableRequest, DescribeTableResponse,
     DescribeTableVersionRequest, DescribeTableVersionResponse, DescribeTransactionRequest,
@@ -27,7 +29,8 @@ use lance_namespace_reqwest_client::models::{
     ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableStatsResponse,
     GetTableTagVersionRequest, GetTableTagVersionResponse, InsertIntoTableRequest,
     InsertIntoTableResponse, ListNamespacesRequest, ListNamespacesResponse,
-    ListTableIndicesRequest, ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
+    ListTableBranchesRequest, ListTableBranchesResponse, ListTableIndicesRequest,
+    ListTableIndicesResponse, ListTableTagsRequest, ListTableTagsResponse,
     ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, ListTablesResponse,
     MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest,
     QueryTableRequest, RefreshMaterializedViewRequest, RefreshMaterializedViewResponse,
@@ -500,6 +503,44 @@ pub trait LanceNamespace: Send + Sync + std::fmt::Debug {
         Err(Error::not_supported("update_table_tag not implemented"))
     }
 
+    /// Create a branch for a table.
+    ///
+    /// The new branch forks from the source ref selected by `from_branch` and
+    /// `from_version`, defaulting to the latest version of the main branch when
+    /// both are omitted.
+    ///
+    /// # Errors
+    ///
+    /// - Returns [`crate::ErrorCode::TableBranchAlreadyExists`] if a branch with the same name already exists.
+    /// - Returns [`crate::ErrorCode::TableNotFound`] if the table does not exist.
+    /// - Returns [`crate::ErrorCode::InvalidInput`] if `from_branch` or `from_version` references a source that does not exist.
+    async fn create_table_branch(
+        &self,
+        _request: CreateTableBranchRequest,
+    ) -> Result<CreateTableBranchResponse> {
+        Err(Error::not_supported("create_table_branch not implemented"))
+    }
+
+    /// List all branches for a table.
+    async fn list_table_branches(
+        &self,
+        _request: ListTableBranchesRequest,
+    ) -> Result<ListTableBranchesResponse> {
+        Err(Error::not_supported("list_table_branches not implemented"))
+    }
+
+    /// Delete a branch from a table.
+    ///
+    /// # Errors
+    ///
+    /// Returns [`crate::ErrorCode::TableBranchNotFound`] if the branch does not exist.
+    async fn delete_table_branch(
+        &self,
+        _request: DeleteTableBranchRequest,
+    ) -> Result<DeleteTableBranchResponse> {
+        Err(Error::not_supported("delete_table_branch not implemented"))
+    }
+
     /// Return a human-readable unique identifier for this namespace instance.
     ///
     /// This is used for equality comparison and hashing when the namespace is
diff --git a/rust/lance-select/Cargo.toml b/rust/lance-select/Cargo.toml
index 4d72b55b5e2..4cba7f082a8 100644
--- a/rust/lance-select/Cargo.toml
+++ b/rust/lance-select/Cargo.toml
@@ -18,7 +18,6 @@ arrow-schema = { workspace = true }
 byteorder = { workspace = true }
 tracing = { workspace = true }
 bytes = { workspace = true }
-deepsize = { workspace = true }
 itertools = { workspace = true }
 lance-core = { workspace = true }
 roaring = { workspace = true }
diff --git a/rust/lance-select/src/mask.rs b/rust/lance-select/src/mask.rs
index 86a475866c9..f9df7720441 100644
--- a/rust/lance-select/src/mask.rs
+++ b/rust/lance-select/src/mask.rs
@@ -9,11 +9,11 @@ use std::{collections::BTreeMap, io::Read};
 use arrow_array::{Array, BinaryArray, GenericBinaryArray};
 use arrow_buffer::{Buffer, NullBuffer, OffsetBuffer};
 use byteorder::{ReadBytesExt, WriteBytesExt};
-use deepsize::DeepSizeOf;
 use itertools::Itertools;
+use lance_core::deepsize::DeepSizeOf;
 use roaring::{MultiOps, RoaringBitmap, RoaringTreemap};
 
-use lance_core::cache::CacheCodecImpl;
+use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
 use lance_core::utils::address::RowAddress;
 use lance_core::{Error, Result};
 
@@ -78,6 +78,13 @@ impl RowAddrMask {
         }
     }
 
+    /// True if every row_id is selected. Lets callers (e.g. the FTS wand
+    /// loop) skip per-row mask checks entirely, which in turn lets the
+    /// deferred-row_id scoring path skip loading the row_id column.
+    pub fn is_select_all(&self) -> bool {
+        matches!(self, Self::BlockList(b) if b.is_empty())
+    }
+
     /// Return the indices of the input row ids that were valid
     pub fn selected_indices<'a>(&self, row_ids: impl Iterator<Item = &'a u64> + 'a) -> Vec<u64> {
         row_ids
@@ -301,7 +308,7 @@ pub enum RowAddrSelection {
 }
 
 impl DeepSizeOf for RowAddrSelection {
-    fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
         match self {
             Self::Full => 0,
             Self::Partial(bitmap) => bitmap.serialized_size(),
@@ -685,12 +692,17 @@ impl RowAddrTreeMap {
 }
 
 impl CacheCodecImpl for RowAddrTreeMap {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        self.serialize_into(writer)
+    const TYPE_ID: &'static str = "lance.RowAddrTreeMap";
+    const CURRENT_VERSION: u32 = 1;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        // A roaring bitmap has its own stable, portable serialization; it is
+        // the whole body, so write it raw rather than length-prefixed.
+        self.serialize_into(w.raw_writer())
     }
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
-        Self::deserialize_from(data.as_ref())
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        Self::deserialize_from(r.body().as_ref())
     }
 }
 
@@ -1859,7 +1871,7 @@ mod tests {
 
     #[test]
     fn test_row_addr_selection_deep_size_of() {
-        use deepsize::DeepSizeOf;
+        use lance_core::deepsize::DeepSizeOf;
 
         // Test Full variant - should have minimal size (just the enum discriminant)
         let full = RowAddrSelection::Full;
diff --git a/rust/lance-select/src/mask/nullable.rs b/rust/lance-select/src/mask/nullable.rs
index f76838170f3..dd9eb341e6a 100644
--- a/rust/lance-select/src/mask/nullable.rs
+++ b/rust/lance-select/src/mask/nullable.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 use super::{RowAddrMask, RowAddrTreeMap, RowSetOps};
 
@@ -62,32 +62,52 @@ impl NullableRowAddrSet {
         &self.nulls
     }
 
+    /// Get the raw `selected` bitmap.
+    ///
+    /// This is the backing field, **not** a semantic "TRUE ∪ NULL" set: a NULL
+    /// row may be stored only in `nulls` without appearing in `selected`. Use
+    /// this when you want a zero-copy view of the raw representation (e.g.
+    /// wire serialization that sends `selected` and `nulls` as separate sets).
+    /// For "TRUE rows only", use [`Self::true_rows`].
+    pub fn selected_rows(&self) -> &RowAddrTreeMap {
+        &self.selected
+    }
+
     /// Get the TRUE rows (selected but not null)
     pub fn true_rows(&self) -> RowAddrTreeMap {
         self.selected.clone() - self.nulls.clone()
     }
 
     pub fn union_all(selections: &[Self]) -> Self {
-        let true_rows = selections
-            .iter()
-            .map(|s| s.true_rows())
-            .collect::<Vec<RowAddrTreeMap>>();
-        let true_rows_refs = true_rows.iter().collect::<Vec<&RowAddrTreeMap>>();
-        let selected = RowAddrTreeMap::union_all(&true_rows_refs);
+        let selected = RowAddrTreeMap::union_all(
+            &selections
+                .iter()
+                .map(|s| &s.selected)
+                .collect::<Vec<&RowAddrTreeMap>>(),
+        );
         let nulls = RowAddrTreeMap::union_all(
             &selections
                 .iter()
                 .map(|s| &s.nulls)
                 .collect::<Vec<&RowAddrTreeMap>>(),
         );
-        // TRUE | NULL = TRUE, so remove any TRUE rows from nulls
-        let nulls = nulls - &selected;
+        // TRUE | NULL = TRUE, so remove any TRUE rows from nulls.
+        // A row is TRUE in some input iff it's in that input's (selected - nulls).
+        let any_true = selections
+            .iter()
+            .map(|s| s.selected.clone() - &s.nulls)
+            .fold(RowAddrTreeMap::new(), |acc, t| acc | t);
+        let nulls = nulls - &any_true;
         Self { selected, nulls }
     }
 }
 
 impl PartialEq for NullableRowAddrSet {
     fn eq(&self, other: &Self) -> bool {
+        // Semantic equality: two sets are equal iff they decode to the same
+        // Kleene state on every row. Comparing raw `selected` would be wrong
+        // because a NULL row can be represented either inside or outside the
+        // `selected` bitmap.
         self.true_rows() == other.true_rows() && self.nulls == other.nulls
     }
 }
@@ -473,6 +493,135 @@ mod tests {
         assert_eq!(result.null_rows(), &rows(&[5, 6, 8]));
     }
 
+    #[test]
+    fn test_partial_eq_semantic_equivalence() {
+        // Two representations of "row 5 is NULL, nothing is TRUE":
+        //   a: selected={5}, nulls={5}  (NULL row also in selected)
+        //   b: selected={},  nulls={5}  (NULL row only in nulls)
+        // Both decode to the same Kleene state on every row, so they must
+        // compare equal under semantic PartialEq.
+        let a = NullableRowAddrSet::new(rows(&[5]), rows(&[5]));
+        let b = NullableRowAddrSet::new(rows(&[]), rows(&[5]));
+        assert_eq!(a, b);
+        assert_eq!(a.true_rows(), b.true_rows());
+        assert_eq!(a.null_rows(), b.null_rows());
+    }
+
+    #[test]
+    fn test_union_all_true_overrides_null() {
+        // Critical conflict case: row 5 is TRUE in set1 but NULL in set2.
+        // Kleene: TRUE ∨ NULL = TRUE → row 5 must end up TRUE, not NULL.
+        let set1 = nullable_set(&[5], &[]);
+        let set2 = nullable_set(&[5], &[5]);
+
+        let result = NullableRowAddrSet::union_all(&[set1, set2]);
+
+        assert_eq!(result.true_rows(), rows(&[5]));
+        assert!(result.null_rows().is_empty());
+    }
+
+    #[test]
+    fn test_union_all_null_only_input() {
+        // Input where a row is NULL but NOT in `selected` (the type allows
+        // `selected` to be a strict subset of TRUE ∪ NULL).
+        let set1 = NullableRowAddrSet::new(rows(&[]), rows(&[5]));
+        let set2 = NullableRowAddrSet::new(rows(&[1]), rows(&[]));
+
+        let result = NullableRowAddrSet::union_all(&[set1, set2]);
+
+        assert_eq!(result.true_rows(), rows(&[1]));
+        assert_eq!(result.null_rows(), &rows(&[5]));
+    }
+
+    #[test]
+    fn test_union_all_all_null_for_a_row() {
+        // Every input marks row 7 as NULL; nothing makes it TRUE.
+        let set1 = nullable_set(&[7], &[7]);
+        let set2 = nullable_set(&[7], &[7]);
+        let set3 = NullableRowAddrSet::new(rows(&[]), rows(&[7]));
+
+        let result = NullableRowAddrSet::union_all(&[set1, set2, set3]);
+
+        assert!(result.true_rows().is_empty());
+        assert_eq!(result.null_rows(), &rows(&[7]));
+    }
+
+    #[test]
+    fn test_union_all_empty_inputs() {
+        let result = NullableRowAddrSet::union_all(&[]);
+        assert!(result.true_rows().is_empty());
+        assert!(result.null_rows().is_empty());
+    }
+
+    #[test]
+    fn test_union_all_single_input() {
+        // One input → state of every row preserved.
+        let set = nullable_set(&[1, 2, 3, 4], &[2, 4]);
+        let result = NullableRowAddrSet::union_all(std::slice::from_ref(&set));
+
+        assert_eq!(result.true_rows(), rows(&[1, 3]));
+        assert_eq!(result.null_rows(), &rows(&[2, 4]));
+    }
+
+    #[test]
+    fn test_union_all_all_empty_inputs() {
+        let inputs = [
+            NullableRowAddrSet::empty(),
+            NullableRowAddrSet::empty(),
+            NullableRowAddrSet::empty(),
+        ];
+        let result = NullableRowAddrSet::union_all(&inputs);
+        assert!(result.true_rows().is_empty());
+        assert!(result.null_rows().is_empty());
+    }
+
+    #[test]
+    fn test_union_all_disjoint_inputs() {
+        // No row appears in more than one input.
+        let set1 = nullable_set(&[1, 2], &[2]);
+        let set2 = nullable_set(&[10, 11], &[11]);
+        let set3 = nullable_set(&[20], &[]);
+
+        let result = NullableRowAddrSet::union_all(&[set1, set2, set3]);
+
+        assert_eq!(result.true_rows(), rows(&[1, 10, 20]));
+        assert_eq!(result.null_rows(), &rows(&[2, 11]));
+    }
+
+    #[test]
+    fn test_union_all_three_state_row() {
+        // Same row 42 across three inputs in three different states:
+        //   set1: TRUE   set2: NULL   set3: FALSE
+        // Kleene OR: TRUE ∨ NULL ∨ FALSE = TRUE.
+        let set1 = nullable_set(&[42], &[]);
+        let set2 = nullable_set(&[42], &[42]);
+        let set3 = NullableRowAddrSet::empty();
+
+        let result = NullableRowAddrSet::union_all(&[set1, set2, set3]);
+
+        assert_eq!(result.true_rows(), rows(&[42]));
+        assert!(result.null_rows().is_empty());
+    }
+
+    #[test]
+    fn test_union_all_matches_repeated_bitor() {
+        // union_all(a, b, c) must equal ((a | b) | c) — same Kleene operator,
+        // applied pairwise via the independently-implemented BitOrAssign.
+        let set1 = nullable_set(&[1, 2, 3, 4], &[4, 5, 6]);
+        let set2 = nullable_set(&[1, 4, 7, 8], &[2, 5, 8]);
+        let set3 = nullable_set(&[2, 6, 10], &[6, 10]);
+
+        let via_union_all =
+            NullableRowAddrSet::union_all(&[set1.clone(), set2.clone(), set3.clone()]);
+
+        let mut via_bitor = set1;
+        via_bitor |= &set2;
+        via_bitor |= &set3;
+
+        assert_eq!(via_union_all.true_rows(), via_bitor.true_rows());
+        assert_eq!(via_union_all.null_rows(), via_bitor.null_rows());
+    }
+
     #[test]
     fn test_nullable_row_addr_set_with_nulls() {
         let set = NullableRowAddrSet::new(rows(&[1, 2, 3]), RowAddrTreeMap::new());
diff --git a/rust/lance-table/Cargo.toml b/rust/lance-table/Cargo.toml
index 836b54e942f..042ae92c618 100644
--- a/rust/lance-table/Cargo.toml
+++ b/rust/lance-table/Cargo.toml
@@ -28,7 +28,6 @@ aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false
 byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
-deepsize.workspace = true
 futures.workspace = true
 log.workspace = true
 object_store.workspace = true
diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs
index aa798a0bfc2..81bd57c1a22 100644
--- a/rust/lance-table/benches/manifest_intern.rs
+++ b/rust/lance-table/benches/manifest_intern.rs
@@ -11,7 +11,7 @@
 //! `RowDatasetVersionMeta::Inline` bytes across many fragments.
 
 use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use prost::Message;
 
 use lance_table::format::pb;
@@ -59,6 +59,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec<pb::DataFragment>
                 file_size_bytes: 0,
                 base_id: None,
             }],
+            overlays: vec![],
             deletion_file: None,
             row_id_sequence: None,
             physical_rows: 1000,
@@ -135,6 +136,7 @@ fn make_diverse_pb_fragments(
                     file_size_bytes: 0,
                     base_id: None,
                 }],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_sequence: None,
                 physical_rows: 1000,
diff --git a/rust/lance-table/src/feature_flags.rs b/rust/lance-table/src/feature_flags.rs
index 096f0da79e5..41b8e415f8e 100644
--- a/rust/lance-table/src/feature_flags.rs
+++ b/rust/lance-table/src/feature_flags.rs
@@ -20,8 +20,22 @@ pub const FLAG_TABLE_CONFIG: u64 = 8;
 pub const FLAG_BASE_PATHS: u64 = 16;
 /// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest
 pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32;
+/// Fragments contain data overlay files, which supply new values for a subset of
+/// cells without rewriting base data files. A reader that does not understand
+/// overlays must refuse the dataset, since ignoring an overlay would silently
+/// return stale base values.
+///
+/// Data overlay files are not yet a released feature: in release builds this flag
+/// is treated as unknown (so a release reader/writer refuses an overlay dataset)
+/// unless [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set, which lets benchmarks opt in.
+/// Debug builds always understand it so tests exercise the path.
+pub const FLAG_UNSTABLE_DATA_OVERLAY_FILES: u64 = 64;
 /// The first bit that is unknown as a feature flag
-pub const FLAG_UNKNOWN: u64 = 64;
+pub const FLAG_UNKNOWN: u64 = 128;
+
+/// Environment variable that opts a release build into reading and writing data
+/// overlay files before the feature is generally released.
+pub const ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV: &str = "LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES";
 
 /// Set the reader and writer feature flags in the manifest based on the contents of the manifest.
 pub fn apply_feature_flags(
@@ -71,18 +85,62 @@ pub fn apply_feature_flags(
         manifest.writer_feature_flags |= FLAG_BASE_PATHS;
     }
 
+    // Overlay files change cell values on read, so a reader that ignores them
+    // would return stale base values. Both readers and writers must understand
+    // them.
+    let has_overlays = manifest
+        .fragments
+        .iter()
+        .any(|frag| !frag.overlays.is_empty());
+    if has_overlays {
+        manifest.reader_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
+        manifest.writer_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES;
+    }
+
     if disable_transaction_file {
         manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE;
     }
     Ok(())
 }
 
+/// Whether this build understands data overlay files: always in debug builds,
+/// and in release builds only when [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set.
+fn data_overlay_files_enabled() -> bool {
+    cfg!(debug_assertions) || std::env::var_os(ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV).is_some()
+}
+
+/// Clear `flag` from `flags` when its gating feature is not enabled in this
+/// build; leave it set otherwise. One call per unstable flag, so support for
+/// several unstable features chains cleanly.
+fn mark_supported(flags: &mut u64, flag: u64, feature_enabled: bool) {
+    if !feature_enabled {
+        *flags &= !flag;
+    }
+}
+
+/// The feature-flag bits this build understands, given whether overlay support
+/// is enabled. Split out from [`supported_flags`] so the policy is testable
+/// without toggling the build profile or environment.
+fn supported_flags_when(overlay_enabled: bool) -> u64 {
+    let mut supported = FLAG_UNKNOWN - 1;
+    mark_supported(
+        &mut supported,
+        FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+        overlay_enabled,
+    );
+    supported
+}
+
+fn supported_flags() -> u64 {
+    supported_flags_when(data_overlay_files_enabled())
+}
+
 pub fn can_read_dataset(reader_flags: u64) -> bool {
-    reader_flags < FLAG_UNKNOWN
+    reader_flags & !supported_flags() == 0
 }
 
 pub fn can_write_dataset(writer_flags: u64) -> bool {
-    writer_flags < FLAG_UNKNOWN
+    writer_flags & !supported_flags() == 0
 }
 
 pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool {
@@ -103,6 +161,13 @@ mod tests {
         assert!(can_read_dataset(super::FLAG_TABLE_CONFIG));
         assert!(can_read_dataset(super::FLAG_BASE_PATHS));
         assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
+        // Overlay support is gated on the build profile / env opt-in, so the
+        // flag is readable exactly when overlays are enabled (see
+        // test_data_overlay_flag_release_gating for the full policy).
+        assert_eq!(
+            can_read_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
+            data_overlay_files_enabled()
+        );
         assert!(can_read_dataset(
             super::FLAG_DELETION_FILES
                 | super::FLAG_STABLE_ROW_IDS
@@ -111,6 +176,58 @@ mod tests {
         assert!(!can_read_dataset(super::FLAG_UNKNOWN));
     }
 
+    #[test]
+    fn test_data_overlay_flag_release_gating() {
+        // Release default (overlays disabled): the overlay flag is treated as
+        // unknown so the dataset is refused, while other known flags still pass.
+        let supported = supported_flags_when(false);
+        assert_eq!(supported & FLAG_UNSTABLE_DATA_OVERLAY_FILES, 0);
+        assert_eq!(FLAG_DELETION_FILES & !supported, 0);
+        assert_ne!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
+        // Enabled (debug or env opt-in): the overlay flag is understood.
+        let supported = supported_flags_when(true);
+        assert_eq!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0);
+    }
+
+    #[test]
+    fn test_apply_feature_flags_sets_overlay_flag() {
+        use crate::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use crate::format::{DataFile, DataStorageFormat, Fragment};
+        use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
+        use lance_core::datatypes::Schema;
+        use roaring::RoaringBitmap;
+        use std::collections::HashMap;
+        use std::sync::Arc;
+
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            arrow_schema::DataType::Int64,
+            false,
+        )]);
+        let schema = Schema::try_from(&arrow_schema).unwrap();
+        let mut fragment = Fragment::new(0);
+        fragment.overlays = vec![DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![0], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: 1,
+        }];
+        let mut manifest = Manifest::new(
+            schema,
+            Arc::new(vec![fragment]),
+            DataStorageFormat::default(),
+            HashMap::new(),
+        );
+        apply_feature_flags(&mut manifest, false, false).unwrap();
+        assert_ne!(
+            manifest.reader_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+            0
+        );
+        assert_ne!(
+            manifest.writer_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES,
+            0
+        );
+    }
+
     #[test]
     fn test_write_check() {
         assert!(can_write_dataset(0));
@@ -120,6 +237,13 @@ mod tests {
         assert!(can_write_dataset(super::FLAG_TABLE_CONFIG));
         assert!(can_write_dataset(super::FLAG_BASE_PATHS));
         assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE));
+        // Overlay support is gated on the build profile / env opt-in, so the
+        // flag is writable exactly when overlays are enabled (see
+        // test_data_overlay_flag_release_gating for the full policy).
+        assert_eq!(
+            can_write_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES),
+            data_overlay_files_enabled()
+        );
         assert!(can_write_dataset(
             super::FLAG_DELETION_FILES
                 | super::FLAG_STABLE_ROW_IDS
diff --git a/rust/lance-table/src/format.rs b/rust/lance-table/src/format.rs
index 842c76f1e58..5a5db7919d3 100644
--- a/rust/lance-table/src/format.rs
+++ b/rust/lance-table/src/format.rs
@@ -7,6 +7,7 @@ use uuid::Uuid;
 mod fragment;
 mod index;
 mod manifest;
+pub mod overlay;
 mod transaction;
 
 pub use crate::rowids::version::{
diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs
index dc5c94b388a..4929f54b7ff 100644
--- a/rust/lance-table/src/format/fragment.rs
+++ b/rust/lance-table/src/format/fragment.rs
@@ -5,14 +5,15 @@ use std::collections::HashMap;
 use std::num::NonZero;
 use std::sync::Arc;
 
-use deepsize::DeepSizeOf;
 use lance_core::Error;
+use lance_core::deepsize::DeepSizeOf;
 use lance_file::format::{MAJOR_VERSION, MINOR_VERSION};
 use lance_file::version::LanceFileVersion;
 use lance_io::utils::CachedFileSize;
 use object_store::path::Path;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
+use super::overlay::{DataOverlayFile, sort_overlays_newest_last};
 use crate::format::pb;
 
 use crate::rowids::version::{
@@ -375,6 +376,15 @@ impl DataFileFieldInterner {
                 .into_iter()
                 .map(|f| self.intern_data_file(f))
                 .collect::<Result<_>>()?,
+            overlays: {
+                let mut overlays = p
+                    .overlays
+                    .into_iter()
+                    .map(DataOverlayFile::try_from)
+                    .collect::<Result<Vec<_>>>()?;
+                sort_overlays_newest_last(&mut overlays);
+                overlays
+            },
             deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?,
             row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?,
             physical_rows,
@@ -483,6 +493,12 @@ pub struct Fragment {
     /// Files within the fragment.
     pub files: Vec<DataFile>,
 
+    /// Overlay files supplying new values for a subset of cells without
+    /// rewriting the base data files. Order is significant: a later entry is
+    /// newer than an earlier one. See [`DataOverlayFile`] for resolution rules.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub overlays: Vec<DataOverlayFile>,
+
     /// Optional file with deleted local row offsets.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub deletion_file: Option<DeletionFile>,
@@ -510,6 +526,7 @@ impl Fragment {
         Self {
             id,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: None,
@@ -549,6 +566,7 @@ impl Fragment {
         Self {
             id,
             files: vec![DataFile::new_legacy(path, schema, None, None)],
+            overlays: vec![],
             deletion_file: None,
             physical_rows,
             row_id_meta: None,
@@ -669,6 +687,15 @@ impl TryFrom<pb::DataFragment> for Fragment {
                 .into_iter()
                 .map(DataFile::try_from)
                 .collect::<Result<_>>()?,
+            overlays: {
+                let mut overlays = p
+                    .overlays
+                    .into_iter()
+                    .map(DataOverlayFile::try_from)
+                    .collect::<Result<Vec<_>>>()?;
+                sort_overlays_newest_last(&mut overlays);
+                overlays
+            },
             deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?,
             row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?,
             physical_rows,
@@ -716,6 +743,7 @@ impl From<&Fragment> for pb::DataFragment {
         Self {
             id: f.id,
             files: f.files.iter().map(pb::DataFile::from).collect(),
+            overlays: f.overlays.iter().map(pb::DataOverlayFile::from).collect(),
             deletion_file,
             row_id_sequence,
             physical_rows: f.physical_rows.unwrap_or_default() as u64,
@@ -728,12 +756,108 @@ impl From<&Fragment> for pb::DataFragment {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::format::overlay::OverlayCoverage;
     use arrow_schema::{
         DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema,
     };
     use object_store::path::Path;
+    use roaring::RoaringBitmap;
     use serde_json::{Value, json};
 
+    #[test]
+    fn test_data_overlay_roundtrip() {
+        // A fragment carrying a dense overlay round-trips through protobuf and
+        // back, and the parsed coverage bitmap is recovered per field.
+        let mut bitmap = RoaringBitmap::new();
+        bitmap.insert(1);
+        bitmap.insert(3);
+
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(bitmap.clone()),
+            committed_version: 7,
+        };
+        let mut fragment = Fragment::new(0);
+        fragment.files = vec![DataFile::new_legacy_from_fields(
+            "base.lance",
+            vec![1, 3],
+            None,
+        )];
+        fragment.overlays = vec![overlay];
+
+        let proto = pb::DataFragment::from(&fragment);
+        assert_eq!(proto.overlays.len(), 1);
+        let round_tripped = Fragment::try_from(proto).unwrap();
+        assert_eq!(round_tripped, fragment);
+
+        // Dense coverage applies to every field.
+        let recovered = round_tripped.overlays[0].coverage_for_field(0).unwrap();
+        assert_eq!(*recovered, bitmap);
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(5).unwrap(),
+            bitmap
+        );
+    }
+
+    #[test]
+    fn test_data_overlay_sparse_per_field_coverage() {
+        // A sparse overlay carries one bitmap per field, recovered by position.
+        let name_coverage = RoaringBitmap::from_iter([2u32, 3]);
+        let embedding_coverage = RoaringBitmap::from_iter([1u32]);
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-1.lance", vec![2, 4], None),
+            coverage: OverlayCoverage::sparse(vec![
+                name_coverage.clone(),
+                embedding_coverage.clone(),
+            ]),
+            committed_version: 3,
+        };
+        let mut fragment = Fragment::new(1);
+        fragment.overlays = vec![overlay];
+
+        let round_tripped = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap();
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(0).unwrap(),
+            name_coverage
+        );
+        assert_eq!(
+            *round_tripped.overlays[0].coverage_for_field(1).unwrap(),
+            embedding_coverage
+        );
+    }
+
+    #[test]
+    fn test_overlays_sorted_newest_last_on_load() {
+        // Overlays load stable-sorted by committed_version (newest last), with
+        // list position preserved as the tiebreak for equal versions.
+        let mk = |version: u64, field: i32| DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: version,
+        };
+        let mut fragment = Fragment::new(0);
+        // Written out of order: v5, v2, v2 (second), v3.
+        fragment.overlays = vec![mk(5, 1), mk(2, 2), mk(2, 3), mk(3, 4)];
+
+        let loaded = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap();
+        let versions: Vec<u64> = loaded
+            .overlays
+            .iter()
+            .map(|o| o.committed_version)
+            .collect();
+        assert_eq!(versions, vec![2, 2, 3, 5]);
+        // Stable: the two v2 overlays keep their original relative order (field 2
+        // before field 3).
+        assert_eq!(
+            loaded.overlays[0].data_file.fields.as_ref(),
+            [2i32].as_slice()
+        );
+        assert_eq!(
+            loaded.overlays[1].data_file.fields.as_ref(),
+            [3i32].as_slice()
+        );
+    }
+
     #[test]
     fn test_new_fragment() {
         let path = "foobar.lance";
diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs
index 945d8364123..f603536a3eb 100644
--- a/rust/lance-table/src/format/index.rs
+++ b/rust/lance-table/src/format/index.rs
@@ -7,14 +7,15 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use chrono::{DateTime, Utc};
-use deepsize::DeepSizeOf;
 use futures::StreamExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_io::object_store::ObjectStore;
 use object_store::path::Path;
 use roaring::RoaringBitmap;
 use uuid::Uuid;
 
 use super::pb;
+use lance_core::cache::{CacheEntryReader, CacheEntryWriter};
 use lance_core::{Error, Result};
 
 /// Metadata about a single file within an index segment.
@@ -121,7 +122,7 @@ impl IndexMetadata {
 }
 
 impl DeepSizeOf for IndexMetadata {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.uuid.as_bytes().deep_size_of_children(context)
             + self.fields.deep_size_of_children(context)
             + self.name.deep_size_of_children(context)
@@ -235,24 +236,26 @@ impl From<&IndexMetadata> for pb::IndexMetadata {
 /// orphan rule prevents `impl CacheCodecImpl for Vec<IndexMetadata>`.
 type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
 
+/// Stable type identifier for the `Vec<IndexMetadata>` cache entry.
+const INDEX_METADATA_TYPE_ID: &str = "lance.table.IndexMetadataList";
+/// Body schema version written by this build.
+const INDEX_METADATA_VERSION: u32 = 1;
+
 fn serialize_index_metadata(
     any: &ArcAny,
-    writer: &mut dyn std::io::Write,
+    writer: &mut CacheEntryWriter<'_>,
 ) -> lance_core::Result<()> {
-    use prost::Message;
     let vec = any
         .downcast_ref::<Vec<IndexMetadata>>()
         .expect("index_metadata_codec: wrong type (this is a bug in the cache layer)");
     let section = pb::IndexSection {
         indices: vec.iter().map(pb::IndexMetadata::from).collect(),
     };
-    writer.write_all(&section.encode_to_vec())?;
-    Ok(())
+    writer.write_header(&section)
 }
 
-fn deserialize_index_metadata(data: &bytes::Bytes) -> lance_core::Result<ArcAny> {
-    use prost::Message;
-    let section = pb::IndexSection::decode(data.as_ref())?;
+fn deserialize_index_metadata(reader: &mut CacheEntryReader<'_>) -> lance_core::Result<ArcAny> {
+    let section: pb::IndexSection = reader.read_header()?;
     let indices: Vec<IndexMetadata> = section
         .indices
         .into_iter()
@@ -262,7 +265,12 @@ fn deserialize_index_metadata(data: &bytes::Bytes) -> lance_core::Result<ArcAny>
 }
 
 pub fn index_metadata_codec() -> lance_core::cache::CacheCodec {
-    lance_core::cache::CacheCodec::new(serialize_index_metadata, deserialize_index_metadata)
+    lance_core::cache::CacheCodec::new(
+        INDEX_METADATA_TYPE_ID,
+        INDEX_METADATA_VERSION,
+        serialize_index_metadata,
+        deserialize_index_metadata,
+    )
 }
 
 /// List all files in an index directory with their sizes.
@@ -348,7 +356,8 @@ mod tests {
         let bytes = store.get(&key).unwrap();
         let recovered = codec
             .deserialize(&bytes::Bytes::copy_from_slice(bytes))
-            .unwrap();
+            .hit()
+            .expect("entry should decode as a hit");
         let recovered = recovered
             .downcast::<Vec<IndexMetadata>>()
             .expect("downcast should succeed");
diff --git a/rust/lance-table/src/format/manifest.rs b/rust/lance-table/src/format/manifest.rs
index d2b5f2d31c6..83c9643e600 100644
--- a/rust/lance-table/src/format/manifest.rs
+++ b/rust/lance-table/src/format/manifest.rs
@@ -3,7 +3,7 @@
 
 use async_trait::async_trait;
 use chrono::prelude::*;
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 use lance_file::datatypes::{Fields, FieldsWithMeta, populate_schema_dictionary};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::version::{LEGACY_FORMAT_VERSION, LanceFileVersion};
@@ -588,7 +588,7 @@ impl BasePath {
 }
 
 impl DeepSizeOf for BasePath {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.name.deep_size_of_children(context)
             + self.path.deep_size_of_children(context) * 2
             + size_of::<bool>()
@@ -900,7 +900,7 @@ impl TryFrom<pb::Manifest> for Manifest {
             Some(format) => DataStorageFormat::from(format),
         };
 
-        let schema = Schema::from(fields_with_meta);
+        let schema = Schema::try_from(fields_with_meta)?;
 
         Ok(Self {
             schema,
@@ -1316,6 +1316,7 @@ mod tests {
                     vec![0, 1, 2],
                     None,
                 )],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1328,6 +1329,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 43], None),
                     DataFile::new_legacy_from_fields("path3", vec![2], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
diff --git a/rust/lance-table/src/format/overlay.rs b/rust/lance-table/src/format/overlay.rs
new file mode 100644
index 00000000000..5e729411502
--- /dev/null
+++ b/rust/lance-table/src/format/overlay.rs
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Data overlay files.
+//!
+//! An overlay file supplies new values for a subset of `(physical offset, field)`
+//! cells within a fragment, without rewriting the fragment's base data files. See
+//! the Data Overlay Files specification for the full rules; the invariants this
+//! module relies on are:
+//!
+//! - **Physical-offset coverage.** Coverage bitmaps index *physical* row offsets
+//!   (positions in the base data files, counting deleted rows), so they are stable
+//!   across deletions, like deletion vectors.
+//! - **Rank-based values.** The overlay's `data_file` stores one value column per
+//!   field, with no row-offset key column. Within a value column, a covered
+//!   offset's value sits at its **rank** — the 0-based count of set bits below it
+//!   in that field's coverage bitmap.
+//! - **Dense vs. sparse coverage.** A dense overlay shares one bitmap across every
+//!   field ([`OverlayCoverage::Shared`]); a sparse overlay carries one bitmap per
+//!   field ([`OverlayCoverage::PerField`]).
+//! - **Parse once.** Bitmaps are parsed from their 32-bit Roaring encoding a single
+//!   time when the fragment loads and held behind an `Arc`, so cloning a fragment
+//!   is cheap.
+//! - **Newest-last ordering.** A fragment's overlays are stored newest-last and
+//!   stable-sorted by `committed_version` on load (see [`sort_overlays_newest_last`]),
+//!   with list position breaking ties for equal versions. When two overlays cover
+//!   the same `(offset, field)`, the higher `committed_version` wins.
+//! - **Field tombstones.** When new base values are written for a field (a
+//!   DataReplacement, or an in-place column rewrite), any overlay value for that
+//!   field is stale and must stop shadowing the fresh base. The field is marked
+//!   obsolete in the overlay's `data_file.fields` with [`TOMBSTONE_FIELD_ID`]
+//!   (the same sentinel used for obsolete base columns) rather than physically
+//!   removed, so the overlay's other fields — and its coverage positions — stay
+//!   intact (see [`tombstone_overlay_fields`]).
+
+use std::sync::Arc;
+
+use lance_core::Error;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::error::Result;
+use roaring::RoaringBitmap;
+use serde::{Deserialize, Serialize};
+
+use object_store::path::Path;
+
+use super::DataFile;
+use crate::format::pb;
+
+/// Field-id sentinel marking a tombstoned (obsolete) field within an overlay's
+/// `data_file.fields`. Matches the tombstone convention for obsolete columns in
+/// base data files; a tombstoned field's values are ignored on read.
+pub const TOMBSTONE_FIELD_ID: i32 = -2;
+
+/// Which `(physical offset, field)` cells a [`DataOverlayFile`] provides values
+/// for.
+///
+/// Bitmaps are parsed from their 32-bit Roaring encoding once when the fragment
+/// is loaded and held behind an `Arc` so cloning a fragment is cheap; use
+/// [`DataOverlayFile::coverage_for_field`] to obtain the one that applies to a
+/// given field.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(into = "OverlayCoverageBytes", try_from = "OverlayCoverageBytes")]
+pub enum OverlayCoverage {
+    /// A single bitmap that applies to every field in the overlay's
+    /// `data_file.fields` (a dense / rectangular overlay): every covered offset
+    /// has a value for every field.
+    Shared(Arc<RoaringBitmap>),
+    /// One bitmap per field, in the same order as the overlay's
+    /// `data_file.fields` (a sparse overlay): different fields may cover
+    /// different offset sets.
+    PerField(Vec<Arc<RoaringBitmap>>),
+}
+
+/// Serialized form of [`OverlayCoverage`] — each bitmap as its 32-bit Roaring
+/// byte encoding. The in-memory form parses these once at load.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+enum OverlayCoverageBytes {
+    Shared(Vec<u8>),
+    PerField(Vec<Vec<u8>>),
+}
+
+// The bytes come from a persisted overlay (the protobuf manifest or a
+// serialized fragment), so a decode failure is on-disk corruption, not caller
+// input. `path` locates the overlay's data file when known (empty on the serde
+// path, which deserializes coverage in isolation).
+fn deserialize_roaring(bytes: &[u8], path: &Path) -> Result<RoaringBitmap> {
+    RoaringBitmap::deserialize_from(bytes).map_err(|e| {
+        Error::corrupt_file(
+            path.clone(),
+            format!("failed to deserialize overlay coverage bitmap: {e}"),
+        )
+    })
+}
+
+fn serialize_roaring(bitmap: &RoaringBitmap) -> Vec<u8> {
+    let mut bytes = Vec::with_capacity(bitmap.serialized_size());
+    // Writing to a Vec is infallible.
+    bitmap.serialize_into(&mut bytes).unwrap();
+    bytes
+}
+
+impl From<OverlayCoverage> for OverlayCoverageBytes {
+    fn from(coverage: OverlayCoverage) -> Self {
+        match coverage {
+            OverlayCoverage::Shared(bitmap) => Self::Shared(serialize_roaring(&bitmap)),
+            OverlayCoverage::PerField(bitmaps) => {
+                Self::PerField(bitmaps.iter().map(|b| serialize_roaring(b)).collect())
+            }
+        }
+    }
+}
+
+impl TryFrom<OverlayCoverageBytes> for OverlayCoverage {
+    type Error = Error;
+
+    fn try_from(bytes: OverlayCoverageBytes) -> Result<Self> {
+        // Serde deserializes the coverage in isolation, so the owning data
+        // file's path is not available here.
+        let path = Path::default();
+        Ok(match bytes {
+            OverlayCoverageBytes::Shared(b) => {
+                Self::Shared(Arc::new(deserialize_roaring(&b, &path)?))
+            }
+            OverlayCoverageBytes::PerField(bs) => Self::PerField(
+                bs.iter()
+                    .map(|b| deserialize_roaring(b, &path).map(Arc::new))
+                    .collect::<Result<_>>()?,
+            ),
+        })
+    }
+}
+
+impl DeepSizeOf for OverlayCoverage {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // The same `Arc<RoaringBitmap>` is shared across every clone of a
+        // fragment, so mark each Arc's pointer and count its heap only the first
+        // time it is seen — otherwise walking many fragments double-counts the
+        // shared bitmaps. RoaringBitmap does not expose its allocation size; its
+        // serialized size is a cheap, close proxy for the heap it holds.
+        let bitmap_heap = |bitmap: &Arc<RoaringBitmap>,
+                           context: &mut lance_core::deepsize::Context| {
+            if context.mark_seen(Arc::as_ptr(bitmap) as usize) {
+                std::mem::size_of::<RoaringBitmap>() + bitmap.serialized_size()
+            } else {
+                0
+            }
+        };
+        match self {
+            Self::Shared(bitmap) => bitmap_heap(bitmap, context),
+            Self::PerField(bitmaps) => {
+                bitmaps.capacity() * std::mem::size_of::<Arc<RoaringBitmap>>()
+                    + bitmaps
+                        .iter()
+                        .map(|b| bitmap_heap(b, context))
+                        .sum::<usize>()
+            }
+        }
+    }
+}
+
+impl OverlayCoverage {
+    /// Build a dense coverage from a single bitmap shared across every field.
+    pub fn dense(bitmap: RoaringBitmap) -> Self {
+        Self::Shared(Arc::new(bitmap))
+    }
+
+    /// Build a sparse coverage from one bitmap per field.
+    pub fn sparse(bitmaps: Vec<RoaringBitmap>) -> Self {
+        Self::PerField(bitmaps.into_iter().map(Arc::new).collect())
+    }
+}
+
+/// An overlay file supplies new values for a subset of `(physical offset, field)`
+/// cells within a fragment, without rewriting the fragment's base data files. See
+/// the [module documentation](self) for the coverage, rank, and versioning rules.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct DataOverlayFile {
+    /// The data file storing the overlay's new cell values.
+    pub data_file: DataFile,
+    /// Which cells this overlay provides values for.
+    pub coverage: OverlayCoverage,
+    /// The dataset version at which this overlay became effective (the version of
+    /// the commit that introduced it, stamped at commit time and re-stamped on
+    /// retry). Higher wins when two overlays cover the same `(offset, field)`.
+    pub committed_version: u64,
+}
+
+impl DataOverlayFile {
+    /// The parsed coverage bitmap that applies to the field stored at
+    /// `field_pos` within `data_file.fields`.
+    ///
+    /// For a dense overlay the same shared bitmap is returned for every field;
+    /// for a sparse overlay the per-field bitmap at `field_pos` is returned. The
+    /// bitmap is already parsed, so this is a cheap `Arc` clone.
+    pub fn coverage_for_field(&self, field_pos: usize) -> Result<Arc<RoaringBitmap>> {
+        match &self.coverage {
+            OverlayCoverage::Shared(bitmap) => Ok(bitmap.clone()),
+            OverlayCoverage::PerField(bitmaps) => {
+                bitmaps.get(field_pos).cloned().ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "overlay per-field coverage has {} bitmaps but field position {} was requested",
+                        bitmaps.len(),
+                        field_pos
+                    ))
+                })
+            }
+        }
+    }
+}
+
+/// Stable-sort a fragment's overlays newest-last by `committed_version`. The
+/// stable sort preserves list position as the tiebreak for equal versions, so
+/// resolution can rely on the ordering without re-checking. See the [module
+/// documentation](self) for the ordering invariant.
+pub fn sort_overlays_newest_last(overlays: &mut [DataOverlayFile]) {
+    overlays.sort_by_key(|overlay| overlay.committed_version);
+}
+
+/// Verify a fragment's overlays are stored newest-last (non-decreasing
+/// `committed_version`), the ordering invariant readers rely on for
+/// resolution. Returns an error identifying the first out-of-order pair.
+///
+/// [`sort_overlays_newest_last`] normalizes on load; this is the write-side
+/// guard that rejects any commit path that assembled overlays out of order. See
+/// the [module documentation](self) for the ordering invariant.
+pub fn verify_overlays_newest_last(overlays: &[DataOverlayFile]) -> Result<()> {
+    for pair in overlays.windows(2) {
+        if pair[0].committed_version > pair[1].committed_version {
+            return Err(Error::invalid_input(format!(
+                "overlay files must be stored newest-last, but committed_version {} precedes {}",
+                pair[0].committed_version, pair[1].committed_version
+            )));
+        }
+    }
+    Ok(())
+}
+
+/// Tombstone `fields` across a fragment's `overlays`, dropping any overlay left
+/// with no live fields.
+///
+/// Called when new base values are written for those fields (a DataReplacement,
+/// or an in-place column rewrite): the stale overlay values must stop shadowing
+/// the fresh base. Each matching field id is replaced with [`TOMBSTONE_FIELD_ID`]
+/// in place, preserving the overlay's remaining fields and its coverage positions
+/// (a per-field coverage bitmap stays aligned with `data_file.fields`). An overlay
+/// whose fields are now all tombstoned is removed entirely. See the [module
+/// documentation](self) for the tombstone invariant.
+pub fn tombstone_overlay_fields(overlays: &mut Vec<DataOverlayFile>, fields: &[u32]) {
+    for overlay in overlays.iter_mut() {
+        let tombstoned: Vec<i32> = overlay
+            .data_file
+            .fields
+            .iter()
+            .map(|&field| {
+                if field >= 0 && fields.contains(&(field as u32)) {
+                    TOMBSTONE_FIELD_ID
+                } else {
+                    field
+                }
+            })
+            .collect();
+        overlay.data_file.fields = tombstoned.into();
+    }
+    overlays.retain(|overlay| {
+        overlay
+            .data_file
+            .fields
+            .iter()
+            .any(|&field| field != TOMBSTONE_FIELD_ID)
+    });
+}
+
+impl From<&DataOverlayFile> for pb::DataOverlayFile {
+    fn from(overlay: &DataOverlayFile) -> Self {
+        let coverage = match &overlay.coverage {
+            OverlayCoverage::Shared(bitmap) => {
+                pb::data_overlay_file::Coverage::SharedOffsetBitmap(serialize_roaring(bitmap))
+            }
+            OverlayCoverage::PerField(bitmaps) => {
+                pb::data_overlay_file::Coverage::FieldCoverage(pb::FieldCoverage {
+                    offset_bitmaps: bitmaps.iter().map(|b| serialize_roaring(b)).collect(),
+                })
+            }
+        };
+        Self {
+            data_file: Some(pb::DataFile::from(&overlay.data_file)),
+            coverage: Some(coverage),
+            committed_version: overlay.committed_version,
+        }
+    }
+}
+
+impl TryFrom<pb::DataOverlayFile> for DataOverlayFile {
+    type Error = Error;
+
+    fn try_from(proto: pb::DataOverlayFile) -> Result<Self> {
+        let data_file = proto
+            .data_file
+            .ok_or_else(|| Error::invalid_input("DataOverlayFile is missing its data_file"))?;
+        let path = Path::from(data_file.path.as_str());
+        let coverage = match proto.coverage {
+            Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(bytes)) => {
+                OverlayCoverage::Shared(Arc::new(deserialize_roaring(&bytes, &path)?))
+            }
+            Some(pb::data_overlay_file::Coverage::FieldCoverage(fc)) => OverlayCoverage::PerField(
+                fc.offset_bitmaps
+                    .iter()
+                    .map(|b| deserialize_roaring(b, &path).map(Arc::new))
+                    .collect::<Result<_>>()?,
+            ),
+            None => {
+                return Err(Error::invalid_input(
+                    "DataOverlayFile is missing its coverage",
+                ));
+            }
+        };
+        Ok(Self {
+            data_file: DataFile::try_from(data_file)?,
+            coverage,
+            committed_version: proto.committed_version,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_data_overlay_missing_fields_error() {
+        // A DataOverlayFile proto missing its coverage or data_file is rejected.
+        let no_coverage = pb::DataOverlayFile {
+            data_file: Some(pb::DataFile::from(&DataFile::new_legacy_from_fields(
+                "overlay.lance",
+                vec![3],
+                None,
+            ))),
+            coverage: None,
+            committed_version: 1,
+        };
+        let err = DataOverlayFile::try_from(no_coverage).unwrap_err();
+        assert!(err.to_string().contains("missing its coverage"), "{err}");
+
+        let no_data_file = pb::DataOverlayFile {
+            data_file: None,
+            coverage: Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(
+                serialize_roaring(&RoaringBitmap::from_iter([0u32])),
+            )),
+            committed_version: 1,
+        };
+        let err = DataOverlayFile::try_from(no_data_file).unwrap_err();
+        assert!(err.to_string().contains("missing its data_file"), "{err}");
+    }
+
+    #[test]
+    fn test_overlay_coverage_serde_json_roundtrip() {
+        // The custom serde impl round-trips through JSON for dense/sparse,
+        // including empty bitmaps and a zero-bitmap sparse coverage.
+        for coverage in [
+            OverlayCoverage::dense(RoaringBitmap::from_iter([1u32, 5, 100])),
+            OverlayCoverage::dense(RoaringBitmap::new()),
+            OverlayCoverage::sparse(vec![
+                RoaringBitmap::from_iter([2u32, 3]),
+                RoaringBitmap::new(),
+            ]),
+            OverlayCoverage::sparse(vec![]),
+        ] {
+            let json = serde_json::to_string(&coverage).unwrap();
+            let back: OverlayCoverage = serde_json::from_str(&json).unwrap();
+            assert_eq!(back, coverage);
+        }
+    }
+
+    #[test]
+    fn test_tombstone_overlay_fields() {
+        // An overlay covering fields [3, 5]: replacing field 5 tombstones just
+        // field 5's slot and keeps field 3. An overlay covering only field 5 is
+        // dropped entirely. An overlay touching no replaced field is untouched.
+        let mut overlays = vec![
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("a.lance", vec![3, 5], None),
+                coverage: OverlayCoverage::sparse(vec![
+                    RoaringBitmap::from_iter([0u32]),
+                    RoaringBitmap::from_iter([1u32]),
+                ]),
+                committed_version: 1,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("b.lance", vec![5], None),
+                coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                committed_version: 1,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("c.lance", vec![7], None),
+                coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                committed_version: 1,
+            },
+        ];
+
+        tombstone_overlay_fields(&mut overlays, &[5]);
+
+        // The single-field overlay on field 5 is gone; the others remain.
+        assert_eq!(overlays.len(), 2);
+        // Field 3 preserved, field 5 tombstoned in place (coverage stays aligned).
+        assert_eq!(
+            overlays[0].data_file.fields.as_ref(),
+            &[3, TOMBSTONE_FIELD_ID]
+        );
+        // The untouched overlay keeps its field.
+        assert_eq!(overlays[1].data_file.fields.as_ref(), &[7]);
+    }
+
+    #[test]
+    fn test_verify_overlays_newest_last() {
+        let mk = |version: u64| DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+            committed_version: version,
+        };
+        // Non-decreasing (including equal versions) is accepted.
+        assert!(verify_overlays_newest_last(&[]).is_ok());
+        assert!(verify_overlays_newest_last(&[mk(1), mk(2), mk(2), mk(5)]).is_ok());
+        // A newer version before an older one is rejected.
+        let err = verify_overlays_newest_last(&[mk(2), mk(1)]).unwrap_err();
+        assert!(err.to_string().contains("newest-last"), "{err}");
+    }
+
+    #[test]
+    fn test_coverage_for_field_out_of_bounds() {
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None),
+            coverage: OverlayCoverage::sparse(vec![
+                RoaringBitmap::from_iter([1u32]),
+                RoaringBitmap::from_iter([2u32]),
+            ]),
+            committed_version: 1,
+        };
+        assert!(overlay.coverage_for_field(0).is_ok());
+        assert!(overlay.coverage_for_field(1).is_ok());
+        let err = overlay.coverage_for_field(5).unwrap_err();
+        assert!(err.to_string().contains("field position"), "{err}");
+    }
+}
diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs
index 3784e84a785..d682bf3a4a2 100644
--- a/rust/lance-table/src/io/commit.rs
+++ b/rust/lance-table/src/io/commit.rs
@@ -798,6 +798,26 @@ pub trait CommitHandler: Debug + Send + Sync {
         default_resolve_version(base_path, version, object_store).await
     }
 
+    /// Check whether an attached manifest version exists without loading it.
+    ///
+    /// The default implementation probes the deterministic manifest path for
+    /// the given naming scheme. Commit handlers with an external source of
+    /// truth should override this method.
+    async fn version_exists(
+        &self,
+        base_path: &Path,
+        version: u64,
+        object_store: &dyn OSObjectStore,
+        naming_scheme: ManifestNamingScheme,
+    ) -> Result<bool> {
+        let path = naming_scheme.manifest_path(base_path, version);
+        match object_store.head(&path).await {
+            Ok(_) => Ok(true),
+            Err(ObjectStoreError::NotFound { .. }) => Ok(false),
+            Err(e) => Err(e.into()),
+        }
+    }
+
     /// List detached manifest locations.
     ///
     /// Returns a stream of detached manifest locations in arbitrary order.
@@ -1071,7 +1091,7 @@ pub async fn commit_handler_from_url(
 
     match url.scheme() {
         "file" | "file-object-store" => Ok(local_handler),
-        "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "shared-memory" => {
+        "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "tos" | "shared-memory" => {
             Ok(Arc::new(ConditionalPutCommitHandler))
         }
         #[cfg(not(feature = "dynamodb"))]
@@ -1946,19 +1966,27 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_commit_handler_from_url_memory_schemes() {
-        // Both `memory://` and `shared-memory://` must route to
-        // ConditionalPutCommitHandler — otherwise concurrent writers fall
-        // through to UnsafeCommitHandler and silently clobber each other's
-        // manifests.
-        for url in ["memory://bucket-a/ds", "shared-memory://bucket-a/ds"] {
-            let handler = commit_handler_from_url(url, &None).await.unwrap();
-            assert_eq!(
-                format!("{:?}", handler),
-                "ConditionalPutCommitHandler",
-                "{url} should route to ConditionalPutCommitHandler",
-            );
-        }
+    #[rstest::rstest]
+    #[case::memory("memory://bucket-a/ds")]
+    #[case::shared_memory("shared-memory://bucket-a/ds")]
+    #[case::s3("s3://bucket-a/ds")]
+    #[case::gs("gs://bucket-a/ds")]
+    #[case::az("az://bucket-a/ds")]
+    #[case::abfss("abfss://bucket-a/ds")]
+    #[case::oss("oss://bucket-a/ds")]
+    #[case::cos("cos://bucket-a/ds")]
+    #[case::tos("tos://bucket-a/ds")]
+    async fn test_commit_handler_from_url_conditional_put_schemes(#[case] url: &str) {
+        // Every scheme whose store supports atomic put-if-not-exists must
+        // route to ConditionalPutCommitHandler — otherwise concurrent writers
+        // fall through to UnsafeCommitHandler and silently clobber each
+        // other's manifests.
+        let handler = commit_handler_from_url(url, &None).await.unwrap();
+        assert_eq!(
+            format!("{:?}", handler),
+            "ConditionalPutCommitHandler",
+            "{url} should route to ConditionalPutCommitHandler",
+        );
     }
 
     /// A [CommitLock] whose lease records whether it was released, so we can
diff --git a/rust/lance-table/src/io/commit/external_manifest.rs b/rust/lance-table/src/io/commit/external_manifest.rs
index 75993ca8d1f..22ebaa10b4a 100644
--- a/rust/lance-table/src/io/commit/external_manifest.rs
+++ b/rust/lance-table/src/io/commit/external_manifest.rs
@@ -8,6 +8,8 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use bytes::Bytes;
+use futures::StreamExt;
 use lance_core::utils::tracing::{
     AUDIT_MODE_CREATE, AUDIT_MODE_DELETE, AUDIT_TYPE_MANIFEST, TRACE_FILE_AUDIT,
 };
@@ -123,7 +125,7 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync {
 
         // Step 2: Copy staging to final path
         let final_path = naming_scheme.manifest_path(base_path, version);
-        let copied = match object_store.copy(staging_path, &final_path).await {
+        let copied = match copy_size_aware(object_store, staging_path, &final_path, size).await {
             Ok(_) => true,
             Err(ObjectStoreError::NotFound { .. }) => false,
             Err(e) => return Err(e.into()),
@@ -213,6 +215,129 @@ pub(crate) fn detect_naming_scheme_from_path(path: &Path) -> Result<ManifestNami
         })
 }
 
+/// The most conservative server-side-copy size limit across the object
+/// stores we support. This is not S3-specific: S3's `CopyObject` and GCS's
+/// single-shot `Objects: copy` both reject sources above ~5 GiB, so we use
+/// 5 GiB as a backend-agnostic threshold. Above it we stream the source
+/// through the client and re-upload via multipart instead of relying on a
+/// server-side copy. Stores that have no such cap (e.g. local filesystem)
+/// also take the fallback above this size — correctness is preserved; only
+/// the rare >5 GiB copy is slower than a native copy would be.
+const MAX_SERVER_SIDE_COPY_BYTES: u64 = 5 * 1024 * 1024 * 1024;
+
+/// Part size for the read+rewrite fallback. Multipart-capable stores
+/// (S3, GCS) require every part except the last to be ≥5 MB and allow up to
+/// 10,000 parts. 100 MB sits comfortably inside both bounds and keeps the
+/// part count low (~140 parts for a 14 GB manifest) without large per-part
+/// RAM.
+const COPY_REWRITE_PART_SIZE: usize = 100 * 1024 * 1024;
+
+/// Copy `from` to `to`, falling back to a multipart-equivalent read+rewrite
+/// when the source exceeds the server-side-copy size limit
+/// (`MAX_SERVER_SIDE_COPY_BYTES`).
+///
+/// For sources below the limit, this is the same fast server-side
+/// `store.copy()` as before. For larger sources, the source is streamed
+/// through the client and re-uploaded as a multipart upload at `to`. This
+/// doubles bytes-on-the-wire for the rare large case while preserving the
+/// cheap fast path for the common small case.
+///
+/// `size` is the known source size. It is required: the only caller already
+/// has it, and the alternative (an extra `head(from)` round-trip) is work
+/// the caller can avoid by passing what it already knows.
+///
+/// `NotFound` errors on `from` propagate unchanged so callers can keep
+/// existing `Err(NotFound { .. })` arms.
+///
+/// This is a workaround for the missing `UploadPartCopy` primitive in the
+/// upstream `object_store` crate. Once that lands, this helper can be
+/// deleted and the call sites can go back to plain `store.copy()`.
+async fn copy_size_aware(
+    store: &dyn OSObjectStore,
+    from: &Path,
+    to: &Path,
+    size: u64,
+) -> std::result::Result<(), ObjectStoreError> {
+    if size < MAX_SERVER_SIDE_COPY_BYTES {
+        store.copy(from, to).await
+    } else {
+        copy_via_read_rewrite(store, from, to).await
+    }
+}
+
+// NOTE: parts are uploaded sequentially. This could be parallelized (a
+// bounded JoinSet, like lance-io/src/object_writer.rs's
+// LANCE_UPLOAD_CONCURRENCY) or sidestepped entirely by switching to
+// `object_store::WriteMultipart` (which also handles abort-on-drop). Left
+// sequential here: this is a cold path (only >5 GiB manifests) and the
+// helper is itself a stopgap until `object_store` exposes UploadPartCopy.
+async fn copy_via_read_rewrite(
+    store: &dyn OSObjectStore,
+    from: &Path,
+    to: &Path,
+) -> std::result::Result<(), ObjectStoreError> {
+    // NotFound here propagates upward unchanged.
+    let mut stream = store.get(from).await?.into_stream();
+
+    // From here on, errors must `abort()` the upload to avoid leaving an
+    // orphan multipart upload on stores that support them (e.g. S3, GCS),
+    // which would otherwise incur storage charges until the bucket's
+    // lifecycle policy cleans it up.
+    //
+    // Note: this does NOT cover task cancellation — `MultipartUpload`'s
+    // upstream Drop is documented as a no-op for S3/GCS. Callers that
+    // need cancellation cleanliness should run this with a guard or
+    // switch to `object_store::WriteMultipart` (planned follow-up).
+    let mut upload = store.put_multipart(to).await?;
+    let mut part_buf: Vec<u8> = Vec::with_capacity(COPY_REWRITE_PART_SIZE);
+
+    while let Some(chunk) = stream.next().await {
+        let chunk = match chunk {
+            Ok(b) => b,
+            Err(e) => {
+                let _ = upload.abort().await;
+                return Err(e);
+            }
+        };
+        // Append the chunk in COPY_REWRITE_PART_SIZE-bounded slices so a
+        // single oversized chunk (e.g., LocalFileSystem returning a whole
+        // file) cannot push part_buf past the backend's per-part size limit
+        // (5 GiB on S3/GCS). COPY_REWRITE_PART_SIZE is well under every
+        // backend's cap, so each flushed part is always valid.
+        let mut offset = 0;
+        while offset < chunk.len() {
+            let want = COPY_REWRITE_PART_SIZE - part_buf.len();
+            let take = want.min(chunk.len() - offset);
+            part_buf.extend_from_slice(&chunk[offset..offset + take]);
+            offset += take;
+
+            if part_buf.len() >= COPY_REWRITE_PART_SIZE {
+                let payload =
+                    std::mem::replace(&mut part_buf, Vec::with_capacity(COPY_REWRITE_PART_SIZE));
+                if let Err(e) = upload.put_part(Bytes::from(payload).into()).await {
+                    let _ = upload.abort().await;
+                    return Err(e);
+                }
+            }
+        }
+    }
+
+    // Flush the final (possibly-short) part. The last part of a multipart
+    // upload is exempt from the per-part minimum on S3/GCS.
+    if !part_buf.is_empty()
+        && let Err(e) = upload.put_part(Bytes::from(part_buf).into()).await
+    {
+        let _ = upload.abort().await;
+        return Err(e);
+    }
+
+    if let Err(e) = upload.complete().await {
+        let _ = upload.abort().await;
+        return Err(e);
+    }
+    Ok(())
+}
+
 /// External manifest commit handler
 /// This handler is used to commit a manifest to an external store
 /// for detailed design, see <https://github.com/lance-format/lance/issues/1183>
@@ -245,14 +370,12 @@ impl ExternalManifestCommitHandler {
         // step 1: copy the manifest to the final location
         let final_manifest_path = naming_scheme.manifest_path(base_path, version);
 
-        let copied = match store
-            .copy(staging_manifest_path, &final_manifest_path)
-            .await
-        {
-            Ok(_) => true,
-            Err(ObjectStoreError::NotFound { .. }) => false, // Another writer beat us to it.
-            Err(e) => return Err(e.into()),
-        };
+        let copied =
+            match copy_size_aware(store, staging_manifest_path, &final_manifest_path, size).await {
+                Ok(_) => true,
+                Err(ObjectStoreError::NotFound { .. }) => false, // Another writer beat us to it.
+                Err(e) => return Err(e.into()),
+            };
         if copied {
             info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_manifest_path.as_ref());
         }
@@ -456,6 +579,31 @@ impl CommitHandler for ExternalManifestCommitHandler {
         .await
     }
 
+    async fn version_exists(
+        &self,
+        base_path: &Path,
+        version: u64,
+        object_store: &dyn OSObjectStore,
+        naming_scheme: ManifestNamingScheme,
+    ) -> Result<bool> {
+        match self
+            .external_manifest_store
+            .get_manifest_location(base_path.as_ref(), version)
+            .await
+        {
+            Ok(_) => Ok(true),
+            Err(Error::NotFound { .. }) => {
+                let path = naming_scheme.manifest_path(base_path, version);
+                match object_store.head(&path).await {
+                    Ok(_) => Ok(true),
+                    Err(ObjectStoreError::NotFound { .. }) => Ok(false),
+                    Err(e) => Err(e.into()),
+                }
+            }
+            Err(e) => Err(e),
+        }
+    }
+
     async fn commit(
         &self,
         manifest: &mut Manifest,
diff --git a/rust/lance-table/src/io/manifest.rs b/rust/lance-table/src/io/manifest.rs
index 4b918177176..6a6bfd2724c 100644
--- a/rust/lance-table/src/io/manifest.rs
+++ b/rust/lance-table/src/io/manifest.rs
@@ -115,14 +115,18 @@ pub async fn read_manifest_indexes(
     manifest: &Manifest,
 ) -> Result<Vec<IndexMetadata>> {
     if let Some(pos) = manifest.index_section.as_ref() {
-        let reader = if let Some(size) = location.size {
-            object_store
-                .open_with_size(&location.path, size as usize)
-                .await?
-        } else {
-            object_store.open(&location.path).await?
+        let result = read_index_section(object_store, &location.path, location.size, *pos).await;
+        // A stale cached size makes the index offset fall outside the sized view,
+        // so the read fails as "file size is too small". Retry once with the true
+        // size; surface any other error unchanged.
+        let section = match result {
+            Err(e)
+                if location.size.is_some() && e.to_string().contains("file size is too small") =>
+            {
+                read_index_section(object_store, &location.path, None, *pos).await?
+            }
+            other => other?,
         };
-        let section: pb::IndexSection = read_message(reader.as_ref(), *pos).await?;
 
         let indices = section
             .indices
@@ -135,6 +139,22 @@ pub async fn read_manifest_indexes(
     }
 }
 
+/// Read the index section message at `pos`, opening the manifest with a known
+/// size when one is provided.
+async fn read_index_section(
+    object_store: &ObjectStore,
+    path: &Path,
+    size: Option<u64>,
+    pos: usize,
+) -> Result<pb::IndexSection> {
+    let reader = if let Some(size) = size {
+        object_store.open_with_size(path, size as usize).await?
+    } else {
+        object_store.open(path).await?
+    };
+    read_message(reader.as_ref(), pos).await
+}
+
 async fn do_write_manifest(
     writer: &mut dyn Writer,
     manifest: &mut Manifest,
diff --git a/rust/lance-table/src/lib.rs b/rust/lance-table/src/lib.rs
index ebe892ba534..89b424adc61 100644
--- a/rust/lance-table/src/lib.rs
+++ b/rust/lance-table/src/lib.rs
@@ -5,4 +5,5 @@ pub mod feature_flags;
 pub mod format;
 pub mod io;
 pub mod rowids;
+pub mod system_index;
 pub mod utils;
diff --git a/rust/lance-table/src/rowids.rs b/rust/lance-table/src/rowids.rs
index 0b56be84f56..6cdc907cda5 100644
--- a/rust/lance-table/src/rowids.rs
+++ b/rust/lance-table/src/rowids.rs
@@ -22,7 +22,7 @@ pub mod segment;
 mod serde;
 pub mod version;
 
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 // These are the public API.
 pub use index::FragmentRowIdIndex;
 pub use index::RowIdIndex;
@@ -355,11 +355,7 @@ impl RowIdSequence {
             while (index - rows_passed) >= cur_seg_len {
                 rows_passed += cur_seg_len;
                 cur_seg = seg_iter.next();
-                if let Some(cur_seg) = cur_seg {
-                    cur_seg_len = cur_seg.len();
-                } else {
-                    return None;
-                }
+                cur_seg_len = cur_seg?.len();
             }
 
             Some(cur_seg.unwrap().get(index - rows_passed).unwrap())
@@ -530,30 +526,32 @@ impl From<&RowIdSequence> for RowAddrTreeMap {
     fn from(row_ids: &RowIdSequence) -> Self {
         let mut tree_map = Self::new();
         for segment in &row_ids.0 {
+            let mut seg = Self::new();
             match segment {
                 U64Segment::Range(range) => {
-                    tree_map.insert_range(range.clone());
+                    seg.insert_range(range.clone());
                 }
                 U64Segment::RangeWithBitmap { range, bitmap } => {
-                    tree_map.insert_range(range.clone());
+                    seg.insert_range(range.clone());
                     for (i, val) in range.clone().enumerate() {
                         if !bitmap.get(i) {
-                            tree_map.remove(val);
+                            seg.remove(val);
                         }
                     }
                 }
                 U64Segment::RangeWithHoles { range, holes } => {
-                    tree_map.insert_range(range.clone());
+                    seg.insert_range(range.clone());
                     for hole in holes.iter() {
-                        tree_map.remove(hole);
+                        seg.remove(hole);
                     }
                 }
                 U64Segment::SortedArray(array) | U64Segment::Array(array) => {
                     for val in array.iter() {
-                        tree_map.insert(val);
+                        seg.insert(val);
                     }
                 }
             }
+            tree_map |= seg;
         }
         tree_map
     }
@@ -1040,6 +1038,27 @@ mod test {
         assert_eq!(tree_map, expected);
     }
 
+    #[test]
+    fn test_row_id_sequence_to_treemap_overlapping_segments() {
+        // Compaction can concatenate segments whose ranges overlap but whose
+        // selected ids are disjoint (here: even ids, then odd ids over 0..6).
+        // The tree map must contain every id the sequence yields.
+        let sequence = RowIdSequence(vec![
+            U64Segment::RangeWithBitmap {
+                range: 0..6,
+                bitmap: [true, false, true, false, true, false].as_slice().into(),
+            },
+            U64Segment::RangeWithBitmap {
+                range: 0..6,
+                bitmap: [false, true, false, true, false, true].as_slice().into(),
+            },
+        ]);
+
+        let expected = sequence.iter().collect::<RowAddrTreeMap>();
+        assert_eq!(expected, (0..6).collect::<RowAddrTreeMap>());
+        assert_eq!(RowAddrTreeMap::from(&sequence), expected);
+    }
+
     #[test]
     fn test_row_addr_mask() {
         // 0, 1, 2, 3, 4
diff --git a/rust/lance-table/src/rowids/bitmap.rs b/rust/lance-table/src/rowids/bitmap.rs
index 9001c04c3a0..ce7eadd5634 100644
--- a/rust/lance-table/src/rowids/bitmap.rs
+++ b/rust/lance-table/src/rowids/bitmap.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 #[derive(PartialEq, Eq, Clone, DeepSizeOf)]
 pub struct Bitmap {
diff --git a/rust/lance-table/src/rowids/encoded_array.rs b/rust/lance-table/src/rowids/encoded_array.rs
index 06614765af1..7564cb6bb21 100644
--- a/rust/lance-table/src/rowids/encoded_array.rs
+++ b/rust/lance-table/src/rowids/encoded_array.rs
@@ -3,7 +3,7 @@
 
 use std::ops::Range;
 
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 /// Encoded array of u64 values.
 ///
diff --git a/rust/lance-table/src/rowids/index.rs b/rust/lance-table/src/rowids/index.rs
index 718f2b8f2a9..4fed0e651c5 100644
--- a/rust/lance-table/src/rowids/index.rs
+++ b/rust/lance-table/src/rowids/index.rs
@@ -5,10 +5,10 @@ use std::ops::RangeInclusive;
 use std::sync::Arc;
 
 use super::{RowIdSequence, U64Segment};
-use deepsize::DeepSizeOf;
-use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::address::RowAddress;
 use lance_core::utils::deletion::DeletionVector;
+use lance_core::{Error, Result};
 use rangemap::RangeInclusiveMap;
 
 /// An index of row ids
@@ -46,18 +46,10 @@ impl RowIdIndex {
                 RawIndexChunk::NonOverlapping(chunk) => {
                     final_chunks.push(chunk);
                 }
-                RawIndexChunk::Overlapping(range, overlapping_chunks) => {
-                    debug_assert_eq!(
-                        range.end() - range.start() + 1,
-                        overlapping_chunks
-                            .iter()
-                            .map(|(_, (seq, _))| seq.len() as u64)
-                            .sum::<u64>(),
-                        "Wrong range for {:?}, chunks: {:?}",
-                        range,
-                        overlapping_chunks,
-                    );
-                    // Merge overlapping chunks.
+                RawIndexChunk::Overlapping(_range, overlapping_chunks) => {
+                    // Intersecting row-id ranges don't imply intersecting id sets;
+                    // sparse ids and deletion holes leave the union short of the span.
+                    // The real invariant (no id in two fragments) is checked in the merge.
                     let merged_chunk = merge_overlapping_chunks(overlapping_chunks)?;
                     final_chunks.push(merged_chunk);
                 }
@@ -120,7 +112,7 @@ impl RowIdIndex {
 }
 
 impl DeepSizeOf for RowIdIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.0
             .iter()
             .map(|(_, (row_id_segment, address_segment))| {
@@ -357,6 +349,14 @@ fn merge_overlapping_chunks(overlapping_chunks: Vec<IndexChunk>) -> Result<Index
         values.extend(row_ids.iter().zip(row_addrs.iter()));
     }
     values.sort_by_key(|(row_id, _)| *row_id);
+    // A duplicate row id here means two fragments claim the same live id: a
+    // corrupt index, not a resolvable sparse-coverage case.
+    if let Some(w) = values.windows(2).find(|w| w[0].0 == w[1].0) {
+        return Err(Error::internal(format!(
+            "row id index corrupt: stable row id {} is live in multiple fragments",
+            w[0].0
+        )));
+    }
     let row_id_segment = U64Segment::from_iter(values.iter().map(|(row_id, _)| *row_id));
     let address_segment = U64Segment::from_iter(values.iter().map(|(_, row_addr)| *row_addr));
 
@@ -524,6 +524,41 @@ mod tests {
         assert_eq!(index.get(99), Some(RowAddress::new_from_parts(0, 98)));
     }
 
+    #[test]
+    fn test_overlapping_chunks_sparse_with_deletions() {
+        // Interleaved (overlapping) id ranges plus a deletion that leaves a hole,
+        // so the union doesn't tile the span. Every live id must still resolve.
+        let fragment_indices = vec![
+            FragmentRowIdIndex {
+                fragment_id: 10,
+                row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray(
+                    vec![1, 3, 5, 7, 9].into(),
+                )])),
+                deletion_vector: Arc::new(DeletionVector::default()),
+            },
+            FragmentRowIdIndex {
+                fragment_id: 20,
+                row_id_sequence: Arc::new(RowIdSequence(vec![U64Segment::SortedArray(
+                    vec![0, 2, 4, 6, 8].into(),
+                )])),
+                // Delete offset 2 (id 4) -> a hole in the span.
+                deletion_vector: Arc::new(DeletionVector::from_iter(vec![2])),
+            },
+        ];
+
+        let index = RowIdIndex::new(&fragment_indices).unwrap();
+
+        assert_eq!(index.get(0), Some(RowAddress::new_from_parts(20, 0)));
+        assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 0)));
+        assert_eq!(index.get(2), Some(RowAddress::new_from_parts(20, 1)));
+        assert_eq!(index.get(3), Some(RowAddress::new_from_parts(10, 1)));
+        assert_eq!(index.get(4), None);
+        // Surviving ids keep their original offsets (the hole is not compacted).
+        assert_eq!(index.get(6), Some(RowAddress::new_from_parts(20, 3)));
+        assert_eq!(index.get(8), Some(RowAddress::new_from_parts(20, 4)));
+        assert_eq!(index.get(9), Some(RowAddress::new_from_parts(10, 4)));
+    }
+
     #[test]
     fn test_index_with_deletion_vector() {
         let deletion_vector = DeletionVector::from_iter(vec![2, 3]);
diff --git a/rust/lance-table/src/rowids/segment.rs b/rust/lance-table/src/rowids/segment.rs
index a02acd8a573..6fba8599016 100644
--- a/rust/lance-table/src/rowids/segment.rs
+++ b/rust/lance-table/src/rowids/segment.rs
@@ -4,7 +4,7 @@
 use std::ops::{Range, RangeInclusive};
 
 use super::{bitmap::Bitmap, encoded_array::EncodedU64Array};
-use deepsize::DeepSizeOf;
+use lance_core::deepsize::DeepSizeOf;
 
 /// Convert an estimated serialized byte cost from `u128` to `usize`, saturating
 /// at [`usize::MAX`] when the value does not fit (infeasible encodings).
@@ -70,7 +70,7 @@ pub enum U64Segment {
 }
 
 impl DeepSizeOf for U64Segment {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         match self {
             Self::Range(_) => 0,
             Self::RangeWithHoles { holes, .. } => holes.deep_size_of_children(context),
diff --git a/rust/lance-table/src/rowids/version.rs b/rust/lance-table/src/rowids/version.rs
index 80f3d06db60..6ddb083c36e 100644
--- a/rust/lance-table/src/rowids/version.rs
+++ b/rust/lance-table/src/rowids/version.rs
@@ -9,9 +9,9 @@
 
 use std::sync::Arc;
 
-use deepsize::DeepSizeOf;
 use lance_core::Error;
 use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use prost::Message;
 use serde::de::Deserializer;
 use serde::ser::Serializer;
diff --git a/rust/lance-table/src/system_index.rs b/rust/lance-table/src/system_index.rs
new file mode 100644
index 00000000000..021c01a5e52
--- /dev/null
+++ b/rust/lance-table/src/system_index.rs
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! System indices: table-level structure persisted as indices.
+//!
+//! Unlike normal indices, whose internals stay opaque behind
+//! [`crate::format::IndexMetadata::index_details`], the table format genuinely
+//! interprets the contents of these indices (fragment remapping, row
+//! visibility). They therefore live at the table layer.
+//!
+//! The `Index`-trait adapters for these structs live in `lance-index`, which
+//! re-exports the structs defined here.
+
+pub mod frag_reuse;
+pub mod mem_wal;
diff --git a/rust/lance-table/src/system_index/frag_reuse.rs b/rust/lance-table/src/system_index/frag_reuse.rs
new file mode 100644
index 00000000000..40bbc4f58b6
--- /dev/null
+++ b/rust/lance-table/src/system_index/frag_reuse.rs
@@ -0,0 +1,480 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{collections::HashMap, sync::Arc};
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt64Type;
+use arrow_array::{Array, ArrayRef, PrimitiveArray, RecordBatch, UInt64Array};
+use lance_core::deepsize::{Context, DeepSizeOf};
+use lance_core::{Error, Result};
+use lance_select::RowAddrTreeMap;
+use roaring::{RoaringBitmap, RoaringTreemap};
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::format::pb::fragment_reuse_index_details::InlineContent;
+use crate::format::{ExternalFile, Fragment, pb};
+
+pub const FRAG_REUSE_INDEX_NAME: &str = "__lance_frag_reuse";
+pub const FRAG_REUSE_DETAILS_FILE_NAME: &str = "details.binpb";
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct FragDigest {
+    pub id: u64,
+    pub physical_rows: usize,
+    pub num_deleted_rows: usize,
+}
+
+impl From<&FragDigest> for pb::fragment_reuse_index_details::FragmentDigest {
+    fn from(digest: &FragDigest) -> Self {
+        Self {
+            id: digest.id,
+            physical_rows: digest.physical_rows as u64,
+            num_deleted_rows: digest.num_deleted_rows as u64,
+        }
+    }
+}
+
+impl From<&Fragment> for FragDigest {
+    fn from(fragment: &Fragment) -> Self {
+        Self {
+            id: fragment.id,
+            physical_rows: fragment
+                .physical_rows
+                .expect("Fragment doesn't have physical rows recorded"),
+            num_deleted_rows: fragment
+                .deletion_file
+                .as_ref()
+                .and_then(|d| d.num_deleted_rows)
+                .unwrap_or(0),
+        }
+    }
+}
+
+impl TryFrom<pb::fragment_reuse_index_details::FragmentDigest> for FragDigest {
+    type Error = Error;
+
+    fn try_from(digest: pb::fragment_reuse_index_details::FragmentDigest) -> Result<Self> {
+        Ok(Self {
+            id: digest.id,
+            physical_rows: digest.physical_rows as usize,
+            num_deleted_rows: digest.num_deleted_rows as usize,
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct FragReuseGroup {
+    pub changed_row_addrs: Vec<u8>,
+    pub old_frags: Vec<FragDigest>,
+    pub new_frags: Vec<FragDigest>,
+}
+
+impl From<&FragReuseGroup> for pb::fragment_reuse_index_details::Group {
+    fn from(group: &FragReuseGroup) -> Self {
+        Self {
+            changed_row_addrs: group.changed_row_addrs.clone(),
+            old_fragments: group.old_frags.iter().map(|f| f.into()).collect(),
+            new_fragments: group.new_frags.iter().map(|f| f.into()).collect(),
+        }
+    }
+}
+
+impl TryFrom<pb::fragment_reuse_index_details::Group> for FragReuseGroup {
+    type Error = Error;
+
+    fn try_from(group: pb::fragment_reuse_index_details::Group) -> Result<Self> {
+        Ok(Self {
+            changed_row_addrs: group.changed_row_addrs,
+            old_frags: group
+                .old_fragments
+                .into_iter()
+                .map(FragDigest::try_from)
+                .collect::<Result<_>>()?,
+            new_frags: group
+                .new_fragments
+                .into_iter()
+                .map(FragDigest::try_from)
+                .collect::<Result<_>>()?,
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct FragReuseVersion {
+    pub dataset_version: u64,
+    pub groups: Vec<FragReuseGroup>,
+}
+
+impl From<&FragReuseVersion> for pb::fragment_reuse_index_details::Version {
+    fn from(version: &FragReuseVersion) -> Self {
+        Self {
+            dataset_version: version.dataset_version,
+            groups: version.groups.iter().map(|g| g.into()).collect(),
+        }
+    }
+}
+
+impl TryFrom<pb::fragment_reuse_index_details::Version> for FragReuseVersion {
+    type Error = Error;
+
+    fn try_from(version: pb::fragment_reuse_index_details::Version) -> Result<Self> {
+        Ok(Self {
+            dataset_version: version.dataset_version,
+            groups: version
+                .groups
+                .into_iter()
+                .map(FragReuseGroup::try_from)
+                .collect::<Result<_>>()?,
+        })
+    }
+}
+
+impl FragReuseVersion {
+    pub fn old_frag_ids(&self) -> Vec<u64> {
+        self.groups
+            .iter()
+            .flat_map(|g| g.old_frags.iter().map(|f| f.id))
+            .collect::<Vec<_>>()
+    }
+
+    pub fn new_frag_ids(&self) -> Vec<u64> {
+        self.groups
+            .iter()
+            .flat_map(|g| g.new_frags.iter().map(|f| f.id))
+            .collect::<Vec<_>>()
+    }
+
+    pub fn new_frag_bitmap(&self) -> RoaringBitmap {
+        RoaringBitmap::from_iter(self.new_frag_ids().iter().map(|&id| id as u32))
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub enum FragReuseIndexDetailsContentType {
+    Inline(FragReuseIndexDetails),
+    External(ExternalFile),
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct FragReuseIndexDetails {
+    pub versions: Vec<FragReuseVersion>,
+}
+
+impl From<&FragReuseIndexDetails> for InlineContent {
+    fn from(details: &FragReuseIndexDetails) -> Self {
+        let mut versions: Vec<pb::fragment_reuse_index_details::Version> =
+            details.versions.iter().map(|m| m.into()).collect();
+        // sort from oldest to latest version
+        versions.sort_by_key(|v| v.dataset_version);
+        Self { versions }
+    }
+}
+
+impl TryFrom<InlineContent> for FragReuseIndexDetails {
+    type Error = Error;
+
+    fn try_from(content: InlineContent) -> Result<Self> {
+        Ok(Self {
+            versions: content
+                .versions
+                .into_iter()
+                .map(|m| m.try_into())
+                .collect::<Result<Vec<_>>>()?,
+        })
+    }
+}
+
+impl FragReuseIndexDetails {
+    pub fn new_frag_bitmap(&self) -> RoaringBitmap {
+        RoaringBitmap::from_iter(
+            self.versions
+                .iter()
+                .flat_map(|v| v.new_frag_ids().into_iter().map(|id| id as u32)),
+        )
+    }
+}
+
+/// An index that stores row ID maps.
+/// A row ID map describes the mapping from old row address to new address after compactions.
+/// Each version contains the mapping for one round of compaction.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct FragReuseIndex {
+    pub uuid: Uuid,
+    pub row_id_maps: Vec<HashMap<u64, Option<u64>>>,
+    pub details: FragReuseIndexDetails,
+}
+
+impl DeepSizeOf for FragReuseIndex {
+    fn deep_size_of_children(&self, cx: &mut Context) -> usize {
+        self.row_id_maps.deep_size_of_children(cx) + self.details.deep_size_of_children(cx)
+    }
+}
+
+impl FragReuseIndex {
+    pub fn new(
+        uuid: Uuid,
+        row_id_maps: Vec<HashMap<u64, Option<u64>>>,
+        details: FragReuseIndexDetails,
+    ) -> Self {
+        Self {
+            uuid,
+            row_id_maps,
+            details,
+        }
+    }
+
+    pub fn remap_row_id(&self, row_id: u64) -> Option<u64> {
+        let mut mapped_value = Some(row_id);
+        for row_id_map in self.row_id_maps.iter() {
+            if mapped_value.is_some() {
+                mapped_value = row_id_map
+                    .get(&mapped_value.unwrap())
+                    .copied()
+                    .unwrap_or(mapped_value);
+            }
+        }
+
+        mapped_value
+    }
+
+    pub fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap {
+        RowAddrTreeMap::from_iter(row_addrs.row_addrs().unwrap().filter_map(|addr| {
+            let addr_as_u64 = u64::from(addr);
+            self.remap_row_id(addr_as_u64)
+        }))
+    }
+
+    pub fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap {
+        RoaringTreemap::from_iter(row_ids.iter().filter_map(|addr| self.remap_row_id(addr)))
+    }
+
+    /// Remap a record batch that contains a row_id column at index `row_id_idx`
+    /// Currently this assumes there are only 2 columns in the schema,
+    /// which is the case for all indexes.
+    /// For example, for btree, the schema is (value, row_id).
+    /// For vector index storage, the schema is (row_id, vector).
+    pub fn remap_row_ids_record_batch(
+        &self,
+        batch: RecordBatch,
+        row_id_idx: usize,
+    ) -> Result<RecordBatch> {
+        assert_eq!(batch.schema().fields().len(), 2);
+        let other_column_idx = 1 - row_id_idx;
+        let row_ids = batch.column(row_id_idx).as_primitive::<UInt64Type>();
+        let (val_indices, new_row_ids): (Vec<u64>, Vec<u64>) = row_ids
+            .values()
+            .iter()
+            .enumerate()
+            .filter_map(|(idx, old_id)| {
+                self.remap_row_id(*old_id)
+                    .map(|new_id| (idx as u64, new_id))
+            })
+            .unzip();
+        let new_val_indices = UInt64Array::from_iter_values(val_indices);
+        let new_vals =
+            arrow::compute::take(batch.column(other_column_idx), &new_val_indices, None)?;
+
+        let mut batch_data: Vec<(usize, ArrayRef)> = vec![
+            (
+                row_id_idx,
+                Arc::new(UInt64Array::from_iter_values(new_row_ids)) as ArrayRef,
+            ),
+            (other_column_idx, Arc::new(new_vals)),
+        ];
+        batch_data.sort_by_key(|(i, _)| *i);
+        Ok(RecordBatch::try_new(
+            batch.schema(),
+            batch_data.into_iter().map(|(_, item)| item).collect(),
+        )?)
+    }
+
+    pub fn remap_row_ids_array(&self, array: ArrayRef) -> PrimitiveArray<UInt64Type> {
+        let primitive_array = array
+            .as_any()
+            .downcast_ref::<PrimitiveArray<UInt64Type>>()
+            .expect("expected row IDs to be uint64 array");
+        (0..primitive_array.len())
+            .map(|i| {
+                if primitive_array.is_null(i) {
+                    None
+                } else {
+                    self.remap_row_id(primitive_array.value(i))
+                }
+            })
+            .collect()
+    }
+
+    pub fn remap_fragment_bitmap(&self, fragment_bitmap: &mut RoaringBitmap) -> Result<()> {
+        for version in self.details.versions.iter() {
+            for group in version.groups.iter() {
+                let mut removed = 0;
+                for old_frag in group.old_frags.iter() {
+                    if fragment_bitmap.remove(old_frag.id as u32) {
+                        removed += 1;
+                    }
+                }
+
+                if removed > 0 {
+                    if removed != group.old_frags.len() {
+                        // Straddle: the index covered only part of this rewrite
+                        // group. Caused by the bug fixed in
+                        // <https://github.com/lance-format/lance/pull/6610>.
+                        // We've already removed the indexed old_frags from the
+                        // bitmap above; deliberately do NOT insert new_frags,
+                        // since the merged fragment also contains rows that
+                        // were never indexed. Affected rows fall through to
+                        // flat scan until the next optimize_indices. The fix
+                        // is persisted on the next write via build_manifest.
+                        tracing::warn!(
+                            "Healing straddling fragment-reuse rewrite group in index bitmap: \
+                             group {:?} was only partially indexed ({} of {} old fragments). \
+                             Affected rows will use flat scan until the next optimize_indices.",
+                            group.old_frags,
+                            removed,
+                            group.old_frags.len(),
+                        );
+                        continue;
+                    }
+
+                    for new_frag in group.new_frags.iter() {
+                        fragment_bitmap.insert(new_frag.id as u32);
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_serialize_deserialize_index_details() {
+        // Create sample FragReuseVersions with different dataset versions
+        let version1 = FragReuseVersion {
+            dataset_version: 2,
+            groups: vec![FragReuseGroup {
+                changed_row_addrs: vec![1, 2, 3],
+                old_frags: vec![FragDigest {
+                    id: 1,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                }],
+                new_frags: vec![
+                    FragDigest {
+                        id: 2,
+                        physical_rows: 1,
+                        num_deleted_rows: 0,
+                    },
+                    FragDigest {
+                        id: 3,
+                        physical_rows: 1,
+                        num_deleted_rows: 0,
+                    },
+                ],
+            }],
+        };
+
+        let version2 = FragReuseVersion {
+            dataset_version: 1,
+            groups: vec![FragReuseGroup {
+                changed_row_addrs: vec![4, 5, 6],
+                old_frags: vec![FragDigest {
+                    id: 2,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                }],
+                new_frags: vec![
+                    FragDigest {
+                        id: 4,
+                        physical_rows: 1,
+                        num_deleted_rows: 0,
+                    },
+                    FragDigest {
+                        id: 5,
+                        physical_rows: 1,
+                        num_deleted_rows: 0,
+                    },
+                ],
+            }],
+        };
+
+        // Create FragReuseIndexDetails with versions in reverse order
+        let details = FragReuseIndexDetails {
+            versions: vec![version1, version2],
+        };
+
+        // Convert to protobuf format
+        let inline_content: InlineContent = (&details).into();
+
+        // Convert back to FragReuseIndexDetails
+        let roundtrip_details = FragReuseIndexDetails::try_from(inline_content).unwrap();
+
+        // Verify the roundtrip
+        assert_eq!(roundtrip_details.versions.len(), 2);
+
+        // Verify versions are sorted by dataset_version (oldest to latest)
+        assert_eq!(roundtrip_details.versions[0].dataset_version, 1);
+        assert_eq!(
+            roundtrip_details.versions[0].groups[0].changed_row_addrs,
+            vec![4, 5, 6]
+        );
+        assert_eq!(
+            roundtrip_details.versions[0].groups[0].new_frags,
+            vec![
+                FragDigest {
+                    id: 4,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                },
+                FragDigest {
+                    id: 5,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                }
+            ]
+        );
+        assert_eq!(
+            roundtrip_details.versions[0].groups[0].old_frags,
+            vec![FragDigest {
+                id: 2,
+                physical_rows: 1,
+                num_deleted_rows: 0,
+            }]
+        );
+
+        assert_eq!(roundtrip_details.versions[1].dataset_version, 2);
+        assert_eq!(
+            roundtrip_details.versions[1].groups[0].changed_row_addrs,
+            vec![1, 2, 3]
+        );
+        assert_eq!(
+            roundtrip_details.versions[1].groups[0].new_frags,
+            vec![
+                FragDigest {
+                    id: 2,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                },
+                FragDigest {
+                    id: 3,
+                    physical_rows: 1,
+                    num_deleted_rows: 0,
+                }
+            ]
+        );
+        assert_eq!(
+            roundtrip_details.versions[1].groups[0].old_frags,
+            vec![FragDigest {
+                id: 1,
+                physical_rows: 1,
+                num_deleted_rows: 0,
+            }]
+        );
+    }
+}
diff --git a/rust/lance-table/src/system_index/mem_wal.rs b/rust/lance-table/src/system_index/mem_wal.rs
new file mode 100644
index 00000000000..1d82fd9e44f
--- /dev/null
+++ b/rust/lance-table/src/system_index/mem_wal.rs
@@ -0,0 +1,438 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+
+use lance_core::Error;
+use lance_core::deepsize::DeepSizeOf;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+use crate::format::pb;
+
+pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal";
+
+/// Type alias for shard identifier (UUID v4).
+pub type ShardId = Uuid;
+
+/// A flushed MemTable generation and its storage location.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct FlushedGeneration {
+    pub generation: u64,
+    pub path: String,
+}
+
+impl From<&FlushedGeneration> for pb::FlushedGeneration {
+    fn from(fg: &FlushedGeneration) -> Self {
+        Self {
+            generation: fg.generation,
+            path: fg.path.clone(),
+        }
+    }
+}
+
+impl From<pb::FlushedGeneration> for FlushedGeneration {
+    fn from(fg: pb::FlushedGeneration) -> Self {
+        Self {
+            generation: fg.generation,
+            path: fg.path,
+        }
+    }
+}
+
+/// A shard's merged generation, used in MemWalIndexDetails.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)]
+pub struct MergedGeneration {
+    pub shard_id: Uuid,
+    pub generation: u64,
+}
+
+impl DeepSizeOf for MergedGeneration {
+    fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
+        0 // UUID is 16 bytes fixed size, no heap allocations
+    }
+}
+
+impl MergedGeneration {
+    pub fn new(shard_id: Uuid, generation: u64) -> Self {
+        Self {
+            shard_id,
+            generation,
+        }
+    }
+}
+
+impl From<&MergedGeneration> for pb::MergedGeneration {
+    fn from(mg: &MergedGeneration) -> Self {
+        Self {
+            shard_id: Some((&mg.shard_id).into()),
+            generation: mg.generation,
+        }
+    }
+}
+
+impl TryFrom<pb::MergedGeneration> for MergedGeneration {
+    type Error = Error;
+
+    fn try_from(mg: pb::MergedGeneration) -> lance_core::Result<Self> {
+        let shard_id = mg
+            .shard_id
+            .as_ref()
+            .map(Uuid::try_from)
+            .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??;
+        Ok(Self {
+            shard_id,
+            generation: mg.generation,
+        })
+    }
+}
+
+/// Tracks which merged generation a base table index has been rebuilt to cover.
+/// Used to determine whether to read from flushed MemTable indexes or base table.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct IndexCatchupProgress {
+    pub index_name: String,
+    pub caught_up_generations: Vec<MergedGeneration>,
+}
+
+impl IndexCatchupProgress {
+    pub fn new(index_name: String, caught_up_generations: Vec<MergedGeneration>) -> Self {
+        Self {
+            index_name,
+            caught_up_generations,
+        }
+    }
+
+    /// Get the caught up generation for a specific shard.
+    /// Returns None if the shard is not present (assumed fully caught up).
+    pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
+        self.caught_up_generations
+            .iter()
+            .find(|mg| &mg.shard_id == shard_id)
+            .map(|mg| mg.generation)
+    }
+}
+
+impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress {
+    fn from(icp: &IndexCatchupProgress) -> Self {
+        Self {
+            index_name: icp.index_name.clone(),
+            caught_up_generations: icp
+                .caught_up_generations
+                .iter()
+                .map(|mg| mg.into())
+                .collect(),
+        }
+    }
+}
+
+impl TryFrom<pb::IndexCatchupProgress> for IndexCatchupProgress {
+    type Error = Error;
+
+    fn try_from(icp: pb::IndexCatchupProgress) -> lance_core::Result<Self> {
+        Ok(Self {
+            index_name: icp.index_name,
+            caught_up_generations: icp
+                .caught_up_generations
+                .into_iter()
+                .map(MergedGeneration::try_from)
+                .collect::<lance_core::Result<_>>()?,
+        })
+    }
+}
+
+/// Lifecycle status of a WAL shard, persisted in [`ShardManifest`].
+///
+/// `Sealed` is the durable in-doubt record for drop-table two-phase
+/// commit: a sealed shard refuses new writer claims (enforced in
+/// `claim_epoch`) but is reversible back to `Active` on rollback.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ShardStatus {
+    /// Normal: the shard accepts writer claims.
+    #[default]
+    Active,
+    /// A drop is in flight: claims are refused. Reversible.
+    Sealed,
+}
+
+impl ShardStatus {
+    /// Map to the protobuf enum discriminant (`pb::ShardStatus`).
+    fn to_i32(self) -> i32 {
+        match self {
+            Self::Active => 0,
+            Self::Sealed => 1,
+        }
+    }
+
+    /// Map from the protobuf enum discriminant; unknown values decode as
+    /// `Active` (forward-compatible default).
+    fn from_i32(v: i32) -> Self {
+        match v {
+            1 => Self::Sealed,
+            _ => Self::Active,
+        }
+    }
+}
+
+/// Shard manifest containing epoch-based fencing and WAL state.
+/// Each shard has exactly one active writer at any time.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ShardManifest {
+    pub shard_id: Uuid,
+    pub version: u64,
+    pub shard_spec_id: u32,
+    /// Computed shard field values as raw Arrow scalar bytes, keyed by field id.
+    /// The byte encoding follows Arrow's little-endian convention: int32 is 4 LE
+    /// bytes, utf8 is raw UTF-8 bytes, etc. The result_type in the corresponding
+    /// ShardingField from the ShardingSpec determines how to interpret each value.
+    pub shard_field_values: HashMap<String, Vec<u8>>,
+    pub writer_epoch: u64,
+    /// The most recent WAL entry position flushed to a MemTable.
+    /// Recovery replays from `replay_after_wal_entry_position + 1`. The
+    /// default value 0 means "no flush has ever stamped this shard" — WAL
+    /// positions themselves are 1-based, so 0 is never a valid covered
+    /// position.
+    pub replay_after_wal_entry_position: u64,
+    /// The most recent WAL entry position observed at manifest write time.
+    /// Default 0 means "no entry has been written yet"; WAL positions are
+    /// 1-based.
+    pub wal_entry_position_last_seen: u64,
+    pub current_generation: u64,
+    pub flushed_generations: Vec<FlushedGeneration>,
+    /// Lifecycle status (drop-table 2PC). Defaults to `Active`; preserved
+    /// across claims via `..base` so only fresh constructions set it.
+    pub status: ShardStatus,
+}
+
+impl DeepSizeOf for ShardManifest {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        self.shard_field_values.deep_size_of_children(context)
+            + self.flushed_generations.deep_size_of_children(context)
+    }
+}
+
+impl From<&ShardManifest> for pb::ShardManifest {
+    fn from(rm: &ShardManifest) -> Self {
+        Self {
+            shard_id: Some((&rm.shard_id).into()),
+            version: rm.version,
+            shard_spec_id: rm.shard_spec_id,
+            shard_field_entries: rm
+                .shard_field_values
+                .iter()
+                .map(|(k, v)| pb::ShardFieldEntry {
+                    field_id: k.clone(),
+                    value: v.clone(),
+                })
+                .collect(),
+            writer_epoch: rm.writer_epoch,
+            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
+            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
+            current_generation: rm.current_generation,
+            flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(),
+            status: rm.status.to_i32(),
+        }
+    }
+}
+
+impl TryFrom<pb::ShardManifest> for ShardManifest {
+    type Error = Error;
+
+    fn try_from(rm: pb::ShardManifest) -> lance_core::Result<Self> {
+        let shard_id = rm
+            .shard_id
+            .as_ref()
+            .map(Uuid::try_from)
+            .ok_or_else(|| Error::invalid_input("Missing shard_id in ShardManifest"))??;
+        let shard_field_values = rm
+            .shard_field_entries
+            .into_iter()
+            .map(|e| (e.field_id, e.value))
+            .collect();
+        Ok(Self {
+            shard_id,
+            version: rm.version,
+            shard_spec_id: rm.shard_spec_id,
+            shard_field_values,
+            writer_epoch: rm.writer_epoch,
+            replay_after_wal_entry_position: rm.replay_after_wal_entry_position,
+            wal_entry_position_last_seen: rm.wal_entry_position_last_seen,
+            current_generation: rm.current_generation,
+            flushed_generations: rm
+                .flushed_generations
+                .into_iter()
+                .map(FlushedGeneration::from)
+                .collect(),
+            status: ShardStatus::from_i32(rm.status),
+        })
+    }
+}
+
+/// Sharding field definition.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct ShardingField {
+    pub field_id: String,
+    pub source_ids: Vec<i32>,
+    pub transform: Option<String>,
+    pub expression: Option<String>,
+    pub result_type: String,
+    pub parameters: HashMap<String, String>,
+}
+
+impl From<&ShardingField> for pb::ShardingField {
+    fn from(rf: &ShardingField) -> Self {
+        Self {
+            field_id: rf.field_id.clone(),
+            source_ids: rf.source_ids.clone(),
+            transform: rf.transform.clone(),
+            expression: rf.expression.clone(),
+            result_type: rf.result_type.clone(),
+            parameters: rf.parameters.clone(),
+        }
+    }
+}
+
+impl From<pb::ShardingField> for ShardingField {
+    fn from(rf: pb::ShardingField) -> Self {
+        Self {
+            field_id: rf.field_id,
+            source_ids: rf.source_ids,
+            transform: rf.transform,
+            expression: rf.expression,
+            result_type: rf.result_type,
+            parameters: rf.parameters,
+        }
+    }
+}
+
+/// Sharding spec definition.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct ShardingSpec {
+    pub spec_id: u32,
+    pub fields: Vec<ShardingField>,
+}
+
+impl From<&ShardingSpec> for pb::ShardingSpec {
+    fn from(rs: &ShardingSpec) -> Self {
+        Self {
+            spec_id: rs.spec_id,
+            fields: rs.fields.iter().map(|f| f.into()).collect(),
+        }
+    }
+}
+
+impl From<pb::ShardingSpec> for ShardingSpec {
+    fn from(rs: pb::ShardingSpec) -> Self {
+        Self {
+            spec_id: rs.spec_id,
+            fields: rs.fields.into_iter().map(ShardingField::from).collect(),
+        }
+    }
+}
+
+/// Index details for MemWAL Index, stored in IndexMetadata.index_details.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)]
+pub struct MemWalIndexDetails {
+    pub snapshot_ts_millis: i64,
+    pub num_shards: u32,
+    pub inline_snapshots: Option<Vec<u8>>,
+    pub sharding_specs: Vec<ShardingSpec>,
+    pub maintained_indexes: Vec<String>,
+    pub merged_generations: Vec<MergedGeneration>,
+    pub index_catchup: Vec<IndexCatchupProgress>,
+    /// Default `ShardWriter` configuration values for this MemWAL index.
+    ///
+    /// Persisted so every writer — across processes and restarts — starts
+    /// from the same default writer configuration. These are defaults only;
+    /// an individual writer may still override any value at runtime in its
+    /// own (non-persisted) `ShardWriterConfig`.
+    pub writer_config_defaults: HashMap<String, String>,
+}
+
+impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails {
+    fn from(details: &MemWalIndexDetails) -> Self {
+        Self {
+            snapshot_ts_millis: details.snapshot_ts_millis,
+            num_shards: details.num_shards,
+            inline_snapshots: details.inline_snapshots.clone(),
+            sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(),
+            maintained_indexes: details.maintained_indexes.clone(),
+            merged_generations: details
+                .merged_generations
+                .iter()
+                .map(|mg| mg.into())
+                .collect(),
+            index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(),
+            writer_config_defaults: details.writer_config_defaults.clone(),
+        }
+    }
+}
+
+impl TryFrom<pb::MemWalIndexDetails> for MemWalIndexDetails {
+    type Error = Error;
+
+    fn try_from(details: pb::MemWalIndexDetails) -> lance_core::Result<Self> {
+        Ok(Self {
+            snapshot_ts_millis: details.snapshot_ts_millis,
+            num_shards: details.num_shards,
+            inline_snapshots: details.inline_snapshots,
+            sharding_specs: details
+                .sharding_specs
+                .into_iter()
+                .map(ShardingSpec::from)
+                .collect(),
+            maintained_indexes: details.maintained_indexes,
+            merged_generations: details
+                .merged_generations
+                .into_iter()
+                .map(MergedGeneration::try_from)
+                .collect::<lance_core::Result<_>>()?,
+            index_catchup: details
+                .index_catchup
+                .into_iter()
+                .map(IndexCatchupProgress::try_from)
+                .collect::<lance_core::Result<_>>()?,
+            writer_config_defaults: details.writer_config_defaults,
+        })
+    }
+}
+
+/// MemWAL Index provides access to MemWAL configuration and state.
+#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)]
+pub struct MemWalIndex {
+    pub details: MemWalIndexDetails,
+}
+
+impl MemWalIndex {
+    pub fn new(details: MemWalIndexDetails) -> Self {
+        Self { details }
+    }
+
+    pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option<u64> {
+        self.details
+            .merged_generations
+            .iter()
+            .find(|mg| &mg.shard_id == shard_id)
+            .map(|mg| mg.generation)
+    }
+
+    /// Get the caught up generation for a specific index and shard.
+    /// Returns None if the index is not tracked (assumed fully caught up).
+    pub fn index_caught_up_generation(&self, index_name: &str, shard_id: &Uuid) -> Option<u64> {
+        self.details
+            .index_catchup
+            .iter()
+            .find(|icp| icp.index_name == index_name)
+            .and_then(|icp| icp.caught_up_generation_for_shard(shard_id))
+    }
+
+    /// Check if an index is fully caught up for a shard.
+    /// Returns true if the index covers all merged data for the shard.
+    pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool {
+        let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0);
+        let caught_up_gen = self.index_caught_up_generation(index_name, shard_id);
+
+        // If not tracked in index_catchup, assumed fully caught up
+        caught_up_gen.is_none_or(|generation| generation >= merged_gen)
+    }
+}
diff --git a/rust/lance-table/src/utils.rs b/rust/lance-table/src/utils.rs
index 01c64f78710..0c37ef1e001 100644
--- a/rust/lance-table/src/utils.rs
+++ b/rust/lance-table/src/utils.rs
@@ -45,3 +45,5 @@ impl<I: Iterator> Iterator for ExactSize<I> {
         (self.size, Some(self.size))
     }
 }
+
+impl<T: Iterator> ExactSizeIterator for ExactSize<T> {}
diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml
index 5edfe4a9f16..e1006cd93c7 100644
--- a/rust/lance-tokenizer/Cargo.toml
+++ b/rust/lance-tokenizer/Cargo.toml
@@ -17,6 +17,7 @@ jieba-rs = { workspace = true, optional = true }
 lindera = { workspace = true, optional = true }
 rust-stemmers = "1.2.0"
 serde = { workspace = true, features = ["derive"] }
+stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] }
 unicode-normalization = "0.1.25"
 
 [features]
diff --git a/rust/lance-tokenizer/src/ascii_folding_filter.rs b/rust/lance-tokenizer/src/ascii_folding_filter.rs
index 376c0e1ebdb..8800545f1fb 100644
--- a/rust/lance-tokenizer/src/ascii_folding_filter.rs
+++ b/rust/lance-tokenizer/src/ascii_folding_filter.rs
@@ -49,9 +49,10 @@ impl<T: TokenStream> TokenStream for AsciiFoldingFilterTokenStream<'_, T> {
         if !self.tail.advance() {
             return false;
         }
-        if !self.token_mut().text.is_ascii() {
-            to_ascii(&self.tail.token().text, self.buffer);
-            mem::swap(&mut self.tail.token_mut().text, self.buffer);
+        let token = self.tail.token_mut();
+        if !token.text.is_ascii() {
+            to_ascii(&token.text, self.buffer);
+            mem::swap(&mut token.text, self.buffer);
         }
         true
     }
@@ -67,6 +68,7 @@ impl<T: TokenStream> TokenStream for AsciiFoldingFilterTokenStream<'_, T> {
 
 fn to_ascii(text: &str, output: &mut String) {
     output.clear();
+    output.reserve(text.len());
     for ch in text.chars() {
         if ch.is_ascii() {
             output.push(ch);
@@ -149,4 +151,10 @@ mod tests {
         let tokens = collect_tokens("straße");
         assert_eq!(tokens[0].text, "strasse");
     }
+
+    #[test]
+    fn test_ascii_folding_cjk_unchanged() {
+        let tokens = collect_tokens("こんにちは世界");
+        assert_eq!(tokens[0].text, "こんにちは世界");
+    }
 }
diff --git a/rust/lance-tokenizer/src/icu.rs b/rust/lance-tokenizer/src/icu.rs
index 4e36f23115f..eb9cd874c76 100644
--- a/rust/lance-tokenizer/src/icu.rs
+++ b/rust/lance-tokenizer/src/icu.rs
@@ -8,17 +8,25 @@ use crate::{TextAnalyzer, TextAnalyzerBuilder, Token, TokenStream, Tokenizer};
 #[derive(Clone)]
 pub struct IcuTokenizer {
     segmenter: WordSegmenterBorrowed<'static>,
+    split_on_non_alphanumeric: bool,
 }
 
 impl Default for IcuTokenizer {
     fn default() -> Self {
         Self {
             segmenter: WordSegmenter::new_dictionary(WordBreakInvariantOptions::default()),
+            split_on_non_alphanumeric: false,
         }
     }
 }
 
 impl IcuTokenizer {
+    /// Split ICU word segments again on simple-tokenizer delimiters.
+    pub fn with_simple_split(mut self) -> Self {
+        self.split_on_non_alphanumeric = true;
+        self
+    }
+
     pub fn analyzer(self) -> TextAnalyzer {
         TextAnalyzer::builder(self).build()
     }
@@ -33,6 +41,40 @@ pub struct IcuTokenStream {
     index: usize,
 }
 
+fn push_token(tokens: &mut Vec<Token>, text: &str, offset_from: usize, offset_to: usize) {
+    if offset_from == offset_to {
+        return;
+    }
+
+    let token_text = &text[offset_from..offset_to];
+    if token_text.chars().any(char::is_alphanumeric) {
+        tokens.push(Token {
+            offset_from,
+            offset_to,
+            position: tokens.len(),
+            text: token_text.to_owned(),
+            position_length: 1,
+        });
+    }
+}
+
+fn push_tokens_split_on_non_alphanumeric(
+    tokens: &mut Vec<Token>,
+    text: &str,
+    offset_from: usize,
+    offset_to: usize,
+) {
+    let mut part_start = offset_from;
+    for (relative_offset, c) in text[offset_from..offset_to].char_indices() {
+        if !c.is_alphanumeric() {
+            let delimiter_offset = offset_from + relative_offset;
+            push_token(tokens, text, part_start, delimiter_offset);
+            part_start = delimiter_offset + c.len_utf8();
+        }
+    }
+    push_token(tokens, text, part_start, offset_to);
+}
+
 impl TokenStream for IcuTokenStream {
     fn advance(&mut self) -> bool {
         if self.index < self.tokens.len() {
@@ -63,15 +105,10 @@ impl Tokenizer for IcuTokenizer {
         };
 
         for offset_to in boundaries {
-            let token_text = &text[offset_from..offset_to];
-            if token_text.chars().any(char::is_alphanumeric) {
-                tokens.push(Token {
-                    offset_from,
-                    offset_to,
-                    position: tokens.len(),
-                    text: token_text.to_owned(),
-                    position_length: 1,
-                });
+            if self.split_on_non_alphanumeric {
+                push_tokens_split_on_non_alphanumeric(&mut tokens, text, offset_from, offset_to);
+            } else {
+                push_token(&mut tokens, text, offset_from, offset_to);
             }
             offset_from = offset_to;
         }
@@ -84,14 +121,21 @@ impl Tokenizer for IcuTokenizer {
 mod tests {
     use crate::{IcuTokenizer, Token, TokenStream, Tokenizer};
 
-    fn collect_tokens(text: &str) -> Vec<Token> {
+    fn collect_tokens_with_split(text: &str, split_on_non_alphanumeric: bool) -> Vec<Token> {
         let mut tokenizer = IcuTokenizer::default();
+        if split_on_non_alphanumeric {
+            tokenizer = tokenizer.with_simple_split();
+        }
         let mut stream = tokenizer.token_stream(text);
         let mut tokens = Vec::new();
         stream.process(&mut |token| tokens.push(token.clone()));
         tokens
     }
 
+    fn collect_tokens(text: &str) -> Vec<Token> {
+        collect_tokens_with_split(text, false)
+    }
+
     #[test]
     fn test_icu_tokenizer_segments_mixed_text() {
         let tokens = collect_tokens("Hello, こんにちは世界!");
@@ -124,4 +168,37 @@ mod tests {
             vec!["Mark'd", "ye", "his", "words"]
         );
     }
+
+    #[test]
+    fn test_icu_tokenizer_splits_on_non_alphanumeric_when_enabled() {
+        let tokens = collect_tokens_with_split("foo_bar__baz-alpha.beta", true);
+
+        assert_eq!(
+            tokens
+                .iter()
+                .map(|token| token.text.as_str())
+                .collect::<Vec<_>>(),
+            vec!["foo", "bar", "baz", "alpha", "beta"]
+        );
+        assert_eq!(
+            tokens
+                .iter()
+                .map(|token| (token.offset_from, token.offset_to, token.position))
+                .collect::<Vec<_>>(),
+            vec![(0, 3, 0), (4, 7, 1), (9, 12, 2), (13, 18, 3), (19, 23, 4)]
+        );
+    }
+
+    #[test]
+    fn test_icu_tokenizer_split_control_keeps_icu_segmentation() {
+        let tokens = collect_tokens_with_split("hello_world こんにちは世界", true);
+
+        assert_eq!(
+            tokens
+                .iter()
+                .map(|token| token.text.as_str())
+                .collect::<Vec<_>>(),
+            vec!["hello", "world", "こんにちは", "世界"]
+        );
+    }
 }
diff --git a/rust/lance-tokenizer/src/lower_caser.rs b/rust/lance-tokenizer/src/lower_caser.rs
index a041ac04e1f..3ad430f2f5a 100644
--- a/rust/lance-tokenizer/src/lower_caser.rs
+++ b/rust/lance-tokenizer/src/lower_caser.rs
@@ -47,22 +47,30 @@ pub struct LowerCaserTokenStream<'a, T> {
 
 fn to_lowercase_unicode(text: &str, output: &mut String) {
     output.clear();
-    output.reserve(50);
+    output.reserve(text.len());
     for ch in text.chars() {
         output.extend(ch.to_lowercase());
     }
 }
 
+fn is_lowercase_stable(text: &str) -> bool {
+    text.chars().all(|ch| {
+        let mut lower = ch.to_lowercase();
+        lower.next() == Some(ch) && lower.next().is_none()
+    })
+}
+
 impl<T: TokenStream> TokenStream for LowerCaserTokenStream<'_, T> {
     fn advance(&mut self) -> bool {
         if !self.tail.advance() {
             return false;
         }
-        if self.token_mut().text.is_ascii() {
-            self.token_mut().text.make_ascii_lowercase();
-        } else {
-            to_lowercase_unicode(&self.tail.token().text, self.buffer);
-            mem::swap(&mut self.tail.token_mut().text, self.buffer);
+        let token = self.tail.token_mut();
+        if token.text.is_ascii() {
+            token.text.make_ascii_lowercase();
+        } else if !is_lowercase_stable(&token.text) {
+            to_lowercase_unicode(&token.text, self.buffer);
+            mem::swap(&mut token.text, self.buffer);
         }
         true
     }
@@ -75,3 +83,30 @@ impl<T: TokenStream> TokenStream for LowerCaserTokenStream<'_, T> {
         self.tail.token_mut()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::{LowerCaser, RawTokenizer, TextAnalyzer, Token};
+
+    fn collect_tokens(text: &str) -> Vec<Token> {
+        let mut analyzer = TextAnalyzer::builder(RawTokenizer::default())
+            .filter(LowerCaser)
+            .build();
+        let mut stream = analyzer.token_stream(text);
+        let mut tokens = Vec::new();
+        stream.process(&mut |token| tokens.push(token.clone()));
+        tokens
+    }
+
+    #[test]
+    fn test_lower_caser_unicode_changed() {
+        let tokens = collect_tokens("İSTANBUL");
+        assert_eq!(tokens[0].text, "i\u{307}stanbul");
+    }
+
+    #[test]
+    fn test_lower_caser_unicode_unchanged() {
+        let tokens = collect_tokens("こんにちは世界");
+        assert_eq!(tokens[0].text, "こんにちは世界");
+    }
+}
diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs
index 0c49330a619..9a690b0ec06 100644
--- a/rust/lance-tokenizer/src/stop_word_filter.rs
+++ b/rust/lance-tokenizer/src/stop_word_filter.rs
@@ -12,6 +12,39 @@ use std::sync::Arc;
 
 use crate::{Language, Token, TokenFilter, TokenStream, Tokenizer};
 
+fn all_stop_words() -> impl Iterator<Item = &'static str> {
+    [
+        stop_words::get("ar"),
+        stopwords::DANISH,
+        stopwords::DUTCH,
+        // Use the fuller `stop-words` crate English list (~198 words) rather
+        // than the local Tantivy-style list (~33 words), which omits extremely
+        // common pronouns/function words (you, my, your, we, she, what, ...).
+        // Those omissions let the highest-frequency English tokens through the
+        // ICU stop-word path and build pathologically large posting lists.
+        stop_words::get("en"),
+        stopwords::FINNISH,
+        stopwords::FRENCH,
+        stopwords::GERMAN,
+        stop_words::get("el"),
+        stopwords::HUNGARIAN,
+        stopwords::ITALIAN,
+        stopwords::NORWEGIAN,
+        stopwords::PORTUGUESE,
+        stop_words::get("ro"),
+        stopwords::RUSSIAN,
+        stopwords::SPANISH,
+        stopwords::SWEDISH,
+        stop_words::get("ta"),
+        stop_words::get("tr"),
+        stop_words::get("zh"),
+        stop_words::get("ja"),
+        stop_words::get("ko"),
+    ]
+    .into_iter()
+    .flat_map(|words| words.iter().copied())
+}
+
 #[derive(Clone)]
 pub struct StopWordFilter {
     words: Arc<HashSet<String>>,
@@ -20,28 +53,36 @@ pub struct StopWordFilter {
 impl StopWordFilter {
     pub fn new(language: Language) -> Option<Self> {
         let words = match language {
+            Language::Arabic => stop_words::get("ar"),
             Language::Danish => stopwords::DANISH,
             Language::Dutch => stopwords::DUTCH,
-            Language::English => &[
-                "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into",
-                "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then",
-                "there", "these", "they", "this", "to", "was", "will", "with",
-            ],
+            // Use the fuller `stop-words` crate English list (~198 words); the
+            // local Tantivy-style list (~33 words) omits common pronouns/function
+            // words (you, my, your, we, ...) that would otherwise leak through
+            // stop-word removal and build pathologically large posting lists.
+            Language::English => stop_words::get("en"),
             Language::Finnish => stopwords::FINNISH,
             Language::French => stopwords::FRENCH,
             Language::German => stopwords::GERMAN,
+            Language::Greek => stop_words::get("el"),
             Language::Hungarian => stopwords::HUNGARIAN,
             Language::Italian => stopwords::ITALIAN,
             Language::Norwegian => stopwords::NORWEGIAN,
             Language::Portuguese => stopwords::PORTUGUESE,
+            Language::Romanian => stop_words::get("ro"),
             Language::Russian => stopwords::RUSSIAN,
             Language::Spanish => stopwords::SPANISH,
             Language::Swedish => stopwords::SWEDISH,
-            _ => return None,
+            Language::Tamil => stop_words::get("ta"),
+            Language::Turkish => stop_words::get("tr"),
         };
         Some(Self::remove(words.iter().map(|word| (*word).to_owned())))
     }
 
+    pub fn all() -> Self {
+        Self::remove(all_stop_words().map(str::to_owned))
+    }
+
     pub fn remove<W: IntoIterator<Item = String>>(words: W) -> Self {
         Self {
             words: Arc::new(words.into_iter().collect()),
@@ -49,6 +90,42 @@ impl StopWordFilter {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::all_stop_words;
+    use crate::StopWordFilter;
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_external_stop_word_lists_are_available() {
+        let words = all_stop_words().collect::<HashSet<_>>();
+        for word in ["إلى", "και", "acesta", "அவர்", "ama", "的", "ある", "그리고"]
+        {
+            assert!(
+                words.contains(word),
+                "built-in stop words should contain {word}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_language_stop_word_lists_are_available() {
+        for (language, word) in [
+            (crate::Language::Arabic, "إلى"),
+            (crate::Language::Greek, "και"),
+            (crate::Language::Romanian, "acesta"),
+            (crate::Language::Tamil, "அவர்"),
+            (crate::Language::Turkish, "ama"),
+        ] {
+            let filter = StopWordFilter::new(language).unwrap();
+            assert!(
+                filter.words.contains(word),
+                "{language:?} should contain {word}"
+            );
+        }
+    }
+}
+
 impl TokenFilter for StopWordFilter {
     type Tokenizer<T: Tokenizer> = StopWordFilterWrapper<T>;
 
diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml
index e83353e9a63..762ca513a19 100644
--- a/rust/lance/Cargo.toml
+++ b/rust/lance/Cargo.toml
@@ -32,6 +32,7 @@ lance-namespace = { workspace = true }
 lance-select = { workspace = true }
 lance-tokenizer = { workspace = true }
 lance-table = { workspace = true }
+lance-bitpacking = { workspace = true }
 arrow-arith = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
@@ -43,7 +44,6 @@ arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
 async-recursion.workspace = true
 async-trait.workspace = true
-bitpacking = { workspace = true }
 byteorder.workspace = true
 bytes.workspace = true
 chrono.workspace = true
@@ -56,7 +56,6 @@ crossbeam-queue = { workspace = true }
 crossbeam-skiplist.workspace = true
 # This is already used by datafusion
 dashmap = "6"
-deepsize.workspace = true
 # matches arrow-rs use
 half.workspace = true
 # Fast non-cryptographic hasher for the hot FTS mem-index insert path.
@@ -104,6 +103,7 @@ prost-build.workspace = true
 protobuf-src = { version = "2.1", optional = true }
 
 [target.'cfg(target_os = "linux")'.dev-dependencies]
+pprof.workspace = true
 # Need this so we can prevent dynamic linking in binaries (see cli feature)
 lzma-sys = { version = "0.1" }
 
@@ -138,7 +138,8 @@ parquet = { version = "58", default-features = false, features = ["arrow", "asyn
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] }
 
 [features]
-default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "geo"]
+default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"]
+backtrace = ["lance-core/backtrace"]
 fp16kernels = ["lance-linalg/fp16kernels"]
 # Prevent dynamic linking of lzma, which comes from datafusion
 cli = ["dep:clap", "lzma-sys/static"]
@@ -157,7 +158,11 @@ gcp = ["lance-io/gcp"]
 azure = ["lance-io/azure"]
 oss = ["lance-io/oss"]
 tencent = ["lance-io/tencent"]
+goosefs = ["lance-io/goosefs"]
+tos = ["lance-io/tos"]
 huggingface = ["lance-io/huggingface"]
+# Publish object store metrics via the `metrics` crate.
+metrics = ["lance-io/metrics"]
 geo = ["lance-datafusion/geo", "lance-index/geo"]
 # Enable slow integration tests (disabled by default in CI)
 slow_tests = []
@@ -169,10 +174,22 @@ slow_tests = []
 name = "lq"
 required-features = ["cli"]
 
+[[bin]]
+name = "fm_contains_bench"
+required-features = ["cli"]
+
+[[bin]]
+name = "fm_index_tool"
+required-features = ["cli"]
+
 [[bench]]
 name = "scalar_index"
 harness = false
 
+[[bench]]
+name = "regex_ngram"
+harness = false
+
 [[bench]]
 name = "merge_insert"
 harness = false
@@ -181,6 +198,14 @@ harness = false
 name = "scan"
 harness = false
 
+[[bench]]
+name = "s3_file_reader_diagnostics"
+harness = false
+
+[[bench]]
+name = "count_pushdown"
+harness = false
+
 [[bench]]
 name = "vector_index"
 harness = false
@@ -294,5 +319,9 @@ harness = false
 name = "concurrent_append"
 harness = false
 
+[[bench]]
+name = "hamming"
+harness = false
+
 [lints]
 workspace = true
diff --git a/rust/lance/benches/count_pushdown.rs b/rust/lance/benches/count_pushdown.rs
new file mode 100644
index 00000000000..4f633d489dc
--- /dev/null
+++ b/rust/lance/benches/count_pushdown.rs
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Benchmarks for `COUNT(*)` via the scanner aggregate plan (the path the
+//! `count_pushdown` rule rewrites into `CountFromMaskExec`).
+//!
+//! The dataset uses stable row ids, multiple fragments, and scattered
+//! cross-fragment deletions, with a BTree scalar index on the filter column.
+//! Run on two revisions to compare (e.g. before/after a change to the rule):
+//!
+//! ```text
+//! cargo bench -p lance --bench count_pushdown
+//! ```
+
+use std::sync::Arc;
+
+use arrow_array::types::UInt32Type;
+use criterion::{Criterion, criterion_group, criterion_main};
+use lance::Dataset;
+use lance::dataset::WriteParams;
+use lance::index::DatasetIndexExt;
+use lance_core::utils::tempfile::TempStrDir;
+use lance_datagen::{BatchCount, RowCount, array, gen_batch};
+use lance_index::IndexType;
+use lance_index::scalar::ScalarIndexParams;
+#[cfg(target_os = "linux")]
+use lance_testing::pprof::{Output, PProfProfiler};
+
+const ROWS_PER_FRAGMENT: usize = 100_000;
+const NUM_FRAGMENTS: usize = 50;
+const TOTAL_ROWS: u32 = (ROWS_PER_FRAGMENT * NUM_FRAGMENTS) as u32; // 5,000,000
+
+struct Fixture {
+    _datadir: TempStrDir,
+    dataset: Arc<Dataset>,
+}
+
+impl Fixture {
+    async fn open() -> Self {
+        let datadir = TempStrDir::default();
+        // `value` steps 0..TOTAL_ROWS, so `value < k` selects exactly k rows
+        // (before deletions) and gives precise control over selectivity.
+        let reader = gen_batch()
+            .col("value", array::step::<UInt32Type>())
+            .into_reader_rows(
+                RowCount::from(ROWS_PER_FRAGMENT as u64),
+                BatchCount::from(NUM_FRAGMENTS as u32),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            datadir.as_str(),
+            Some(WriteParams {
+                max_rows_per_file: ROWS_PER_FRAGMENT,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Scatter deletions across every fragment (~1%) to exercise the
+        // deletion mask in stable-id space.
+        dataset.delete("value % 100 = 0").await.unwrap();
+
+        dataset
+            .create_index(
+                &["value"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        Self {
+            _datadir: datadir,
+            dataset: Arc::new(dataset),
+        }
+    }
+}
+
+async fn count_unfiltered(dataset: &Dataset) -> u64 {
+    dataset.scan().count_rows().await.unwrap()
+}
+
+async fn count_filtered(dataset: &Dataset, filter: &str) -> u64 {
+    let mut scanner = dataset.scan();
+    scanner.filter(filter).unwrap();
+    scanner.count_rows().await.unwrap()
+}
+
+fn bench_count(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let fixture = rt.block_on(Fixture::open());
+    let ds = &fixture.dataset;
+
+    c.bench_function("count_unfiltered", |b| {
+        b.iter(|| rt.block_on(count_unfiltered(ds)))
+    });
+
+    // ~1% of rows match.
+    let filter_1pct = format!("value < {}", TOTAL_ROWS / 100);
+    c.bench_function("count_filtered_1pct", |b| {
+        b.iter(|| rt.block_on(count_filtered(ds, &filter_1pct)))
+    });
+
+    // ~50% of rows match.
+    let filter_50pct = format!("value < {}", TOTAL_ROWS / 2);
+    c.bench_function("count_filtered_50pct", |b| {
+        b.iter(|| rt.block_on(count_filtered(ds, &filter_50pct)))
+    });
+}
+
+#[cfg(target_os = "linux")]
+criterion_group!(
+    name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(10)
+        .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    targets = bench_count);
+
+#[cfg(not(target_os = "linux"))]
+criterion_group!(
+    name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(10);
+    targets = bench_count);
+
+criterion_main!(benches);
diff --git a/rust/lance/benches/distributed_vector_build.rs b/rust/lance/benches/distributed_vector_build.rs
index 94ae9007a5d..1e66537c8b4 100644
--- a/rust/lance/benches/distributed_vector_build.rs
+++ b/rust/lance/benches/distributed_vector_build.rs
@@ -290,7 +290,7 @@ async fn build_partial_fixture(dataset: &mut Dataset, bench_case: BenchCase) ->
         builder = builder
             .name("distributed_merge_only".to_string())
             .fragments(fragments)
-            .index_uuid(fixture_uuid.to_string());
+            .index_uuid(fixture_uuid);
         Box::pin(builder.execute_uncommitted()).await.unwrap();
     }
 
@@ -416,7 +416,7 @@ fn bench_distributed_merge_only(c: &mut Criterion) {
                     || prepare_iteration_target(&source_index_dir_fs, &target_index_dir_fs),
                     |_| {
                         rt.block_on(dataset.merge_index_metadata(
-                            &target_uuid.to_string(),
+                            &target_uuid,
                             IndexType::IvfPq,
                             None,
                             noop_progress(),
diff --git a/rust/lance/benches/hamming.rs b/rust/lance/benches/hamming.rs
new file mode 100644
index 00000000000..7e926a795db
--- /dev/null
+++ b/rust/lance/benches/hamming.rs
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Benchmark for hamming distance clustering.
+//!
+//! This benchmark tests the pairwise hamming distance computation and clustering
+//! performance at various scales.
+//!
+//! Run with: cargo bench -p lance --bench hamming
+//!
+//! Environment variables:
+//!   - DATASET_URI: Path to a dataset with a hash column (optional, generates random if not set)
+//!   - HASH_COLUMN: Name of the hash column (default: "hash")
+//!   - SAMPLE_SIZE: Number of rows to sample (default: 10000)
+//!   - THRESHOLD: Hamming distance threshold (default: 10)
+
+#![allow(clippy::print_stdout)]
+
+use std::env;
+use std::sync::Arc;
+use std::time::Instant;
+
+use arrow_array::{FixedSizeListArray, RecordBatch, RecordBatchIterator, UInt8Array};
+use arrow_schema::{DataType, Field, FieldRef, Schema};
+use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
+use lance_arrow::FixedSizeListArrayExt;
+use rand::Rng;
+
+use lance::index::vector::hamming::{
+    hamming_clustering_for_sample, hamming_clustering_from_hashes,
+};
+use lance::{Dataset, dataset::WriteParams};
+use lance_linalg::distance::pairwise_hamming_distance_parallel;
+
+#[cfg(target_os = "linux")]
+use lance_testing::pprof::{Output, PProfProfiler};
+
+/// Generate random 64-bit hashes.
+fn generate_random_hashes(n: usize) -> Vec<u64> {
+    let mut rng = rand::rng();
+    (0..n).map(|_| rng.random()).collect()
+}
+
+/// Generate random hash dataset as Arrow arrays.
+fn generate_hash_batch(num_rows: usize) -> RecordBatch {
+    let mut rng = rand::rng();
+
+    // Generate random bytes for the hashes (8 bytes per hash)
+    let bytes: Vec<u8> = (0..num_rows * 8).map(|_| rng.random()).collect();
+    let values = UInt8Array::from(bytes);
+
+    let hash_array = FixedSizeListArray::try_new_from_values(values, 8).unwrap();
+
+    let schema = Arc::new(Schema::new(vec![Field::new(
+        "hash",
+        DataType::FixedSizeList(FieldRef::new(Field::new("item", DataType::UInt8, true)), 8),
+        false,
+    )]));
+
+    RecordBatch::try_new(schema, vec![Arc::new(hash_array)]).unwrap()
+}
+
+/// Create a test dataset with random hashes.
+async fn create_hash_dataset(path: &std::path::Path, num_rows: usize) {
+    let batch = generate_hash_batch(num_rows);
+    let schema = batch.schema();
+
+    let write_params = WriteParams {
+        max_rows_per_file: num_rows,
+        max_rows_per_group: 10_000,
+        ..Default::default()
+    };
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+    Dataset::write(reader, path.to_str().unwrap(), Some(write_params))
+        .await
+        .unwrap();
+}
+
+/// Benchmark pure pairwise hamming computation (no I/O).
+fn bench_pairwise_compute(c: &mut Criterion) {
+    let mut group = c.benchmark_group("hamming_pairwise_compute");
+
+    for size in [1_000, 5_000, 10_000, 20_000] {
+        let hashes = generate_random_hashes(size);
+        let total_pairs = (size as u64) * (size as u64 - 1) / 2;
+
+        group.throughput(Throughput::Elements(total_pairs));
+        group.bench_with_input(BenchmarkId::new("parallel", size), &hashes, |b, hashes| {
+            b.iter(|| {
+                pairwise_hamming_distance_parallel(hashes, None, Some(10));
+            });
+        });
+    }
+
+    group.finish();
+}
+
+/// Benchmark full clustering pipeline (compute + cluster).
+fn bench_cluster_hashes(c: &mut Criterion) {
+    let mut group = c.benchmark_group("hamming_cluster");
+
+    for size in [1_000, 5_000, 10_000] {
+        let hashes = generate_random_hashes(size);
+
+        group.bench_with_input(
+            BenchmarkId::new("full_pipeline", size),
+            &hashes,
+            |b, hashes| {
+                b.iter(|| {
+                    hamming_clustering_from_hashes(hashes, None, 10);
+                });
+            },
+        );
+    }
+
+    group.finish();
+}
+
+/// Benchmark with dataset I/O (if DATASET_URI is set).
+fn bench_dataset_cluster(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    // Check if we should use an external dataset
+    let dataset_uri = env::var("DATASET_URI").ok();
+    let hash_column = env::var("HASH_COLUMN").unwrap_or_else(|_| "hash".to_string());
+    let sample_size: usize = env::var("SAMPLE_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(10_000);
+    let threshold: u32 = env::var("THRESHOLD")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(10);
+
+    let mut group = c.benchmark_group("hamming_dataset");
+
+    if let Some(uri) = dataset_uri {
+        // Use external dataset
+        println!("Using external dataset: {}", uri);
+        println!(
+            "Column: {}, Sample: {}, Threshold: {}",
+            hash_column, sample_size, threshold
+        );
+
+        let dataset = rt.block_on(async { Dataset::open(&uri).await.unwrap() });
+
+        group.bench_function(format!("external_sample_{}", sample_size), |b| {
+            b.to_async(&rt).iter(|| async {
+                hamming_clustering_for_sample(&dataset, &hash_column, Some(sample_size), threshold)
+                    .await
+                    .unwrap()
+            });
+        });
+    } else {
+        // Create temporary dataset with random hashes
+        let temp_dir = tempfile::tempdir().unwrap();
+        let uri = temp_dir.path().join("bench_hashes.lance");
+
+        rt.block_on(async {
+            create_hash_dataset(&uri, 100_000).await;
+        });
+
+        let dataset = rt.block_on(async { Dataset::open(uri.to_str().unwrap()).await.unwrap() });
+
+        for sample in [1_000, 5_000, 10_000] {
+            group.bench_function(format!("generated_sample_{}", sample), |b| {
+                let ds = dataset.clone();
+                b.to_async(&rt).iter(|| {
+                    let ds = ds.clone();
+                    async move {
+                        hamming_clustering_for_sample(&ds, "hash", Some(sample), 10)
+                            .await
+                            .unwrap()
+                    }
+                });
+            });
+        }
+    }
+
+    group.finish();
+}
+
+/// Quick standalone benchmark that prints results (for quick testing).
+#[allow(dead_code)]
+fn run_quick_bench() {
+    println!("=== Hamming Distance Clustering Benchmark ===\n");
+
+    let sizes = [1_000, 5_000, 10_000, 20_000];
+
+    for &size in &sizes {
+        let hashes = generate_random_hashes(size);
+        let total_pairs = (size as u64) * (size as u64 - 1) / 2;
+
+        println!("Size: {} rows, {} pairs", size, total_pairs);
+        let start = Instant::now();
+        let reader = hamming_clustering_from_hashes(&hashes, None, 10);
+        // Consume the reader to count clusters
+        let cluster_count: usize = reader.map(|b| b.unwrap().num_rows()).sum();
+        let elapsed = start.elapsed();
+
+        let pairs_per_sec = total_pairs as f64 / elapsed.as_secs_f64();
+        println!(
+            "  Total time: {:?} ({:.2}M pairs/sec)",
+            elapsed,
+            pairs_per_sec / 1_000_000.0
+        );
+        println!("  Total clusters: {}", cluster_count);
+        println!();
+    }
+}
+
+#[cfg(target_os = "linux")]
+criterion_group! {
+    name = benches;
+    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    targets = bench_pairwise_compute, bench_cluster_hashes, bench_dataset_cluster
+}
+
+#[cfg(not(target_os = "linux"))]
+criterion_group!(
+    benches,
+    bench_pairwise_compute,
+    bench_cluster_hashes,
+    bench_dataset_cluster
+);
+
+criterion_main!(benches);
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
index 53bfc61fe23..63f9368c3c5 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs
@@ -706,10 +706,14 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result<serde_jso
         if *is_phrase {
             scanner.full_text_phrase(TEXT_COL, q, 0);
         } else {
-            scanner.full_text_search(TEXT_COL, q);
+            scanner.full_text_search(
+                FullTextSearchQuery::new(q.to_string())
+                    .with_column(TEXT_COL.to_string())
+                    .unwrap(),
+            )?;
         }
-        scanner.project(&["id"]);
-        scanner.limit(args.top_k, None);
+        scanner.project(&["id"])?;
+        scanner.limit(Some(args.top_k as i64), None)?;
         let t0 = Instant::now();
         let stream = scanner.try_into_stream().await?;
         let batches: Vec<RecordBatch> = stream.try_collect().await?;
diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
index 578eb0bc0ba..08fdb76b95f 100644
--- a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
+++ b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs
@@ -524,7 +524,7 @@ async fn run_local(planner: &LsmFtsSearchPlanner, queries: &[String], k: usize)
     for q in queries {
         let t0 = Instant::now();
         let plan = planner
-            .plan_search(TEXT_COL, FullTextSearchQuery::new(q.clone()), k, None)
+            .plan_search(TEXT_COL, FullTextSearchQuery::new(q.clone()), Some(k), None)
             .await?;
         let stream = plan.execute(0, ctx.task_ctx())?;
         let batches: Vec<RecordBatch> = stream.try_collect().await?;
@@ -684,6 +684,8 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
     let batches_per_gen = (args.max_memtable_rows / args.batch_rows).max(1);
     let config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: false,
diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
index d99ea97a70a..757c2539642 100644
--- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
+++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs
@@ -788,6 +788,8 @@ async fn run_lance(
     let big = args.rows.saturating_mul(args.value_size + 256).max(1 << 30);
     let config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: true,
         sync_indexed_write: true,
@@ -817,8 +819,12 @@ async fn run_lance(
         };
         while lo < part_end {
             let hi = (lo + args.batch_rows).min(part_end);
-            let batch =
-                make_batch(schema.clone(), &insert_order[lo..hi], args.value_size, key_type);
+            let batch = make_batch(
+                schema.clone(),
+                &insert_order[lo..hi],
+                args.value_size,
+                key_type,
+            );
             writer.put(vec![batch]).await?;
             lo = hi;
         }
@@ -1519,7 +1525,14 @@ async fn run_lance_flushed(
     let peak_rss_mb = sampler.stop();
     println!(
         "[lance] read p50={:.2}us p95={:.2}us p99={:.2}us mean={:.2}us qps_1t={:.0} qps_{}t={:.0} (hits={hits} miss={misses_resolved}) peak_rss={:.0}MB",
-        stats.p50_us, stats.p95_us, stats.p99_us, stats.mean_us, read_qps_1t, args.threads, read_qps_nt, peak_rss_mb
+        stats.p50_us,
+        stats.p95_us,
+        stats.p99_us,
+        stats.mean_us,
+        read_qps_1t,
+        args.threads,
+        read_qps_nt,
+        peak_rss_mb
     );
 
     Ok(EngineResult {
diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
index ca57b475560..cb9e6413ac1 100644
--- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
+++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs
@@ -334,6 +334,8 @@ async fn run_lookup(args: &Args) -> Result<serde_json::Value> {
     let max_memtable_rows = args.max_memtable_rows;
     let config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: false,
diff --git a/rust/lance/benches/mem_wal/read/memtable_read.rs b/rust/lance/benches/mem_wal/read/memtable_read.rs
index 5d24943e4cc..27d21e95763 100644
--- a/rust/lance/benches/mem_wal/read/memtable_read.rs
+++ b/rust/lance/benches/mem_wal/read/memtable_read.rs
@@ -773,8 +773,14 @@ fn bench_fts(c: &mut Criterion) {
                 async move {
                     let mut total_found = 0usize;
                     for (mut scanner, term) in scanners.into_iter().zip(terms.iter()) {
+                        scanner
+                            .full_text_search(
+                                FullTextSearchQuery::new(term.to_string())
+                                    .with_column("text".to_string())
+                                    .unwrap(),
+                            )
+                            .unwrap();
                         let batches: Vec<RecordBatch> = scanner
-                            .full_text_search("text", term)
                             .try_into_stream()
                             .await
                             .unwrap()
@@ -947,11 +953,14 @@ fn bench_vector_search(c: &mut Criterion) {
             let query_array: Arc<dyn arrow_array::Array> = Arc::new(Float32Array::from(q.clone()));
             b.to_async(&rt).iter(|| {
                 let query_array = query_array.clone();
-                async {
+                let memtable = &memtable;
+                async move {
                     let mut scanner = memtable.scan();
+                    scanner
+                        .nearest("vector", query_array.as_ref(), k)
+                        .unwrap()
+                        .nprobes(8);
                     let batches: Vec<RecordBatch> = scanner
-                        .nearest("vector", query_array, k)
-                        .nprobes(8)
                         .try_into_stream()
                         .await
                         .unwrap()
diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
index fc0d6984b53..7b9f35b73c9 100644
--- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
+++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs
@@ -385,6 +385,8 @@ async fn run_checkpoint(
     }
     let writer_config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: true,
@@ -542,8 +544,7 @@ async fn run_checkpoint(
         let uuid = idx_metas
             .first()
             .ok_or_else(|| lance_core::Error::io("flushed gen has no vector index".to_string()))?
-            .uuid
-            .to_string();
+            .uuid;
         let vidx = gen_ds
             .open_vector_index(VECTOR_COL, &uuid, &NoOpMetricsCollector)
             .await?;
@@ -590,6 +591,7 @@ async fn run_checkpoint(
                     use_index: true,
                     query_parallelism: 1,
                     dist_q_c: 0.0,
+                    approx_mode: Default::default(),
                 };
                 // IVFIndex::search is intentionally unimplemented (top-level does
                 // partition-aware search); replicate the ANN exec node: pick the
@@ -665,7 +667,7 @@ async fn run_checkpoint(
             active.schema.clone(),
         );
         let q_arr: ArrayRef = Arc::new(q_fsl);
-        scanner.nearest(VECTOR_COL, q_arr, k);
+        scanner.nearest(VECTOR_COL, q_arr.as_ref(), k)?;
         let h_t = Instant::now();
         let stream = scanner.try_into_stream().await?;
         let batches: Vec<RecordBatch> = stream.try_collect().await?;
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
index 57658637b33..8bff64dfcca 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs
@@ -208,6 +208,8 @@ async fn main() -> lance_core::Result<()> {
     let total_batches_max = max_rows.div_ceil(batch_size);
     let writer_config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write,
         sync_indexed_write: true,
@@ -298,7 +300,7 @@ async fn main() -> lance_core::Result<()> {
                     active.schema.clone(),
                 );
                 let q_arr: ArrayRef = Arc::new(q_fsl);
-                scanner.nearest(VECTOR_COL, q_arr, 10);
+                scanner.nearest(VECTOR_COL, q_arr.as_ref(), 10)?;
                 let t = Instant::now();
                 let stream = scanner.try_into_stream().await?;
                 let _: Vec<RecordBatch> = stream.try_collect().await?;
diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
index 07c58a72bc4..632bd37626c 100644
--- a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
+++ b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs
@@ -516,6 +516,8 @@ async fn run_search(args: &Args) -> Result<serde_json::Value> {
     let row_bytes = VECTOR_DIM * 4 + 8; // embedding + id
     let config = ShardWriterConfig {
         shard_id,
+        max_wal_persist_retries: 3,
+        wal_persist_retry_base_delay: std::time::Duration::from_millis(50),
         shard_spec_id: 0,
         durable_write: false,
         sync_indexed_write: false,
diff --git a/rust/lance/benches/mem_wal/write/mem_wal_write.rs b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
index 24f3a0d7c8f..db28c5b33ab 100644
--- a/rust/lance/benches/mem_wal/write/mem_wal_write.rs
+++ b/rust/lance/benches/mem_wal/write/mem_wal_write.rs
@@ -630,6 +630,9 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                 let config = ShardWriterConfig {
                                     shard_id,
                                     shard_spec_id: 0,
+                                    max_wal_persist_retries: 3,
+                                    wal_persist_retry_base_delay:
+                                        std::time::Duration::from_millis(50),
                                     durable_write: durable,
                                     sync_indexed_write: indexed,
                                     max_wal_buffer_size: max_wal_buffer_size
@@ -649,8 +652,12 @@ fn bench_lance_memwal_write(c: &mut Criterion) {
                                     backpressure_log_interval: default_config
                                         .backpressure_log_interval,
                                     stats_log_interval: default_config.stats_log_interval,
+                                    frozen_memtable_grace: default_config.frozen_memtable_grace,
                                     enable_memtable,
                                     hnsw_params: default_config.hnsw_params,
+                                    warmer: None,
+                                    store_params: default_config.store_params,
+                                    session: default_config.session,
                                 };
 
                                 // Get writer through Dataset API (index configs loaded automatically)
diff --git a/rust/lance/benches/merge_insert.rs b/rust/lance/benches/merge_insert.rs
index b003d03de44..5de190ffde2 100644
--- a/rust/lance/benches/merge_insert.rs
+++ b/rust/lance/benches/merge_insert.rs
@@ -15,6 +15,9 @@
 //! computation. The other shapes (`clean`, `with_new_rows_only`,
 //! `with_deletions_only`) skip that branch and serve as controls.
 //!
+//! The `composite_key_indexed` shape covers merge_insert joined on two
+//! indexed key columns, which probes every index and AND-folds the results.
+//!
 //! Run with `cargo bench --bench merge_insert`.
 
 use std::sync::Arc;
@@ -127,6 +130,90 @@ async fn one_merge_insert(ds: Arc<Dataset>, base_existing: i64, base_new: i64) {
     let _ = job.execute_reader(reader).await.unwrap();
 }
 
+// --- Composite-key shape: join on (a, b) with a BTree index on each key ---
+//
+// Exercises the multi-index probe path: one `IsIn` query per indexed key
+// column, AND-folded inside a single `MapIndexExec`, followed by the
+// composite hash join.  `b` is a deterministic function of `a` so source
+// rows can target existing composite keys without tracking extra state.
+
+fn composite_schema() -> Arc<ArrowSchema> {
+    Arc::new(ArrowSchema::new(vec![
+        Field::new("a", DataType::Int64, false),
+        Field::new("b", DataType::Int64, false),
+    ]))
+}
+
+fn composite_b(a: i64) -> i64 {
+    a * 7 + 3
+}
+
+fn make_composite_batch(start: i64, n: usize) -> RecordBatch {
+    let a = Int64Array::from_iter_values(start..start + n as i64);
+    let b = Int64Array::from_iter_values((start..start + n as i64).map(composite_b));
+    RecordBatch::try_new(composite_schema(), vec![Arc::new(a), Arc::new(b)]).unwrap()
+}
+
+async fn build_composite_key_indexed() -> (TempStrDir, Arc<Dataset>) {
+    let dir = TempStrDir::default();
+    let path = dir.as_str().to_string();
+    let total = ROWS_PER_FRAG * NUM_FRAGS;
+    let mut batches = Vec::new();
+    let mut start = 0i64;
+    while (start as u64) < total {
+        let n = (total - start as u64).min(ROWS_PER_FRAG) as usize;
+        batches.push(make_composite_batch(start, n));
+        start += n as i64;
+    }
+    let params = WriteParams {
+        max_rows_per_file: ROWS_PER_FRAG as usize,
+        max_rows_per_group: ROWS_PER_FRAG as usize,
+        mode: WriteMode::Create,
+        ..Default::default()
+    };
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), composite_schema());
+    Dataset::write(reader, &path, Some(params)).await.unwrap();
+
+    let mut ds = Dataset::open(&path).await.unwrap();
+    for col in ["a", "b"] {
+        ds.create_index(
+            &[col],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+    }
+    (dir, Arc::new(ds))
+}
+
+/// Composite-key merge_insert op: 30 updates of existing (a, b) pairs +
+/// 70 inserts of new pairs, joined on both columns.
+async fn one_composite_merge_insert(ds: Arc<Dataset>, base_existing: i64, base_new: i64) {
+    let mut a_vals: Vec<i64> = (0..30).map(|i| base_existing + i).collect();
+    a_vals.extend(base_new..base_new + 70);
+    let b_vals: Vec<i64> = a_vals.iter().copied().map(composite_b).collect();
+    let batch = RecordBatch::try_new(
+        composite_schema(),
+        vec![
+            Arc::new(Int64Array::from(a_vals)),
+            Arc::new(Int64Array::from(b_vals)),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(std::iter::once(Ok(batch)), composite_schema());
+
+    let mut builder =
+        MergeInsertBuilder::try_new(ds, vec!["a".to_string(), "b".to_string()]).unwrap();
+    builder
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll);
+    let job = builder.try_build().unwrap();
+    let _ = job.execute_reader(reader).await.unwrap();
+}
+
 async fn build_clean() -> (TempStrDir, Arc<Dataset>) {
     let dir = TempStrDir::default();
     let path = dir.as_str().to_string();
@@ -161,10 +248,12 @@ async fn build_with_new_rows_and_deletions() -> (TempStrDir, Arc<Dataset>) {
     (dir, Arc::new(ds))
 }
 
-fn bench_one_shape<F, Fut>(c: &mut Criterion, name: &str, builder: F)
+fn bench_one_shape<F, Fut, M, MFut>(c: &mut Criterion, name: &str, builder: F, merge: M)
 where
     F: FnOnce() -> Fut,
     Fut: std::future::Future<Output = (TempStrDir, Arc<Dataset>)>,
+    M: Fn(Arc<Dataset>, i64, i64) -> MFut + Copy,
+    MFut: std::future::Future<Output = ()>,
 {
     let rt = tokio::runtime::Runtime::new().unwrap();
     let (_dir, ds) = rt.block_on(builder());
@@ -183,29 +272,40 @@ where
                 bench_ds.restore().await.unwrap();
                 let arc = Arc::new(bench_ds);
                 // base_existing in the indexed range, base_new beyond all data so it's an insert.
-                one_merge_insert(arc, 100, total + 1_000_000).await;
+                merge(arc, 100, total + 1_000_000).await;
             })
         })
     });
 }
 
 fn bench_merge_insert(c: &mut Criterion) {
-    bench_one_shape(c, "merge_insert/clean", build_clean);
+    bench_one_shape(c, "merge_insert/clean", build_clean, one_merge_insert);
     bench_one_shape(
         c,
         "merge_insert/with_new_rows_only",
         build_with_new_rows_only,
+        one_merge_insert,
     );
     bench_one_shape(
         c,
         "merge_insert/with_deletions_only",
         build_with_deletions_only,
+        one_merge_insert,
     );
     // The shape that exercises the AllowList(Full) - BlockList(Partial) path.
     bench_one_shape(
         c,
         "merge_insert/with_new_rows_and_deletions",
         build_with_new_rows_and_deletions,
+        one_merge_insert,
+    );
+    // Composite key joined on (a, b), both BTree-indexed: exercises the
+    // AND-folded multi-index probe inside MapIndexExec.
+    bench_one_shape(
+        c,
+        "merge_insert/composite_key_indexed",
+        build_composite_key_indexed,
+        one_composite_merge_insert,
     );
 }
 
diff --git a/rust/lance/benches/regex_ngram.rs b/rust/lance/benches/regex_ngram.rs
new file mode 100644
index 00000000000..76f597ad9cb
--- /dev/null
+++ b/rust/lance/benches/regex_ngram.rs
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Benchmark: regex predicate scans over an ngram-indexed string column.
+//!
+//! Each query is a `regexp_match(doc, '...')` filter against a dataset that has
+//! an NGram index on `doc`. The query set spans a selective AND pattern, an
+//! alternation, a plain literal (rewritten to an infix LIKE before it reaches
+//! the index), and a deliberately non-accelerable pattern (`a.b`, which yields
+//! no trigram) that serves as a regression guard.
+//!
+//! On `main` none of these use the index (regex falls through to a full scan +
+//! recheck); with the ngram-regex acceleration the index prunes candidates for
+//! the first three while `a.b` stays a full scan. Capture a baseline on `main`
+//! with `--save-baseline before_7130`, then compare after the change with
+//! `--baseline before_7130`.
+
+use std::hint::black_box;
+use std::sync::Arc;
+use std::time::Duration;
+
+use arrow::array::AsArray;
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use criterion::{Criterion, criterion_group, criterion_main};
+use futures::TryStreamExt;
+use lance::Dataset;
+use lance::index::DatasetIndexExt;
+use lance_core::utils::tempfile::TempStrDir;
+use lance_datagen::{RowCount, array};
+use lance_index::IndexType;
+use lance_index::scalar::ScalarIndexParams;
+#[cfg(target_os = "linux")]
+use lance_testing::pprof::{Output, PProfProfiler};
+
+const TOTAL: usize = 200_000;
+
+/// Build the `doc` column: random sentences with rare markers injected into a
+/// small fraction of rows so the regex queries have controlled selectivity.
+/// The markers (`zqxwvu`, `needlexyz`, `qwerasdf`) are unlikely to appear in
+/// the generated English-word sentences.
+fn build_docs() -> StringArray {
+    let mut sentence_gen = array::random_sentence(1, 30, false);
+    let base = sentence_gen
+        .generate_default(RowCount::from(TOTAL as u64))
+        .unwrap();
+    let base = base.as_string::<i32>();
+    let docs = (0..TOTAL).map(|i| {
+        let sentence = base.value(i);
+        if i % 200 == 0 {
+            // ~0.5% of rows match `zqxwvu.*needlexyz` and `zqxwvu`.
+            format!("{sentence} zqxwvu needlexyz")
+        } else if i % 211 == 0 {
+            // A second marker for the alternation query.
+            format!("{sentence} qwerasdf")
+        } else {
+            sentence.to_string()
+        }
+    });
+    StringArray::from_iter_values(docs)
+}
+
+async fn build_dataset(tempdir: &TempStrDir) -> Arc<Dataset> {
+    let schema = Arc::new(Schema::new(vec![Field::new("doc", DataType::Utf8, false)]));
+    let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(build_docs())]).unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+    let mut dataset = Dataset::write(reader, tempdir.as_str(), None)
+        .await
+        .unwrap();
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::NGram,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+    Arc::new(dataset)
+}
+
+async fn scan_filter(dataset: &Dataset, filter: &str) -> usize {
+    let mut scanner = dataset.scan();
+    scanner.filter(filter).unwrap();
+    let stream = scanner.try_into_stream().await.unwrap();
+    let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+    batches.iter().map(|b| b.num_rows()).sum()
+}
+
+fn bench_regex_ngram(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let tempdir = TempStrDir::default();
+    let dataset = rt.block_on(build_dataset(&tempdir));
+
+    let queries = [
+        ("selective_and", "regexp_match(doc, 'zqxwvu.*needlexyz')"),
+        (
+            "alternation",
+            "regexp_match(doc, '(zqxwvu|qwerasdf|needlexyz)')",
+        ),
+        ("plain_literal", "regexp_match(doc, 'zqxwvu')"),
+        ("non_accelerable_a_dot_b", "regexp_match(doc, 'a.b')"),
+    ];
+
+    let mut group = c.benchmark_group("regex_ngram");
+    group
+        .sample_size(10)
+        .measurement_time(Duration::from_secs(15));
+    for (name, filter) in queries {
+        group.bench_function(name, |b| {
+            b.iter(|| black_box(rt.block_on(scan_filter(&dataset, filter))));
+        });
+    }
+    group.finish();
+}
+
+#[cfg(target_os = "linux")]
+criterion_group!(
+    name = benches;
+    config = Criterion::default()
+        .significance_level(0.1)
+        .sample_size(10)
+        .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    targets = bench_regex_ngram);
+
+#[cfg(not(target_os = "linux"))]
+criterion_group!(
+    name = benches;
+    config = Criterion::default().significance_level(0.1).sample_size(10);
+    targets = bench_regex_ngram);
+
+criterion_main!(benches);
diff --git a/rust/lance/benches/s3_file_reader_diagnostics.rs b/rust/lance/benches/s3_file_reader_diagnostics.rs
new file mode 100644
index 00000000000..5c661f447e3
--- /dev/null
+++ b/rust/lance/benches/s3_file_reader_diagnostics.rs
@@ -0,0 +1,2356 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#![allow(clippy::print_stdout)]
+#![recursion_limit = "256"]
+
+use std::collections::BTreeMap;
+use std::env;
+use std::fs;
+use std::hint::black_box;
+use std::ops::Range;
+use std::sync::{
+    Arc, Mutex,
+    atomic::{AtomicU64, Ordering},
+};
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use arrow_array::RecordBatch;
+use futures::future::BoxFuture;
+use futures::stream::{FuturesOrdered, FuturesUnordered};
+use futures::{FutureExt, StreamExt, TryStreamExt};
+use lance::dataset::ProjectionRequest;
+use lance::dataset::builder::DatasetBuilder;
+use lance::dataset::fragment::{FileFragment, FragReadConfig};
+use lance::dataset::scanner::{ExecutionStatsCallback, ExecutionSummaryCounts};
+use lance_core::datatypes::Schema;
+use lance_encoding::decoder::PageEncoding;
+use lance_encoding::format::pb21;
+use lance_file::reader::{
+    DEFAULT_READ_CHUNK_SIZE, FileReader as LanceFileReader, FileReaderOptions,
+};
+use lance_io::object_store::ObjectStore as LanceObjectStore;
+use lance_io::scheduler::{FileScheduler, ScanScheduler, ScanStats, SchedulerConfig};
+use lance_io::utils::CachedFileSize;
+use serde_json::{Value, json};
+use tracing::field::{Field, Visit};
+use tracing::subscriber::Interest;
+use tracing::{Event, Metadata, Subscriber};
+use tracing_subscriber::layer::{Context, Layer};
+use tracing_subscriber::prelude::*;
+
+type Error = Box<dyn std::error::Error + Send + Sync>;
+type Result<T> = std::result::Result<T, Error>;
+
+const GIB: u64 = 1024 * 1024 * 1024;
+const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state";
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum SchedulerQueueKind {
+    Standard,
+    Lite,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct SchedulerDiagnostics {
+    kind: SchedulerQueueKind,
+    stats: ScanStats,
+    io_capacity: u64,
+    iops_available: u64,
+    active_iops: u64,
+    pending_iops: u64,
+    pending_bytes: u64,
+    bytes_available: i64,
+    bytes_reserved: i64,
+    io_buffer_size_bytes: u64,
+    priorities_in_flight: u64,
+    no_backpressure: bool,
+    head_task_bytes: Option<u64>,
+    head_task_priority_high: Option<u64>,
+    head_task_priority_low: Option<u64>,
+    min_in_flight_priority_high: Option<u64>,
+    min_in_flight_priority_low: Option<u64>,
+    head_task_can_deliver: Option<bool>,
+    head_task_priority_bypass: Option<bool>,
+    head_task_blocked_by_iops: Option<bool>,
+    head_task_blocked_by_bytes: Option<bool>,
+}
+
+#[derive(Debug, Clone)]
+struct Config {
+    backend: Backend,
+    uri: String,
+    dataset_version: u64,
+    columns: Option<Vec<String>>,
+    limit_rows: u64,
+    target_bytes: Option<u64>,
+    raw_range_size_bytes: u64,
+    raw_range_mode: RawRangeMode,
+    raw_column_indices: Option<Vec<u32>>,
+    raw_submit_mode: RawSubmitMode,
+    raw_completion_mode: RawCompletionMode,
+    take_repetitions: u64,
+    io_buffer_gib: Vec<Option<u64>>,
+    batch_size: u32,
+    batch_size_bytes: Option<u64>,
+    skip_batch_byte_accounting: bool,
+    read_chunk_size: Option<u64>,
+    fragment_concurrency: usize,
+    batch_concurrency: usize,
+    sample_ms: u64,
+    out_dir: String,
+    case_name: String,
+    describe_layout: bool,
+    detach_fragment_streams: bool,
+    drop_read_tasks: bool,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum Backend {
+    FileReader,
+    Scanner,
+    SchedulerRaw,
+    DatasetTake,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawSubmitMode {
+    Single,
+    SplitNoConcat,
+    SplitConcat,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawRangeMode {
+    FileSequential,
+    MetadataPages,
+    MetadataPagesRoundRobin,
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+enum RawCompletionMode {
+    Unordered,
+    Ordered,
+}
+
+impl RawRangeMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::FileSequential => "file-sequential",
+            Self::MetadataPages => "metadata-pages",
+            Self::MetadataPagesRoundRobin => "metadata-pages-round-robin",
+        }
+    }
+}
+
+impl RawSubmitMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Single => "single",
+            Self::SplitNoConcat => "split-no-concat",
+            Self::SplitConcat => "split-concat",
+        }
+    }
+}
+
+impl RawCompletionMode {
+    fn name(self) -> &'static str {
+        match self {
+            Self::Unordered => "unordered",
+            Self::Ordered => "ordered",
+        }
+    }
+}
+
+impl Backend {
+    fn name(self) -> &'static str {
+        match self {
+            Self::FileReader => "lance-file-reader",
+            Self::Scanner => "lance-scanner",
+            Self::SchedulerRaw => "lance-scheduler-raw",
+            Self::DatasetTake => "lance-dataset-take",
+        }
+    }
+
+    fn layer(self) -> &'static str {
+        match self {
+            Self::FileReader => "file-reader",
+            Self::Scanner => "scanner",
+            Self::SchedulerRaw => "scheduler",
+            Self::DatasetTake => "dataset-take",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct CpuSample {
+    idle: u64,
+    total: u64,
+}
+
+#[derive(Debug, Default)]
+struct SharedCounters {
+    fragments_started: AtomicU64,
+    fragments_completed: AtomicU64,
+    batch_futures_emitted: AtomicU64,
+    batch_futures_received: AtomicU64,
+    batches_completed: AtomicU64,
+    rows_completed: AtomicU64,
+    arrow_bytes: AtomicU64,
+    open_reader_ns: AtomicU64,
+    read_stream_create_ns: AtomicU64,
+    next_batch_poll_ns: AtomicU64,
+    channel_send_wait_ns: AtomicU64,
+    decode_ns: AtomicU64,
+    raw_reassemble_ns: AtomicU64,
+}
+
+#[derive(Debug)]
+struct CaseStats {
+    rows: u64,
+    batches: u64,
+    arrow_bytes: u64,
+    planned_fragments: usize,
+    planned_rows: u64,
+    elapsed: Duration,
+    producer_finished_at: Option<Duration>,
+    peak_decode_in_flight: usize,
+    cpu_avg: Option<f64>,
+    scheduler_diagnostics: SchedulerDiagnostics,
+    counters: Arc<SharedCounters>,
+    samples: Vec<Value>,
+}
+
+#[derive(Debug)]
+struct LastSample {
+    elapsed: Duration,
+    scheduler_stats: ScanStats,
+    rows: u64,
+    batches: u64,
+    arrow_bytes: u64,
+}
+
+fn usage() -> &'static str {
+    "usage: s3_file_reader_diagnostics --uri <s3-dataset-uri> \
+     [--backend <file-reader|scanner|scheduler-raw>] \
+     [--dataset-version <n>] [--columns <all|col[,col...]>] \
+     [--limit-rows <n>] [--target-bytes <n>] [--raw-range-size-bytes <n>] \
+     [--raw-range-mode <file-sequential|metadata-pages|metadata-pages-round-robin>] \
+     [--raw-column-indices <all|n[,n...]>] \
+     [--raw-submit-mode <single|split-no-concat|split-concat>] \
+     [--raw-completion-mode <unordered|ordered>] \
+     [--take-repetitions <n>] \
+     [--io-buffer-gib <auto|n[,n...]>] \
+     [--batch-size <n>] [--batch-size-bytes <n>] [--read-chunk-size <n>] \
+     [--skip-batch-byte-accounting] \
+     [--fragment-concurrency <n>] [--batch-concurrency <n>] \
+     [--sample-ms <n>] [--out-dir <path>] [--case <name>] \
+     [--detach-fragment-streams] [--drop-read-tasks] [--describe-layout]"
+}
+
+fn parse_args() -> Result<Config> {
+    let mut backend = Backend::FileReader;
+    let mut uri = None;
+    let mut dataset_version = 1u64;
+    let mut columns = Some(vec!["vector".to_string()]);
+    let mut limit_rows = 67_108_864u64;
+    let mut target_bytes = None;
+    let mut raw_range_size_bytes = 16 * 1024 * 1024;
+    let mut raw_range_mode = RawRangeMode::FileSequential;
+    let mut raw_column_indices = None;
+    let mut raw_submit_mode = RawSubmitMode::Single;
+    let mut raw_completion_mode = RawCompletionMode::Unordered;
+    let mut take_repetitions = 100u64;
+    let mut io_buffer_gib = vec![Some(8)];
+    let mut batch_size = 8192u32;
+    let mut batch_size_bytes = None;
+    let mut skip_batch_byte_accounting = false;
+    let mut read_chunk_size = None;
+    let mut fragment_concurrency = 256usize;
+    let mut batch_concurrency = 256usize;
+    let mut sample_ms = 1000u64;
+    let mut out_dir = "/tmp/lance-s3-bottleneck-results".to_string();
+    let mut case_name = "lance-file-reader-diagnostics".to_string();
+    let mut describe_layout = false;
+    let mut detach_fragment_streams = false;
+    let mut drop_read_tasks = false;
+
+    let mut args = env::args().skip(1);
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--backend" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --backend. {}", usage()))?;
+                backend = parse_backend(&value)?;
+            }
+            "--uri" => uri = args.next(),
+            "--dataset-version" => {
+                dataset_version = parse_required_value(&mut args, "--dataset-version")?;
+            }
+            "--columns" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --columns. {}", usage()))?;
+                columns = parse_columns(&value)?;
+            }
+            "--limit-rows" => {
+                limit_rows = parse_required_value(&mut args, "--limit-rows")?;
+            }
+            "--target-bytes" => {
+                target_bytes = Some(parse_required_value(&mut args, "--target-bytes")?);
+            }
+            "--raw-range-size-bytes" => {
+                raw_range_size_bytes = parse_required_value(&mut args, "--raw-range-size-bytes")?;
+            }
+            "--raw-range-mode" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --raw-range-mode. {}", usage()))?;
+                raw_range_mode = parse_raw_range_mode(&value)?;
+            }
+            "--raw-column-indices" => {
+                let value = args.next().ok_or_else(|| {
+                    format!("missing value for --raw-column-indices. {}", usage())
+                })?;
+                raw_column_indices = parse_raw_column_indices(&value)?;
+            }
+            "--raw-submit-mode" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --raw-submit-mode. {}", usage()))?;
+                raw_submit_mode = parse_raw_submit_mode(&value)?;
+            }
+            "--raw-completion-mode" => {
+                let value = args.next().ok_or_else(|| {
+                    format!("missing value for --raw-completion-mode. {}", usage())
+                })?;
+                raw_completion_mode = parse_raw_completion_mode(&value)?;
+            }
+            "--take-repetitions" => {
+                take_repetitions = parse_required_value(&mut args, "--take-repetitions")?;
+            }
+            "--io-buffer-gib" => {
+                let value = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --io-buffer-gib. {}", usage()))?;
+                io_buffer_gib = parse_io_buffer_gib(&value)?;
+            }
+            "--batch-size" => {
+                batch_size = parse_required_value(&mut args, "--batch-size")?;
+            }
+            "--batch-size-bytes" => {
+                batch_size_bytes = Some(parse_required_value(&mut args, "--batch-size-bytes")?);
+            }
+            "--skip-batch-byte-accounting" => {
+                skip_batch_byte_accounting = true;
+            }
+            "--read-chunk-size" => {
+                read_chunk_size = Some(parse_required_value(&mut args, "--read-chunk-size")?);
+            }
+            "--fragment-concurrency" => {
+                fragment_concurrency = parse_required_value(&mut args, "--fragment-concurrency")?;
+            }
+            "--batch-concurrency" => {
+                batch_concurrency = parse_required_value(&mut args, "--batch-concurrency")?;
+            }
+            "--sample-ms" => {
+                sample_ms = parse_required_value(&mut args, "--sample-ms")?;
+            }
+            "--out-dir" => {
+                out_dir = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --out-dir. {}", usage()))?;
+            }
+            "--case" => {
+                case_name = args
+                    .next()
+                    .ok_or_else(|| format!("missing value for --case. {}", usage()))?;
+            }
+            "--describe-layout" => {
+                describe_layout = true;
+            }
+            "--detach-fragment-streams" => {
+                detach_fragment_streams = true;
+            }
+            "--drop-read-tasks" => {
+                drop_read_tasks = true;
+            }
+            "--help" | "-h" => {
+                println!("{}", usage());
+                std::process::exit(0);
+            }
+            "--bench" => {
+                // Cargo appends this flag when running harness-free benches.
+            }
+            other => {
+                return Err(format!("unknown argument {other}. {}", usage()).into());
+            }
+        }
+    }
+
+    let uri = uri.ok_or_else(|| format!("missing required --uri. {}", usage()))?;
+    if limit_rows == 0 {
+        return Err("--limit-rows must be greater than zero".into());
+    }
+    if matches!(target_bytes, Some(0)) {
+        return Err("--target-bytes must be greater than zero".into());
+    }
+    if raw_range_size_bytes == 0 {
+        return Err("--raw-range-size-bytes must be greater than zero".into());
+    }
+    if take_repetitions == 0 {
+        return Err("--take-repetitions must be greater than zero".into());
+    }
+    if io_buffer_gib.is_empty() {
+        return Err("--io-buffer-gib must not be empty".into());
+    }
+    if batch_size == 0 {
+        return Err("--batch-size must be greater than zero".into());
+    }
+    if matches!(batch_size_bytes, Some(0)) {
+        return Err("--batch-size-bytes must be greater than zero".into());
+    }
+    if matches!(read_chunk_size, Some(0)) {
+        return Err("--read-chunk-size must be greater than zero".into());
+    }
+    if fragment_concurrency == 0 && !matches!(backend, Backend::Scanner) {
+        return Err("--fragment-concurrency must be greater than zero".into());
+    }
+    if batch_concurrency == 0 && !matches!(backend, Backend::Scanner) {
+        return Err("--batch-concurrency must be greater than zero".into());
+    }
+    if sample_ms == 0 {
+        return Err("--sample-ms must be greater than zero".into());
+    }
+
+    Ok(Config {
+        backend,
+        uri,
+        dataset_version,
+        columns,
+        limit_rows,
+        target_bytes,
+        raw_range_size_bytes,
+        raw_range_mode,
+        raw_column_indices,
+        raw_submit_mode,
+        raw_completion_mode,
+        take_repetitions,
+        io_buffer_gib,
+        batch_size,
+        batch_size_bytes,
+        skip_batch_byte_accounting,
+        read_chunk_size,
+        fragment_concurrency,
+        batch_concurrency,
+        sample_ms,
+        out_dir,
+        case_name,
+        describe_layout,
+        detach_fragment_streams,
+        drop_read_tasks,
+    })
+}
+
+fn parse_backend(value: &str) -> Result<Backend> {
+    match value {
+        "file-reader" | "lance-file-reader" => Ok(Backend::FileReader),
+        "scanner" | "lance-scanner" => Ok(Backend::Scanner),
+        "scheduler-raw" | "lance-scheduler-raw" => Ok(Backend::SchedulerRaw),
+        "dataset-take" | "take" | "lance-dataset-take" => Ok(Backend::DatasetTake),
+        other => Err(format!(
+            "invalid --backend value {other}; expected file-reader, scanner, scheduler-raw, or dataset-take"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_submit_mode(value: &str) -> Result<RawSubmitMode> {
+    match value {
+        "single" => Ok(RawSubmitMode::Single),
+        "split-no-concat" => Ok(RawSubmitMode::SplitNoConcat),
+        "split-concat" => Ok(RawSubmitMode::SplitConcat),
+        other => Err(format!(
+            "invalid --raw-submit-mode value {other}; expected single, split-no-concat, or split-concat"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_range_mode(value: &str) -> Result<RawRangeMode> {
+    match value {
+        "file-sequential" => Ok(RawRangeMode::FileSequential),
+        "metadata-pages" => Ok(RawRangeMode::MetadataPages),
+        "metadata-pages-round-robin" => Ok(RawRangeMode::MetadataPagesRoundRobin),
+        other => Err(format!(
+            "invalid --raw-range-mode value {other}; expected file-sequential, metadata-pages, or metadata-pages-round-robin"
+        )
+        .into()),
+    }
+}
+
+fn parse_raw_column_indices(value: &str) -> Result<Option<Vec<u32>>> {
+    if value == "all" {
+        return Ok(None);
+    }
+
+    let indices = value
+        .split(',')
+        .map(str::trim)
+        .filter(|part| !part.is_empty())
+        .map(|part| {
+            part.parse::<u32>()
+                .map_err(|err| format!("invalid raw column index {part}: {err}").into())
+        })
+        .collect::<Result<Vec<_>>>()?;
+    if indices.is_empty() {
+        return Err("--raw-column-indices must specify at least one column index or all".into());
+    }
+    Ok(Some(indices))
+}
+
+fn parse_raw_completion_mode(value: &str) -> Result<RawCompletionMode> {
+    match value {
+        "unordered" => Ok(RawCompletionMode::Unordered),
+        "ordered" => Ok(RawCompletionMode::Ordered),
+        other => Err(format!(
+            "invalid --raw-completion-mode value {other}; expected unordered or ordered"
+        )
+        .into()),
+    }
+}
+
+fn parse_required_value<T>(args: &mut impl Iterator<Item = String>, name: &str) -> Result<T>
+where
+    T: std::str::FromStr,
+    T::Err: std::fmt::Display + Send + Sync + 'static,
+{
+    let value = args
+        .next()
+        .ok_or_else(|| format!("missing value for {name}. {}", usage()))?;
+    value
+        .parse()
+        .map_err(|err| format!("invalid {name} value {value}: {err}").into())
+}
+
+fn parse_columns(value: &str) -> Result<Option<Vec<String>>> {
+    match value {
+        "all" => Ok(None),
+        "empty" => Err("FileReader benchmark requires at least one data column".into()),
+        _ => Ok(Some(
+            value
+                .split(',')
+                .map(str::trim)
+                .filter(|column| !column.is_empty())
+                .map(ToString::to_string)
+                .collect(),
+        )),
+    }
+}
+
+fn parse_io_buffer_gib(value: &str) -> Result<Vec<Option<u64>>> {
+    value
+        .split(',')
+        .map(|part| {
+            let part = part.trim();
+            if part == "auto" {
+                Ok(None)
+            } else {
+                part.parse::<u64>()
+                    .map(Some)
+                    .map_err(|err| format!("invalid --io-buffer-gib value {part}: {err}").into())
+            }
+        })
+        .collect()
+}
+
+fn projection_name(columns: &Option<Vec<String>>) -> String {
+    match columns {
+        None => "all".to_string(),
+        Some(columns) => columns.join(","),
+    }
+}
+
+fn page_layout_kind(encoding: &PageEncoding) -> &'static str {
+    match encoding {
+        PageEncoding::Legacy(_) => "legacy",
+        PageEncoding::Structural(layout) => match layout.layout.as_ref() {
+            Some(pb21::page_layout::Layout::MiniBlockLayout(_)) => "miniblock",
+            Some(pb21::page_layout::Layout::ConstantLayout(_)) => "constant",
+            Some(pb21::page_layout::Layout::FullZipLayout(_)) => "fullzip",
+            Some(pb21::page_layout::Layout::BlobLayout(_)) => "blob",
+            None => "missing",
+        },
+    }
+}
+
+fn summarize_u64(values: &[u64]) -> Value {
+    if values.is_empty() {
+        return json!({
+            "count": 0,
+            "min": null,
+            "p50": null,
+            "p90": null,
+            "max": null,
+            "sum": 0,
+        });
+    }
+    let mut sorted = values.to_vec();
+    sorted.sort_unstable();
+    let percentile = |p: f64| {
+        let idx = ((sorted.len() - 1) as f64 * p).round() as usize;
+        sorted[idx]
+    };
+    json!({
+        "count": sorted.len(),
+        "min": sorted[0],
+        "p50": percentile(0.5),
+        "p90": percentile(0.9),
+        "max": *sorted.last().unwrap(),
+        "sum": values.iter().sum::<u64>(),
+    })
+}
+
+async fn describe_layout(config: &Config) -> Result<()> {
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let fragment = dataset
+        .fragments()
+        .first()
+        .ok_or("dataset has no fragments")?;
+    let data_file = fragment
+        .files
+        .first()
+        .ok_or("first fragment has no data files")?;
+    if data_file.base_id.is_some() {
+        return Err("layout diagnostics do not support external base data files yet".into());
+    }
+
+    let data_path = dataset.data_dir().join(data_file.path.as_str());
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler = ScanScheduler::new(object_store, SchedulerConfig::new(8 * GIB));
+    let file_scheduler = scheduler
+        .open_file(&data_path, &CachedFileSize::unknown())
+        .await?;
+    let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?;
+
+    let columns = metadata
+        .column_infos
+        .iter()
+        .map(|column| {
+            let mut layout_counts = BTreeMap::new();
+            let mut page_rows = Vec::with_capacity(column.page_infos.len());
+            let mut page_bytes = Vec::with_capacity(column.page_infos.len());
+            for page in column.page_infos.iter() {
+                *layout_counts
+                    .entry(page_layout_kind(&page.encoding))
+                    .or_insert(0usize) += 1;
+                page_rows.push(page.num_rows);
+                page_bytes.push(
+                    page.buffer_offsets_and_sizes
+                        .iter()
+                        .map(|(_, size)| *size)
+                        .sum::<u64>(),
+                );
+            }
+            json!({
+                "column_index": column.index,
+                "num_pages": column.page_infos.len(),
+                "layout_counts": layout_counts,
+                "page_rows": summarize_u64(&page_rows),
+                "page_bytes": summarize_u64(&page_bytes),
+            })
+        })
+        .collect::<Vec<_>>();
+
+    println!(
+        "{}",
+        serde_json::to_string_pretty(&json!({
+            "dataset_uri": config.uri,
+            "dataset_version": config.dataset_version,
+            "fragment_id": fragment.id,
+            "data_file_path": data_file.path,
+            "resolved_data_path": data_path.to_string(),
+            "file_version": metadata.version().to_string(),
+            "num_rows": metadata.num_rows,
+            "num_data_bytes": metadata.num_data_bytes,
+            "columns": columns,
+        }))?
+    );
+    Ok(())
+}
+
+fn projected_schema(dataset_schema: &Schema, columns: &Option<Vec<String>>) -> Result<Schema> {
+    Ok(match columns {
+        None => dataset_schema.clone(),
+        Some(columns) => dataset_schema.project(columns)?,
+    })
+}
+
+fn file_reader_options(config: &Config) -> Option<FileReaderOptions> {
+    if config.batch_size_bytes.is_none() && config.read_chunk_size.is_none() {
+        return None;
+    }
+    Some(FileReaderOptions {
+        batch_size_bytes: config.batch_size_bytes,
+        read_chunk_size: config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE),
+        ..Default::default()
+    })
+}
+
+fn add_duration(counter: &AtomicU64, duration: Duration) {
+    let nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
+    counter.fetch_add(nanos, Ordering::Relaxed);
+}
+
+fn ns_to_seconds(ns: u64) -> f64 {
+    ns as f64 / 1_000_000_000.0
+}
+
+fn diff_u64(current: u64, previous: u64) -> u64 {
+    current.saturating_sub(previous)
+}
+
+fn scheduler_kind_name(kind: SchedulerQueueKind) -> &'static str {
+    match kind {
+        SchedulerQueueKind::Standard => "standard",
+        SchedulerQueueKind::Lite => "lite",
+    }
+}
+
+fn diagnostics_json(diagnostics: SchedulerDiagnostics) -> Value {
+    json!({
+        "queue_kind": scheduler_kind_name(diagnostics.kind),
+        "scheduler_iops": diagnostics.stats.iops,
+        "scheduler_requests": diagnostics.stats.requests,
+        "scheduler_bytes_read": diagnostics.stats.bytes_read,
+        "io_capacity": diagnostics.io_capacity,
+        "iops_available": diagnostics.iops_available,
+        "active_iops": diagnostics.active_iops,
+        "pending_iops": diagnostics.pending_iops,
+        "pending_bytes": diagnostics.pending_bytes,
+        "bytes_available": diagnostics.bytes_available,
+        "bytes_reserved": diagnostics.bytes_reserved,
+        "io_buffer_size_bytes": diagnostics.io_buffer_size_bytes,
+        "priorities_in_flight": diagnostics.priorities_in_flight,
+        "no_backpressure": diagnostics.no_backpressure,
+        "head_task_bytes": diagnostics.head_task_bytes,
+        "head_task_priority_high": diagnostics.head_task_priority_high,
+        "head_task_priority_low": diagnostics.head_task_priority_low,
+        "min_in_flight_priority_high": diagnostics.min_in_flight_priority_high,
+        "min_in_flight_priority_low": diagnostics.min_in_flight_priority_low,
+        "head_task_can_deliver": diagnostics.head_task_can_deliver,
+        "head_task_priority_bypass": diagnostics.head_task_priority_bypass,
+        "head_task_blocked_by_iops": diagnostics.head_task_blocked_by_iops,
+        "head_task_blocked_by_bytes": diagnostics.head_task_blocked_by_bytes,
+    })
+}
+
+#[derive(Debug, Default)]
+struct ExecutionStatsHolder {
+    collected_stats: Arc<Mutex<Option<ExecutionSummaryCounts>>>,
+}
+
+impl ExecutionStatsHolder {
+    fn get_setter(&self) -> ExecutionStatsCallback {
+        let collected_stats = self.collected_stats.clone();
+        Arc::new(move |stats| {
+            *collected_stats.lock().unwrap() = Some(stats.clone());
+        })
+    }
+
+    fn consume(self) -> Option<ExecutionSummaryCounts> {
+        self.collected_stats.lock().unwrap().take()
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+struct SchedulerDiagnosticsCollector {
+    latest: Arc<Mutex<Option<SchedulerDiagnostics>>>,
+}
+
+impl SchedulerDiagnosticsCollector {
+    fn clear(&self) {
+        *self.latest.lock().unwrap() = None;
+    }
+
+    fn observe(&self, diagnostics: SchedulerDiagnostics) {
+        *self.latest.lock().unwrap() = Some(diagnostics);
+    }
+
+    fn snapshot(&self, io_buffer_gib: Option<u64>) -> SchedulerDiagnostics {
+        self.latest
+            .lock()
+            .unwrap()
+            .as_ref()
+            .copied()
+            .unwrap_or_else(|| diagnostics_from_scan_stats(ScanStats::default(), io_buffer_gib))
+    }
+}
+
+#[derive(Debug, Clone)]
+struct SchedulerDiagnosticsLayer {
+    collector: SchedulerDiagnosticsCollector,
+}
+
+impl SchedulerDiagnosticsLayer {
+    fn new(collector: SchedulerDiagnosticsCollector) -> Self {
+        Self { collector }
+    }
+}
+
+fn is_scheduler_state_metadata(metadata: &Metadata<'_>) -> bool {
+    // The scheduler uses `tracing::enabled!` before constructing the event;
+    // that guard registers a HINT callsite, not an EVENT callsite.
+    metadata.target() == SCHEDULER_STATE_EVENT_TARGET && *metadata.level() == tracing::Level::TRACE
+}
+
+impl<S> Layer<S> for SchedulerDiagnosticsLayer
+where
+    S: Subscriber,
+{
+    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {
+        if is_scheduler_state_metadata(metadata) {
+            Interest::always()
+        } else {
+            Interest::never()
+        }
+    }
+
+    fn enabled(&self, metadata: &Metadata<'_>, _ctx: Context<'_, S>) -> bool {
+        is_scheduler_state_metadata(metadata)
+    }
+
+    fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
+        if !is_scheduler_state_metadata(event.metadata()) {
+            return;
+        }
+        let mut visitor = SchedulerDiagnosticsVisitor::default();
+        event.record(&mut visitor);
+        if let Some(diagnostics) = visitor.into_diagnostics() {
+            self.collector.observe(diagnostics);
+        }
+    }
+}
+
+#[derive(Debug, Default)]
+struct SchedulerDiagnosticsVisitor {
+    kind: Option<SchedulerQueueKind>,
+    scheduler_iops: Option<u64>,
+    scheduler_requests: Option<u64>,
+    scheduler_bytes_read: Option<u64>,
+    io_capacity: Option<u64>,
+    iops_available: Option<u64>,
+    active_iops: Option<u64>,
+    pending_iops: Option<u64>,
+    pending_bytes: Option<u64>,
+    bytes_available: Option<i64>,
+    bytes_reserved: Option<i64>,
+    io_buffer_size_bytes: Option<u64>,
+    priorities_in_flight: Option<u64>,
+    no_backpressure: Option<bool>,
+    head_task_bytes_present: bool,
+    head_task_bytes: Option<u64>,
+    head_task_priority_high_present: bool,
+    head_task_priority_high: Option<u64>,
+    head_task_priority_low_present: bool,
+    head_task_priority_low: Option<u64>,
+    min_in_flight_priority_high_present: bool,
+    min_in_flight_priority_high: Option<u64>,
+    min_in_flight_priority_low_present: bool,
+    min_in_flight_priority_low: Option<u64>,
+    head_task_can_deliver_present: bool,
+    head_task_can_deliver: Option<bool>,
+    head_task_priority_bypass_present: bool,
+    head_task_priority_bypass: Option<bool>,
+    head_task_blocked_by_iops_present: bool,
+    head_task_blocked_by_iops: Option<bool>,
+    head_task_blocked_by_bytes_present: bool,
+    head_task_blocked_by_bytes: Option<bool>,
+}
+
+impl SchedulerDiagnosticsVisitor {
+    fn into_diagnostics(self) -> Option<SchedulerDiagnostics> {
+        Some(SchedulerDiagnostics {
+            kind: self.kind?,
+            stats: ScanStats {
+                iops: self.scheduler_iops.unwrap_or_default(),
+                requests: self.scheduler_requests.unwrap_or_default(),
+                bytes_read: self.scheduler_bytes_read.unwrap_or_default(),
+            },
+            io_capacity: self.io_capacity.unwrap_or_default(),
+            iops_available: self.iops_available.unwrap_or_default(),
+            active_iops: self.active_iops.unwrap_or_default(),
+            pending_iops: self.pending_iops.unwrap_or_default(),
+            pending_bytes: self.pending_bytes.unwrap_or_default(),
+            bytes_available: self.bytes_available.unwrap_or_default(),
+            bytes_reserved: self.bytes_reserved.unwrap_or_default(),
+            io_buffer_size_bytes: self.io_buffer_size_bytes.unwrap_or_default(),
+            priorities_in_flight: self.priorities_in_flight.unwrap_or_default(),
+            no_backpressure: self.no_backpressure.unwrap_or(false),
+            head_task_bytes: optional_u64(self.head_task_bytes_present, self.head_task_bytes),
+            head_task_priority_high: optional_u64(
+                self.head_task_priority_high_present,
+                self.head_task_priority_high,
+            ),
+            head_task_priority_low: optional_u64(
+                self.head_task_priority_low_present,
+                self.head_task_priority_low,
+            ),
+            min_in_flight_priority_high: optional_u64(
+                self.min_in_flight_priority_high_present,
+                self.min_in_flight_priority_high,
+            ),
+            min_in_flight_priority_low: optional_u64(
+                self.min_in_flight_priority_low_present,
+                self.min_in_flight_priority_low,
+            ),
+            head_task_can_deliver: optional_bool(
+                self.head_task_can_deliver_present,
+                self.head_task_can_deliver,
+            ),
+            head_task_priority_bypass: optional_bool(
+                self.head_task_priority_bypass_present,
+                self.head_task_priority_bypass,
+            ),
+            head_task_blocked_by_iops: optional_bool(
+                self.head_task_blocked_by_iops_present,
+                self.head_task_blocked_by_iops,
+            ),
+            head_task_blocked_by_bytes: optional_bool(
+                self.head_task_blocked_by_bytes_present,
+                self.head_task_blocked_by_bytes,
+            ),
+        })
+    }
+}
+
+impl Visit for SchedulerDiagnosticsVisitor {
+    fn record_bool(&mut self, field: &Field, value: bool) {
+        match field.name() {
+            "no_backpressure" => self.no_backpressure = Some(value),
+            "head_task_bytes_present" => self.head_task_bytes_present = value,
+            "head_task_priority_high_present" => self.head_task_priority_high_present = value,
+            "head_task_priority_low_present" => self.head_task_priority_low_present = value,
+            "min_in_flight_priority_high_present" => {
+                self.min_in_flight_priority_high_present = value;
+            }
+            "min_in_flight_priority_low_present" => {
+                self.min_in_flight_priority_low_present = value;
+            }
+            "head_task_can_deliver_present" => self.head_task_can_deliver_present = value,
+            "head_task_can_deliver" => self.head_task_can_deliver = Some(value),
+            "head_task_priority_bypass_present" => {
+                self.head_task_priority_bypass_present = value;
+            }
+            "head_task_priority_bypass" => self.head_task_priority_bypass = Some(value),
+            "head_task_blocked_by_iops_present" => {
+                self.head_task_blocked_by_iops_present = value;
+            }
+            "head_task_blocked_by_iops" => self.head_task_blocked_by_iops = Some(value),
+            "head_task_blocked_by_bytes_present" => {
+                self.head_task_blocked_by_bytes_present = value;
+            }
+            "head_task_blocked_by_bytes" => self.head_task_blocked_by_bytes = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_i64(&mut self, field: &Field, value: i64) {
+        match field.name() {
+            "bytes_available" => self.bytes_available = Some(value),
+            "bytes_reserved" => self.bytes_reserved = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_u64(&mut self, field: &Field, value: u64) {
+        match field.name() {
+            "scheduler_iops" => self.scheduler_iops = Some(value),
+            "scheduler_requests" => self.scheduler_requests = Some(value),
+            "scheduler_bytes_read" => self.scheduler_bytes_read = Some(value),
+            "io_capacity" => self.io_capacity = Some(value),
+            "iops_available" => self.iops_available = Some(value),
+            "active_iops" => self.active_iops = Some(value),
+            "pending_iops" => self.pending_iops = Some(value),
+            "pending_bytes" => self.pending_bytes = Some(value),
+            "io_buffer_size_bytes" => self.io_buffer_size_bytes = Some(value),
+            "priorities_in_flight" => self.priorities_in_flight = Some(value),
+            "head_task_bytes" => self.head_task_bytes = Some(value),
+            "head_task_priority_high" => self.head_task_priority_high = Some(value),
+            "head_task_priority_low" => self.head_task_priority_low = Some(value),
+            "min_in_flight_priority_high" => self.min_in_flight_priority_high = Some(value),
+            "min_in_flight_priority_low" => self.min_in_flight_priority_low = Some(value),
+            _ => {}
+        }
+    }
+
+    fn record_str(&mut self, field: &Field, value: &str) {
+        if field.name() == "queue_kind" {
+            self.kind = match value {
+                "standard" => Some(SchedulerQueueKind::Standard),
+                "lite" => Some(SchedulerQueueKind::Lite),
+                _ => None,
+            };
+        }
+    }
+
+    fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {}
+}
+
+fn optional_u64(present: bool, value: Option<u64>) -> Option<u64> {
+    present.then(|| value.unwrap_or_default())
+}
+
+fn optional_bool(present: bool, value: Option<bool>) -> Option<bool> {
+    present.then(|| value.unwrap_or(false))
+}
+
+fn diagnostics_from_scan_stats(
+    stats: ScanStats,
+    io_buffer_gib: Option<u64>,
+) -> SchedulerDiagnostics {
+    SchedulerDiagnostics {
+        kind: SchedulerQueueKind::Standard,
+        stats,
+        io_capacity: 0,
+        iops_available: 0,
+        active_iops: 0,
+        pending_iops: 0,
+        pending_bytes: 0,
+        bytes_available: 0,
+        bytes_reserved: 0,
+        io_buffer_size_bytes: io_buffer_gib.map(|value| value * GIB).unwrap_or_default(),
+        priorities_in_flight: 0,
+        no_backpressure: false,
+        head_task_bytes: None,
+        head_task_priority_high: None,
+        head_task_priority_low: None,
+        min_in_flight_priority_high: None,
+        min_in_flight_priority_low: None,
+        head_task_can_deliver: None,
+        head_task_priority_bypass: None,
+        head_task_blocked_by_iops: None,
+        head_task_blocked_by_bytes: None,
+    }
+}
+
+fn scan_stats_from_execution_summary(summary: &ExecutionSummaryCounts) -> ScanStats {
+    ScanStats {
+        iops: summary.iops as u64,
+        requests: summary.requests as u64,
+        bytes_read: summary.bytes_read as u64,
+    }
+}
+
+fn sample_json(
+    started: Instant,
+    counters: &SharedCounters,
+    diagnostics: SchedulerDiagnostics,
+    decode_in_flight: usize,
+    channel_buffered: usize,
+    last: &mut LastSample,
+) -> Value {
+    let elapsed = started.elapsed();
+    let interval = elapsed.saturating_sub(last.elapsed);
+    let interval_secs = interval.as_secs_f64();
+    let rows = counters.rows_completed.load(Ordering::Relaxed);
+    let batches = counters.batches_completed.load(Ordering::Relaxed);
+    let arrow_bytes = counters.arrow_bytes.load(Ordering::Relaxed);
+    let scheduler_stats = diagnostics.stats;
+    let delta_scheduler_bytes =
+        diff_u64(scheduler_stats.bytes_read, last.scheduler_stats.bytes_read);
+    let delta_rows = diff_u64(rows, last.rows);
+    let delta_arrow_bytes = diff_u64(arrow_bytes, last.arrow_bytes);
+    let physical_gbps = if interval_secs > 0.0 {
+        delta_scheduler_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0
+    } else {
+        0.0
+    };
+    let logical_gbps = if interval_secs > 0.0 {
+        delta_arrow_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0
+    } else {
+        0.0
+    };
+    let rows_per_second = if interval_secs > 0.0 {
+        delta_rows as f64 / interval_secs
+    } else {
+        0.0
+    };
+
+    last.elapsed = elapsed;
+    last.scheduler_stats = scheduler_stats;
+    last.rows = rows;
+    last.batches = batches;
+    last.arrow_bytes = arrow_bytes;
+
+    json!({
+        "elapsed_seconds": elapsed.as_secs_f64(),
+        "interval_seconds": interval_secs,
+        "physical_gbps": physical_gbps,
+        "logical_gbps": logical_gbps,
+        "rows_per_second": rows_per_second,
+        "rows": rows,
+        "batches": batches,
+        "arrow_bytes": arrow_bytes,
+        "delta_rows": delta_rows,
+        "delta_arrow_bytes": delta_arrow_bytes,
+        "delta_scheduler_bytes": delta_scheduler_bytes,
+        "fragments_started": counters.fragments_started.load(Ordering::Relaxed),
+        "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed),
+        "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed),
+        "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed),
+        "batches_completed": counters.batches_completed.load(Ordering::Relaxed),
+        "decode_in_flight": decode_in_flight,
+        "channel_buffered": channel_buffered,
+        "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)),
+        "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)),
+        "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)),
+        "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)),
+        "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)),
+        "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)),
+        "scheduler": diagnostics_json(diagnostics),
+    })
+}
+
+async fn run_scanner_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+
+    let mut remaining_rows = config.limit_rows;
+    let mut planned_fragments = 0usize;
+    let mut planned_rows = 0u64;
+    for fragment in dataset.fragments().iter() {
+        if remaining_rows == 0 {
+            break;
+        }
+        let fragment_rows = fragment
+            .num_rows()
+            .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))?
+            as u64;
+        let rows = fragment_rows.min(remaining_rows);
+        planned_fragments += 1;
+        planned_rows += rows;
+        remaining_rows -= rows;
+    }
+    if planned_fragments == 0 {
+        return Err("no fragments selected".into());
+    }
+
+    let counters = Arc::new(SharedCounters::default());
+    let stats_holder = ExecutionStatsHolder::default();
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+
+    let mut scanner = dataset.scan();
+    if let Some(columns) = config.columns.as_ref() {
+        scanner.project(columns)?;
+    }
+    scanner
+        .batch_size(config.batch_size as usize)
+        .scan_in_order(false)
+        .scan_stats_callback(stats_holder.get_setter());
+    if config.batch_concurrency > 0 {
+        scanner
+            .batch_readahead(config.batch_concurrency)
+            .target_parallelism(config.batch_concurrency);
+    }
+    if config.fragment_concurrency > 0 {
+        scanner.fragment_readahead(config.fragment_concurrency);
+    }
+    if let Some(file_reader_options) = file_reader_options(config) {
+        scanner.with_file_reader_options(file_reader_options);
+    }
+    if let Some(batch_size_bytes) = config.batch_size_bytes {
+        scanner.batch_size_bytes(batch_size_bytes);
+    }
+    if let Some(io_buffer_gib) = io_buffer_gib {
+        scanner.io_buffer_size(io_buffer_gib * GIB);
+    }
+    let limit_rows = i64::try_from(config.limit_rows)
+        .map_err(|_| "--limit-rows is too large for scanner limit")?;
+    scanner.limit(Some(limit_rows), None)?;
+
+    let mut stream = scanner.try_into_stream().await?;
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+    loop {
+        tokio::select! {
+            maybe_batch = stream.next() => {
+                let Some(batch) = maybe_batch else {
+                    break;
+                };
+                let batch = batch?;
+                let batch_bytes = if config.skip_batch_byte_accounting {
+                    0
+                } else {
+                    batch.get_array_memory_size() as u64
+                };
+                rows += batch.num_rows() as u64;
+                batches += 1;
+                arrow_bytes += batch_bytes;
+                counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                counters
+                    .rows_completed
+                    .fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+                counters.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed);
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    0,
+                    0,
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    drop(stream);
+
+    let elapsed = started.elapsed();
+    let summary = stats_holder
+        .consume()
+        .ok_or("scanner execution stats callback did not run")?;
+    let scheduler_stats = scan_stats_from_execution_summary(&summary);
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler_stats;
+    let cpu_after = read_cpu_sample();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        0,
+        0,
+        &mut last_sample,
+    ));
+
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments,
+        planned_rows,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: 0,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+async fn run_dataset_take_case(
+    config: &Config,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = DatasetBuilder::from_uri(&config.uri)
+        .with_version(config.dataset_version)
+        .load()
+        .await?;
+    let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?);
+    let total_rows = dataset
+        .fragments()
+        .iter()
+        .map(|fragment| {
+            fragment
+                .num_rows()
+                .map(|rows| rows as u64)
+                .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))
+        })
+        .collect::<std::result::Result<Vec<_>, _>>()?
+        .into_iter()
+        .sum::<u64>();
+    if total_rows == 0 {
+        return Err("dataset has no rows".into());
+    }
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    const STRIDE: u64 = 104_729;
+
+    for repetition in 0..config.take_repetitions {
+        let row_ids = (0..config.limit_rows)
+            .map(|offset| {
+                repetition
+                    .wrapping_mul(STRIDE)
+                    .wrapping_add(offset.wrapping_mul(STRIDE))
+                    % total_rows
+            })
+            .collect::<Vec<_>>();
+        let batch = dataset
+            .take(&row_ids, ProjectionRequest::Schema(projection.clone()))
+            .await?;
+        if batch.num_rows() as u64 != config.limit_rows {
+            return Err(format!(
+                "take_rows returned {} rows, expected {}",
+                batch.num_rows(),
+                config.limit_rows
+            )
+            .into());
+        }
+        black_box(&batch);
+        rows += batch.num_rows() as u64;
+        batches += 1;
+        let batch_bytes = if config.skip_batch_byte_accounting {
+            0
+        } else {
+            batch.get_array_memory_size() as u64
+        };
+        arrow_bytes += batch_bytes;
+        counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+        counters
+            .rows_completed
+            .fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+        counters
+            .arrow_bytes
+            .fetch_add(batch_bytes, Ordering::Relaxed);
+    }
+
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments: dataset.fragments().len(),
+        planned_rows: total_rows,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: 0,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: scheduler_diagnostics.snapshot(None),
+        counters,
+        samples: Vec::new(),
+    })
+}
+
+enum RawInFlight {
+    Unordered(FuturesUnordered<BoxFuture<'static, lance_core::Result<usize>>>),
+    Ordered(FuturesOrdered<BoxFuture<'static, lance_core::Result<usize>>>),
+}
+
+impl RawInFlight {
+    fn new(mode: RawCompletionMode) -> Self {
+        match mode {
+            RawCompletionMode::Unordered => Self::Unordered(FuturesUnordered::new()),
+            RawCompletionMode::Ordered => Self::Ordered(FuturesOrdered::new()),
+        }
+    }
+
+    fn len(&self) -> usize {
+        match self {
+            Self::Unordered(in_flight) => in_flight.len(),
+            Self::Ordered(in_flight) => in_flight.len(),
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        match self {
+            Self::Unordered(in_flight) => in_flight.is_empty(),
+            Self::Ordered(in_flight) => in_flight.is_empty(),
+        }
+    }
+
+    fn push(&mut self, future: BoxFuture<'static, lance_core::Result<usize>>) {
+        match self {
+            Self::Unordered(in_flight) => in_flight.push(future),
+            Self::Ordered(in_flight) => in_flight.push_back(future),
+        }
+    }
+
+    async fn next(&mut self) -> Option<lance_core::Result<usize>> {
+        match self {
+            Self::Unordered(in_flight) => in_flight.next().await,
+            Self::Ordered(in_flight) => in_flight.next().await,
+        }
+    }
+}
+
+fn raw_read_future(
+    file_scheduler: FileScheduler,
+    range: Range<u64>,
+    priority: u64,
+    raw_submit_mode: RawSubmitMode,
+    read_chunk_size: u64,
+    counters: Arc<SharedCounters>,
+) -> BoxFuture<'static, lance_core::Result<usize>> {
+    async move {
+        match raw_submit_mode {
+            RawSubmitMode::Single => {
+                let bytes = file_scheduler.submit_single(range, priority).await?;
+                Ok(bytes.len())
+            }
+            RawSubmitMode::SplitNoConcat => {
+                let ranges = split_range_by_size(range, read_chunk_size);
+                let bytes = file_scheduler.submit_request(ranges, priority).await?;
+                Ok(bytes.iter().map(bytes::Bytes::len).sum())
+            }
+            RawSubmitMode::SplitConcat => {
+                let ranges = split_range_by_size(range, read_chunk_size);
+                let bytes = file_scheduler.submit_request(ranges, priority).await?;
+                let reassemble_started = Instant::now();
+                let total_size = bytes.iter().map(bytes::Bytes::len).sum();
+                let mut combined = Vec::with_capacity(total_size);
+                for chunk in bytes {
+                    combined.extend_from_slice(&chunk);
+                }
+                add_duration(&counters.raw_reassemble_ns, reassemble_started.elapsed());
+                let len = combined.len();
+                black_box(&combined);
+                Ok(len)
+            }
+        }
+    }
+    .boxed()
+}
+
+fn split_range_by_size(range: Range<u64>, chunk_size: u64) -> Vec<Range<u64>> {
+    let range_size = range.end - range.start;
+    if range_size <= chunk_size {
+        return vec![range];
+    }
+
+    let num_chunks = range_size.div_ceil(chunk_size);
+    let per_chunk = range_size / num_chunks;
+    let mut ranges = Vec::with_capacity(num_chunks as usize);
+    for idx in 0..num_chunks {
+        let start = range.start + idx * per_chunk;
+        let end = if idx == num_chunks - 1 {
+            range.end
+        } else {
+            start + per_chunk
+        };
+        ranges.push(start..end);
+    }
+    ranges
+}
+
+fn push_split_planned_ranges(
+    planned: &mut Vec<(usize, Range<u64>)>,
+    file_idx: usize,
+    range: Range<u64>,
+    chunk_size: u64,
+    remaining: &mut u64,
+) {
+    let mut start = range.start;
+    while start < range.end && *remaining > 0 {
+        let bytes_to_read = chunk_size.min(range.end - start).min(*remaining);
+        if bytes_to_read == 0 {
+            break;
+        }
+        let end = start + bytes_to_read;
+        planned.push((file_idx, start..end));
+        *remaining -= bytes_to_read;
+        start = end;
+    }
+}
+
+fn push_split_ranges(ranges: &mut Vec<Range<u64>>, range: Range<u64>, chunk_size: u64) {
+    let mut start = range.start;
+    while start < range.end {
+        let bytes_to_read = chunk_size.min(range.end - start);
+        if bytes_to_read == 0 {
+            break;
+        }
+        let end = start + bytes_to_read;
+        ranges.push(start..end);
+        start = end;
+    }
+}
+
+async fn run_scheduler_raw_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let target_bytes = config
+        .target_bytes
+        .ok_or("--target-bytes is required for --backend scheduler-raw")?;
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler_config = io_buffer_gib
+        .map(|gib| SchedulerConfig::new(gib * GIB))
+        .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref()));
+    let scheduler = ScanScheduler::new(object_store, scheduler_config);
+
+    let (selected_files, planned) = match config.raw_range_mode {
+        RawRangeMode::FileSequential => {
+            let mut selected_files = Vec::new();
+            let mut selected_file_bytes = 0u64;
+            for fragment in dataset.fragments().iter() {
+                for data_file in &fragment.files {
+                    if data_file.base_id.is_some() {
+                        continue;
+                    }
+                    let Some(file_size) = data_file.file_size_bytes.get() else {
+                        continue;
+                    };
+                    let path = dataset.data_dir().join(data_file.path.as_str());
+                    let file_scheduler = scheduler
+                        .open_file_with_priority(&path, 0, &data_file.file_size_bytes)
+                        .await?;
+                    selected_file_bytes += file_size.get();
+                    selected_files.push((file_scheduler, file_size.get()));
+                    if selected_file_bytes >= target_bytes {
+                        break;
+                    }
+                }
+                if selected_file_bytes >= target_bytes {
+                    break;
+                }
+            }
+            if selected_files.is_empty() {
+                return Err("scheduler-raw found no data files with known sizes".into());
+            }
+
+            let mut offsets = vec![0u64; selected_files.len()];
+            let mut planned = Vec::new();
+            let mut remaining = target_bytes;
+            let mut file_idx = 0usize;
+            while remaining > 0 {
+                let idx = file_idx % selected_files.len();
+                let file_size = selected_files[idx].1;
+                if offsets[idx] >= file_size {
+                    offsets[idx] = 0;
+                }
+                let available = file_size - offsets[idx];
+                let bytes_to_read = config.raw_range_size_bytes.min(available).min(remaining);
+                let start = offsets[idx];
+                let end = start + bytes_to_read;
+                planned.push((idx, start..end));
+                offsets[idx] = end;
+                remaining -= bytes_to_read;
+                file_idx += 1;
+            }
+            (selected_files, planned)
+        }
+        RawRangeMode::MetadataPages | RawRangeMode::MetadataPagesRoundRobin => {
+            let mut selected_files = Vec::new();
+            let mut per_file_ranges = Vec::<Vec<Range<u64>>>::new();
+            let mut candidate_bytes = 0u64;
+
+            'fragments: for fragment in dataset.fragments().iter() {
+                for data_file in &fragment.files {
+                    if data_file.base_id.is_some() {
+                        continue;
+                    }
+                    if data_file.file_size_bytes.get().is_none() {
+                        continue;
+                    }
+                    let path = dataset.data_dir().join(data_file.path.as_str());
+                    let file_scheduler = scheduler
+                        .open_file_with_priority(&path, 0, &data_file.file_size_bytes)
+                        .await?;
+                    let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?;
+                    let mut file_ranges = Vec::new();
+
+                    let raw_column_indices = config
+                        .raw_column_indices
+                        .clone()
+                        .unwrap_or_else(|| (0..metadata.column_infos.len() as u32).collect());
+                    for column_index in raw_column_indices {
+                        let column_info = metadata
+                            .column_infos
+                            .get(column_index as usize)
+                            .ok_or_else(|| {
+                                format!(
+                                    "raw metadata-pages requested column index {column_index} but file has {} columns",
+                                    metadata.column_infos.len()
+                                )
+                            })?;
+                        for page in column_info.page_infos.iter() {
+                            for (offset, size) in page.buffer_offsets_and_sizes.iter() {
+                                if *size == 0 {
+                                    continue;
+                                }
+                                push_split_ranges(
+                                    &mut file_ranges,
+                                    *offset..(*offset + *size),
+                                    config.raw_range_size_bytes,
+                                );
+                            }
+                        }
+                    }
+
+                    if !file_ranges.is_empty() {
+                        candidate_bytes += file_ranges
+                            .iter()
+                            .map(|range| range.end - range.start)
+                            .sum::<u64>();
+                        selected_files.push((
+                            file_scheduler,
+                            data_file.file_size_bytes.get().unwrap().get(),
+                        ));
+                        per_file_ranges.push(file_ranges);
+                        if candidate_bytes >= target_bytes {
+                            break 'fragments;
+                        }
+                    }
+                }
+            }
+            if selected_files.is_empty() || per_file_ranges.is_empty() {
+                return Err("scheduler-raw metadata-pages found no readable page buffers".into());
+            }
+
+            let mut planned = Vec::new();
+            let mut remaining = target_bytes;
+            match config.raw_range_mode {
+                RawRangeMode::MetadataPages => {
+                    'ranges: for (file_idx, ranges) in per_file_ranges.iter().enumerate() {
+                        for range in ranges {
+                            push_split_planned_ranges(
+                                &mut planned,
+                                file_idx,
+                                range.clone(),
+                                config.raw_range_size_bytes,
+                                &mut remaining,
+                            );
+                            if remaining == 0 {
+                                break 'ranges;
+                            }
+                        }
+                    }
+                }
+                RawRangeMode::MetadataPagesRoundRobin => {
+                    let mut positions = vec![0usize; per_file_ranges.len()];
+                    while remaining > 0 {
+                        let mut made_progress = false;
+                        for (file_idx, ranges) in per_file_ranges.iter().enumerate() {
+                            if positions[file_idx] >= ranges.len() {
+                                continue;
+                            }
+                            let range = ranges[positions[file_idx]].clone();
+                            positions[file_idx] += 1;
+                            made_progress = true;
+                            push_split_planned_ranges(
+                                &mut planned,
+                                file_idx,
+                                range,
+                                config.raw_range_size_bytes,
+                                &mut remaining,
+                            );
+                            if remaining == 0 {
+                                break;
+                            }
+                        }
+                        if !made_progress {
+                            break;
+                        }
+                    }
+                }
+                RawRangeMode::FileSequential => unreachable!(),
+            }
+
+            if remaining > 0 {
+                return Err(format!(
+                    "scheduler-raw metadata-pages planned {} bytes but target is {target_bytes}",
+                    target_bytes - remaining
+                )
+                .into());
+            }
+            (selected_files, planned)
+        }
+    };
+    let planned_bytes = planned
+        .iter()
+        .map(|(_, range)| range.end - range.start)
+        .sum::<u64>();
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+    let mut in_flight = RawInFlight::new(config.raw_completion_mode);
+    let mut next_range = 0usize;
+    let read_chunk_size = config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE);
+    while next_range < planned.len() && in_flight.len() < config.batch_concurrency {
+        let (idx, range) = planned[next_range].clone();
+        in_flight.push(raw_read_future(
+            selected_files[idx].0.clone(),
+            range,
+            next_range as u64,
+            config.raw_submit_mode,
+            read_chunk_size,
+            counters.clone(),
+        ));
+        next_range += 1;
+    }
+
+    let mut bytes_read = 0u64;
+    let mut requests_completed = 0u64;
+    while !in_flight.is_empty() {
+        tokio::select! {
+            maybe_bytes = in_flight.next() => {
+                let bytes = maybe_bytes.expect("raw read future disappeared")?;
+                let bytes = bytes as u64;
+                bytes_read += bytes;
+                requests_completed += 1;
+                counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                counters.arrow_bytes.fetch_add(bytes, Ordering::Relaxed);
+                if next_range < planned.len() {
+                    let (idx, range) = planned[next_range].clone();
+                    in_flight.push(raw_read_future(
+                        selected_files[idx].0.clone(),
+                        range,
+                        next_range as u64,
+                        config.raw_submit_mode,
+                        read_chunk_size,
+                        counters.clone(),
+                    ));
+                    next_range += 1;
+                }
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    in_flight.len(),
+                    planned.len().saturating_sub(next_range),
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    counters.arrow_bytes.store(bytes_read, Ordering::Relaxed);
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler.stats();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        in_flight.len(),
+        0,
+        &mut last_sample,
+    ));
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+
+    Ok(CaseStats {
+        rows: 0,
+        batches: requests_completed,
+        arrow_bytes: bytes_read,
+        planned_fragments: selected_files.len(),
+        planned_rows: planned_bytes,
+        elapsed,
+        producer_finished_at: Some(elapsed),
+        peak_decode_in_flight: config.batch_concurrency,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+async fn run_case(
+    config: &Config,
+    io_buffer_gib: Option<u64>,
+    scheduler_diagnostics: &SchedulerDiagnosticsCollector,
+) -> Result<CaseStats> {
+    scheduler_diagnostics.clear();
+    let dataset = Arc::new(
+        DatasetBuilder::from_uri(&config.uri)
+            .with_version(config.dataset_version)
+            .load()
+            .await?,
+    );
+    let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?);
+    let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?;
+    let scheduler_config = io_buffer_gib
+        .map(|gib| SchedulerConfig::new(gib * GIB))
+        .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref()));
+    let scheduler = ScanScheduler::new(object_store, scheduler_config);
+
+    let mut planned = Vec::new();
+    let mut remaining_rows = config.limit_rows;
+    for fragment in dataset.fragments().iter() {
+        if remaining_rows == 0 {
+            break;
+        }
+        let fragment_rows = fragment
+            .num_rows()
+            .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))?
+            as u64;
+        let rows = fragment_rows.min(remaining_rows);
+        planned.push((fragment.clone(), rows));
+        remaining_rows -= rows;
+    }
+    if planned.is_empty() {
+        return Err("no fragments selected".into());
+    }
+    let planned_rows: u64 = planned.iter().map(|(_, rows)| *rows).sum();
+    let planned_fragments = planned.len();
+
+    let counters = Arc::new(SharedCounters::default());
+    let cpu_before = read_cpu_sample();
+    let started = Instant::now();
+
+    let (tx, mut rx) = tokio::sync::mpsc::channel::<
+        BoxFuture<'static, lance_core::Result<RecordBatch>>,
+    >(config.batch_concurrency * 2);
+    let producer = if config.detach_fragment_streams {
+        tokio::spawn({
+            let dataset = dataset.clone();
+            let projection = projection.clone();
+            let scheduler = scheduler.clone();
+            let counters = counters.clone();
+            let batch_size = config.batch_size;
+            let file_reader_options = file_reader_options(config);
+            let fragment_concurrency = config.fragment_concurrency;
+            async move {
+                let drainers = futures::stream::iter(planned.into_iter().enumerate())
+                    .map({
+                        move |(priority, (fragment, rows))| {
+                            let dataset = dataset.clone();
+                            let projection = projection.clone();
+                            let scheduler = scheduler.clone();
+                            let tx = tx.clone();
+                            let counters = counters.clone();
+                            let file_reader_options = file_reader_options.clone();
+                            async move {
+                                counters.fragments_started.fetch_add(1, Ordering::Relaxed);
+                                let file_fragment = FileFragment::new(dataset, fragment);
+                                let read_config = FragReadConfig::default()
+                                    .with_scan_scheduler(scheduler)
+                                    .with_reader_priority(priority as u32);
+                                let read_config = if let Some(file_reader_options) =
+                                    file_reader_options.clone()
+                                {
+                                    read_config.with_file_reader_options(file_reader_options)
+                                } else {
+                                    read_config
+                                };
+
+                                let open_started = Instant::now();
+                                let reader =
+                                    file_fragment.open(projection.as_ref(), read_config).await?;
+                                add_duration(&counters.open_reader_ns, open_started.elapsed());
+
+                                let create_stream_started = Instant::now();
+                                let mut read_stream =
+                                    reader.read_ranges(vec![0..rows].into(), batch_size).await?;
+                                add_duration(
+                                    &counters.read_stream_create_ns,
+                                    create_stream_started.elapsed(),
+                                );
+
+                                let drainer = tokio::spawn(async move {
+                                    loop {
+                                        let next_started = Instant::now();
+                                        let maybe_batch_fut = read_stream.next().await;
+                                        add_duration(
+                                            &counters.next_batch_poll_ns,
+                                            next_started.elapsed(),
+                                        );
+                                        let Some(batch_fut) = maybe_batch_fut else {
+                                            break;
+                                        };
+                                        counters
+                                            .batch_futures_emitted
+                                            .fetch_add(1, Ordering::Relaxed);
+                                        let send_started = Instant::now();
+                                        tx.send(batch_fut)
+                                            .await
+                                            .map_err(|_| "batch consumer dropped")?;
+                                        add_duration(
+                                            &counters.channel_send_wait_ns,
+                                            send_started.elapsed(),
+                                        );
+                                    }
+                                    counters.fragments_completed.fetch_add(1, Ordering::Relaxed);
+                                    Ok::<_, Error>(())
+                                });
+                                Ok::<_, Error>(drainer)
+                            }
+                        }
+                    })
+                    .buffer_unordered(fragment_concurrency)
+                    .try_collect::<Vec<_>>()
+                    .await?;
+
+                for drainer in drainers {
+                    drainer.await.map_err(Error::from)??;
+                }
+                Ok::<_, Error>(())
+            }
+        })
+    } else {
+        tokio::spawn({
+            let dataset = dataset.clone();
+            let projection = projection.clone();
+            let scheduler = scheduler.clone();
+            let counters = counters.clone();
+            let batch_size = config.batch_size;
+            let file_reader_options = file_reader_options(config);
+            let fragment_concurrency = config.fragment_concurrency;
+            async move {
+                futures::stream::iter(planned.into_iter().enumerate())
+                    .map({
+                        move |(priority, (fragment, rows))| {
+                            let dataset = dataset.clone();
+                            let projection = projection.clone();
+                            let scheduler = scheduler.clone();
+                            let tx = tx.clone();
+                            let counters = counters.clone();
+                            let file_reader_options = file_reader_options.clone();
+                            async move {
+                                counters.fragments_started.fetch_add(1, Ordering::Relaxed);
+                                let file_fragment = FileFragment::new(dataset, fragment);
+                                let read_config = FragReadConfig::default()
+                                    .with_scan_scheduler(scheduler)
+                                    .with_reader_priority(priority as u32);
+                                let read_config = if let Some(file_reader_options) =
+                                    file_reader_options.clone()
+                                {
+                                    read_config.with_file_reader_options(file_reader_options)
+                                } else {
+                                    read_config
+                                };
+
+                                let open_started = Instant::now();
+                                let reader =
+                                    file_fragment.open(projection.as_ref(), read_config).await?;
+                                add_duration(&counters.open_reader_ns, open_started.elapsed());
+
+                                let create_stream_started = Instant::now();
+                                let mut read_stream =
+                                    reader.read_ranges(vec![0..rows].into(), batch_size).await?;
+                                add_duration(
+                                    &counters.read_stream_create_ns,
+                                    create_stream_started.elapsed(),
+                                );
+
+                                loop {
+                                    let next_started = Instant::now();
+                                    let maybe_batch_fut = read_stream.next().await;
+                                    add_duration(
+                                        &counters.next_batch_poll_ns,
+                                        next_started.elapsed(),
+                                    );
+                                    let Some(batch_fut) = maybe_batch_fut else {
+                                        break;
+                                    };
+                                    counters
+                                        .batch_futures_emitted
+                                        .fetch_add(1, Ordering::Relaxed);
+                                    let send_started = Instant::now();
+                                    tx.send(batch_fut)
+                                        .await
+                                        .map_err(|_| "batch consumer dropped")?;
+                                    add_duration(
+                                        &counters.channel_send_wait_ns,
+                                        send_started.elapsed(),
+                                    );
+                                }
+                                counters.fragments_completed.fetch_add(1, Ordering::Relaxed);
+                                Ok::<_, Error>(())
+                            }
+                        }
+                    })
+                    .buffer_unordered(fragment_concurrency)
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                Ok::<_, Error>(())
+            }
+        })
+    };
+
+    let mut in_flight = FuturesUnordered::new();
+    let skip_batch_byte_accounting = config.skip_batch_byte_accounting;
+    let drop_read_tasks = config.drop_read_tasks;
+    let mut producer_done = false;
+    let mut producer_finished_at = None;
+    let mut rows = 0u64;
+    let mut batches = 0u64;
+    let mut arrow_bytes = 0u64;
+    let mut peak_decode_in_flight = 0usize;
+    let mut samples = Vec::new();
+    let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms));
+    sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut last_sample = LastSample {
+        elapsed: Duration::default(),
+        scheduler_stats: ScanStats::default(),
+        rows: 0,
+        batches: 0,
+        arrow_bytes: 0,
+    };
+
+    loop {
+        if producer_done && in_flight.is_empty() {
+            break;
+        }
+        tokio::select! {
+            maybe_batch_fut = rx.recv(), if !producer_done && in_flight.len() < config.batch_concurrency => {
+                if let Some(batch_fut) = maybe_batch_fut {
+                    counters.batch_futures_received.fetch_add(1, Ordering::Relaxed);
+                    if drop_read_tasks {
+                        drop(batch_fut);
+                        counters.batches_completed.fetch_add(1, Ordering::Relaxed);
+                        batches += 1;
+                        continue;
+                    }
+                    let counters_for_task = counters.clone();
+                    in_flight.push(async move {
+                        let decode_started = Instant::now();
+                        let batch = batch_fut.await?;
+                        let batch_bytes = if skip_batch_byte_accounting {
+                            0
+                        } else {
+                            batch.get_array_memory_size() as u64
+                        };
+                        add_duration(&counters_for_task.decode_ns, decode_started.elapsed());
+                        counters_for_task.batches_completed.fetch_add(1, Ordering::Relaxed);
+                        counters_for_task.rows_completed.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
+                        counters_for_task.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed);
+                        Ok::<_, lance_core::Error>((batch, batch_bytes))
+                    });
+                    peak_decode_in_flight = peak_decode_in_flight.max(in_flight.len());
+                } else {
+                    producer_done = true;
+                    producer_finished_at = Some(started.elapsed());
+                }
+            }
+            maybe_batch = in_flight.next(), if !in_flight.is_empty() => {
+                let (batch, batch_bytes) = maybe_batch.expect("in-flight batch future disappeared")?;
+                rows += batch.num_rows() as u64;
+                batches += 1;
+                arrow_bytes += batch_bytes;
+            }
+            _ = sample_interval.tick() => {
+                samples.push(sample_json(
+                    started,
+                    counters.as_ref(),
+                    scheduler_diagnostics.snapshot(io_buffer_gib),
+                    in_flight.len(),
+                    rx.len(),
+                    &mut last_sample,
+                ));
+            }
+        }
+    }
+    producer.await??;
+
+    let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib);
+    final_diagnostics.stats = scheduler.stats();
+    samples.push(sample_json(
+        started,
+        counters.as_ref(),
+        final_diagnostics,
+        in_flight.len(),
+        rx.len(),
+        &mut last_sample,
+    ));
+
+    let elapsed = started.elapsed();
+    let cpu_after = read_cpu_sample();
+
+    Ok(CaseStats {
+        rows,
+        batches,
+        arrow_bytes,
+        planned_fragments,
+        planned_rows,
+        elapsed,
+        producer_finished_at,
+        peak_decode_in_flight,
+        cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| {
+            let total = after.total.checked_sub(before.total)?;
+            let idle = after.idle.checked_sub(before.idle)?;
+            if total == 0 {
+                return None;
+            }
+            Some((total - idle) as f64 / total as f64 * 100.0)
+        }),
+        scheduler_diagnostics: final_diagnostics,
+        counters,
+        samples,
+    })
+}
+
+fn read_cpu_sample() -> Option<CpuSample> {
+    let contents = fs::read_to_string("/proc/stat").ok()?;
+    let line = contents.lines().next()?;
+    let values = line
+        .split_whitespace()
+        .skip(1)
+        .map(|value| value.parse::<u64>())
+        .collect::<std::result::Result<Vec<_>, _>>()
+        .ok()?;
+    if values.len() < 5 {
+        return None;
+    }
+
+    let idle = values[3] + values[4];
+    let total = values.iter().sum();
+    Some(CpuSample { idle, total })
+}
+
+fn now_unix_secs() -> u64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|duration| duration.as_secs())
+        .unwrap_or_default()
+}
+
+fn current_commit() -> String {
+    option_env!("LANCE_BENCH_COMMIT")
+        .or_else(|| option_env!("GIT_COMMIT"))
+        .unwrap_or("unknown")
+        .to_string()
+}
+
+fn env_var(name: &str) -> Option<String> {
+    env::var(name).ok()
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let config = parse_args()?;
+    if config.describe_layout {
+        return describe_layout(&config).await;
+    }
+    let scheduler_diagnostics = SchedulerDiagnosticsCollector::default();
+    let subscriber = tracing_subscriber::registry().with(SchedulerDiagnosticsLayer::new(
+        scheduler_diagnostics.clone(),
+    ));
+    tracing::subscriber::set_global_default(subscriber).map_err(|error| {
+        std::io::Error::other(format!(
+            "failed to install scheduler diagnostics subscriber: {error}"
+        ))
+    })?;
+
+    fs::create_dir_all(&config.out_dir)?;
+    let output_path = format!(
+        "{}/s3_file_reader_diagnostics_{}.jsonl",
+        config.out_dir,
+        now_unix_secs()
+    );
+    let mut jsonl = String::new();
+    let commit = current_commit();
+    let instance = env_var("EC2_INSTANCE_TYPE").unwrap_or_else(|| "unknown".to_string());
+    let region = env_var("AWS_REGION")
+        .or_else(|| env_var("AWS_DEFAULT_REGION"))
+        .unwrap_or_else(|| "unknown".to_string());
+    let projection = projection_name(&config.columns);
+
+    for io_buffer_gib in &config.io_buffer_gib {
+        println!(
+            "running case={} backend={} projection={} limit_rows={} io_buffer_gib={} batch_size={} fragment_concurrency={} batch_concurrency={} sample_ms={}",
+            config.case_name,
+            config.backend.name(),
+            projection,
+            config.limit_rows,
+            io_buffer_gib
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "auto".to_string()),
+            config.batch_size,
+            config.fragment_concurrency,
+            config.batch_concurrency,
+            config.sample_ms
+        );
+        let stats = match config.backend {
+            Backend::FileReader => {
+                run_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::Scanner => {
+                run_scanner_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::SchedulerRaw => {
+                run_scheduler_raw_case(&config, *io_buffer_gib, &scheduler_diagnostics).await?
+            }
+            Backend::DatasetTake => run_dataset_take_case(&config, &scheduler_diagnostics).await?,
+        };
+        let elapsed_secs = stats.elapsed.as_secs_f64();
+        let scheduler_stats = stats.scheduler_diagnostics.stats;
+        let logical_gbps = if elapsed_secs > 0.0 {
+            stats.arrow_bytes as f64 * 8.0 / elapsed_secs / 1_000_000_000.0
+        } else {
+            0.0
+        };
+        let physical_gbps = if elapsed_secs > 0.0 {
+            scheduler_stats.bytes_read as f64 * 8.0 / elapsed_secs / 1_000_000_000.0
+        } else {
+            0.0
+        };
+        let rows_per_second = if elapsed_secs > 0.0 {
+            stats.rows as f64 / elapsed_secs
+        } else {
+            0.0
+        };
+        let bytes_per_row = stats.arrow_bytes.checked_div(stats.rows).unwrap_or(0);
+        let avg_bytes_per_scheduler_request = scheduler_stats
+            .bytes_read
+            .checked_div(scheduler_stats.requests)
+            .unwrap_or(0);
+        let avg_bytes_per_scheduler_iop = scheduler_stats
+            .bytes_read
+            .checked_div(scheduler_stats.iops)
+            .unwrap_or(0);
+        let counters = stats.counters.as_ref();
+        let record = json!({
+            "case": config.case_name,
+            "instance": instance,
+            "region": region,
+            "layer": config.backend.layer(),
+            "backend": config.backend.name(),
+            "dataset_uri": config.uri,
+            "dataset_version": config.dataset_version,
+            "lance_commit": commit,
+            "projection": projection,
+            "limit_rows": config.limit_rows,
+            "target_bytes": config.target_bytes,
+            "raw_range_size_bytes": config.raw_range_size_bytes,
+            "raw_range_mode": config.raw_range_mode.name(),
+            "raw_column_indices": config.raw_column_indices.clone(),
+            "raw_submit_mode": config.raw_submit_mode.name(),
+            "raw_completion_mode": config.raw_completion_mode.name(),
+            "take_repetitions": config.take_repetitions,
+            "raw_read_chunk_size_bytes": config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE),
+            "planned_rows": stats.planned_rows,
+            "planned_fragments": stats.planned_fragments,
+            "rows": stats.rows,
+            "batches": stats.batches,
+            "batch_size": config.batch_size,
+            "batch_size_bytes": config.batch_size_bytes,
+            "skip_batch_byte_accounting": config.skip_batch_byte_accounting,
+            "read_chunk_size": config.read_chunk_size,
+            "fragment_concurrency": config.fragment_concurrency,
+            "batch_concurrency": config.batch_concurrency,
+            "detach_fragment_streams": config.detach_fragment_streams,
+            "drop_read_tasks": config.drop_read_tasks,
+            "sample_ms": config.sample_ms,
+            "io_buffer_bytes": io_buffer_gib.map(|value| value * GIB),
+            "io_buffer_mode": if io_buffer_gib.is_some() { "explicit" } else { "auto" },
+            "lance_io_threads": env_var("LANCE_IO_THREADS").or_else(|| env_var("IO_THREADS")),
+            "lance_default_io_buffer_size": env_var("LANCE_DEFAULT_IO_BUFFER_SIZE"),
+            "lance_max_iop_size": env_var("LANCE_MAX_IOP_SIZE"),
+            "lance_use_lite_scheduler": env_var("LANCE_USE_LITE_SCHEDULER"),
+            "lance_inline_scheduling_threshold": env_var("LANCE_INLINE_SCHEDULING_THRESHOLD"),
+            "elapsed_seconds": elapsed_secs,
+            "producer_finished_seconds": stats.producer_finished_at.map(|duration| duration.as_secs_f64()),
+            "logical_gbps": logical_gbps,
+            "physical_gbps": physical_gbps,
+            "rows_per_second": rows_per_second,
+            "arrow_bytes": stats.arrow_bytes,
+            "bytes_per_row": bytes_per_row,
+            "avg_bytes_per_scheduler_request": avg_bytes_per_scheduler_request,
+            "avg_bytes_per_scheduler_iop": avg_bytes_per_scheduler_iop,
+            "scheduler_iops": scheduler_stats.iops,
+            "scheduler_requests": scheduler_stats.requests,
+            "scheduler_bytes_read": scheduler_stats.bytes_read,
+            "scheduler_diagnostics": diagnostics_json(stats.scheduler_diagnostics),
+            "fragments_started": counters.fragments_started.load(Ordering::Relaxed),
+            "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed),
+            "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed),
+            "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed),
+            "batches_completed": counters.batches_completed.load(Ordering::Relaxed),
+            "peak_decode_in_flight": stats.peak_decode_in_flight,
+            "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)),
+            "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)),
+            "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)),
+            "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)),
+            "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)),
+            "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)),
+            "cpu_avg": stats.cpu_avg,
+            "samples": stats.samples,
+        });
+        println!(
+            "case={} backend={} projection={} io_buffer_gib={} elapsed={:.3}s logical_gbps={:.2} physical_gbps={:.2} rows={} batches={} scheduler_bytes_read={} scheduler_iops={} scheduler_requests={} active_iops={} pending_iops={} cpu_avg={}",
+            config.case_name,
+            config.backend.name(),
+            projection,
+            io_buffer_gib
+                .map(|value| value.to_string())
+                .unwrap_or_else(|| "auto".to_string()),
+            elapsed_secs,
+            logical_gbps,
+            physical_gbps,
+            stats.rows,
+            stats.batches,
+            scheduler_stats.bytes_read,
+            scheduler_stats.iops,
+            scheduler_stats.requests,
+            stats.scheduler_diagnostics.active_iops,
+            stats.scheduler_diagnostics.pending_iops,
+            stats
+                .cpu_avg
+                .map(|value| format!("{value:.1}%"))
+                .unwrap_or_else(|| "unknown".to_string())
+        );
+        jsonl.push_str(&serde_json::to_string(&record)?);
+        jsonl.push('\n');
+        fs::write(&output_path, &jsonl)?;
+    }
+
+    println!("wrote {output_path}");
+    Ok(())
+}
diff --git a/rust/lance/src/bin/fm_contains_bench.rs b/rust/lance/src/bin/fm_contains_bench.rs
new file mode 100644
index 00000000000..b62ad8699f2
--- /dev/null
+++ b/rust/lance/src/bin/fm_contains_bench.rs
@@ -0,0 +1,610 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#![allow(clippy::print_stdout)]
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use arrow_array::{Array, LargeStringArray, RecordBatch, StringArray, UInt64Array};
+use clap::Parser;
+use futures::future::join_all;
+use lance::dataset::builder::DatasetBuilder;
+use lance::dataset::{Dataset, ReadParams};
+use lance::index::DatasetIndexExt;
+use lance::{Error, Result};
+use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor};
+use serde::Serialize;
+use tokio::sync::Semaphore;
+use tokio::time::timeout;
+
+const ROW_ID_COLUMN: &str = "_rowid";
+
+#[derive(Parser, Debug, Clone)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    #[arg(
+        long,
+        default_value = "az://datasets/mmlb/mmlb_100m_fts_en_fm_20260626.lance"
+    )]
+    uri: String,
+
+    #[arg(long, default_value = "full_content_idx")]
+    index_name: String,
+
+    #[arg(long, default_value = "full_content")]
+    text_column: String,
+
+    #[arg(long, default_value = "summary_in_image")]
+    pattern_source_column: String,
+
+    #[arg(long, default_value = "oailancepub")]
+    storage_account: String,
+
+    #[arg(long, value_parser = parse_storage_option)]
+    storage_option: Vec<(String, String)>,
+
+    #[arg(long, default_value_t = 1_099_511_627_776)]
+    index_cache_bytes: usize,
+
+    #[arg(long, default_value_t = 8_589_934_592)]
+    metadata_cache_bytes: usize,
+
+    #[arg(long, default_value_t = 5_000)]
+    sample_rows: usize,
+
+    #[arg(long, default_value_t = 5)]
+    query_term_count: usize,
+
+    #[arg(long, default_value_t = 100)]
+    k: i64,
+
+    #[arg(long, default_value_t = 0)]
+    warmup: usize,
+
+    #[arg(long, default_value_t = 4)]
+    queries: usize,
+
+    #[arg(long, default_value = "1 2 4 8")]
+    threads: String,
+
+    #[arg(long, default_value_t = 60.0)]
+    query_timeout_secs: f64,
+
+    #[arg(long, default_value_t = false)]
+    skip_prewarm: bool,
+
+    #[arg(long, default_value_t = true)]
+    explain: bool,
+
+    #[arg(long)]
+    output_json: String,
+
+    #[arg(long)]
+    output_csv: String,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct QueryPattern {
+    query_id: String,
+    pattern: String,
+    filter_sql: String,
+}
+
+#[derive(Debug, Serialize)]
+struct QueryResult {
+    query_id: String,
+    pattern: String,
+    filter_sql: String,
+    success: bool,
+    error: Option<String>,
+    elapsed_ms: f64,
+    result_count: usize,
+    checksum: u64,
+}
+
+#[derive(Debug, Serialize)]
+struct BatchSummary {
+    round_type: String,
+    threads: usize,
+    queries: usize,
+    successes: usize,
+    success_rate: f64,
+    wall_s: f64,
+    qps: f64,
+    mean_ms: f64,
+    p50_ms: f64,
+    p95_ms: f64,
+    p99_ms: f64,
+    avg_result_count: f64,
+    max_result_count: usize,
+    checksum: u64,
+    results: Vec<QueryResult>,
+}
+
+#[derive(Debug, Serialize)]
+struct BenchmarkOutput {
+    uri: String,
+    index_name: String,
+    text_column: String,
+    pattern_source_column: String,
+    sample_rows: usize,
+    query_term_count: usize,
+    k: i64,
+    warmup: usize,
+    queries: usize,
+    threads: Vec<usize>,
+    query_timeout_secs: f64,
+    index_cache_bytes: usize,
+    metadata_cache_bytes: usize,
+    prewarm_ms: Option<f64>,
+    explain_plan: Option<String>,
+    summaries: Vec<BatchSummary>,
+}
+
+fn parse_threads(value: &str) -> std::result::Result<Vec<usize>, String> {
+    let mut threads = Vec::new();
+    for part in value.replace(',', " ").split_whitespace() {
+        let thread_count = part
+            .parse::<usize>()
+            .map_err(|err| format!("invalid thread count {part:?}: {err}"))?;
+        if thread_count == 0 {
+            return Err("thread counts must be greater than zero".to_string());
+        }
+        threads.push(thread_count);
+    }
+    if threads.is_empty() {
+        return Err("at least one thread count is required".to_string());
+    }
+    Ok(threads)
+}
+
+fn parse_storage_option(value: &str) -> std::result::Result<(String, String), String> {
+    let Some((key, val)) = value.split_once('=') else {
+        return Err("storage options must be key=value".to_string());
+    };
+    if key.is_empty() {
+        return Err("storage option key cannot be empty".to_string());
+    }
+    Ok((key.to_string(), val.to_string()))
+}
+
+fn sql_quote(value: &str) -> String {
+    format!("'{}'", value.replace('\'', "''"))
+}
+
+fn query_terms_from_text(text: &str, term_count: usize) -> Result<String> {
+    let terms = text.split_whitespace().take(term_count).collect::<Vec<_>>();
+    if terms.len() < term_count {
+        return Err(Error::invalid_input(format!(
+            "sampled query text has {} terms, but requested {}",
+            terms.len(),
+            term_count
+        )));
+    }
+    Ok(terms.join(" "))
+}
+
+fn string_value(batch: &RecordBatch, column: &str, row: usize) -> Result<Option<String>> {
+    let array = batch
+        .column_by_name(column)
+        .ok_or_else(|| Error::invalid_input(format!("column {column:?} not found")))?;
+    if array.is_null(row) {
+        return Ok(None);
+    }
+    if let Some(strings) = array.as_any().downcast_ref::<StringArray>() {
+        return Ok(Some(strings.value(row).to_string()));
+    }
+    if let Some(strings) = array.as_any().downcast_ref::<LargeStringArray>() {
+        return Ok(Some(strings.value(row).to_string()));
+    }
+    Err(Error::invalid_input(format!(
+        "column {column:?} must be Utf8 or LargeUtf8, got {:?}",
+        array.data_type()
+    )))
+}
+
+async fn open_dataset(args: &Args) -> Result<Dataset> {
+    let mut options = HashMap::from([("account_name".to_string(), args.storage_account.clone())]);
+    for (key, val) in &args.storage_option {
+        options.insert(key.clone(), val.clone());
+    }
+    let read_params = ReadParams {
+        index_cache_size_bytes: args.index_cache_bytes,
+        metadata_cache_size_bytes: args.metadata_cache_bytes,
+        store_options: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options,
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    DatasetBuilder::from_uri(&args.uri)
+        .with_read_params(read_params)
+        .load()
+        .await
+}
+
+async fn sample_patterns(dataset: &Dataset, args: &Args) -> Result<Vec<QueryPattern>> {
+    let total = args.warmup + args.queries;
+    let mut scanner = dataset.scan();
+    scanner
+        .project(&[args.pattern_source_column.as_str()])?
+        .limit(Some(args.sample_rows as i64), None)?;
+    let batch = scanner.try_into_batch().await?;
+    let mut patterns = Vec::with_capacity(total);
+    let mut seen = std::collections::HashSet::new();
+
+    for row in 0..batch.num_rows() {
+        let Some(text) = string_value(&batch, &args.pattern_source_column, row)? else {
+            continue;
+        };
+        let pattern = query_terms_from_text(&text, args.query_term_count)?;
+        if seen.insert(pattern.clone()) {
+            let filter_sql = format!(
+                "contains({}, {})",
+                args.text_column,
+                sql_quote(pattern.as_str())
+            );
+            patterns.push(QueryPattern {
+                query_id: format!("Q{:06}", patterns.len() + 1),
+                pattern,
+                filter_sql,
+            });
+            if patterns.len() == total {
+                return Ok(patterns);
+            }
+        }
+    }
+
+    Err(Error::invalid_input(format!(
+        "sampled only {} unique patterns, need {}; increase --sample-rows",
+        patterns.len(),
+        total
+    )))
+}
+
+async fn explain_first_query(
+    dataset: &Dataset,
+    pattern: &QueryPattern,
+    args: &Args,
+) -> Result<String> {
+    let mut scanner = dataset.scan();
+    scanner
+        .with_row_id()
+        .project::<&str>(&[])?
+        .filter(&pattern.filter_sql)?
+        .limit(Some(args.k), None)?;
+    scanner.explain_plan(false).await
+}
+
+async fn execute_query(
+    dataset: Arc<Dataset>,
+    pattern: &QueryPattern,
+    args: &Args,
+) -> Result<QueryResult> {
+    let start = Instant::now();
+    let mut scanner = dataset.scan();
+    scanner
+        .with_row_id()
+        .project::<&str>(&[])?
+        .filter(&pattern.filter_sql)?
+        .limit(Some(args.k), None)?;
+    let batch = scanner.try_into_batch().await?;
+    let elapsed_ms = start.elapsed().as_secs_f64() * 1000.0;
+    let row_ids = batch
+        .column_by_name(ROW_ID_COLUMN)
+        .ok_or_else(|| Error::invalid_input(format!("{ROW_ID_COLUMN} column missing")))?;
+    let row_ids = row_ids
+        .as_any()
+        .downcast_ref::<UInt64Array>()
+        .ok_or_else(|| Error::invalid_input(format!("{ROW_ID_COLUMN} must be UInt64")))?;
+    let mut checksum = 0_u64;
+    for row in 0..row_ids.len() {
+        if !row_ids.is_null(row) {
+            checksum ^= row_ids.value(row);
+        }
+    }
+    Ok(QueryResult {
+        query_id: pattern.query_id.clone(),
+        pattern: pattern.pattern.clone(),
+        filter_sql: pattern.filter_sql.clone(),
+        success: true,
+        error: None,
+        elapsed_ms,
+        result_count: batch.num_rows(),
+        checksum,
+    })
+}
+
+async fn run_one(dataset: Arc<Dataset>, pattern: QueryPattern, args: Arc<Args>) -> QueryResult {
+    let start = Instant::now();
+    let result = if args.query_timeout_secs > 0.0 {
+        timeout(
+            Duration::from_secs_f64(args.query_timeout_secs),
+            execute_query(dataset, &pattern, &args),
+        )
+        .await
+        .map_err(|_| {
+            Error::io(format!(
+                "query timed out after {}s",
+                args.query_timeout_secs
+            ))
+        })
+        .and_then(|inner| inner)
+    } else {
+        execute_query(dataset, &pattern, &args).await
+    };
+
+    match result {
+        Ok(result) => result,
+        Err(err) => QueryResult {
+            query_id: pattern.query_id,
+            pattern: pattern.pattern,
+            filter_sql: pattern.filter_sql,
+            success: false,
+            error: Some(err.to_string()),
+            elapsed_ms: start.elapsed().as_secs_f64() * 1000.0,
+            result_count: 0,
+            checksum: 0,
+        },
+    }
+}
+
+async fn run_batch(
+    dataset: Arc<Dataset>,
+    args: Arc<Args>,
+    patterns: &[QueryPattern],
+    thread_count: usize,
+    round_type: &str,
+) -> BatchSummary {
+    let wall_start = Instant::now();
+    let semaphore = Arc::new(Semaphore::new(thread_count));
+    let tasks = patterns
+        .iter()
+        .map(|pattern| {
+            let pattern = pattern.clone();
+            let dataset = Arc::clone(&dataset);
+            let args = Arc::clone(&args);
+            let semaphore = Arc::clone(&semaphore);
+            tokio::spawn(async move {
+                let _permit = semaphore.acquire_owned().await.expect("semaphore closed");
+                run_one(dataset, pattern, args).await
+            })
+        })
+        .collect::<Vec<_>>();
+
+    let results = join_all(tasks)
+        .await
+        .into_iter()
+        .map(|result| match result {
+            Ok(query_result) => query_result,
+            Err(err) => QueryResult {
+                query_id: "join_error".to_string(),
+                pattern: String::new(),
+                filter_sql: String::new(),
+                success: false,
+                error: Some(err.to_string()),
+                elapsed_ms: 0.0,
+                result_count: 0,
+                checksum: 0,
+            },
+        })
+        .collect::<Vec<_>>();
+    summarize(round_type, thread_count, results, wall_start.elapsed())
+}
+
+fn percentile(values: &[f64], pct: usize) -> f64 {
+    if values.is_empty() {
+        return 0.0;
+    }
+    let mut ordered = values.to_vec();
+    ordered.sort_by(|left, right| left.total_cmp(right));
+    let idx = ((ordered.len() * pct).div_ceil(100)).saturating_sub(1);
+    ordered[idx.min(ordered.len() - 1)]
+}
+
+fn summarize(
+    round_type: &str,
+    threads: usize,
+    results: Vec<QueryResult>,
+    wall: Duration,
+) -> BatchSummary {
+    let successes = results.iter().filter(|result| result.success).count();
+    let latencies = results
+        .iter()
+        .filter(|result| result.success)
+        .map(|result| result.elapsed_ms)
+        .collect::<Vec<_>>();
+    let result_counts = results
+        .iter()
+        .filter(|result| result.success)
+        .map(|result| result.result_count)
+        .collect::<Vec<_>>();
+    let wall_s = wall.as_secs_f64();
+    BatchSummary {
+        round_type: round_type.to_string(),
+        threads,
+        queries: results.len(),
+        successes,
+        success_rate: if results.is_empty() {
+            0.0
+        } else {
+            successes as f64 / results.len() as f64
+        },
+        wall_s,
+        qps: if wall_s == 0.0 {
+            0.0
+        } else {
+            successes as f64 / wall_s
+        },
+        mean_ms: if latencies.is_empty() {
+            0.0
+        } else {
+            latencies.iter().sum::<f64>() / latencies.len() as f64
+        },
+        p50_ms: percentile(&latencies, 50),
+        p95_ms: percentile(&latencies, 95),
+        p99_ms: percentile(&latencies, 99),
+        avg_result_count: if result_counts.is_empty() {
+            0.0
+        } else {
+            result_counts.iter().sum::<usize>() as f64 / result_counts.len() as f64
+        },
+        max_result_count: result_counts.into_iter().max().unwrap_or(0),
+        checksum: results
+            .iter()
+            .filter(|result| result.success)
+            .map(|result| result.checksum)
+            .fold(0, |acc, checksum| acc ^ checksum),
+        results,
+    }
+}
+
+fn write_csv(path: &str, summaries: &[BatchSummary]) -> Result<()> {
+    let file = File::create(path)?;
+    let mut writer = BufWriter::new(file);
+    writeln!(
+        writer,
+        "round_type,threads,queries,successes,success_rate,wall_s,qps,mean_ms,p50_ms,p95_ms,p99_ms,avg_result_count,max_result_count,checksum"
+    )?;
+    for summary in summaries {
+        writeln!(
+            writer,
+            "{},{},{},{},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{},{}",
+            csv_escape(&summary.round_type),
+            summary.threads,
+            summary.queries,
+            summary.successes,
+            summary.success_rate,
+            summary.wall_s,
+            summary.qps,
+            summary.mean_ms,
+            summary.p50_ms,
+            summary.p95_ms,
+            summary.p99_ms,
+            summary.avg_result_count,
+            summary.max_result_count,
+            summary.checksum
+        )?;
+    }
+    Ok(())
+}
+
+fn csv_escape(value: &str) -> String {
+    if value.contains([',', '"', '\n']) {
+        format!("\"{}\"", value.replace('"', "\"\""))
+    } else {
+        value.to_string()
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let args = Arc::new(Args::parse());
+    let thread_counts = parse_threads(&args.threads).map_err(Error::invalid_input)?;
+    if args.k <= 0 {
+        return Err(Error::invalid_input("--k must be greater than zero"));
+    }
+    if args.query_term_count == 0 {
+        return Err(Error::invalid_input(
+            "--query-term-count must be greater than zero",
+        ));
+    }
+    if args.query_timeout_secs < 0.0 {
+        return Err(Error::invalid_input(
+            "--query-timeout-secs must be non-negative",
+        ));
+    }
+
+    println!("opening dataset {}", args.uri);
+    let dataset = Arc::new(open_dataset(&args).await?);
+    println!("dataset version {}", dataset.version().version);
+
+    let patterns = sample_patterns(&dataset, &args).await?;
+    println!("sampled {} patterns", patterns.len());
+
+    let explain_plan = if args.explain {
+        Some(explain_first_query(&dataset, &patterns[0], &args).await?)
+    } else {
+        None
+    };
+
+    let prewarm_ms = if args.skip_prewarm {
+        None
+    } else {
+        println!("prewarming index {}", args.index_name);
+        let start = Instant::now();
+        dataset.prewarm_index(&args.index_name).await?;
+        let elapsed_ms = start.elapsed().as_secs_f64() * 1000.0;
+        println!("prewarm finished in {:.3} ms", elapsed_ms);
+        Some(elapsed_ms)
+    };
+
+    let warmup_patterns = &patterns[..args.warmup];
+    let query_patterns = &patterns[args.warmup..];
+    let mut summaries = Vec::new();
+    for thread_count in thread_counts.iter().copied() {
+        if !warmup_patterns.is_empty() {
+            summaries.push(
+                run_batch(
+                    Arc::clone(&dataset),
+                    Arc::clone(&args),
+                    warmup_patterns,
+                    thread_count,
+                    "warmup",
+                )
+                .await,
+            );
+        }
+        let summary = run_batch(
+            Arc::clone(&dataset),
+            Arc::clone(&args),
+            query_patterns,
+            thread_count,
+            "test",
+        )
+        .await;
+        println!(
+            "test threads={} queries={} successes={} wall_s={:.3} qps={:.3} mean_ms={:.3} p95_ms={:.3}",
+            summary.threads,
+            summary.queries,
+            summary.successes,
+            summary.wall_s,
+            summary.qps,
+            summary.mean_ms,
+            summary.p95_ms
+        );
+        summaries.push(summary);
+    }
+
+    let output = BenchmarkOutput {
+        uri: args.uri.clone(),
+        index_name: args.index_name.clone(),
+        text_column: args.text_column.clone(),
+        pattern_source_column: args.pattern_source_column.clone(),
+        sample_rows: args.sample_rows,
+        query_term_count: args.query_term_count,
+        k: args.k,
+        warmup: args.warmup,
+        queries: args.queries,
+        threads: thread_counts,
+        query_timeout_secs: args.query_timeout_secs,
+        index_cache_bytes: args.index_cache_bytes,
+        metadata_cache_bytes: args.metadata_cache_bytes,
+        prewarm_ms,
+        explain_plan,
+        summaries,
+    };
+
+    let mut json = serde_json::to_string_pretty(&output)?;
+    json.push('\n');
+    std::fs::write(&args.output_json, json)?;
+    write_csv(&args.output_csv, &output.summaries)?;
+    Ok(())
+}
diff --git a/rust/lance/src/bin/fm_index_tool.rs b/rust/lance/src/bin/fm_index_tool.rs
new file mode 100644
index 00000000000..1eb03b78b26
--- /dev/null
+++ b/rust/lance/src/bin/fm_index_tool.rs
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#![allow(clippy::print_stdout)]
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Instant;
+
+use clap::{Parser, ValueEnum};
+use lance::dataset::ReadParams;
+use lance::dataset::builder::DatasetBuilder;
+use lance::index::{CreateIndexBuilder, DatasetIndexExt};
+use lance::{Error, Result};
+use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor};
+use uuid::Uuid;
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    #[arg(value_enum)]
+    action: Action,
+
+    #[arg(long)]
+    uri: String,
+
+    #[arg(long, default_value = "full_content_idx")]
+    index_name: String,
+
+    #[arg(long, default_value = "full_content")]
+    column: String,
+
+    #[arg(long, default_value = "oailancepub")]
+    storage_account: String,
+
+    #[arg(long, value_parser = parse_storage_option)]
+    storage_option: Vec<(String, String)>,
+
+    #[arg(long, default_value_t = 1)]
+    num_segments: u32,
+
+    #[arg(long)]
+    index_uuid: Option<Uuid>,
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, ValueEnum)]
+enum Action {
+    List,
+    Drop,
+    Create,
+}
+
+fn parse_storage_option(value: &str) -> std::result::Result<(String, String), String> {
+    let Some((key, val)) = value.split_once('=') else {
+        return Err("storage options must be key=value".to_string());
+    };
+    if key.is_empty() {
+        return Err("storage option key cannot be empty".to_string());
+    }
+    Ok((key.to_string(), val.to_string()))
+}
+
+async fn open_dataset(args: &Args) -> Result<lance::Dataset> {
+    let mut options = HashMap::from([("account_name".to_string(), args.storage_account.clone())]);
+    for (key, val) in &args.storage_option {
+        options.insert(key.clone(), val.clone());
+    }
+    let read_params = ReadParams {
+        store_options: Some(ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                options,
+            ))),
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+
+    DatasetBuilder::from_uri(&args.uri)
+        .with_read_params(read_params)
+        .load()
+        .await
+}
+
+fn fm_params(num_segments: u32) -> ScalarIndexParams {
+    let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+    if num_segments == 1 {
+        params
+    } else {
+        params.with_params(&serde_json::json!({ "num_segments": num_segments }))
+    }
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    let args = Args::parse();
+    if args.num_segments == 0 {
+        return Err(Error::invalid_input(
+            "--num-segments must be greater than 0",
+        ));
+    }
+
+    println!("opening dataset {}", args.uri);
+    let mut dataset = open_dataset(&args).await?;
+    println!("dataset version {}", dataset.version().version);
+
+    match args.action {
+        Action::List => {
+            let indices = dataset.load_indices().await?;
+            println!("indices={}", indices.len());
+            for index in indices.iter() {
+                println!(
+                    "name={} uuid={} version={} fields={:?} fragments={}",
+                    index.name,
+                    index.uuid,
+                    index.index_version,
+                    index.fields,
+                    index
+                        .fragment_bitmap
+                        .as_ref()
+                        .map(|bitmap| bitmap.len())
+                        .unwrap_or(0)
+                );
+            }
+        }
+        Action::Drop => {
+            let start = Instant::now();
+            dataset.drop_index(&args.index_name).await?;
+            println!(
+                "dropped index {} in {:.3}s; new version {}",
+                args.index_name,
+                start.elapsed().as_secs_f64(),
+                dataset.version().version
+            );
+        }
+        Action::Create => {
+            let params = fm_params(args.num_segments);
+            let start = Instant::now();
+            let mut builder = CreateIndexBuilder::new(
+                &mut dataset,
+                &[args.column.as_str()],
+                lance_index::IndexType::Fm,
+                &params,
+            )
+            .name(args.index_name.clone())
+            .replace(true);
+            if let Some(index_uuid) = args.index_uuid {
+                builder = builder.index_uuid(index_uuid);
+            }
+            let metadata = builder.await?;
+            println!(
+                "created index {} uuid={} in {:.3}s; new version {}",
+                metadata.name,
+                metadata.uuid,
+                start.elapsed().as_secs_f64(),
+                dataset.version().version
+            );
+        }
+    }
+    Ok(())
+}
diff --git a/rust/lance/src/blob.rs b/rust/lance/src/blob.rs
index 322bf67a04c..b48a5603be6 100644
--- a/rust/lance/src/blob.rs
+++ b/rust/lance/src/blob.rs
@@ -1,18 +1,42 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-//! Convenience builders for Lance blob v2 input columns.
+//! Builders and file-level writer helpers for Lance blob v2 columns.
 //!
-//! Blob v2 expects a column shaped as `Struct<data: LargeBinary?, uri: Utf8?>` and
-//! tagged with `ARROW:extension:name = "lance.blob.v2"`. This module offers a
-//! type-safe builder to construct that struct without manually wiring metadata
+//! Logical blob input uses `Struct<data: LargeBinary?, uri: Utf8?>`. File-level blob
+//! descriptors use a physical writer-side struct with `kind`, `blob_id`, and range fields.
 
-use std::sync::Arc;
+use std::collections::{HashMap, HashSet};
+use std::num::NonZeroUsize;
+use std::ops::Range;
+use std::sync::{
+    Arc, Mutex,
+    atomic::{AtomicU32, Ordering},
+};
 
-use arrow_array::{ArrayRef, StructArray, builder::LargeBinaryBuilder, builder::StringBuilder};
+use arrow_array::{
+    Array, ArrayRef, StructArray,
+    builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder},
+    cast::AsArray,
+    types::{UInt8Type, UInt32Type, UInt64Type},
+};
 use arrow_buffer::NullBufferBuilder;
-use arrow_schema::{DataType, Field};
-use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME};
+use arrow_schema::{DataType, Field, Fields};
+use bytes::Bytes;
+use lance_arrow::{
+    ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+    BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, FieldExt,
+};
+use lance_core::{
+    datatypes::{BlobKind, Field as LanceField, Schema as LanceSchema},
+    utils::blob::blob_path,
+};
+use lance_io::{
+    object_store::ObjectStore,
+    traits::{Reader, WriteExt, Writer},
+};
+use object_store::path::Path;
+use tokio::io::AsyncWriteExt;
 
 use crate::{Error, Result};
 
@@ -21,9 +45,71 @@ use crate::{Error, Result};
 /// Blob v2 expects a column shaped as `Struct<data: LargeBinary?, uri: Utf8?>` and
 /// tagged with `ARROW:extension:name = "lance.blob.v2"`.
 pub fn blob_field(name: &str, nullable: bool) -> Field {
-    let metadata = [(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]
+    blob_field_with_options(name, nullable, BlobFieldOptions::default())
+}
+
+/// Options for constructing a blob v2 field.
+#[derive(Clone, Debug, Default)]
+pub struct BlobFieldOptions {
+    /// Maximum payload size to keep inline in the data file before using packed blob storage.
+    pub inline_size_threshold: Option<usize>,
+    /// Maximum payload size to store in packed blob storage before using dedicated blob storage.
+    ///
+    /// A zero threshold is invalid because dedicated blob storage is selected when
+    /// the payload size is greater than this value.
+    pub dedicated_size_threshold: Option<NonZeroUsize>,
+}
+
+impl BlobFieldOptions {
+    /// Set the maximum payload size to keep inline in the data file.
+    pub fn with_inline_size_threshold(mut self, threshold: usize) -> Self {
+        self.inline_size_threshold = Some(threshold);
+        self
+    }
+
+    /// Set the maximum payload size to store in packed blob storage.
+    pub fn with_dedicated_size_threshold(mut self, threshold: NonZeroUsize) -> Self {
+        self.dedicated_size_threshold = Some(threshold);
+        self
+    }
+}
+
+/// Construct the Arrow field for a blob v2 column with storage layout options.
+///
+/// Blob v2 expects a column shaped as `Struct<data: LargeBinary?, uri: Utf8?>` and
+/// tagged with `ARROW:extension:name = "lance.blob.v2"`.
+///
+/// ```
+/// # use lance::{BlobFieldOptions, blob_field_with_options};
+/// let field = blob_field_with_options(
+///     "blob",
+///     true,
+///     BlobFieldOptions::default().with_inline_size_threshold(16 * 1024),
+/// );
+/// assert_eq!(
+///     field
+///         .metadata()
+///         .get("lance-encoding:blob-inline-size-threshold")
+///         .map(String::as_str),
+///     Some("16384"),
+/// );
+/// ```
+pub fn blob_field_with_options(name: &str, nullable: bool, options: BlobFieldOptions) -> Field {
+    let mut metadata = [(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]
         .into_iter()
-        .collect();
+        .collect::<std::collections::HashMap<_, _>>();
+    if let Some(threshold) = options.inline_size_threshold {
+        metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            threshold.to_string(),
+        );
+    }
+    if let Some(threshold) = options.dedicated_size_threshold {
+        metadata.insert(
+            BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY.to_string(),
+            threshold.get().to_string(),
+        );
+    }
     Field::new(
         name,
         DataType::Struct(
@@ -38,6 +124,858 @@ pub fn blob_field(name: &str, nullable: bool) -> Field {
     .with_metadata(metadata)
 }
 
+fn prepared_blob_child_fields() -> Fields {
+    Fields::from(vec![
+        Field::new("kind", DataType::UInt8, true),
+        Field::new("data", DataType::LargeBinary, true),
+        Field::new("uri", DataType::Utf8, true),
+        Field::new("blob_id", DataType::UInt32, true),
+        Field::new("blob_size", DataType::UInt64, true),
+        Field::new("position", DataType::UInt64, true),
+    ])
+}
+
+fn prepared_blob_field_with_metadata(
+    name: &str,
+    nullable: bool,
+    metadata: HashMap<String, String>,
+) -> Field {
+    Field::new(
+        name,
+        DataType::Struct(prepared_blob_child_fields()),
+        nullable,
+    )
+    .with_metadata(metadata)
+}
+
+fn logical_blob_lance_children() -> Result<Vec<LanceField>> {
+    [
+        Field::new("data", DataType::LargeBinary, true),
+        Field::new("uri", DataType::Utf8, true),
+    ]
+    .iter()
+    .map(LanceField::try_from)
+    .collect()
+}
+
+fn field_matches(field: &Field, name: &str, data_type: &DataType, nullable: bool) -> bool {
+    field.name() == name && field.data_type() == data_type && field.is_nullable() == nullable
+}
+
+fn blob_v2_shape_error(field: &Field) -> Error {
+    Error::invalid_input(format!(
+        "Blob v2 field '{}' must use either logical struct<data: LargeBinary?, uri: Utf8?> \
+         with optional position/size UInt64 fields or prepared struct<kind: UInt8?, data: \
+         LargeBinary?, uri: Utf8?, blob_id: UInt32?, blob_size: UInt64?, position: UInt64?>",
+        field.name()
+    ))
+}
+
+/// Returns true when `field` is the writer-side prepared blob v2 struct.
+pub(crate) fn is_prepared_blob_v2_field(field: &Field) -> bool {
+    if !field.is_blob_v2() {
+        return false;
+    }
+    let DataType::Struct(fields) = field.data_type() else {
+        return false;
+    };
+    let expected = prepared_blob_child_fields();
+    fields.len() == expected.len()
+        && fields
+            .iter()
+            .zip(expected.iter())
+            .all(|(actual, expected)| actual.as_ref() == expected.as_ref())
+}
+
+/// Returns true when `field` is the logical blob v2 input struct.
+pub(crate) fn is_logical_blob_v2_field(field: &Field) -> bool {
+    if !field.is_blob_v2() {
+        return false;
+    }
+    let DataType::Struct(fields) = field.data_type() else {
+        return false;
+    };
+    match fields.len() {
+        2 => {
+            field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true)
+                && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true)
+        }
+        4 => {
+            field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true)
+                && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true)
+                && fields[2].name() == "position"
+                && fields[2].data_type() == &DataType::UInt64
+                && fields[3].name() == "size"
+                && fields[3].data_type() == &DataType::UInt64
+        }
+        _ => false,
+    }
+}
+
+fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result<LanceField> {
+    if field.is_blob_v2() {
+        let arrow_field = Field::from(field);
+        if is_prepared_blob_v2_field(&arrow_field) {
+            let mut normalized = field.clone();
+            let mut logical_children = logical_blob_lance_children()?;
+            for (logical_child, prepared_child) in
+                logical_children.iter_mut().zip(field.children.iter())
+            {
+                logical_child.id = prepared_child.id;
+                logical_child.parent_id = field.id;
+            }
+            normalized.children = logical_children;
+            return Ok(normalized);
+        }
+        if is_logical_blob_v2_field(&arrow_field) {
+            return Ok(field.clone());
+        }
+        return Err(blob_v2_shape_error(&arrow_field));
+    }
+
+    if field.children.is_empty() {
+        return Ok(field.clone());
+    }
+
+    let normalized_children = field
+        .children
+        .iter()
+        .map(normalize_prepared_blob_lance_field)
+        .collect::<Result<Vec<_>>>()?;
+
+    Ok(LanceField {
+        children: normalized_children,
+        ..field.clone()
+    })
+}
+
+pub(crate) fn normalize_prepared_blob_schema(schema: &LanceSchema) -> Result<LanceSchema> {
+    let fields = schema
+        .fields
+        .iter()
+        .map(normalize_prepared_blob_lance_field)
+        .collect::<Result<Vec<_>>>()?;
+    Ok(LanceSchema {
+        fields,
+        metadata: schema.metadata.clone(),
+    })
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct BlobIdAllocator {
+    inner: Arc<BlobIdAllocatorInner>,
+}
+
+#[derive(Debug)]
+struct BlobIdAllocatorInner {
+    next: AtomicU32,
+    used: Mutex<HashSet<u32>>,
+}
+
+impl BlobIdAllocator {
+    pub(crate) fn new(start: u32) -> Self {
+        Self {
+            inner: Arc::new(BlobIdAllocatorInner {
+                next: AtomicU32::new(start),
+                used: Mutex::new(HashSet::new()),
+            }),
+        }
+    }
+
+    pub(crate) fn next(&self) -> Result<u32> {
+        loop {
+            let id = self.inner.next.load(Ordering::Relaxed);
+            if id == u32::MAX {
+                return Err(Error::invalid_input(
+                    "Blob id allocator exhausted u32 id space",
+                ));
+            }
+            if self
+                .inner
+                .next
+                .compare_exchange(id, id + 1, Ordering::Relaxed, Ordering::Relaxed)
+                .is_err()
+            {
+                continue;
+            }
+            let mut used =
+                self.inner.used.lock().map_err(|_| {
+                    Error::internal("Blob id allocator mutex was poisoned".to_string())
+                })?;
+            if used.insert(id) {
+                return Ok(id);
+            }
+        }
+    }
+}
+
+fn validate_blob_id(blob_id: u32) -> Result<()> {
+    if blob_id == 0 {
+        return Err(Error::invalid_input("Blob id 0 is reserved"));
+    }
+    Ok(())
+}
+
+fn validate_range(offset: u64, size: u64, object_size: u64, label: &str) -> Result<()> {
+    let end = offset.checked_add(size).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "{label} range overflows u64: offset={offset}, size={size}"
+        ))
+    })?;
+    if end > object_size {
+        return Err(Error::invalid_input(format!(
+            "{label} range [{offset}, {end}) exceeds blob object size {object_size}"
+        )));
+    }
+    Ok(())
+}
+
+fn validate_prepared_blob_value_array(field: &Field, array: &ArrayRef) -> Result<()> {
+    if !is_prepared_blob_v2_field(field) {
+        return Err(blob_v2_shape_error(field));
+    }
+
+    let struct_arr = array
+        .as_any()
+        .downcast_ref::<StructArray>()
+        .ok_or_else(|| Error::invalid_input("Prepared blob column was not a struct array"))?;
+    let kind_col = struct_arr
+        .column_by_name("kind")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `kind` field"))?
+        .as_primitive::<UInt8Type>();
+    let data_col = struct_arr
+        .column_by_name("data")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `data` field"))?
+        .as_binary::<i64>();
+    let uri_col = struct_arr
+        .column_by_name("uri")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `uri` field"))?
+        .as_string::<i32>();
+    let blob_id_col = struct_arr
+        .column_by_name("blob_id")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `blob_id` field"))?
+        .as_primitive::<UInt32Type>();
+    let blob_size_col = struct_arr
+        .column_by_name("blob_size")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `blob_size` field"))?
+        .as_primitive::<UInt64Type>();
+    let position_col = struct_arr
+        .column_by_name("position")
+        .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `position` field"))?
+        .as_primitive::<UInt64Type>();
+
+    for row in 0..struct_arr.len() {
+        if struct_arr.is_null(row) {
+            continue;
+        }
+        if kind_col.is_null(row) {
+            return Err(Error::invalid_input(format!(
+                "Prepared blob row {row} is non-null but `kind` is null"
+            )));
+        }
+
+        match BlobKind::try_from(kind_col.value(row))? {
+            BlobKind::Inline => {
+                if data_col.is_null(row) {
+                    return Err(Error::invalid_input(format!(
+                        "Prepared inline blob row {row} must set `data`"
+                    )));
+                }
+            }
+            BlobKind::Packed => {
+                if blob_id_col.is_null(row)
+                    || blob_size_col.is_null(row)
+                    || position_col.is_null(row)
+                {
+                    return Err(Error::invalid_input(format!(
+                        "Prepared packed blob row {row} must set `blob_id`, `blob_size`, and `position`"
+                    )));
+                }
+                validate_blob_id(blob_id_col.value(row))?;
+                let offset = position_col.value(row);
+                let size = blob_size_col.value(row);
+                offset.checked_add(size).ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "Prepared packed blob row {row} range overflows u64: offset={offset}, size={size}"
+                    ))
+                })?;
+            }
+            BlobKind::Dedicated => {
+                if blob_id_col.is_null(row) || blob_size_col.is_null(row) {
+                    return Err(Error::invalid_input(format!(
+                        "Prepared dedicated blob row {row} must set `blob_id` and `blob_size`"
+                    )));
+                }
+                validate_blob_id(blob_id_col.value(row))?;
+            }
+            BlobKind::External => {
+                if uri_col.is_null(row) || uri_col.value(row).is_empty() {
+                    return Err(Error::invalid_input(format!(
+                        "Prepared external blob row {row} must set a non-empty `uri`"
+                    )));
+                }
+                let offset = if position_col.is_null(row) {
+                    0
+                } else {
+                    position_col.value(row)
+                };
+                let size = if blob_size_col.is_null(row) {
+                    0
+                } else {
+                    blob_size_col.value(row)
+                };
+                offset.checked_add(size).ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "Prepared external blob row {row} range overflows u64: offset={offset}, size={size}"
+                    ))
+                })?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Validate a writer-side prepared blob v2 array before it reaches the encoder.
+pub(crate) fn validate_prepared_blob_array(field: &Field, array: &ArrayRef) -> Result<()> {
+    validate_prepared_blob_value_array(field, array)
+}
+
+fn sidecar_path_for_data_file(data_file_path: &Path, blob_id: u32) -> Result<Path> {
+    validate_blob_id(blob_id)?;
+    let file_name = data_file_path.filename().ok_or_else(|| {
+        Error::invalid_input("Data file path must include a file name".to_string())
+    })?;
+    let data_file_key = file_name.strip_suffix(".lance").ok_or_else(|| {
+        Error::invalid_input(format!(
+            "Data file path '{}' must end with '.lance'",
+            data_file_path
+        ))
+    })?;
+    let data_dir = data_file_path.parent().unwrap_or_default();
+    Ok(blob_path(&data_dir, data_file_key, blob_id))
+}
+
+/// Byte range inside a packed or external blob object.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct BlobRange {
+    /// Byte offset relative to the beginning of the blob object.
+    pub offset: u64,
+    /// Number of bytes in this range.
+    pub size: u64,
+}
+
+/// A physical blob descriptor row.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum BlobDescriptor {
+    /// A null blob row.
+    Null,
+    /// Payload bytes embedded into the data file by the blob encoder.
+    Inline { data: Bytes },
+    /// Payload bytes stored as a range in a packed sidecar blob.
+    Packed {
+        blob_id: u32,
+        offset: u64,
+        size: u64,
+    },
+    /// Payload bytes stored as the full contents of a dedicated sidecar blob.
+    Dedicated { blob_id: u32, size: u64 },
+    /// Payload bytes referenced from an external object or registered base.
+    External {
+        base_id: u32,
+        uri: String,
+        offset: u64,
+        size: u64,
+    },
+}
+
+/// A physical blob descriptor column ready to be included in a [`RecordBatch`](arrow_array::RecordBatch).
+pub struct BlobDescriptorColumn {
+    field: Field,
+    array: ArrayRef,
+}
+
+impl BlobDescriptorColumn {
+    /// Return the Arrow field for the descriptor column.
+    pub fn field(&self) -> &Field {
+        &self.field
+    }
+
+    /// Return the Arrow array for the descriptor column.
+    pub fn array(&self) -> &ArrayRef {
+        &self.array
+    }
+
+    /// Consume this column into `(field, array)` parts.
+    pub fn into_parts(self) -> (Field, ArrayRef) {
+        (self.field, self.array)
+    }
+}
+
+/// Builds physical blob descriptors for one blob v2 column.
+///
+/// This builder only produces the writer-side descriptor struct array. It does not allocate blob ids,
+/// choose sidecar paths, write blob objects, or commit data files.
+pub struct BlobDescriptorArrayBuilder {
+    field: Field,
+    values: Vec<BlobDescriptor>,
+}
+
+impl BlobDescriptorArrayBuilder {
+    /// Create a descriptor array builder for one blob column.
+    pub fn new(column: impl Into<String>) -> Self {
+        let mut metadata = HashMap::with_capacity(1);
+        metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string());
+        Self::new_with_metadata(column, true, metadata)
+    }
+
+    pub(crate) fn new_with_metadata(
+        column: impl Into<String>,
+        nullable: bool,
+        metadata: HashMap<String, String>,
+    ) -> Self {
+        Self {
+            field: prepared_blob_field_with_metadata(&column.into(), nullable, metadata),
+            values: Vec::new(),
+        }
+    }
+
+    /// Append one packed blob descriptor.
+    pub fn push_packed(&mut self, blob_id: u32, range: BlobRange) -> Result<()> {
+        self.push(BlobDescriptor::Packed {
+            blob_id,
+            offset: range.offset,
+            size: range.size,
+        })
+    }
+
+    /// Append multiple packed blob descriptors for the same blob object.
+    pub fn extend_packed(
+        &mut self,
+        blob_id: u32,
+        ranges: impl IntoIterator<Item = BlobRange>,
+    ) -> Result<()> {
+        for range in ranges {
+            self.push_packed(blob_id, range)?;
+        }
+        Ok(())
+    }
+
+    /// Append one dedicated blob descriptor.
+    pub fn push_dedicated(&mut self, blob_id: u32, size: u64) -> Result<()> {
+        self.push(BlobDescriptor::Dedicated { blob_id, size })
+    }
+
+    /// Append one blob descriptor to this column.
+    pub fn push(&mut self, value: BlobDescriptor) -> Result<()> {
+        validate_blob_descriptor(&value)?;
+        self.values.push(value);
+        Ok(())
+    }
+
+    /// Append multiple blob descriptors to this column.
+    pub fn extend(&mut self, values: impl IntoIterator<Item = BlobDescriptor>) -> Result<()> {
+        for value in values {
+            self.push(value)?;
+        }
+        Ok(())
+    }
+
+    /// Append an inline blob value.
+    pub fn push_inline(&mut self, data: impl Into<Bytes>) -> Result<()> {
+        self.push(BlobDescriptor::Inline { data: data.into() })
+    }
+
+    /// Append an external blob reference.
+    pub fn push_external(
+        &mut self,
+        uri: impl Into<String>,
+        range: Option<BlobRange>,
+    ) -> Result<()> {
+        let range = range.unwrap_or(BlobRange { offset: 0, size: 0 });
+        self.push(BlobDescriptor::External {
+            base_id: 0,
+            uri: uri.into(),
+            offset: range.offset,
+            size: range.size,
+        })
+    }
+
+    /// Append a null blob row.
+    pub fn push_null(&mut self) -> Result<()> {
+        self.push(BlobDescriptor::Null)
+    }
+
+    /// Return the descriptor Arrow field for this blob column.
+    pub fn field(&self) -> &Field {
+        &self.field
+    }
+
+    /// Finish this column and return the writer-side descriptor struct array.
+    pub fn finish(self) -> Result<BlobDescriptorColumn> {
+        let mut kind_builder = PrimitiveBuilder::<UInt8Type>::with_capacity(self.values.len());
+        let mut data_builder = LargeBinaryBuilder::with_capacity(self.values.len(), 0);
+        let mut uri_builder = StringBuilder::with_capacity(self.values.len(), 0);
+        let mut blob_id_builder = PrimitiveBuilder::<UInt32Type>::with_capacity(self.values.len());
+        let mut blob_size_builder =
+            PrimitiveBuilder::<UInt64Type>::with_capacity(self.values.len());
+        let mut position_builder = PrimitiveBuilder::<UInt64Type>::with_capacity(self.values.len());
+        let mut validity = NullBufferBuilder::new(self.values.len());
+
+        for value in self.values {
+            match value {
+                BlobDescriptor::Null => {
+                    validity.append_null();
+                    kind_builder.append_null();
+                    data_builder.append_null();
+                    uri_builder.append_null();
+                    blob_id_builder.append_null();
+                    blob_size_builder.append_null();
+                    position_builder.append_null();
+                }
+                BlobDescriptor::Inline { data } => {
+                    validity.append_non_null();
+                    kind_builder.append_value(BlobKind::Inline as u8);
+                    data_builder.append_value(data.as_ref());
+                    uri_builder.append_null();
+                    blob_id_builder.append_null();
+                    blob_size_builder.append_null();
+                    position_builder.append_null();
+                }
+                BlobDescriptor::Packed {
+                    blob_id,
+                    offset,
+                    size,
+                } => {
+                    validity.append_non_null();
+                    kind_builder.append_value(BlobKind::Packed as u8);
+                    data_builder.append_null();
+                    uri_builder.append_null();
+                    blob_id_builder.append_value(blob_id);
+                    blob_size_builder.append_value(size);
+                    position_builder.append_value(offset);
+                }
+                BlobDescriptor::Dedicated { blob_id, size } => {
+                    validity.append_non_null();
+                    kind_builder.append_value(BlobKind::Dedicated as u8);
+                    data_builder.append_null();
+                    uri_builder.append_null();
+                    blob_id_builder.append_value(blob_id);
+                    blob_size_builder.append_value(size);
+                    position_builder.append_null();
+                }
+                BlobDescriptor::External {
+                    base_id,
+                    uri,
+                    offset,
+                    size,
+                } => {
+                    validity.append_non_null();
+                    kind_builder.append_value(BlobKind::External as u8);
+                    data_builder.append_null();
+                    uri_builder.append_value(uri);
+                    blob_id_builder.append_value(base_id);
+                    blob_size_builder.append_value(size);
+                    position_builder.append_value(offset);
+                }
+            }
+        }
+
+        let array = Arc::new(StructArray::try_new(
+            prepared_blob_child_fields(),
+            vec![
+                Arc::new(kind_builder.finish()),
+                Arc::new(data_builder.finish()),
+                Arc::new(uri_builder.finish()),
+                Arc::new(blob_id_builder.finish()),
+                Arc::new(blob_size_builder.finish()),
+                Arc::new(position_builder.finish()),
+            ],
+            validity.finish(),
+        )?) as ArrayRef;
+        validate_prepared_blob_array(&self.field, &array)?;
+
+        Ok(BlobDescriptorColumn {
+            field: self.field,
+            array,
+        })
+    }
+}
+
+fn validate_blob_descriptor(value: &BlobDescriptor) -> Result<()> {
+    match value {
+        BlobDescriptor::Null => Ok(()),
+        BlobDescriptor::Inline { .. } => Ok(()),
+        BlobDescriptor::Packed {
+            blob_id,
+            offset,
+            size,
+        } => {
+            validate_blob_id(*blob_id)?;
+            offset.checked_add(*size).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Packed blob range overflows u64: offset={offset}, size={size}"
+                ))
+            })?;
+            Ok(())
+        }
+        BlobDescriptor::Dedicated { blob_id, .. } => validate_blob_id(*blob_id),
+        BlobDescriptor::External {
+            uri, offset, size, ..
+        } => {
+            if uri.is_empty() {
+                return Err(Error::invalid_input("External blob URI cannot be empty"));
+            }
+            offset.checked_add(*size).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "External blob range overflows u64: offset={offset}, size={size}"
+                ))
+            })?;
+            Ok(())
+        }
+    }
+}
+
+fn packed_descriptor(blob_id: u32, offset: u64, size: u64) -> Result<(BlobDescriptor, u64)> {
+    let next_offset = offset.checked_add(size).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "Packed blob writer offset overflowed: offset={offset}, size={size}"
+        ))
+    })?;
+    Ok((
+        BlobDescriptor::Packed {
+            blob_id,
+            offset,
+            size,
+        },
+        next_offset,
+    ))
+}
+
+/// Writes a Lance-owned packed sidecar blob for one data file and returns descriptors.
+pub struct PackedBlobWriter {
+    object_store: ObjectStore,
+    path: Path,
+    blob_id: u32,
+    writer: Option<Box<dyn Writer>>,
+    offset: u64,
+    values: Vec<BlobDescriptor>,
+}
+
+impl PackedBlobWriter {
+    /// Create a packed blob writer for `data_file_path` and `blob_id`.
+    pub async fn try_new(
+        object_store: ObjectStore,
+        data_file_path: Path,
+        blob_id: u32,
+    ) -> Result<Self> {
+        let path = sidecar_path_for_data_file(&data_file_path, blob_id)?;
+        let writer = object_store.create(&path).await?;
+        Ok(Self {
+            object_store,
+            path,
+            blob_id,
+            writer: Some(writer),
+            offset: 0,
+            values: Vec::new(),
+        })
+    }
+
+    /// Return the blob id for this packed sidecar.
+    pub fn blob_id(&self) -> u32 {
+        self.blob_id
+    }
+
+    /// Return the derived sidecar path for this packed blob.
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    /// Append one logical blob payload to the packed sidecar.
+    pub async fn write_blob(&mut self, bytes: impl AsRef<[u8]>) -> Result<()> {
+        let bytes = bytes.as_ref();
+        self.write_blob_bytes(bytes).await?;
+        Ok(())
+    }
+
+    /// Append multiple logical blobs, one per iterator item.
+    ///
+    /// Each `Some(bytes)` is appended to the sidecar and records a packed
+    /// descriptor; an empty slice records a valid zero-length blob. Each `None`
+    /// records a [`BlobDescriptor::Null`] without writing any bytes, so the
+    /// descriptors returned by [`Self::finish`] stay row-aligned with the input.
+    ///
+    /// If writing fails or the future is cancelled after a partial write, no
+    /// descriptors from this call are recorded, the active writer is dropped,
+    /// and this instance cannot be reused.
+    ///
+    /// ```
+    /// # use lance::{PackedBlobWriter, Result};
+    /// # async fn write(mut writer: PackedBlobWriter) -> Result<()> {
+    /// writer
+    ///     .write_packed_blobs([Some(b"first".as_slice()), None, Some(b"second".as_slice())])
+    ///     .await?;
+    /// let descriptors = writer.finish().await?;
+    /// assert_eq!(descriptors.len(), 3);
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn write_packed_blobs<'a>(
+        &mut self,
+        blobs: impl IntoIterator<Item = Option<&'a [u8]>>,
+    ) -> Result<()> {
+        let mut writer = self.take_writer()?;
+        let mut descriptors = Vec::new();
+        let mut next_offset = self.offset;
+        for blob in blobs {
+            let Some(blob) = blob else {
+                descriptors.push(BlobDescriptor::Null);
+                continue;
+            };
+            let (descriptor, following_offset) =
+                packed_descriptor(self.blob_id, next_offset, blob.len() as u64)?;
+            if !blob.is_empty() {
+                writer.write_all(blob).await?;
+            }
+            descriptors.push(descriptor);
+            next_offset = following_offset;
+        }
+        self.writer = Some(writer);
+        self.offset = next_offset;
+        self.values.extend(descriptors);
+        Ok(())
+    }
+
+    pub(crate) async fn write_blob_bytes(&mut self, bytes: &[u8]) -> Result<BlobDescriptor> {
+        let size = bytes.len() as u64;
+        let offset = self.offset;
+        let mut writer = self.take_writer()?;
+        writer.write_all(bytes).await?;
+        self.writer = Some(writer);
+        self.record_written_blob(offset, size)
+    }
+
+    pub(crate) async fn write_blob_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<BlobDescriptor> {
+        let size = range.len() as u64;
+        let offset = self.offset;
+        let mut writer = self.take_writer()?;
+        writer.copy_range_from_reader(reader, range).await?;
+        self.writer = Some(writer);
+        self.record_written_blob(offset, size)
+    }
+
+    fn record_written_blob(&mut self, offset: u64, size: u64) -> Result<BlobDescriptor> {
+        let (value, next_offset) = packed_descriptor(self.blob_id, offset, size)?;
+        self.offset = next_offset;
+        self.values.push(value.clone());
+        Ok(value)
+    }
+
+    fn take_writer(&mut self) -> Result<Box<dyn Writer>> {
+        self.writer.take().ok_or_else(|| {
+            Error::io(format!(
+                "Packed blob writer for '{}' has no active upload",
+                self.path
+            ))
+        })
+    }
+
+    /// Finish the packed sidecar and return descriptors in write order.
+    pub async fn finish(mut self) -> Result<Vec<BlobDescriptor>> {
+        let mut writer = self.take_writer()?;
+        Writer::shutdown(writer.as_mut()).await?;
+        let object_size = self.object_store.size(&self.path).await?;
+        validate_range(0, self.offset, object_size, "Packed blob")?;
+        Ok(self.values)
+    }
+}
+
+/// Writes a Lance-owned dedicated sidecar blob for one data file and returns its descriptor.
+pub struct DedicatedBlobWriter {
+    object_store: ObjectStore,
+    path: Path,
+    blob_id: u32,
+    writer: Box<dyn Writer>,
+    size: u64,
+}
+
+impl DedicatedBlobWriter {
+    /// Create a dedicated blob writer for `data_file_path` and `blob_id`.
+    pub async fn try_new(
+        object_store: ObjectStore,
+        data_file_path: Path,
+        blob_id: u32,
+    ) -> Result<Self> {
+        let path = sidecar_path_for_data_file(&data_file_path, blob_id)?;
+        let writer = object_store.create(&path).await?;
+        Ok(Self {
+            object_store,
+            path,
+            blob_id,
+            writer,
+            size: 0,
+        })
+    }
+
+    /// Return the blob id for this dedicated sidecar.
+    pub fn blob_id(&self) -> u32 {
+        self.blob_id
+    }
+
+    /// Return the derived sidecar path for this dedicated blob.
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
+    /// Append bytes to the dedicated sidecar.
+    pub async fn write(&mut self, bytes: impl AsRef<[u8]>) -> Result<()> {
+        let bytes = bytes.as_ref();
+        let size = bytes.len() as u64;
+        self.writer.write_all(bytes).await?;
+        self.record_written_bytes(size)
+    }
+
+    pub(crate) async fn write_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<()> {
+        let size = range.len() as u64;
+        self.writer.copy_range_from_reader(reader, range).await?;
+        self.record_written_bytes(size)
+    }
+
+    fn record_written_bytes(&mut self, size: u64) -> Result<()> {
+        self.size = self.size.checked_add(size).ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Dedicated blob writer size overflowed: current={}, append={size}",
+                self.size
+            ))
+        })?;
+        Ok(())
+    }
+
+    /// Finish the dedicated sidecar and return its descriptor.
+    pub async fn finish(mut self) -> Result<BlobDescriptor> {
+        Writer::shutdown(self.writer.as_mut()).await?;
+        let object_size = self.object_store.size(&self.path).await?;
+        if object_size != self.size {
+            return Err(Error::io(format!(
+                "Dedicated blob sidecar '{}' has size {}, expected {}",
+                self.path, object_size, self.size
+            )));
+        }
+        Ok(BlobDescriptor::Dedicated {
+            blob_id: self.blob_id,
+            size: self.size,
+        })
+    }
+}
+
 /// Builder for blob v2 input struct columns.
 ///
 /// The builder enforces that each row contains exactly one of `data` or `uri` (or is null).
@@ -142,9 +1080,105 @@ impl BlobArrayBuilder {
 
 #[cfg(test)]
 mod tests {
+    use std::future::Future;
+    use std::io;
+    use std::num::NonZeroUsize;
+    use std::pin::Pin;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+    use std::task::{Context, Poll};
+
     use super::*;
-    use arrow_array::Array;
     use arrow_array::cast::AsArray;
+    use arrow_array::{Array, StringArray};
+    use arrow_schema::Schema as ArrowSchema;
+    use async_trait::async_trait;
+    use futures::task::noop_waker;
+    use lance_core::utils::tempfile::TempDir;
+    use lance_io::object_writer::WriteResult;
+    use tokio::io::AsyncWrite;
+
+    #[derive(Clone, Copy)]
+    enum WriteTerminal {
+        Error,
+        Pending,
+    }
+
+    struct PartialWriter {
+        bytes_before_terminal: usize,
+        terminal: WriteTerminal,
+        bytes_written: Arc<AtomicUsize>,
+        dropped: Arc<AtomicBool>,
+    }
+
+    impl AsyncWrite for PartialWriter {
+        fn poll_write(
+            mut self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            bytes: &[u8],
+        ) -> Poll<io::Result<usize>> {
+            if self.bytes_before_terminal > 0 {
+                let written = self.bytes_before_terminal.min(bytes.len());
+                self.bytes_before_terminal -= written;
+                self.bytes_written.fetch_add(written, Ordering::SeqCst);
+                return Poll::Ready(Ok(written));
+            }
+            match self.terminal {
+                WriteTerminal::Error => {
+                    Poll::Ready(Err(io::Error::other("injected write failure")))
+                }
+                WriteTerminal::Pending => Poll::Pending,
+            }
+        }
+
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[async_trait]
+    impl Writer for PartialWriter {
+        async fn tell(&mut self) -> Result<usize> {
+            Ok(self.bytes_written.load(Ordering::SeqCst))
+        }
+
+        async fn shutdown(&mut self) -> Result<WriteResult> {
+            Ok(WriteResult::default())
+        }
+    }
+
+    impl Drop for PartialWriter {
+        fn drop(&mut self) {
+            self.dropped.store(true, Ordering::SeqCst);
+        }
+    }
+
+    fn partial_writer(
+        terminal: WriteTerminal,
+    ) -> (Box<dyn Writer>, Arc<AtomicUsize>, Arc<AtomicBool>) {
+        let bytes_written = Arc::new(AtomicUsize::new(0));
+        let dropped = Arc::new(AtomicBool::new(false));
+        (
+            Box::new(PartialWriter {
+                bytes_before_terminal: 2,
+                terminal,
+                bytes_written: bytes_written.clone(),
+                dropped: dropped.clone(),
+            }),
+            bytes_written,
+            dropped,
+        )
+    }
+
+    fn cancel_pending<F: Future>(future: F) {
+        let mut future = Box::pin(future);
+        let waker = noop_waker();
+        let mut context = Context::from_waker(&waker);
+        assert!(future.as_mut().poll(&mut context).is_pending());
+    }
 
     #[test]
     fn test_field_metadata() {
@@ -156,6 +1190,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_field_metadata_with_options() {
+        let field = blob_field_with_options(
+            "blob",
+            true,
+            BlobFieldOptions::default()
+                .with_inline_size_threshold(16 * 1024)
+                .with_dedicated_size_threshold(NonZeroUsize::new(2 * 1024 * 1024).unwrap()),
+        );
+        assert_eq!(
+            field
+                .metadata()
+                .get(BLOB_INLINE_SIZE_THRESHOLD_META_KEY)
+                .unwrap(),
+            "16384"
+        );
+        assert_eq!(
+            field
+                .metadata()
+                .get(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY)
+                .unwrap(),
+            "2097152"
+        );
+    }
+
     #[test]
     fn test_builder_basic() {
         let mut b = BlobArrayBuilder::new(4);
@@ -194,4 +1253,280 @@ mod tests {
         let err = b.push_uri("").unwrap_err();
         assert!(err.to_string().contains("URI cannot be empty"));
     }
+
+    #[test]
+    fn test_prepared_blob_column_finish() {
+        let mut writer = BlobDescriptorArrayBuilder::new("blob");
+        writer.push_inline(Bytes::from_static(b"hello")).unwrap();
+        writer
+            .push_packed(7, BlobRange { offset: 3, size: 5 })
+            .unwrap();
+        writer.push_dedicated(8, 9).unwrap();
+        writer.push_null().unwrap();
+
+        let column = writer.finish().unwrap();
+        assert!(is_prepared_blob_v2_field(column.field()));
+        let struct_arr = column.array().as_struct();
+        let kinds = struct_arr
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<UInt8Type>();
+        let blob_ids = struct_arr
+            .column_by_name("blob_id")
+            .unwrap()
+            .as_primitive::<UInt32Type>();
+        let sizes = struct_arr
+            .column_by_name("blob_size")
+            .unwrap()
+            .as_primitive::<UInt64Type>();
+        let positions = struct_arr
+            .column_by_name("position")
+            .unwrap()
+            .as_primitive::<UInt64Type>();
+
+        assert_eq!(kinds.value(0), BlobKind::Inline as u8);
+        assert_eq!(kinds.value(1), BlobKind::Packed as u8);
+        assert_eq!(blob_ids.value(1), 7);
+        assert_eq!(positions.value(1), 3);
+        assert_eq!(sizes.value(1), 5);
+        assert_eq!(kinds.value(2), BlobKind::Dedicated as u8);
+        assert_eq!(blob_ids.value(2), 8);
+        assert_eq!(sizes.value(2), 9);
+        assert!(struct_arr.is_null(3));
+    }
+
+    #[test]
+    fn test_prepared_blob_array_rejects_range_overflow() {
+        let mut metadata = HashMap::new();
+        metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string());
+        let field = prepared_blob_field_with_metadata("blob", true, metadata);
+
+        for (kind, uri) in [
+            (BlobKind::Packed, None),
+            (BlobKind::External, Some("file:///external.bin")),
+        ] {
+            let array = Arc::new(
+                StructArray::try_new(
+                    prepared_blob_child_fields(),
+                    vec![
+                        Arc::new(arrow_array::UInt8Array::from(vec![kind as u8])) as ArrayRef,
+                        Arc::new(arrow_array::LargeBinaryArray::from_iter([None::<&[u8]>])),
+                        Arc::new(arrow_array::StringArray::from_iter([uri])),
+                        Arc::new(arrow_array::UInt32Array::from(vec![1])),
+                        Arc::new(arrow_array::UInt64Array::from(vec![4])),
+                        Arc::new(arrow_array::UInt64Array::from(vec![u64::MAX - 1])),
+                    ],
+                    None,
+                )
+                .unwrap(),
+            ) as ArrayRef;
+
+            let err = validate_prepared_blob_array(&field, &array).unwrap_err();
+            assert!(err.to_string().contains("range overflows u64"));
+        }
+    }
+
+    #[test]
+    fn test_normalize_prepared_blob_schema_preserves_non_blob_fields() {
+        let mut metadata = HashMap::new();
+        metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string());
+        let prepared_field = prepared_blob_field_with_metadata("blob", true, metadata);
+        let dict_field = Field::new(
+            "dict",
+            DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
+            true,
+        );
+        let mut schema =
+            LanceSchema::try_from(&ArrowSchema::new(vec![dict_field, prepared_field])).unwrap();
+        schema.fields[0].id = 42;
+        schema.fields[1].id = 7;
+
+        let dictionary_values = Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef;
+        schema.fields[0].set_dictionary_values(&dictionary_values);
+
+        let normalized = normalize_prepared_blob_schema(&schema).unwrap();
+
+        assert_eq!(normalized.fields[0].id, 42);
+        assert_eq!(
+            normalized.fields[0]
+                .dictionary
+                .as_ref()
+                .and_then(|dict| dict.values.as_ref())
+                .map(|values| values.len()),
+            Some(2)
+        );
+        assert_eq!(normalized.fields[1].id, 7);
+        assert_eq!(normalized.fields[1].children.len(), 2);
+        assert_eq!(normalized.fields[1].children[0].name, "data");
+        assert_eq!(normalized.fields[1].children[1].name, "uri");
+        assert!(normalized.fields[1].children[0].id >= 0);
+        assert!(normalized.fields[1].children[1].id >= 0);
+    }
+
+    #[tokio::test]
+    async fn test_sidecar_writers_return_prepared_values() {
+        let temp_dir = TempDir::default();
+        let data_dir = Path::from_absolute_path(temp_dir.std_path().join("data")).unwrap();
+        let data_file_key = "data-file".to_string();
+        let data_file_path = data_dir.clone().join(format!("{data_file_key}.lance"));
+        let object_store = ObjectStore::local();
+
+        let packed_id = 7;
+        let packed_path = blob_path(&data_dir, &data_file_key, packed_id);
+        let mut packed =
+            PackedBlobWriter::try_new(object_store.clone(), data_file_path.clone(), packed_id)
+                .await
+                .unwrap();
+        assert_eq!(packed.path(), &packed_path);
+        packed.write_blob(b"abc").await.unwrap();
+        packed.write_blob(b"de").await.unwrap();
+        let packed_values = packed.finish().await.unwrap();
+        assert_eq!(
+            packed_values,
+            vec![
+                BlobDescriptor::Packed {
+                    blob_id: packed_id,
+                    offset: 0,
+                    size: 3,
+                },
+                BlobDescriptor::Packed {
+                    blob_id: packed_id,
+                    offset: 3,
+                    size: 2,
+                },
+            ]
+        );
+
+        let dedicated_id = 8;
+        let dedicated_path = blob_path(&data_dir, &data_file_key, dedicated_id);
+        let mut dedicated =
+            DedicatedBlobWriter::try_new(object_store.clone(), data_file_path, dedicated_id)
+                .await
+                .unwrap();
+        assert_eq!(dedicated.path(), &dedicated_path);
+        dedicated.write(b"abcdef").await.unwrap();
+        assert_eq!(
+            dedicated.finish().await.unwrap(),
+            BlobDescriptor::Dedicated {
+                blob_id: dedicated_id,
+                size: 6,
+            }
+        );
+
+        let mut builder = BlobDescriptorArrayBuilder::new("blob");
+        builder
+            .extend_packed(
+                42,
+                vec![
+                    BlobRange { offset: 1, size: 2 },
+                    BlobRange { offset: 4, size: 1 },
+                ],
+            )
+            .unwrap();
+        builder.push_dedicated(43, 3).unwrap();
+        let column = builder.finish().unwrap();
+        assert_eq!(column.array().len(), 3);
+    }
+
+    #[tokio::test]
+    async fn test_packed_blob_writer_bulk_bytes() {
+        let temp_dir = TempDir::default();
+        let data_dir = Path::from_absolute_path(temp_dir.std_path().join("data")).unwrap();
+        let data_file_path = data_dir.join("data-file.lance");
+        let mut writer = PackedBlobWriter::try_new(ObjectStore::local(), data_file_path, 7)
+            .await
+            .unwrap();
+
+        writer
+            .write_packed_blobs([
+                Some(b"a".as_slice()),
+                Some(b"".as_slice()),
+                None,
+                Some(b"bc".as_slice()),
+            ])
+            .await
+            .unwrap();
+
+        assert_eq!(
+            writer.finish().await.unwrap(),
+            vec![
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 0,
+                    size: 1,
+                },
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 1,
+                    size: 0,
+                },
+                BlobDescriptor::Null,
+                BlobDescriptor::Packed {
+                    blob_id: 7,
+                    offset: 1,
+                    size: 2,
+                },
+            ]
+        );
+    }
+
+    #[tokio::test]
+    async fn test_packed_blob_writer_bulk_drops_after_partial_write_error() {
+        let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Error);
+        let previous_descriptor = BlobDescriptor::Packed {
+            blob_id: 7,
+            offset: 0,
+            size: 3,
+        };
+        let mut writer = PackedBlobWriter {
+            object_store: ObjectStore::local(),
+            path: Path::from("packed.blob"),
+            blob_id: 7,
+            writer: Some(partial_writer),
+            offset: 3,
+            values: vec![previous_descriptor.clone()],
+        };
+
+        let error = writer
+            .write_packed_blobs([Some(b"abcdef".as_slice())])
+            .await
+            .unwrap_err();
+
+        assert!(matches!(error, Error::IO { .. }));
+        assert!(error.to_string().contains("injected write failure"));
+        assert_eq!(bytes_written.load(Ordering::SeqCst), 2);
+        assert!(dropped.load(Ordering::SeqCst));
+        assert!(writer.writer.is_none());
+        assert_eq!(writer.offset, 3);
+        assert_eq!(writer.values, vec![previous_descriptor]);
+        let retry_error = writer.write_blob(b"retry").await.unwrap_err();
+        assert!(matches!(retry_error, Error::IO { .. }));
+        assert!(retry_error.to_string().contains("no active upload"));
+    }
+
+    #[test]
+    fn test_packed_blob_writer_bulk_drops_if_cancelled() {
+        let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Pending);
+        let previous_descriptor = BlobDescriptor::Packed {
+            blob_id: 7,
+            offset: 0,
+            size: 3,
+        };
+        let mut writer = PackedBlobWriter {
+            object_store: ObjectStore::local(),
+            path: Path::from("packed.blob"),
+            blob_id: 7,
+            writer: Some(partial_writer),
+            offset: 3,
+            values: vec![previous_descriptor.clone()],
+        };
+
+        cancel_pending(writer.write_packed_blobs([Some(b"abcdef".as_slice())]));
+
+        assert_eq!(bytes_written.load(Ordering::SeqCst), 2);
+        assert!(dropped.load(Ordering::SeqCst));
+        assert!(writer.writer.is_none());
+        assert_eq!(writer.offset, 3);
+        assert_eq!(writer.values, vec![previous_descriptor]);
+    }
 }
diff --git a/rust/lance/src/datafusion/dataframe.rs b/rust/lance/src/datafusion/dataframe.rs
index 00db9920bf9..7636177427c 100644
--- a/rust/lance/src/datafusion/dataframe.rs
+++ b/rust/lance/src/datafusion/dataframe.rs
@@ -1,10 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{
-    any::Any,
-    sync::{Arc, Mutex},
-};
+use std::sync::{Arc, Mutex};
 
 use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
@@ -82,10 +79,6 @@ impl LanceTableProvider {
 
 #[async_trait]
 impl TableProvider for LanceTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.full_schema.clone()
     }
diff --git a/rust/lance/src/datafusion/logical_plan.rs b/rust/lance/src/datafusion/logical_plan.rs
index a9fe0ed7750..039aa75864f 100644
--- a/rust/lance/src/datafusion/logical_plan.rs
+++ b/rust/lance/src/datafusion/logical_plan.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{any::Any, borrow::Cow, sync::Arc};
+use std::{borrow::Cow, sync::Arc};
 
 use arrow_schema::Schema as ArrowSchema;
 use async_trait::async_trait;
@@ -19,10 +19,6 @@ use crate::Dataset;
 
 #[async_trait]
 impl TableProvider for Dataset {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> Arc<ArrowSchema> {
         Arc::new(self.schema().into())
     }
@@ -167,17 +163,11 @@ mod tests {
 
         // DataFusion will create a cooperative execution plan, so we need to get its inner plan
         let physical_plan = physical_plan
-            .as_any()
             .downcast_ref::<CooperativeExec>()
             .unwrap()
             .children()[0];
 
-        assert!(
-            physical_plan
-                .as_any()
-                .downcast_ref::<LanceScanExec>()
-                .is_some()
-        );
+        assert!(physical_plan.downcast_ref::<LanceScanExec>().is_some());
 
         let expected_fields = schema
             .fields()
diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs
index 52e21bd7cba..70e96e323ab 100644
--- a/rust/lance/src/dataset.rs
+++ b/rust/lance/src/dataset.rs
@@ -8,10 +8,10 @@ use arrow_array::{RecordBatch, RecordBatchReader};
 use arrow_schema::DataType;
 use byteorder::{ByteOrder, LittleEndian};
 use chrono::{Duration, prelude::*};
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
 use futures::stream::{self, BoxStream, StreamExt, TryStreamExt};
 use futures::{FutureExt, Stream};
+use lance_core::deepsize::DeepSizeOf;
 
 use crate::dataset::metadata::UpdateFieldMetadataBuilder;
 use crate::dataset::transaction::translate_schema_metadata_updates;
@@ -24,8 +24,7 @@ use lance_core::datatypes::{OnMissing, OnTypeMismatch, Projectable, Projection};
 use lance_core::traits::DatasetTakeRows;
 use lance_core::utils::address::RowAddress;
 use lance_core::utils::tracing::{
-    DATASET_CLEANING_EVENT, DATASET_DELETING_EVENT, DATASET_DROPPING_COLUMN_EVENT,
-    TRACE_DATASET_EVENTS,
+    DATASET_DELETING_EVENT, DATASET_DROPPING_COLUMN_EVENT, TRACE_DATASET_EVENTS,
 };
 use lance_datafusion::projection::ProjectionPlan;
 use lance_file::datatypes::populate_schema_dictionary;
@@ -43,7 +42,8 @@ use lance_io::utils::{
 };
 use lance_namespace::LanceNamespace;
 use lance_table::format::{
-    DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, Manifest, RowIdMeta, pb,
+    DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, MAGIC, Manifest, RowIdMeta,
+    pb,
 };
 use lance_table::io::commit::{
     CommitConfig, CommitError, CommitHandler, CommitLock, ManifestLocation, ManifestNamingScheme,
@@ -60,7 +60,7 @@ use roaring::RoaringBitmap;
 use rowids::get_row_id_index;
 use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt::Debug;
 use std::num::NonZero;
 use std::ops::Range;
@@ -69,16 +69,18 @@ use std::sync::Arc;
 use tracing::{info, instrument};
 
 pub(crate) mod blob;
-mod branch_location;
+pub(crate) mod branch_location;
 pub mod builder;
 pub mod cleanup;
 pub mod delta;
+pub mod files;
 pub mod fragment;
 mod hash_joiner;
 pub mod index;
 pub mod mem_wal;
 mod metadata;
 pub mod optimize;
+pub(crate) mod overlay;
 pub mod progress;
 pub mod refs;
 pub(crate) mod rowids;
@@ -100,10 +102,11 @@ use self::cleanup::RemovalStats;
 use self::fragment::FileFragment;
 use self::refs::Refs;
 use self::scanner::{DatasetRecordBatchStream, Scanner};
+use self::statistics::DatasetStatistics;
 use self::transaction::{Operation, Transaction, TransactionBuilder, UpdateMapEntry};
 use self::write::{cleanup_data_fragments, write_fragments_internal};
 use crate::dataset::branch_location::BranchLocation;
-use crate::dataset::cleanup::{CleanupPolicy, CleanupPolicyBuilder};
+use crate::dataset::cleanup::{CleanupOperation, CleanupPolicy, CleanupPolicyBuilder};
 use crate::dataset::refs::{BranchContents, BranchIdentifier, Branches, Tags};
 use crate::dataset::sql::SqlQueryBuilder;
 use crate::datatypes::Schema;
@@ -127,6 +130,7 @@ pub use schema_evolution::{
     BatchInfo, BatchUDF, ColumnAlteration, NewColumnTransform, UDFCheckpointStore,
 };
 pub use take::TakeBuilder;
+use uuid::Uuid;
 pub use write::merge_insert::{
     MergeInsertBuilder, MergeInsertJob, MergeStats, UncommittedMergeInsert, WhenMatched,
     WhenNotMatched, WhenNotMatchedBySource,
@@ -226,6 +230,23 @@ impl From<&Manifest> for Version {
     }
 }
 
+/// The transaction that produced a version of the dataset, along with the
+/// version's commit timestamp.
+///
+/// Returned by [`Dataset::read_version_transaction`], which reads this
+/// information directly from storage without checking out the version.
+#[derive(Debug, Clone)]
+pub struct VersionTransaction {
+    /// Version number.
+    pub version: u64,
+
+    /// Timestamp the version was committed, in UTC.
+    pub timestamp: DateTime<Utc>,
+
+    /// The transaction that produced this version, if one was recorded.
+    pub transaction: Option<Transaction>,
+}
+
 /// Customize read behavior of a dataset.
 #[derive(Clone, Debug)]
 pub struct ReadParams {
@@ -452,6 +473,12 @@ impl Dataset {
         self.refs.tags()
     }
 
+    /// A handle for cheap, index-derived statistics about this dataset (e.g. a
+    /// column's global value range) that never scan data.
+    pub fn statistics(&self) -> DatasetStatistics<'_> {
+        DatasetStatistics::new(self)
+    }
+
     pub fn branches(&self) -> Branches<'_> {
         self.refs.branches()
     }
@@ -499,17 +526,23 @@ impl Dataset {
     ) -> Result<Self> {
         let (source_branch, version_number) = self.resolve_reference(version.into()).await?;
         let branch_location = self.branch_location().find_branch(Some(branch))?;
+        let source_location = self
+            .branch_location()
+            .find_branch(source_branch.as_deref())?;
         let clone_op = Operation::Clone {
             is_shallow: true,
             ref_name: source_branch.clone(),
             ref_version: version_number,
-            ref_path: String::from(self.uri()),
+            ref_path: source_location.uri,
             branch_name: Some(branch.to_string()),
         };
         let transaction = Transaction::new(version_number, clone_op, None);
 
         let builder = CommitBuilder::new(WriteDestination::Uri(branch_location.uri.as_str()))
-            .with_store_params(store_params.unwrap_or_default())
+            // Fall back to the dataset's own store params
+            .with_store_params(
+                store_params.unwrap_or(self.store_params.as_deref().cloned().unwrap_or_default()),
+            )
             .with_object_store(Arc::new(self.object_store.as_ref().clone()))
             .with_commit_handler(self.commit_handler.clone())
             .with_storage_format(self.manifest.data_storage_format.lance_file_version()?);
@@ -555,6 +588,15 @@ impl Dataset {
         version_number: Option<u64>,
         branch: Option<&str>,
     ) -> Result<Self> {
+        // Reject malformed names at the boundary (mirroring the branch CRUD
+        // paths) so they fail as InvalidRef instead of tripping the wrong-chain
+        // check below
+        if let Some(branch_name) = branch
+            && !Branches::is_main_branch(branch)
+        {
+            refs::check_valid_branch(branch_name)?;
+        }
+
         let new_location = self.branch_location().find_branch(branch)?;
 
         let manifest_location = if let Some(version_number) = version_number {
@@ -575,18 +617,33 @@ impl Dataset {
             return Ok(self.clone());
         }
 
-        let manifest = Self::load_manifest(
+        let manifest = Self::get_manifest(
             self.object_store.as_ref(),
             &manifest_location,
             &new_location.uri,
             self.session.as_ref(),
         )
         .await?;
+
+        // The resolved manifest must belong to the requested branch. A mismatch
+        // means the commit handler resolved against a different chain (for
+        // example an external manifest store that ignores branch-qualified
+        // paths); error loudly rather than hand back another branch's data.
+        let requested_branch = branch.and_then(refs::standardize_branch);
+        if manifest.branch.as_deref() != requested_branch.as_deref() {
+            return Err(Error::internal(format!(
+                "checkout of branch '{}' at version {} resolved a manifest belonging to branch '{}'",
+                refs::normalize_branch(branch),
+                manifest.version,
+                refs::normalize_branch(manifest.branch.as_deref()),
+            )));
+        }
+
         Self::checkout_manifest(
             self.object_store.clone(),
             new_location.path,
             new_location.uri,
-            Arc::new(manifest),
+            manifest,
             manifest_location,
             self.session.clone(),
             self.commit_handler.clone(),
@@ -623,6 +680,24 @@ impl Dataset {
                     }
                     _ => Error::io_source(err.into()),
                 })?;
+
+        // A stale cached size yields a bogus footer offset. Detect it (the block
+        // lacks the trailing magic) and retry with the true size, like
+        // read_manifest.
+        if manifest_location.size.is_some() && !last_block.ends_with(MAGIC) {
+            let manifest_location = ManifestLocation {
+                size: None,
+                ..manifest_location.clone()
+            };
+            return Box::pin(Self::load_manifest(
+                object_store,
+                &manifest_location,
+                uri,
+                session,
+            ))
+            .await;
+        }
+
         let offset = read_metadata_offset(&last_block)?;
 
         // If manifest is in the last block, we can decode directly from memory.
@@ -700,6 +775,35 @@ impl Dataset {
         Ok(manifest)
     }
 
+    /// Fetch the manifest for `manifest_location` from the session metadata
+    /// cache, loading and caching it on a miss.
+    pub(crate) async fn get_manifest(
+        object_store: &ObjectStore,
+        manifest_location: &ManifestLocation,
+        uri: &str,
+        session: &Session,
+    ) -> Result<Arc<Manifest>> {
+        if manifest_location.size.is_none() {
+            return Ok(Arc::new(
+                Self::load_manifest(object_store, manifest_location, uri, session).await?,
+            ));
+        }
+        let metadata_cache = session.metadata_cache.for_dataset(uri);
+        let manifest_key = ManifestKey {
+            version: manifest_location.version,
+            e_tag: manifest_location.e_tag.as_deref(),
+        };
+        if let Some(cached) = metadata_cache.get_with_key(&manifest_key).await {
+            return Ok(cached);
+        }
+        let loaded =
+            Arc::new(Self::load_manifest(object_store, manifest_location, uri, session).await?);
+        metadata_cache
+            .insert_with_key(&manifest_key, loaded.clone())
+            .await;
+        Ok(loaded)
+    }
+
     #[allow(clippy::too_many_arguments)]
     fn checkout_manifest(
         object_store: Arc<ObjectStore>,
@@ -725,6 +829,11 @@ impl Dataset {
         let metadata_cache = Arc::new(session.metadata_cache.for_dataset(&uri));
         let index_cache = Arc::new(session.index_cache.for_dataset(&uri));
         let fragment_bitmap = Arc::new(manifest.fragments.iter().map(|f| f.id as u32).collect());
+        write::log_unregistered_base_scoped_options(
+            store_params.as_ref(),
+            &manifest.base_paths,
+            log::Level::Debug,
+        );
         Ok(Self {
             object_store,
             base: base_path,
@@ -778,12 +887,50 @@ impl Dataset {
         batches: impl RecordBatchReader + Send + 'static,
         namespace_client: Arc<dyn LanceNamespace>,
         table_id: Vec<String>,
+        params: Option<WriteParams>,
+    ) -> Result<Self> {
+        Self::write_into_namespace_impl(batches, namespace_client, table_id, None, params).await
+    }
+
+    /// Write into a branch of a namespace client-managed table.
+    ///
+    /// Behaves like [`write_into_namespace`](Self::write_into_namespace), but APPEND and
+    /// OVERWRITE open and commit against `branch` instead of main. CREATE is rejected,
+    /// since a branch forks from an existing version.
+    pub async fn write_into_namespace_on_branch(
+        batches: impl RecordBatchReader + Send + 'static,
+        namespace_client: Arc<dyn LanceNamespace>,
+        table_id: Vec<String>,
+        branch: &str,
+        params: Option<WriteParams>,
+    ) -> Result<Self> {
+        Self::write_into_namespace_impl(
+            batches,
+            namespace_client,
+            table_id,
+            Some(branch.to_string()),
+            params,
+        )
+        .await
+    }
+
+    async fn write_into_namespace_impl(
+        batches: impl RecordBatchReader + Send + 'static,
+        namespace_client: Arc<dyn LanceNamespace>,
+        table_id: Vec<String>,
+        branch: Option<String>,
         mut params: Option<WriteParams>,
     ) -> Result<Self> {
         let mut write_params = params.take().unwrap_or_default();
 
         match write_params.mode {
             WriteMode::Create => {
+                if branch.is_some() {
+                    return Err(Error::not_supported_source(
+                        "cannot create a table on a branch; create on main first, then branch it"
+                            .into(),
+                    ));
+                }
                 let declare_request = DeclareTableRequest {
                     id: Some(table_id.clone()),
                     ..Default::default()
@@ -801,10 +948,13 @@ impl Dataset {
 
                 // Set up commit handler when managed_versioning is enabled
                 if response.managed_versioning == Some(true) {
-                    let external_store = LanceNamespaceExternalManifestStore::new(
+                    // The store derives the branch a request targets from the
+                    // base path it is handed, resolved against the table root.
+                    let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
                         namespace_client.clone(),
                         table_id.clone(),
-                    );
+                        &uri,
+                    )?;
                     let commit_handler: Arc<dyn CommitHandler> =
                         Arc::new(ExternalManifestCommitHandler {
                             external_manifest_store: Arc::new(external_store),
@@ -856,18 +1006,25 @@ impl Dataset {
                     )))
                 })?;
 
-                // Set up commit handler when managed_versioning is enabled
-                if response.managed_versioning == Some(true) {
-                    let external_store = LanceNamespaceExternalManifestStore::new(
-                        namespace_client.clone(),
-                        table_id.clone(),
-                    );
-                    let commit_handler: Arc<dyn CommitHandler> =
-                        Arc::new(ExternalManifestCommitHandler {
+                // Set up commit handler when managed_versioning is enabled.
+                // It must ride on the dataset opened below: InsertBuilder
+                // commits through the destination dataset's handler and does
+                // not consult write params for Dataset destinations.
+                let commit_handler: Option<Arc<dyn CommitHandler>> =
+                    if response.managed_versioning == Some(true) {
+                        // The store derives the branch a request targets from the
+                        // base path it is handed, resolved against the table root.
+                        let external_store = LanceNamespaceExternalManifestStore::for_table_uri(
+                            namespace_client.clone(),
+                            table_id.clone(),
+                            uri.as_str(),
+                        )?;
+                        Some(Arc::new(ExternalManifestCommitHandler {
                             external_manifest_store: Arc::new(external_store),
-                        });
-                    write_params.commit_handler = Some(commit_handler);
-                }
+                        }))
+                    } else {
+                        None
+                    };
 
                 // Set initial credentials and provider from namespace_client
                 if let Some(namespace_storage_options) = response.storage_options {
@@ -906,6 +1063,12 @@ impl Dataset {
                 {
                     builder = builder.with_storage_options_accessor(accessor.clone());
                 }
+                if let Some(commit_handler) = commit_handler {
+                    builder = builder.with_commit_handler(commit_handler);
+                }
+                if let Some(branch) = &branch {
+                    builder = builder.with_branch(branch, None);
+                }
                 let dataset = Arc::new(builder.load().await?);
 
                 Self::write(batches, dataset, Some(write_params)).await
@@ -1041,43 +1204,146 @@ impl Dataset {
             return Ok(Some((*transaction).clone()));
         }
 
+        let transaction = self
+            .read_transaction_from_storage(&self.manifest, &self.manifest_location)
+            .await?;
+
+        if let Some(tx) = transaction.as_ref() {
+            self.metadata_cache
+                .insert_with_key(&transaction_key, Arc::new(tx.clone()))
+                .await;
+        }
+        Ok(transaction)
+    }
+
+    /// Read the transaction recorded by `manifest` directly from storage,
+    /// without consulting or populating any session cache.
+    async fn read_transaction_from_storage(
+        &self,
+        manifest: &Manifest,
+        manifest_location: &ManifestLocation,
+    ) -> Result<Option<Transaction>> {
         // Prefer inline transaction from manifest when available
-        let transaction = if let Some(pos) = self.manifest.transaction_section {
-            let reader = if let Some(size) = self.manifest_location.size {
-                self.object_store
-                    .open_with_size(&self.manifest_location.path, size as usize)
-                    .await?
-            } else {
-                self.object_store.open(&self.manifest_location.path).await?
+        if let Some(pos) = manifest.transaction_section {
+            let reader = match manifest_location.size {
+                Some(size) => {
+                    self.object_store
+                        .open_with_size(&manifest_location.path, size as usize)
+                        .await?
+                }
+                None => self.object_store.open(&manifest_location.path).await?,
             };
 
-            let tx: pb::Transaction = read_message(reader.as_ref(), pos).await?;
-            Transaction::try_from(tx).map(Some)?
-        } else if let Some(path) = &self.manifest.transaction_file {
+            // A concurrent overwrite can leave the listed size too small; retry
+            // once with the true size.
+            let tx: pb::Transaction = match read_message(reader.as_ref(), pos).await {
+                Err(e)
+                    if manifest_location.size.is_some()
+                        && e.to_string().contains("file size is too small") =>
+                {
+                    let reader = self.object_store.open(&manifest_location.path).await?;
+                    read_message(reader.as_ref(), pos).await?
+                }
+                other => other?,
+            };
+            Transaction::try_from(tx).map(Some)
+        } else if let Some(path) = &manifest.transaction_file {
             // Fallback: read external transaction file if present
             let path = self.transactions_dir().join(path.as_str());
             let data = self.object_store.inner.get(&path).await?.bytes().await?;
             let transaction = lance_table::format::pb::Transaction::decode(data)?;
-            Transaction::try_from(transaction).map(Some)?
+            Transaction::try_from(transaction).map(Some)
         } else {
-            None
-        };
+            Ok(None)
+        }
+    }
 
-        if let Some(tx) = transaction.as_ref() {
-            self.metadata_cache
-                .insert_with_key(&transaction_key, Arc::new(tx.clone()))
-                .await;
+    /// Read the transaction (if any) and commit timestamp of a version of the
+    /// dataset. `version` is a version number on this dataset's current branch.
+    ///
+    /// Reads the version's manifest transiently: no historical `Dataset` is
+    /// constructed, no `IndexSection` is decoded, and no session cache is read
+    /// or written, so scanning many historical versions does not fill the
+    /// shared caches.
+    ///
+    /// Returns an error if the version does not exist (for example, if it has
+    /// been cleaned up).
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use lance::{Dataset, Result};
+    /// # async fn example(dataset: &Dataset) -> Result<()> {
+    /// let record = dataset.read_version_transaction(5).await?;
+    /// let committed_at = record.timestamp;
+    /// let operation = record.transaction.as_ref().map(|t| t.operation.name());
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn read_version_transaction(&self, version: u64) -> Result<VersionTransaction> {
+        // Resolve against this dataset's current branch.
+        let manifest_location = self
+            .commit_handler
+            .resolve_version_location(&self.base, version, &self.object_store.inner)
+            .await?;
+
+        // Keep the DatasetNotFound variant callers expect for a missing version.
+        let manifest = read_manifest(
+            &self.object_store,
+            &manifest_location.path,
+            manifest_location.size,
+        )
+        .await
+        .map_err(|e| match &e {
+            Error::NotFound { uri, .. } => Error::dataset_not_found(uri.clone(), box_error(e)),
+            _ => e,
+        })?;
+
+        // The resolved manifest must belong to this dataset's branch. A
+        // mismatch means the commit handler resolved against a different chain
+        // (for example an external manifest store that ignores
+        // branch-qualified paths); error loudly rather than hand back another
+        // branch's transaction.
+        if manifest.branch != self.manifest.branch {
+            return Err(Error::internal(format!(
+                "reading version {} on branch '{}' resolved a manifest belonging to branch '{}'",
+                version,
+                refs::normalize_branch(self.manifest.branch.as_deref()),
+                refs::normalize_branch(manifest.branch.as_deref()),
+            )));
         }
-        Ok(transaction)
+
+        let transaction = self
+            .read_transaction_from_storage(&manifest, &manifest_location)
+            .await?;
+
+        Ok(VersionTransaction {
+            version: manifest.version,
+            timestamp: manifest.timestamp(),
+            transaction,
+        })
     }
 
     /// Read the transaction file for this version of the dataset.
     ///
     /// If there was no transaction file written for this version of the dataset
     /// then this will return None.
+    ///
+    /// Does not populate the session caches; see
+    /// [`Self::read_version_transaction`].
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use lance::{Dataset, Result};
+    /// # async fn example(dataset: &Dataset) -> Result<()> {
+    /// let transaction = dataset.read_transaction_by_version(5).await?;
+    /// let operation = transaction.as_ref().map(|t| t.operation.name());
+    /// # Ok(())
+    /// # }
+    /// ```
     pub async fn read_transaction_by_version(&self, version: u64) -> Result<Option<Transaction>> {
-        let dataset_version = self.checkout_version(version).await?;
-        dataset_version.read_transaction().await
+        Ok(self.read_version_transaction(version).await?.transaction)
     }
 
     /// List transactions for the dataset, up to a maximum number.
@@ -1200,8 +1466,15 @@ impl Dataset {
         &self,
         policy: CleanupPolicy,
     ) -> BoxFuture<'_, Result<RemovalStats>> {
-        info!(target: TRACE_DATASET_EVENTS, event=DATASET_CLEANING_EVENT, uri=&self.uri);
-        cleanup::cleanup_old_versions(self, policy).boxed()
+        async move { self.cleanup(policy).execute().await }.boxed()
+    }
+
+    /// Creates a cleanup operation for this dataset.
+    ///
+    /// The returned operation can be explained without deleting files, or
+    /// executed to re-evaluate the current dataset state and remove files.
+    pub fn cleanup(&self, policy: CleanupPolicy) -> CleanupOperation<'_> {
+        CleanupOperation::new(self, policy)
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -1564,26 +1837,7 @@ impl Dataset {
                     ));
                 }
 
-                let selected_fragment_ids = fragment_ids.iter().copied().collect::<BTreeSet<_>>();
-                let selected_fragments = self
-                    .get_fragments()
-                    .into_iter()
-                    .filter(|fragment| selected_fragment_ids.contains(&(fragment.id() as u32)))
-                    .collect::<Vec<_>>();
-
-                if selected_fragments.len() != selected_fragment_ids.len() {
-                    let present_fragment_ids = selected_fragments
-                        .iter()
-                        .map(|fragment| fragment.id() as u32)
-                        .collect::<HashSet<_>>();
-                    let missing_fragment_ids = selected_fragment_ids
-                        .into_iter()
-                        .filter(|fragment_id| !present_fragment_ids.contains(fragment_id))
-                        .collect::<Vec<_>>();
-                    return Err(Error::invalid_input(format!(
-                        "Dataset::sample received fragment ids that are not part of the current dataset version: {missing_fragment_ids:?}",
-                    )));
-                }
+                let selected_fragments = self.get_fragments_from_ids(fragment_ids)?;
 
                 let num_rows = stream::iter(selected_fragments.iter().cloned())
                     .map(|fragment| async move { fragment.count_rows(None).await })
@@ -1744,21 +1998,26 @@ impl Dataset {
         store_params
     }
 
-    fn store_params_for_base(
+    pub(crate) fn store_params_for_base(
         &self,
         base_path: Option<&lance_table::format::BasePath>,
     ) -> ObjectStoreParams {
         // Base-specific bindings are exact ObjectStoreParams keyed by
-        // `BasePath.path`. If a base has no explicit binding then reads fall back
-        // to the dataset-level default store params.
-        base_path
-            .and_then(|base_path| {
-                self.base_store_params
-                    .as_ref()
-                    .and_then(|params| params.get(&base_path.path))
-            })
-            .cloned()
-            .unwrap_or_else(|| self.store_params.as_deref().cloned().unwrap_or_default())
+        // `BasePath.path` and are used as-is. Otherwise the dataset-level
+        // default params are resolved for the base scope: `base_<id>.<key>`
+        // storage options overlay the shared defaults for that base.
+        if let Some(params) = base_path.and_then(|base_path| {
+            self.base_store_params
+                .as_ref()
+                .and_then(|params| params.get(&base_path.path))
+        }) {
+            return params.clone();
+        }
+        let default_params = self.store_params.as_deref().cloned().unwrap_or_default();
+        match default_params.scoped_to_base(base_path.map(|base_path| base_path.id)) {
+            Cow::Owned(scoped_params) => scoped_params,
+            Cow::Borrowed(_) => default_params,
+        }
     }
 
     /// Returns the initial storage options used when opening this dataset, if any.
@@ -1882,58 +2141,215 @@ impl Dataset {
             file_metadata.minor_version as u32,
         )?;
 
-        // Get top-level column names from file schema in file order
-        let column_names: Vec<&str> = file_metadata
-            .file_schema
-            .fields
-            .iter()
-            .map(|f| f.name.as_str())
-            .collect();
+        let is_structural = file_version >= LanceFileVersion::V2_1;
+        let physical_columns = file_metadata.column_metadatas.len();
+        let has_footer_orphans = file_metadata.file_schema.fields.len() > physical_columns;
+        let dataset_schema = self.schema();
+        let mut represented_columns = 0usize;
+        let mut column_names = Vec::new();
+        let mut consumed_top_level_fields = 0usize;
+
+        fn physical_column_count(
+            field: &lance_core::datatypes::Field,
+            is_structural: bool,
+        ) -> usize {
+            if !is_structural {
+                return 1 + field
+                    .children
+                    .iter()
+                    .map(|child| physical_column_count(child, is_structural))
+                    .sum::<usize>();
+            }
 
-        // Project dataset schema by file column names to get dataset field IDs
-        let projected_ds_schema = self.schema().project(&column_names)?;
+            if field.children.is_empty() || field.is_blob() || field.is_packed_struct() {
+                1
+            } else {
+                field
+                    .children
+                    .iter()
+                    .map(|child| physical_column_count(child, is_structural))
+                    .sum()
+            }
+        }
 
-        // Walk both schemas in parallel to build fields and column_indices
-        let is_structural = file_version >= LanceFileVersion::V2_1;
-        let ds_fields: Vec<_> = projected_ds_schema.fields_pre_order().collect();
-        let file_fields: Vec<_> = file_metadata.file_schema.fields_pre_order().collect();
+        fn field_contains_blob(field: &lance_core::datatypes::Field) -> bool {
+            field.is_blob() || field.children.iter().any(field_contains_blob)
+        }
 
-        if ds_fields.len() != file_fields.len() {
-            return Err(Error::invalid_input(format!(
-                "Schema mismatch: dataset projection has {} fields but file has {} fields",
-                ds_fields.len(),
-                file_fields.len()
-            )));
+        fn field_names_match(
+            fields: &[lance_core::datatypes::Field],
+            start: usize,
+            names: &[&str],
+        ) -> bool {
+            fields
+                .get(start..start + names.len())
+                .is_some_and(|candidate| {
+                    candidate
+                        .iter()
+                        .zip(names)
+                        .all(|(field, name)| field.name == *name)
+                })
         }
 
-        let mut fields = Vec::new();
-        let mut column_indices = Vec::new();
-        let mut curr_column_idx: i32 = 0;
-        let mut packed_struct_fields_num: usize = 0;
+        fn blob_descriptor_orphan_len(
+            fields: &[lance_core::datatypes::Field],
+            start: usize,
+        ) -> usize {
+            const BLOB_V2_DESCRIPTOR_FIELDS: &[&str] =
+                &["kind", "position", "size", "blob_id", "blob_uri"];
+            const BLOB_V1_DESCRIPTOR_FIELDS: &[&str] = &["position", "size"];
+
+            if field_names_match(fields, start, BLOB_V2_DESCRIPTOR_FIELDS) {
+                BLOB_V2_DESCRIPTOR_FIELDS.len()
+            } else if field_names_match(fields, start, BLOB_V1_DESCRIPTOR_FIELDS) {
+                BLOB_V1_DESCRIPTOR_FIELDS.len()
+            } else {
+                0
+            }
+        }
+
+        fn collect_columns(
+            field: &lance_core::datatypes::Field,
+            is_structural: bool,
+            fields: &mut Vec<i32>,
+            column_indices: &mut Vec<i32>,
+            curr_column_idx: &mut i32,
+        ) {
+            let contributes = !is_structural
+                || field.children.is_empty()
+                || field.is_blob()
+                || field.is_packed_struct();
+            let recurse = !is_structural || (!field.is_blob() && !field.is_packed_struct());
+
+            if contributes {
+                fields.push(field.id);
+                column_indices.push(*curr_column_idx);
+                *curr_column_idx += 1;
+            }
 
-        for (ds_field, file_field) in ds_fields.iter().zip(file_fields.iter()) {
-            if ds_field.name != file_field.name {
+            if recurse {
+                for child in &field.children {
+                    collect_columns(
+                        child,
+                        is_structural,
+                        fields,
+                        column_indices,
+                        curr_column_idx,
+                    );
+                }
+            }
+        }
+
+        fn validate_file_field_matches_dataset(
+            dataset_field: &lance_core::datatypes::Field,
+            file_field: &lance_core::datatypes::Field,
+            path: &str,
+        ) -> Result<()> {
+            if dataset_field.name != file_field.name {
                 return Err(Error::invalid_input(format!(
                     "Schema mismatch: expected field '{}' but file has '{}'",
-                    ds_field.name, file_field.name
+                    path, file_field.name
                 )));
             }
 
-            if packed_struct_fields_num > 0 {
-                packed_struct_fields_num -= 1;
-                continue;
+            if dataset_field.is_blob() && file_field.is_blob() {
+                return Ok(());
+            }
+
+            if dataset_field.children.len() != file_field.children.len() {
+                return Err(Error::invalid_input(format!(
+                    "Schema mismatch: field '{}' has {} children in dataset schema but {} children in file schema",
+                    path,
+                    dataset_field.children.len(),
+                    file_field.children.len()
+                )));
             }
 
-            if file_field.is_packed_struct() {
-                fields.push(ds_field.id);
-                column_indices.push(curr_column_idx);
-                curr_column_idx += 1;
-                packed_struct_fields_num = file_field.children.len();
-            } else if file_field.children.is_empty() || !is_structural {
-                fields.push(ds_field.id);
-                column_indices.push(curr_column_idx);
-                curr_column_idx += 1;
+            for (dataset_child, file_child) in
+                dataset_field.children.iter().zip(&file_field.children)
+            {
+                let child_path = format!("{}.{}", path, dataset_child.name);
+                validate_file_field_matches_dataset(dataset_child, file_child, &child_path)?;
             }
+
+            Ok(())
+        }
+
+        let file_schema_fields = &file_metadata.file_schema.fields;
+        let mut idx = 0usize;
+        while represented_columns < physical_columns {
+            let Some(field) = file_schema_fields.get(idx) else {
+                return Err(Error::invalid_input(format!(
+                    "Schema mismatch: file schema ended after representing {} physical columns but file has {} columns",
+                    represented_columns, physical_columns
+                )));
+            };
+
+            let Some(dataset_field) = dataset_schema.field(&field.name) else {
+                return Err(Error::invalid_input(format!(
+                    "Schema mismatch: file has extra field '{}'",
+                    field.name
+                )));
+            };
+            validate_file_field_matches_dataset(dataset_field, field, &field.name)?;
+
+            represented_columns += physical_column_count(field, is_structural);
+            column_names.push(field.name.as_str());
+            consumed_top_level_fields = idx + 1;
+            idx += 1;
+
+            if has_footer_orphans && field_contains_blob(field) {
+                loop {
+                    let skipped = blob_descriptor_orphan_len(file_schema_fields, idx);
+                    if skipped == 0 {
+                        break;
+                    }
+                    consumed_top_level_fields = idx + skipped;
+                    idx += skipped;
+                }
+            }
+        }
+
+        if represented_columns != physical_columns {
+            return Err(Error::invalid_input(format!(
+                "Schema mismatch: file schema represents {} physical columns but file has {} columns",
+                represented_columns, physical_columns
+            )));
+        }
+
+        if let Some(field) = file_schema_fields.get(consumed_top_level_fields) {
+            return Err(Error::invalid_input(format!(
+                "Schema mismatch: file has extra field '{}'",
+                field.name
+            )));
+        }
+
+        let projected_ds_schema = self.schema().project(&column_names)?;
+
+        let mut fields = Vec::new();
+        let mut column_indices = Vec::new();
+        let mut curr_column_idx: i32 = 0;
+        for field in &projected_ds_schema.fields {
+            collect_columns(
+                field,
+                is_structural,
+                &mut fields,
+                &mut column_indices,
+                &mut curr_column_idx,
+            );
+        }
+
+        if curr_column_idx as usize != physical_columns {
+            return Err(Error::invalid_input(format!(
+                "Schema mismatch: dataset projection maps to {} physical columns but file has {} columns",
+                curr_column_idx, physical_columns
+            )));
+        }
+
+        if fields.is_empty() && physical_columns > 0 {
+            return Err(Error::invalid_input(
+                "Schema mismatch: file has columns but none matched the dataset schema",
+            ));
         }
 
         let file_size_nz = NonZero::new(file_size);
@@ -1951,7 +2367,7 @@ impl Dataset {
     /// Resolve the data directory for a given base_id.
     ///
     /// If `base_id` is `None`, returns the default data directory.
-    fn data_file_dir_for_base(&self, base_id: Option<u32>) -> Result<Path> {
+    pub(crate) fn data_file_dir_for_base(&self, base_id: Option<u32>) -> Result<Path> {
         match base_id {
             Some(base_id) => {
                 let base_path = self.manifest.base_paths.get(&base_id).ok_or_else(|| {
@@ -1995,6 +2411,13 @@ impl Dataset {
         }
     }
 
+    /// The `ObjectStoreParams` this dataset was opened with, or `None` when
+    /// opened without explicit params. Lets a caller re-open a derived path
+    /// (e.g. a MemWAL flushed generation) with the same store this dataset used.
+    pub fn store_params(&self) -> Option<&ObjectStoreParams> {
+        self.store_params.as_deref()
+    }
+
     pub(crate) async fn object_store_for_data_file(
         &self,
         data_file: &DataFile,
@@ -2149,6 +2572,39 @@ impl Dataset {
             .version)
     }
 
+    /// Return whether the dataset has a newer committed version.
+    pub async fn is_stale(&self) -> Result<bool> {
+        let latest_version = self.latest_version_id().await?;
+        Ok(latest_version != self.manifest.version)
+    }
+
+    /// Return whether the immediate attached successor manifest exists.
+    ///
+    /// This is a fast contiguous-history probe. It does not resolve the latest
+    /// version and may return `false` if intermediate manifests have been
+    /// removed. Callers that need a general freshness check should use
+    /// [`Self::is_stale`].
+    #[doc(hidden)]
+    pub async fn has_successor_version(&self) -> Result<bool> {
+        let Some(next_version) = self.manifest.version.checked_add(1) else {
+            return Ok(false);
+        };
+        if lance_table::format::is_detached_version(next_version) {
+            return Ok(false);
+        }
+
+        let exists = self
+            .commit_handler
+            .version_exists(
+                &self.base,
+                next_version,
+                self.object_store.inner.as_ref(),
+                self.manifest_location.naming_scheme,
+            )
+            .await?;
+        Ok(exists)
+    }
+
     pub fn count_fragments(&self) -> usize {
         self.manifest.fragments.len()
     }
@@ -2198,37 +2654,108 @@ impl Dataset {
         &self.manifest.fragments
     }
 
-    // Gets a filtered list of fragments from ids in O(N) time instead of using
-    // `get_fragment` which would require O(N^2) time.
-    pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec<Option<FileFragment>> {
-        let mut fragments = Vec::with_capacity(ordered_ids.len());
-        let mut id_iter = ordered_ids.iter();
-        let mut id = id_iter.next();
-        // This field is just used to assert the ids are in order
-        let mut last_id: i64 = -1;
-        for frag in self.manifest.fragments.iter() {
-            let mut the_id = if let Some(id) = id { *id } else { break };
-            // Assert the given ids are, in fact, in order
-            assert!(the_id as i64 > last_id);
-            // For any IDs we've passed we can assume that no fragment exists any longer
-            // with that ID.
-            while the_id < frag.id as u32 {
-                fragments.push(None);
-                last_id = the_id as i64;
-                id = id_iter.next();
-                the_id = if let Some(id) = id { *id } else { break };
-            }
+    pub(crate) fn normalize_fragment_ids(fragment_ids: &[u32]) -> Vec<u32> {
+        let mut ids = fragment_ids.to_vec();
+        ids.sort_unstable();
+        ids.dedup();
+        ids
+    }
 
-            if the_id == frag.id as u32 {
-                fragments.push(Some(FileFragment::new(
-                    Arc::new(self.clone()),
-                    frag.clone(),
-                )));
-                last_id = the_id as i64;
-                id = id_iter.next();
-            }
+    pub(crate) fn get_fragments_from_ids(&self, fragment_ids: &[u32]) -> Result<Vec<FileFragment>> {
+        let ordered_ids = Self::normalize_fragment_ids(fragment_ids);
+        let fragments = self.get_frags_from_ordered_ids(&ordered_ids);
+        if let Some(missing_id) = fragments
+            .iter()
+            .zip(ordered_ids.iter())
+            .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id))
+        {
+            return Err(Error::invalid_input(format!(
+                "Unknown fragment id {missing_id} in fragment filter; not part of the current dataset version"
+            )));
         }
-        fragments
+
+        Ok(fragments.into_iter().flatten().collect())
+    }
+
+    pub(crate) fn get_existing_fragments_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Vec<FileFragment> {
+        let ordered_ids = Self::normalize_fragment_ids(fragment_ids);
+        self.get_frags_from_ordered_ids(&ordered_ids)
+            .into_iter()
+            .flatten()
+            .collect()
+    }
+
+    pub(crate) fn get_fragment_metadata_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Result<Vec<Fragment>> {
+        Ok(self
+            .get_fragments_from_ids(fragment_ids)?
+            .into_iter()
+            .map(|fragment| fragment.metadata().clone())
+            .collect())
+    }
+
+    pub(crate) fn get_existing_fragment_metadata_from_ids(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Vec<Fragment> {
+        self.get_existing_fragments_from_ids(fragment_ids)
+            .into_iter()
+            .map(|fragment| fragment.metadata().clone())
+            .collect()
+    }
+
+    pub(crate) async fn count_rows_in_fragments(&self, fragment_ids: &[u32]) -> Result<usize> {
+        let fragments = self.get_fragments_from_ids(fragment_ids)?;
+        self.count_rows_in_resolved_fragments(fragments).await
+    }
+
+    pub(crate) async fn count_rows_in_existing_fragments(
+        &self,
+        fragment_ids: &[u32],
+    ) -> Result<usize> {
+        let fragments = self.get_existing_fragments_from_ids(fragment_ids);
+        self.count_rows_in_resolved_fragments(fragments).await
+    }
+
+    async fn count_rows_in_resolved_fragments(
+        &self,
+        fragments: Vec<FileFragment>,
+    ) -> Result<usize> {
+        let counts = stream::iter(fragments)
+            .map(|fragment| async move { fragment.count_rows(None).await })
+            .buffer_unordered(16)
+            .try_collect::<Vec<_>>()
+            .await?;
+        Ok(counts.iter().sum())
+    }
+
+    /// Resolves fragments for the given ids without scanning the manifest.
+    ///
+    /// The ids do not need to be sorted or deduplicated. Each id is resolved
+    /// independently via the fragment bitmap.
+    pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec<Option<FileFragment>> {
+        let dataset = Arc::new(self.clone());
+        ordered_ids
+            .iter()
+            .map(|id| {
+                if !self.fragment_bitmap.contains(*id) {
+                    return None;
+                }
+                let fragment_index = self.fragment_bitmap.rank(*id) as usize - 1;
+                let fragment = self.manifest.fragments.get(fragment_index)?;
+                debug_assert_eq!(
+                    fragment.id, *id as u64,
+                    "fragment_bitmap rank({id}) resolved to fragment {}, but fragment_bitmap and manifest.fragments are expected to stay in sync",
+                    fragment.id
+                );
+                Some(FileFragment::new(dataset.clone(), fragment.clone()))
+            })
+            .collect()
     }
 
     // This method filters deleted items from `addr_or_ids` using `addrs` as a reference
@@ -2432,7 +2959,7 @@ impl Dataset {
                     self.manifest_location.path.clone(),
                     format!(
                         "Duplicate index id {} found in dataset {:?}",
-                        &index.uuid, self.base
+                        index.uuid, self.base
                     ),
                 ));
             }
@@ -2509,11 +3036,12 @@ impl Dataset {
         store_params: Option<ObjectStoreParams>,
     ) -> Result<Self> {
         let (ref_name, version_number) = self.resolve_reference(version.into()).await?;
+        let source_location = self.branch_location().find_branch(ref_name.as_deref())?;
         let clone_op = Operation::Clone {
             is_shallow: true,
             ref_name,
             ref_version: version_number,
-            ref_path: self.uri.clone(),
+            ref_path: source_location.uri,
             branch_name: None,
         };
         let transaction = Transaction::new(version_number, clone_op, None);
@@ -2778,30 +3306,13 @@ pub(crate) fn load_new_transactions(dataset: &Dataset) -> NewTransactionResult<'
         .map_ok(move |location| {
             let latest_tx = latest_tx.take();
             async move {
-                let manifest_key = ManifestKey {
-                    version: location.version,
-                    e_tag: location.e_tag.as_deref(),
-                };
-                let manifest = if let Some(cached) =
-                    dataset.metadata_cache.get_with_key(&manifest_key).await
-                {
-                    cached
-                } else {
-                    let loaded = Arc::new(
-                        Dataset::load_manifest(
-                            dataset.object_store.as_ref(),
-                            &location,
-                            &dataset.uri,
-                            dataset.session.as_ref(),
-                        )
-                        .await?,
-                    );
-                    dataset
-                        .metadata_cache
-                        .insert_with_key(&manifest_key, loaded.clone())
-                        .await;
-                    loaded
-                };
+                let manifest = Dataset::get_manifest(
+                    dataset.object_store.as_ref(),
+                    &location,
+                    &dataset.uri,
+                    dataset.session.as_ref(),
+                )
+                .await?;
 
                 if let Some(latest_tx) = latest_tx {
                     // We ignore the error, since we don't care if the receiver is dropped.
@@ -3033,13 +3544,13 @@ impl Dataset {
     /// progress via the supplied callback.
     pub async fn merge_index_metadata(
         &self,
-        index_uuid: &str,
+        index_uuid: &Uuid,
         index_type: IndexType,
         _batch_readhead: Option<usize>,
         progress: Arc<dyn IndexBuildProgress>,
     ) -> Result<()> {
         let store = LanceIndexStore::from_dataset_for_new(self, index_uuid)?;
-        let index_dir = self.indices_dir().join(index_uuid);
+        let index_dir = self.indices_dir().join(index_uuid.to_string());
         match index_type {
             IndexType::Inverted => {
                 // Call merge_index_files function for inverted index
@@ -3054,18 +3565,19 @@ impl Dataset {
             IndexType::BTree => {
                 Err(Error::invalid_input(
                     "BTree distributed indexing no longer supports merge_index_metadata; \
-                     build segments, and commit with commit_existing_index_segments(...)"
+                     build segments, optionally merge groups with merge_existing_index_segments(...), \
+                     and commit with commit_existing_index_segments(...)"
                         .to_string(),
                 ))
             }
             IndexType::Bitmap => {
-                lance_index::scalar::bitmap::merge_index_files(
-                    self.object_store.as_ref(),
-                    &index_dir,
-                    Arc::new(store),
-                    progress,
-                )
-                .await
+                Err(Error::invalid_input(
+                    "Bitmap distributed indexing no longer supports merge_index_metadata; \
+                     build segments with create_index_uncommitted(...), merge them with \
+                     merge_existing_index_segments(...), and commit with \
+                     commit_existing_index_segments(...)"
+                        .to_string(),
+                ))
             }
             IndexType::IvfFlat | IndexType::IvfPq | IndexType::IvfSq | IndexType::Vector => {
                 Err(Error::invalid_input(
diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs
index f2c243367ce..c0559753330 100644
--- a/rust/lance/src/dataset/blob.rs
+++ b/rust/lance/src/dataset/blob.rs
@@ -12,34 +12,108 @@ use std::{
 
 use arrow::array::AsArray;
 use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type};
-use arrow_array::Array;
-use arrow_array::RecordBatch;
-use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder};
-use arrow_schema::DataType as ArrowDataType;
+use arrow_array::{
+    Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, builder::LargeBinaryBuilder,
+};
+use arrow_buffer::{OffsetBuffer, ScalarBuffer};
+use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema};
 use bytes::Bytes;
+use futures::future::BoxFuture;
 use futures::stream::BoxStream;
 use futures::{FutureExt, StreamExt, TryStreamExt, stream};
-use lance_arrow::{BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, FieldExt};
+use lance_arrow::{
+    ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+    BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt,
+    list::ListArrayExt, r#struct::StructArrayExt,
+};
 use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
 use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig};
 use object_store::path::Path;
-use tokio::io::AsyncWriteExt;
 use tokio::sync::{Mutex, OnceCell, oneshot};
 use url::Url;
 
 use super::take::TakeBuilder;
 use super::write::ExternalBlobMode;
 use super::{Dataset, ProjectionRequest};
+use crate::blob::{
+    BlobDescriptor, BlobDescriptorArrayBuilder, BlobIdAllocator, BlobRange, PackedBlobWriter,
+    is_logical_blob_v2_field, is_prepared_blob_v2_field, validate_prepared_blob_array,
+};
 use arrow_array::StructArray;
-use lance_core::datatypes::{BlobKind, BlobVersion, parse_field_path};
+use lance_core::datatypes::{BlobKind, BlobVersion, Field as LanceField, Schema, parse_field_path};
 use lance_core::utils::blob::blob_path;
-use lance_core::{Error, Result, utils::address::RowAddress};
-use lance_io::traits::{Reader, WriteExt, Writer};
+use lance_core::{Error, ROW_ADDR, Result, utils::address::RowAddress};
+use lance_io::traits::Reader;
 use lance_io::utils::CachedFileSize;
 
 const INLINE_MAX: usize = 64 * 1024; // 64KB inline cutoff
 const DEDICATED_THRESHOLD: usize = 4 * 1024 * 1024; // 4MB dedicated cutoff
 const PACK_FILE_MAX_SIZE: usize = 1024 * 1024 * 1024; // 1GiB per .pack sidecar
+
+pub(super) fn blob_inline_threshold_from_metadata(
+    metadata: &HashMap<String, String>,
+    field_name: &str,
+) -> Result<usize> {
+    blob_threshold_from_metadata(
+        metadata,
+        field_name,
+        BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
+        INLINE_MAX,
+        true,
+    )
+}
+
+pub(super) fn blob_dedicated_threshold_from_metadata(
+    metadata: &HashMap<String, String>,
+    field_name: &str,
+) -> Result<usize> {
+    blob_threshold_from_metadata(
+        metadata,
+        field_name,
+        BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+        DEDICATED_THRESHOLD,
+        false,
+    )
+}
+
+pub(super) fn blob_pack_file_threshold_from_metadata(
+    metadata: &HashMap<String, String>,
+    field_name: &str,
+) -> Result<usize> {
+    blob_threshold_from_metadata(
+        metadata,
+        field_name,
+        BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+        PACK_FILE_MAX_SIZE,
+        false,
+    )
+}
+
+fn blob_threshold_from_metadata(
+    metadata: &HashMap<String, String>,
+    field_name: &str,
+    key: &str,
+    default_value: usize,
+    allow_zero: bool,
+) -> Result<usize> {
+    let Some(value) = metadata.get(key) else {
+        return Ok(default_value);
+    };
+    let threshold = value.parse::<usize>().map_err(|_| {
+        Error::invalid_input(format!(
+            "Invalid blob threshold metadata {key}={value:?} for field '{field_name}'; \
+             expected a non-negative integer that fits in usize"
+        ))
+    })?;
+    if !allow_zero && threshold == 0 {
+        return Err(Error::invalid_input(format!(
+            "Invalid blob threshold metadata {key}={value:?} for field '{field_name}'; \
+             expected a positive integer"
+        )));
+    }
+    Ok(threshold)
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct ResolvedExternalBase {
     pub base_id: u32,
@@ -112,84 +186,86 @@ impl ExternalBaseResolver {
     }
 }
 
-// Maintains rolling `.blob` sidecar files for packed blobs.
-// Layout: data/{data_file_key}/{obfuscated_blob_id:032b}.blob where each file is an
-// unframed concatenation of blob payloads; descriptors store (blob_id,
-// position, size) to locate each slice. A dedicated struct keeps path state
-// and rolling size separate from the per-batch preprocessor logic, so we can
-// reuse the same writer across rows and close/roll files cleanly on finish.
-struct PackWriter {
-    object_store: ObjectStore,
-    data_dir: Path,
-    data_file_key: String,
-    max_pack_size: usize,
-    current_blob_id: Option<u32>,
-    writer: Option<Box<dyn lance_io::traits::Writer>>,
+struct RollingPackedBlobWriter {
+    current: Option<PackedBlobWriter>,
     current_size: usize,
+    current_max_pack_size: Option<usize>,
 }
 
-impl PackWriter {
-    fn new(object_store: ObjectStore, data_dir: Path, data_file_key: String) -> Self {
+impl RollingPackedBlobWriter {
+    fn new() -> Self {
         Self {
-            object_store,
-            data_dir,
-            data_file_key,
-            max_pack_size: PACK_FILE_MAX_SIZE,
-            current_blob_id: None,
-            writer: None,
+            current: None,
             current_size: 0,
+            current_max_pack_size: None,
         }
     }
 
-    async fn start_new_pack(&mut self, blob_id: u32) -> Result<()> {
-        let path = blob_path(&self.data_dir, &self.data_file_key, blob_id);
-        let writer = self.object_store.create(&path).await?;
-        self.writer = Some(writer);
-        self.current_blob_id = Some(blob_id);
+    async fn start_new_pack(
+        &mut self,
+        object_store: ObjectStore,
+        data_dir: Path,
+        data_file_key: String,
+        blob_id_allocator: BlobIdAllocator,
+        max_pack_size: usize,
+    ) -> Result<()> {
+        self.finish().await?;
+        let blob_id = blob_id_allocator.next()?;
+        let data_file_path = data_dir.join(format!("{data_file_key}.lance"));
+        self.current =
+            Some(PackedBlobWriter::try_new(object_store, data_file_path, blob_id).await?);
         self.current_size = 0;
+        self.current_max_pack_size = Some(max_pack_size);
         Ok(())
     }
 
-    /// Append `data` to the current `.blob` file, rolling to a new file when
-    /// `max_pack_size` would be exceeded.
-    ///
-    /// alloc_blob_id: called only when a new pack file is opened; returns the
-    /// blob_id used as the file name.
-    ///
-    /// Returns `(blob_id, position)` where
-    /// position is the start offset of this payload in that pack file.
-    async fn write_with_allocator<F>(
+    async fn write(
         &mut self,
-        alloc_blob_id: &mut F,
+        object_store: ObjectStore,
+        data_dir: Path,
+        data_file_key: String,
+        blob_id_allocator: BlobIdAllocator,
+        max_pack_size: usize,
         source: BlobWriteSource<'_>,
-    ) -> Result<(u32, u64)>
-    where
-        F: FnMut() -> u32,
-    {
+    ) -> Result<BlobDescriptor> {
         let len = source.size();
-        if self
-            .current_blob_id
-            .map(|_| self.current_size + len > self.max_pack_size)
-            .unwrap_or(true)
-        {
-            let blob_id = alloc_blob_id();
-            self.finish().await?;
-            self.start_new_pack(blob_id).await?;
+        let needs_new_pack = match (self.current.as_ref(), self.current_max_pack_size) {
+            (Some(_), Some(current_max_pack_size)) => {
+                current_max_pack_size != max_pack_size
+                    || self.current_size + len > current_max_pack_size
+            }
+            _ => true,
+        };
+        if needs_new_pack {
+            self.start_new_pack(
+                object_store,
+                data_dir,
+                data_file_key,
+                blob_id_allocator,
+                max_pack_size,
+            )
+            .await?;
         }
 
-        let writer = self.writer.as_mut().expect("pack writer is initialized");
-        let position = self.current_size as u64;
-        source.write_to(writer.as_mut()).await?;
+        let writer = self.current.as_mut().expect("pack writer is initialized");
+        let value = match source {
+            BlobWriteSource::Bytes(data) => writer.write_blob_bytes(data).await?,
+            BlobWriteSource::External(source) => {
+                writer
+                    .write_blob_from_reader(source.reader.as_ref(), source.reader_range()?)
+                    .await?
+            }
+        };
         self.current_size += len;
-        Ok((self.current_blob_id.expect("pack blob id"), position))
+        Ok(value)
     }
 
     async fn finish(&mut self) -> Result<()> {
-        if let Some(mut writer) = self.writer.take() {
-            Writer::shutdown(writer.as_mut()).await?;
+        if let Some(writer) = self.current.take() {
+            writer.finish().await?;
         }
-        self.current_blob_id = None;
         self.current_size = 0;
+        self.current_max_pack_size = None;
         Ok(())
     }
 }
@@ -203,11 +279,13 @@ pub struct BlobPreprocessor {
     object_store: ObjectStore,
     data_dir: Path,
     data_file_key: String,
-    local_counter: u32,
-    pack_writer: PackWriter,
-    blob_v2_cols: Vec<bool>,
-    dedicated_thresholds: Vec<usize>,
-    writer_metadata: Vec<HashMap<String, String>>,
+    blob_id_allocator: BlobIdAllocator,
+    pack_writer: RollingPackedBlobWriter,
+    /// Write-param override for the pack-file roll size. When set, it takes
+    /// precedence over each field's `blob-pack-file-size-threshold` metadata for
+    /// this write job only; it is not persisted into the dataset schema.
+    pack_file_size_override: Option<usize>,
+    field_processors: Vec<BlobPreprocessField>,
     external_base_resolver: Option<Arc<ExternalBaseResolver>>,
     allow_external_blob_outside_bases: bool,
     external_blob_mode: ExternalBlobMode,
@@ -232,6 +310,97 @@ enum BlobWriteSource<'a> {
     External(&'a ExternalBlobSource),
 }
 
+#[derive(Clone, Debug)]
+struct BlobPreprocessField {
+    kind: BlobPreprocessFieldKind,
+}
+
+#[derive(Clone, Debug)]
+enum BlobPreprocessFieldKind {
+    BlobV2 {
+        inline_threshold: usize,
+        dedicated_threshold: usize,
+        pack_file_threshold: usize,
+        writer_metadata: HashMap<String, String>,
+    },
+    Struct {
+        children: Vec<BlobPreprocessField>,
+    },
+    List {
+        child: Box<BlobPreprocessField>,
+    },
+    Passthrough,
+}
+
+impl BlobPreprocessField {
+    fn new(field: &ArrowField) -> Result<Self> {
+        if field.is_blob_v2() {
+            if is_prepared_blob_v2_field(field) {
+                return Ok(Self {
+                    kind: BlobPreprocessFieldKind::Passthrough,
+                });
+            }
+            if !is_logical_blob_v2_field(field) {
+                return Err(Error::invalid_input(format!(
+                    "Blob v2 field '{}' must use either logical struct<data: LargeBinary?, \
+                     uri: Utf8?> with optional position/size UInt64 fields or prepared \
+                     struct<kind: UInt8?, data: LargeBinary?, uri: Utf8?, blob_id: UInt32?, \
+                     blob_size: UInt64?, position: UInt64?>",
+                    field.name()
+                )));
+            }
+            return Ok(Self {
+                kind: BlobPreprocessFieldKind::BlobV2 {
+                    inline_threshold: blob_inline_threshold_from_metadata(
+                        field.metadata(),
+                        field.name(),
+                    )?,
+                    dedicated_threshold: blob_dedicated_threshold_from_metadata(
+                        field.metadata(),
+                        field.name(),
+                    )?,
+                    pack_file_threshold: blob_pack_file_threshold_from_metadata(
+                        field.metadata(),
+                        field.name(),
+                    )?,
+                    writer_metadata: field.metadata().clone(),
+                },
+            });
+        }
+
+        if let ArrowDataType::Struct(children) = field.data_type() {
+            let children = children
+                .iter()
+                .map(|child| Self::new(child.as_ref()))
+                .collect::<Result<Vec<_>>>()?;
+            if children.iter().any(|child| child.requires_preprocessing()) {
+                return Ok(Self {
+                    kind: BlobPreprocessFieldKind::Struct { children },
+                });
+            }
+        }
+
+        if let ArrowDataType::List(child) | ArrowDataType::LargeList(child) = field.data_type() {
+            let child = Self::new(child.as_ref())?;
+            if child.requires_preprocessing() {
+                return Ok(Self {
+                    kind: BlobPreprocessFieldKind::List {
+                        child: Box::new(child),
+                    },
+                });
+            }
+        }
+
+        Ok(Self {
+            kind: BlobPreprocessFieldKind::Passthrough,
+        })
+    }
+
+    fn requires_preprocessing(&self) -> bool {
+        !matches!(self.kind, BlobPreprocessFieldKind::Passthrough)
+    }
+}
+
 impl ExternalBlobSource {
     /// Return the logical payload size after applying any external slice.
     fn size(&self) -> u64 {
@@ -266,15 +435,6 @@ impl ExternalBlobSource {
         let range = self.reader_range()?;
         self.reader.get_range(range).await.map_err(Into::into)
     }
-
-    /// Stream the slice into a writer for packed or dedicated blob storage.
-    async fn copy_to_writer(&self, writer: &mut dyn Writer) -> Result<()> {
-        let range = self.reader_range()?;
-        writer
-            .copy_range_from_reader(self.reader.as_ref(), range)
-            .await?;
-        Ok(())
-    }
 }
 
 impl BlobWriteSource<'_> {
@@ -282,22 +442,12 @@ impl BlobWriteSource<'_> {
     fn size(&self) -> usize {
         match self {
             Self::Bytes(data) => data.len(),
-            Self::External(source) => usize::try_from(source.size())
+            Self::External(source) => source
+                .size()
+                .try_into()
                 .expect("packed and inline external blobs must fit into usize"),
         }
     }
-
-    /// Write the payload into Lance-managed storage without forcing callers to branch on source
-    /// type.
-    async fn write_to(&self, writer: &mut dyn Writer) -> Result<()> {
-        match self {
-            Self::Bytes(data) => {
-                writer.write_all(data).await?;
-                Ok(())
-            }
-            Self::External(source) => source.copy_to_writer(writer).await,
-        }
-    }
 }
 
 impl BlobPreprocessor {
@@ -312,68 +462,77 @@ impl BlobPreprocessor {
         external_blob_mode: ExternalBlobMode,
         source_store_registry: Arc<ObjectStoreRegistry>,
         source_store_params: ObjectStoreParams,
-        pack_file_size_threshold: Option<usize>,
-    ) -> Self {
-        let mut pack_writer = PackWriter::new(
-            object_store.clone(),
-            data_dir.clone(),
-            data_file_key.clone(),
-        );
-        if let Some(max_bytes) = pack_file_size_threshold {
-            pack_writer.max_pack_size = max_bytes;
-        }
+        pack_file_size_override: Option<usize>,
+    ) -> Result<Self> {
+        let pack_writer = RollingPackedBlobWriter::new();
         let arrow_schema = arrow_schema::Schema::from(schema);
-        let fields = arrow_schema.fields();
-        let blob_v2_cols = fields.iter().map(|field| field.is_blob_v2()).collect();
-        let dedicated_thresholds = fields
-            .iter()
-            .map(|field| dedicated_threshold_from_metadata(field.as_ref()))
-            .collect();
-        let writer_metadata = fields
+        let field_processors = arrow_schema
+            .fields()
             .iter()
-            .map(|field| field.metadata().clone())
-            .collect();
-        Self {
+            .map(|field| BlobPreprocessField::new(field.as_ref()))
+            .collect::<Result<Vec<_>>>()?;
+        Ok(Self {
             object_store,
             data_dir,
             data_file_key,
-            // Start at 1 to avoid a potential all-zero blob_id value.
-            local_counter: 1,
+            blob_id_allocator: BlobIdAllocator::new(1),
             pack_writer,
-            blob_v2_cols,
-            dedicated_thresholds,
-            writer_metadata,
+            pack_file_size_override,
+            field_processors,
             external_base_resolver,
             allow_external_blob_outside_bases,
             external_blob_mode,
             source_store_registry,
             source_store_params,
-        }
+        })
     }
 
-    fn next_blob_id(&mut self) -> u32 {
-        let id = self.local_counter;
-        self.local_counter += 1;
-        id
+    fn blob_writer_with_metadata(
+        &self,
+        field: &ArrowField,
+        metadata: HashMap<String, String>,
+    ) -> BlobDescriptorArrayBuilder {
+        BlobDescriptorArrayBuilder::new_with_metadata(field.name(), field.is_nullable(), metadata)
     }
 
-    async fn write_dedicated(&mut self, blob_id: u32, source: BlobWriteSource<'_>) -> Result<Path> {
-        let path = blob_path(&self.data_dir, &self.data_file_key, blob_id);
-        let mut writer = self.object_store.create(&path).await?;
-        source.write_to(writer.as_mut()).await?;
-        Writer::shutdown(&mut writer).await?;
-        Ok(path)
+    async fn write_dedicated(
+        object_store: ObjectStore,
+        data_dir: Path,
+        data_file_key: String,
+        blob_id_allocator: BlobIdAllocator,
+        source: BlobWriteSource<'_>,
+    ) -> Result<BlobDescriptor> {
+        let blob_id = blob_id_allocator.next()?;
+        let data_file_path = data_dir.join(format!("{data_file_key}.lance"));
+        let mut writer =
+            crate::blob::DedicatedBlobWriter::try_new(object_store, data_file_path, blob_id)
+                .await?;
+        match source {
+            BlobWriteSource::Bytes(data) => writer.write(data).await?,
+            BlobWriteSource::External(source) => {
+                writer
+                    .write_from_reader(source.reader.as_ref(), source.reader_range()?)
+                    .await?;
+            }
+        }
+        writer.finish().await
     }
 
-    async fn write_packed(&mut self, source: BlobWriteSource<'_>) -> Result<(u32, u64)> {
-        let (counter, pack_writer) = (&mut self.local_counter, &mut self.pack_writer);
-        pack_writer
-            .write_with_allocator(
-                &mut || {
-                    let id = *counter;
-                    *counter += 1;
-                    id
-                },
+    async fn write_packed(
+        &mut self,
+        field_pack_file_threshold: usize,
+        source: BlobWriteSource<'_>,
+    ) -> Result<BlobDescriptor> {
+        let max_pack_size = self
+            .pack_file_size_override
+            .unwrap_or(field_pack_file_threshold);
+        self.pack_writer
+            .write(
+                self.object_store.clone(),
+                self.data_dir.clone(),
+                self.data_file_key.clone(),
+                self.blob_id_allocator.clone(),
+                max_pack_size,
                 source,
             )
             .await
@@ -443,7 +602,7 @@ impl BlobPreprocessor {
     }
 
     pub(crate) async fn preprocess_batch(&mut self, batch: &RecordBatch) -> Result<RecordBatch> {
-        let expected_columns = self.blob_v2_cols.len();
+        let expected_columns = self.field_processors.len();
         if batch.num_columns() != expected_columns {
             return Err(Error::invalid_input(format!(
                 "Unexpected number of columns: expected {}, got {}",
@@ -454,245 +613,363 @@ impl BlobPreprocessor {
 
         let batch_schema = batch.schema();
         let batch_fields = batch_schema.fields();
+        let field_processors = self.field_processors.clone();
 
         let mut new_columns = Vec::with_capacity(batch.num_columns());
         let mut new_fields = Vec::with_capacity(batch.num_columns());
 
-        for idx in 0..batch.num_columns() {
-            let array = batch.column(idx);
-            let field = &batch_fields[idx];
-            if !self.blob_v2_cols[idx] {
-                new_columns.push(array.clone());
-                new_fields.push(field.clone());
-                continue;
+        for ((processor, array), field) in field_processors
+            .iter()
+            .zip(batch.columns().iter())
+            .zip(batch_fields.iter())
+        {
+            let (new_column, new_field) = self
+                .preprocess_field(processor, array.clone(), field)
+                .await?;
+            new_columns.push(new_column);
+            new_fields.push(new_field);
+        }
+
+        let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata(
+            new_fields
+                .iter()
+                .map(|f| f.as_ref().clone())
+                .collect::<Vec<_>>(),
+            batch_schema.metadata().clone(),
+        ));
+
+        RecordBatch::try_new(new_schema, new_columns)
+            .map_err(|e| Error::invalid_input(e.to_string()))
+    }
+
+    fn preprocess_field<'a>(
+        &'a mut self,
+        processor: &'a BlobPreprocessField,
+        array: ArrayRef,
+        field: &'a Arc<ArrowField>,
+    ) -> BoxFuture<'a, Result<(ArrayRef, Arc<ArrowField>)>> {
+        async move {
+            if is_prepared_blob_v2_field(field.as_ref()) {
+                validate_prepared_blob_array(field.as_ref(), &array)?;
+                return Ok((array, field.clone()));
             }
 
-            let struct_arr = array
-                .as_any()
-                .downcast_ref::<arrow_array::StructArray>()
-                .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?;
-
-            let data_col = struct_arr
-                .column_by_name("data")
-                .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))?
-                .as_binary::<i64>();
-            let uri_col = struct_arr
-                .column_by_name("uri")
-                .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))?
-                .as_string::<i32>();
-            let position_col = struct_arr
-                .column_by_name("position")
-                .map(|col| col.as_primitive::<UInt64Type>());
-            let size_col = struct_arr
-                .column_by_name("size")
-                .map(|col| col.as_primitive::<UInt64Type>());
-
-            let mut data_builder = LargeBinaryBuilder::with_capacity(struct_arr.len(), 0);
-            let mut uri_builder = StringBuilder::with_capacity(struct_arr.len(), 0);
-            let mut blob_id_builder =
-                PrimitiveBuilder::<arrow_array::types::UInt32Type>::with_capacity(struct_arr.len());
-            let mut blob_size_builder =
-                PrimitiveBuilder::<arrow_array::types::UInt64Type>::with_capacity(struct_arr.len());
-            let mut kind_builder = PrimitiveBuilder::<UInt8Type>::with_capacity(struct_arr.len());
-            let mut position_builder =
-                PrimitiveBuilder::<arrow_array::types::UInt64Type>::with_capacity(struct_arr.len());
-
-            let struct_nulls = struct_arr.nulls();
-
-            for i in 0..struct_arr.len() {
-                if struct_arr.is_null(i) {
-                    data_builder.append_null();
-                    uri_builder.append_null();
-                    blob_id_builder.append_null();
-                    blob_size_builder.append_null();
-                    kind_builder.append_null();
-                    position_builder.append_null();
-                    continue;
+            match &processor.kind {
+                BlobPreprocessFieldKind::Passthrough => Ok((array, field.clone())),
+                BlobPreprocessFieldKind::BlobV2 {
+                    inline_threshold,
+                    dedicated_threshold,
+                    pack_file_threshold,
+                    writer_metadata,
+                } => {
+                    self.preprocess_blob_array(
+                        array,
+                        field.as_ref(),
+                        *inline_threshold,
+                        *dedicated_threshold,
+                        *pack_file_threshold,
+                        writer_metadata,
+                    )
+                    .await
+                }
+                BlobPreprocessFieldKind::Struct { children } => {
+                    self.preprocess_struct_array(array, field.as_ref(), children)
+                        .await
                 }
+                BlobPreprocessFieldKind::List { child } => match field.data_type() {
+                    ArrowDataType::List(_) => {
+                        self.preprocess_list_array::<i32>(array, field.as_ref(), child)
+                            .await
+                    }
+                    ArrowDataType::LargeList(_) => {
+                        self.preprocess_list_array::<i64>(array, field.as_ref(), child)
+                            .await
+                    }
+                    _ => Err(Error::internal(format!(
+                        "Blob list preprocessor received non-list field '{}'",
+                        field.name()
+                    ))),
+                },
+            }
+        }
+        .boxed()
+    }
 
-                let has_data = !data_col.is_null(i);
-                let has_uri = !uri_col.is_null(i);
-                let has_position = position_col
-                    .as_ref()
-                    .map(|col| !col.is_null(i))
-                    .unwrap_or(false);
-                let has_size = size_col
-                    .as_ref()
-                    .map(|col| !col.is_null(i))
-                    .unwrap_or(false);
-                let data_len = if has_data { data_col.value(i).len() } else { 0 };
-
-                let dedicated_threshold = self.dedicated_thresholds[idx];
-                if has_data && data_len > dedicated_threshold {
-                    let blob_id = self.next_blob_id();
-                    self.write_dedicated(blob_id, BlobWriteSource::Bytes(data_col.value(i)))
-                        .await?;
+    async fn preprocess_struct_array(
+        &mut self,
+        array: ArrayRef,
+        field: &ArrowField,
+        children: &[BlobPreprocessField],
+    ) -> Result<(ArrayRef, Arc<ArrowField>)> {
+        let struct_arr = array
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .ok_or_else(|| Error::invalid_input("Struct field was not a struct array"))?;
+        if struct_arr.num_columns() != children.len() {
+            return Err(Error::invalid_input(format!(
+                "Struct field '{}' expected {} children, got {}",
+                field.name(),
+                children.len(),
+                struct_arr.num_columns()
+            )));
+        }
 
-                    kind_builder.append_value(BlobKind::Dedicated as u8);
-                    data_builder.append_null();
-                    uri_builder.append_null();
-                    blob_id_builder.append_value(blob_id);
-                    blob_size_builder.append_value(data_len as u64);
-                    position_builder.append_null();
-                    continue;
-                }
+        let struct_arr = struct_arr.normalize_slicing()?;
+        let parent_nulls = struct_arr.nulls().cloned();
+        let pushed_down = struct_arr.pushdown_nulls()?;
+        let child_fields = pushed_down.fields().clone();
+        let child_columns = pushed_down.columns().to_vec();
 
-                if has_data && data_len > INLINE_MAX {
-                    let (pack_blob_id, position) = self
-                        .write_packed(BlobWriteSource::Bytes(data_col.value(i)))
-                        .await?;
+        let mut new_columns = Vec::with_capacity(children.len());
+        let mut new_fields = Vec::with_capacity(children.len());
+        for ((child_processor, child_array), child_field) in
+            children.iter().zip(child_columns).zip(child_fields.iter())
+        {
+            let (new_column, new_field) = self
+                .preprocess_field(child_processor, child_array, child_field)
+                .await?;
+            new_columns.push(new_column);
+            new_fields.push(new_field);
+        }
 
-                    kind_builder.append_value(BlobKind::Packed as u8);
-                    data_builder.append_null();
-                    uri_builder.append_null();
-                    blob_id_builder.append_value(pack_blob_id);
-                    blob_size_builder.append_value(data_len as u64);
-                    position_builder.append_value(position);
-                    continue;
-                }
+        let struct_array =
+            StructArray::try_new(new_fields.clone().into(), new_columns, parent_nulls)?;
+        let field = Arc::new(
+            ArrowField::new(
+                field.name(),
+                ArrowDataType::Struct(new_fields.into()),
+                field.is_nullable(),
+            )
+            .with_metadata(field.metadata().clone()),
+        );
+        Ok((Arc::new(struct_array), field))
+    }
 
-                if has_uri {
-                    let uri_val = uri_col.value(i);
-                    if self.external_blob_mode == ExternalBlobMode::Ingest {
-                        let position = if has_position {
-                            Some(
-                                position_col
-                                    .as_ref()
-                                    .expect("position column must exist")
-                                    .value(i),
-                            )
-                        } else {
-                            None
-                        };
-                        let size = if has_size {
-                            Some(size_col.as_ref().expect("size column must exist").value(i))
-                        } else {
-                            None
-                        };
-                        let source = self.open_external_source(uri_val, position, size).await?;
-                        let data_len = source.size();
-
-                        if data_len > dedicated_threshold as u64 {
-                            let blob_id = self.next_blob_id();
-                            self.write_dedicated(blob_id, BlobWriteSource::External(&source))
-                                .await?;
-
-                            kind_builder.append_value(BlobKind::Dedicated as u8);
-                            data_builder.append_null();
-                            uri_builder.append_null();
-                            blob_id_builder.append_value(blob_id);
-                            blob_size_builder.append_value(data_len);
-                            position_builder.append_null();
-                            continue;
-                        }
+    async fn preprocess_list_array<O: OffsetSizeTrait>(
+        &mut self,
+        array: ArrayRef,
+        field: &ArrowField,
+        child: &BlobPreprocessField,
+    ) -> Result<(ArrayRef, Arc<ArrowField>)> {
+        let list_arr = array.as_list::<O>();
+        let list_arr = if list_arr.null_count() > 0 {
+            list_arr.filter_garbage_nulls()
+        } else {
+            list_arr.clone()
+        };
 
-                        if data_len > INLINE_MAX as u64 {
-                            let (pack_blob_id, position) = self
-                                .write_packed(BlobWriteSource::External(&source))
-                                .await?;
-
-                            kind_builder.append_value(BlobKind::Packed as u8);
-                            data_builder.append_null();
-                            uri_builder.append_null();
-                            blob_id_builder.append_value(pack_blob_id);
-                            blob_size_builder.append_value(data_len);
-                            position_builder.append_value(position);
-                            continue;
-                        }
+        let first_offset = *list_arr
+            .offsets()
+            .first()
+            .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?;
+        let last_offset = *list_arr
+            .offsets()
+            .last()
+            .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?;
+        let values_len = list_arr.values().len();
+        let needs_trim = first_offset != O::zero()
+            || last_offset.to_usize().ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "List field '{}' offset does not fit into usize",
+                    field.name()
+                ))
+            })? != values_len;
+
+        let (offsets, values) = if needs_trim {
+            let values = list_arr.trimmed_values();
+            let offsets = list_arr
+                .offsets()
+                .iter()
+                .map(|offset| *offset - first_offset)
+                .collect::<Vec<_>>();
+            (OffsetBuffer::new(ScalarBuffer::from(offsets)), values)
+        } else {
+            (list_arr.offsets().clone(), list_arr.values().clone())
+        };
+
+        let child_field = match field.data_type() {
+            ArrowDataType::List(child_field) | ArrowDataType::LargeList(child_field) => {
+                child_field.clone()
+            }
+            other => {
+                return Err(Error::invalid_input(format!(
+                    "Blob list preprocessor expected list field '{}', got {other}",
+                    field.name()
+                )));
+            }
+        };
+        let (new_values, new_child_field) =
+            self.preprocess_field(child, values, &child_field).await?;
+
+        let list_array = GenericListArray::<O>::try_new(
+            new_child_field,
+            offsets,
+            new_values,
+            list_arr.nulls().cloned(),
+        )?;
+        let field = Arc::new(
+            ArrowField::new(
+                field.name(),
+                list_array.data_type().clone(),
+                field.is_nullable(),
+            )
+            .with_metadata(field.metadata().clone()),
+        );
+        Ok((Arc::new(list_array), field))
+    }
+
+    async fn preprocess_blob_array(
+        &mut self,
+        array: ArrayRef,
+        field: &ArrowField,
+        inline_threshold: usize,
+        dedicated_threshold: usize,
+        pack_file_threshold: usize,
+        writer_metadata: &HashMap<String, String>,
+    ) -> Result<(ArrayRef, Arc<ArrowField>)> {
+        let struct_arr = array
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?;
+
+        let data_col = struct_arr
+            .column_by_name("data")
+            .ok_or_else(|| Error::invalid_input("Blob struct missing `data` field"))?
+            .as_binary::<i64>();
+        let uri_col = struct_arr
+            .column_by_name("uri")
+            .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))?
+            .as_string::<i32>();
+        let position_col = struct_arr
+            .column_by_name("position")
+            .map(|col| col.as_primitive::<UInt64Type>());
+        let size_col = struct_arr
+            .column_by_name("size")
+            .map(|col| col.as_primitive::<UInt64Type>());
+
+        let mut blob_writer = self.blob_writer_with_metadata(field, writer_metadata.clone());
+
+        for i in 0..struct_arr.len() {
+            if struct_arr.is_null(i) {
+                blob_writer.push_null()?;
+                continue;
+            }
+
+            let has_data = !data_col.is_null(i);
+            let has_uri = !uri_col.is_null(i);
+            let has_position = position_col
+                .as_ref()
+                .map(|col| !col.is_null(i))
+                .unwrap_or(false);
+            let has_size = size_col
+                .as_ref()
+                .map(|col| !col.is_null(i))
+                .unwrap_or(false);
+            let data_len = if has_data { data_col.value(i).len() } else { 0 };
+
+            if has_data && data_len > dedicated_threshold {
+                let value = Self::write_dedicated(
+                    self.object_store.clone(),
+                    self.data_dir.clone(),
+                    self.data_file_key.clone(),
+                    self.blob_id_allocator.clone(),
+                    BlobWriteSource::Bytes(data_col.value(i)),
+                )
+                .await?;
+                blob_writer.push(value)?;
+                continue;
+            }
 
-                        let data = source.read_all().await?;
+            if has_data && data_len > inline_threshold {
+                let value = self
+                    .write_packed(
+                        pack_file_threshold,
+                        BlobWriteSource::Bytes(data_col.value(i)),
+                    )
+                    .await?;
+                blob_writer.push(value)?;
+                continue;
+            }
 
-                        kind_builder.append_value(BlobKind::Inline as u8);
-                        data_builder.append_value(data.as_ref());
-                        uri_builder.append_null();
-                        blob_id_builder.append_null();
-                        blob_size_builder.append_null();
-                        position_builder.append_null();
+            if has_uri {
+                let uri_val = uri_col.value(i);
+                if self.external_blob_mode == ExternalBlobMode::Ingest {
+                    let position = if has_position {
+                        Some(
+                            position_col
+                                .as_ref()
+                                .expect("position column must exist")
+                                .value(i),
+                        )
+                    } else {
+                        None
+                    };
+                    let size = if has_size {
+                        Some(size_col.as_ref().expect("size column must exist").value(i))
+                    } else {
+                        None
+                    };
+                    let source = self.open_external_source(uri_val, position, size).await?;
+                    let data_len = source.size();
+
+                    if data_len > dedicated_threshold as u64 {
+                        let value = Self::write_dedicated(
+                            self.object_store.clone(),
+                            self.data_dir.clone(),
+                            self.data_file_key.clone(),
+                            self.blob_id_allocator.clone(),
+                            BlobWriteSource::External(&source),
+                        )
+                        .await?;
+                        blob_writer.push(value)?;
                         continue;
                     }
 
-                    let (external_base_id, external_uri_or_path) =
-                        self.resolve_external_reference(uri_val).await?;
-                    kind_builder.append_value(BlobKind::External as u8);
-                    data_builder.append_null();
-                    uri_builder.append_value(external_uri_or_path);
-                    blob_id_builder.append_value(external_base_id);
-                    if has_position && has_size {
-                        let position = position_col
-                            .as_ref()
-                            .expect("position column must exist")
-                            .value(i);
-                        let size = size_col.as_ref().expect("size column must exist").value(i);
-                        blob_size_builder.append_value(size);
-                        position_builder.append_value(position);
-                    } else {
-                        blob_size_builder.append_null();
-                        position_builder.append_null();
+                    if data_len > inline_threshold as u64 {
+                        let value = self
+                            .write_packed(pack_file_threshold, BlobWriteSource::External(&source))
+                            .await?;
+                        blob_writer.push(value)?;
+                        continue;
                     }
+
+                    let data = source.read_all().await?;
+                    blob_writer.push_inline(data)?;
                     continue;
                 }
 
-                if has_data {
-                    kind_builder.append_value(BlobKind::Inline as u8);
-                    let value = data_col.value(i);
-                    data_builder.append_value(value);
-                    uri_builder.append_null();
-                    blob_id_builder.append_null();
-                    blob_size_builder.append_null();
-                    position_builder.append_null();
+                let (external_base_id, external_uri_or_path) =
+                    self.resolve_external_reference(uri_val).await?;
+                let range = if has_position && has_size {
+                    BlobRange {
+                        offset: position_col
+                            .as_ref()
+                            .expect("position column must exist")
+                            .value(i),
+                        size: size_col.as_ref().expect("size column must exist").value(i),
+                    }
                 } else {
-                    data_builder.append_null();
-                    uri_builder.append_null();
-                    blob_id_builder.append_null();
-                    blob_size_builder.append_null();
-                    kind_builder.append_null();
-                    position_builder.append_null();
-                }
+                    BlobRange { offset: 0, size: 0 }
+                };
+                blob_writer.push(BlobDescriptor::External {
+                    base_id: external_base_id,
+                    uri: external_uri_or_path,
+                    offset: range.offset,
+                    size: range.size,
+                })?;
+                continue;
             }
 
-            let child_fields = vec![
-                arrow_schema::Field::new("kind", ArrowDataType::UInt8, true),
-                arrow_schema::Field::new("data", ArrowDataType::LargeBinary, true),
-                arrow_schema::Field::new("uri", ArrowDataType::Utf8, true),
-                arrow_schema::Field::new("blob_id", ArrowDataType::UInt32, true),
-                arrow_schema::Field::new("blob_size", ArrowDataType::UInt64, true),
-                arrow_schema::Field::new("position", ArrowDataType::UInt64, true),
-            ];
-
-            let struct_array = arrow_array::StructArray::try_new(
-                child_fields.clone().into(),
-                vec![
-                    Arc::new(kind_builder.finish()),
-                    Arc::new(data_builder.finish()),
-                    Arc::new(uri_builder.finish()),
-                    Arc::new(blob_id_builder.finish()),
-                    Arc::new(blob_size_builder.finish()),
-                    Arc::new(position_builder.finish()),
-                ],
-                struct_nulls.cloned(),
-            )?;
-
-            new_columns.push(Arc::new(struct_array));
-            new_fields.push(Arc::new(
-                arrow_schema::Field::new(
-                    field.name(),
-                    ArrowDataType::Struct(child_fields.into()),
-                    field.is_nullable(),
-                )
-                .with_metadata(self.writer_metadata[idx].clone()),
-            ));
+            if has_data {
+                blob_writer.push_inline(Bytes::copy_from_slice(data_col.value(i)))?;
+            } else {
+                blob_writer.push_null()?;
+            }
         }
 
-        let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata(
-            new_fields
-                .iter()
-                .map(|f| f.as_ref().clone())
-                .collect::<Vec<_>>(),
-            batch_schema.metadata().clone(),
-        ));
-
-        RecordBatch::try_new(new_schema, new_columns)
-            .map_err(|e| Error::invalid_input(e.to_string()))
+        let column = blob_writer.finish()?;
+        let (field, array) = column.into_parts();
+        Ok((array, Arc::new(field)))
     }
 
     pub(crate) async fn finish(&mut self) -> Result<()> {
@@ -700,16 +977,6 @@ impl BlobPreprocessor {
     }
 }
 
-fn dedicated_threshold_from_metadata(field: &arrow_schema::Field) -> usize {
-    field
-        .metadata()
-        .get(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY)
-        .and_then(|value| value.parse::<i64>().ok())
-        .filter(|value| *value > 0)
-        .and_then(|value| usize::try_from(value).ok())
-        .unwrap_or(DEDICATED_THRESHOLD)
-}
-
 pub async fn preprocess_blob_batches(
     batches: &[RecordBatch],
     pre: &mut BlobPreprocessor,
@@ -1637,6 +1904,27 @@ async fn execute_blob_read_plan(
         .collect())
 }
 
+async fn execute_blob_entries(
+    entries: Vec<BlobEntry>,
+    io_parallelism: usize,
+    io_buffer_size_bytes: Option<u64>,
+) -> Result<Vec<IndexedReadBlob>> {
+    let plans = plan_blob_read_plans(entries);
+    if plans.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let execution = Arc::new(ReadBlobsExecution::new(io_buffer_size_bytes));
+    let batches = stream::iter(plans.into_iter().map(move |plan| {
+        let execution = execution.clone();
+        execute_blob_read_plan(plan, execution)
+    }))
+    .buffer_unordered(io_parallelism.max(1))
+    .try_collect::<Vec<_>>()
+    .await?;
+    Ok(batches.into_iter().flatten().collect())
+}
+
 pub(super) async fn take_blobs(
     dataset: &Arc<Dataset>,
     row_ids: &[u64],
@@ -1770,16 +2058,57 @@ async fn collect_blob_entries_for_selection(
 /// descriptor `StructArray`, descending through nested struct children for
 /// dotted paths.
 fn leaf_descriptor_struct<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a StructArray> {
+    let current = leaf_descriptor_array(batch, column)?;
+    current
+        .as_any()
+        .downcast_ref::<StructArray>()
+        .ok_or_else(|| {
+            Error::invalid_input_source(
+                format!(
+                    "Blob column '{}' expected descriptor struct but got {}",
+                    column,
+                    current.data_type()
+                )
+                .into(),
+            )
+        })
+}
+
+fn leaf_descriptor_array<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a dyn Array> {
     let path = parse_field_path(column)?;
-    let mut current = batch
+    let mut current: &dyn Array = batch
         .column_by_name(&path[0])
-        .expect("validate_blob_column ensured column exists")
-        .as_struct();
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Blob column '{}' was not found in descriptor batch",
+                column
+            ))
+        })?
+        .as_ref();
     for segment in &path[1..] {
-        current = current
+        let struct_array = current
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .ok_or_else(|| {
+                Error::invalid_input_source(
+                    format!(
+                        "Blob column path '{}' expected struct before segment '{}' but got {}",
+                        column,
+                        segment,
+                        current.data_type()
+                    )
+                    .into(),
+                )
+            })?;
+        current = struct_array
             .column_by_name(segment)
-            .expect("validate_blob_column ensured all path segments exist")
-            .as_struct();
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "Blob column path '{}' missing segment '{}'",
+                    column, segment
+                ))
+            })?
+            .as_ref();
     }
     Ok(current)
 }
@@ -1805,15 +2134,47 @@ fn blob_version_from_descriptions(descriptions: &StructArray) -> Result<BlobVers
     .into()))
 }
 
-/// Convert blob v1 descriptors into logical blob entries.
-fn collect_blob_entries_v1(
-    dataset: &Arc<Dataset>,
-    blob_field_id: u32,
-    descriptions: &StructArray,
-    row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
-) -> Result<Vec<BlobEntry>> {
-    let positions = descriptions.column(0).as_primitive::<UInt64Type>();
-    let sizes = descriptions.column(1).as_primitive::<UInt64Type>();
+struct BlobV2DescriptorColumns<'a> {
+    descriptions: &'a StructArray,
+    kinds: &'a arrow::array::PrimitiveArray<UInt8Type>,
+    positions: &'a arrow::array::PrimitiveArray<UInt64Type>,
+    sizes: &'a arrow::array::PrimitiveArray<UInt64Type>,
+    blob_ids: &'a arrow::array::PrimitiveArray<UInt32Type>,
+    blob_uris: &'a arrow::array::GenericStringArray<i32>,
+}
+
+impl<'a> BlobV2DescriptorColumns<'a> {
+    fn new(descriptions: &'a StructArray) -> Self {
+        Self {
+            descriptions,
+            kinds: descriptions.column(0).as_primitive::<UInt8Type>(),
+            positions: descriptions.column(1).as_primitive::<UInt64Type>(),
+            sizes: descriptions.column(2).as_primitive::<UInt64Type>(),
+            blob_ids: descriptions.column(3).as_primitive::<UInt32Type>(),
+            blob_uris: descriptions.column(4).as_string::<i32>(),
+        }
+    }
+
+    fn is_null_blob(&self, idx: usize) -> Result<bool> {
+        if self.descriptions.is_null(idx) || self.kinds.is_null(idx) {
+            return Ok(true);
+        }
+        let kind = BlobKind::try_from(self.kinds.value(idx))?;
+        Ok(matches!(kind, BlobKind::Inline)
+            && self.positions.value(idx) == 0
+            && self.sizes.value(idx) == 0)
+    }
+}
+
+/// Convert blob v1 descriptors into logical blob entries.
+fn collect_blob_entries_v1(
+    dataset: &Arc<Dataset>,
+    blob_field_id: u32,
+    descriptions: &StructArray,
+    row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
+) -> Result<Vec<BlobEntry>> {
+    let positions = descriptions.column(0).as_primitive::<UInt64Type>();
+    let sizes = descriptions.column(1).as_primitive::<UInt64Type>();
     let mut source_cache = HashMap::<BlobSourceKey, Arc<BlobSource>>::new();
     row_addrs
         .values()
@@ -1867,147 +2228,563 @@ async fn collect_blob_entries_v2(
     descriptions: &StructArray,
     row_addrs: &arrow::array::PrimitiveArray<UInt64Type>,
 ) -> Result<Vec<BlobEntry>> {
-    let kinds = descriptions.column(0).as_primitive::<UInt8Type>();
-    let positions = descriptions.column(1).as_primitive::<UInt64Type>();
-    let sizes = descriptions.column(2).as_primitive::<UInt64Type>();
-    let blob_ids = descriptions.column(3).as_primitive::<UInt32Type>();
-    let blob_uris = descriptions.column(4).as_string::<i32>();
-
+    let columns = BlobV2DescriptorColumns::new(descriptions);
     let mut files = Vec::with_capacity(row_addrs.len());
-    let mut fragment_cache = HashMap::<u32, BlobReadLocation>::new();
-    let mut store_cache = HashMap::<u32, Arc<ObjectStore>>::new();
-    let mut external_base_path_cache = HashMap::<u32, Path>::new();
-    let mut source_cache = HashMap::<BlobSourceKey, Arc<BlobSource>>::new();
+    let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id);
     for (selection_index, row_addr) in row_addrs.values().iter().enumerate() {
-        let idx = selection_index;
-        let kind = BlobKind::try_from(kinds.value(idx))?;
+        if let Some(entry) = read_context
+            .collect_entry(&columns, selection_index, selection_index, *row_addr)
+            .await?
+        {
+            files.push(entry);
+        }
+    }
+
+    Ok(files)
+}
+
+fn is_blob_v2_binary_view(field: &LanceField) -> bool {
+    field.is_blob_v2() && matches!(field.data_type(), ArrowDataType::LargeBinary)
+}
+
+fn public_blob_v2_binary_output_field(mut field: LanceField) -> LanceField {
+    if is_blob_v2_binary_view(&field) {
+        field.metadata.remove(ARROW_EXT_NAME_KEY);
+    }
+    field.children = field
+        .children
+        .into_iter()
+        .map(public_blob_v2_binary_output_field)
+        .collect();
+    field
+}
+
+/// Return the public Arrow-facing schema for a blob v2 binary scan.
+///
+/// Scan planning uses a blob v2 extension marker on `LargeBinary` leaves to
+/// identify payloads that need descriptor-based materialization. This helper
+/// removes that internal marker before the schema is exposed to callers.
+pub fn public_blob_v2_binary_output_schema(schema: &Schema) -> Schema {
+    Schema {
+        fields: schema
+            .fields
+            .iter()
+            .cloned()
+            .map(public_blob_v2_binary_output_field)
+            .collect(),
+        metadata: schema.metadata.clone(),
+    }
+}
+
+fn field_has_blob_v2_binary_view(field: &LanceField) -> bool {
+    is_blob_v2_binary_view(field) || field.children.iter().any(field_has_blob_v2_binary_view)
+}
+
+/// Return true if the schema contains a blob v2 leaf in binary payload view.
+///
+/// This detects the internal `LargeBinary` view created by
+/// [`BlobHandling::AllBinary`](lance_core::datatypes::BlobHandling::AllBinary)
+/// or selective binary blob handling.
+pub fn schema_has_blob_v2_binary_view(schema: &Schema) -> bool {
+    schema.fields.iter().any(field_has_blob_v2_binary_view)
+}
+
+fn blob_v2_descriptor_field(mut field: LanceField) -> LanceField {
+    if is_blob_v2_binary_view(&field) {
+        field.unloaded_mut();
+        return field;
+    }
+
+    field.children = field
+        .children
+        .into_iter()
+        .map(blob_v2_descriptor_field)
+        .collect();
+    field
+}
+
+/// Convert blob v2 binary-view leaves back to descriptor-view leaves.
+///
+/// Readers use this schema to fetch stored blob descriptors first. The scan
+/// layer then materializes those descriptors into the caller's binary payload
+/// view after row addresses are available.
+pub fn blob_v2_descriptor_schema(schema: &Schema) -> Schema {
+    Schema {
+        fields: schema
+            .fields
+            .iter()
+            .cloned()
+            .map(blob_v2_descriptor_field)
+            .collect(),
+        metadata: schema.metadata.clone(),
+    }
+}
+
+/// Materialize blob v2 descriptor arrays in a decoded batch into binary arrays.
+///
+/// The input batch must include `_rowaddr`, which is used to resolve packed,
+/// dedicated, inline, and external blob payload locations. `output_schema`
+/// defines the exact returned columns, including requested system columns, with
+/// blob v2 binary leaves exposed as plain `LargeBinary` fields.
+pub async fn materialize_blob_v2_binary_batch(
+    dataset: &Arc<Dataset>,
+    output_schema: &Schema,
+    batch: RecordBatch,
+) -> Result<RecordBatch> {
+    let row_addr_idx = batch
+        .schema()
+        .column_with_name(ROW_ADDR)
+        .ok_or_else(|| {
+            Error::internal(format!(
+                "_rowaddr column missing from blob v2 binary scan batch, columns: {:?}",
+                batch
+                    .schema()
+                    .fields()
+                    .iter()
+                    .map(|field| field.name())
+                    .collect::<Vec<_>>()
+            ))
+        })?
+        .0;
+    let row_addrs = batch
+        .column(row_addr_idx)
+        .as_primitive::<UInt64Type>()
+        .values()
+        .iter()
+        .copied()
+        .collect::<Vec<_>>();
+    let row_addrs: Arc<[u64]> = row_addrs.into();
+
+    let mut columns = Vec::with_capacity(output_schema.fields.len());
+    let mut fields = Vec::with_capacity(output_schema.fields.len());
+
+    for field in &output_schema.fields {
+        let input = batch
+            .column_by_name(&field.name)
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 binary scan batch missing projected column '{}'",
+                    field.name
+                ))
+            })?
+            .clone();
+        let materialized =
+            materialize_blob_v2_binary_array(dataset, field, input, row_addrs.clone()).await?;
+        columns.push(materialized);
+        let output_field = public_blob_v2_binary_output_field(field.clone());
+        fields.push(ArrowField::from(&output_field));
+    }
+
+    Ok(RecordBatch::try_new(
+        Arc::new(ArrowSchema::new_with_metadata(
+            fields,
+            batch.schema().metadata().clone(),
+        )),
+        columns,
+    )?)
+}
+
+fn materialize_blob_v2_binary_array<'a>(
+    dataset: &'a Arc<Dataset>,
+    field: &'a LanceField,
+    array: ArrayRef,
+    row_addrs: Arc<[u64]>,
+) -> BoxFuture<'a, Result<ArrayRef>> {
+    async move {
+        if is_blob_v2_binary_view(field) {
+            let descriptions = array.as_struct();
+            return materialize_blob_v2_descriptors(
+                dataset,
+                field.id as u32,
+                descriptions,
+                row_addrs.as_ref(),
+            )
+            .await;
+        }
+
+        match field.data_type() {
+            ArrowDataType::Struct(_) => {
+                let struct_array = array.as_struct();
+                let mut children = Vec::with_capacity(field.children.len());
+                for (child_field, child_array) in
+                    field.children.iter().zip(struct_array.columns().iter())
+                {
+                    children.push(
+                        materialize_blob_v2_binary_array(
+                            dataset,
+                            child_field,
+                            child_array.clone(),
+                            row_addrs.clone(),
+                        )
+                        .await?,
+                    );
+                }
+                let public_field = public_blob_v2_binary_output_field(field.clone());
+                let ArrowDataType::Struct(fields) = public_field.data_type() else {
+                    unreachable!("public output field preserved struct type")
+                };
+                Ok(Arc::new(StructArray::try_new(
+                    fields,
+                    children,
+                    struct_array.nulls().cloned(),
+                )?) as ArrayRef)
+            }
+            ArrowDataType::List(_) => {
+                let list_array = array.as_list::<i32>();
+                materialize_blob_v2_list_array::<i32>(dataset, field, list_array, row_addrs).await
+            }
+            ArrowDataType::LargeList(_) => {
+                let list_array = array.as_list::<i64>();
+                materialize_blob_v2_list_array::<i64>(dataset, field, list_array, row_addrs).await
+            }
+            _ => Ok(array),
+        }
+    }
+    .boxed()
+}
+
+async fn materialize_blob_v2_list_array<O: OffsetSizeTrait>(
+    dataset: &Arc<Dataset>,
+    field: &LanceField,
+    list_array: &GenericListArray<O>,
+    row_addrs: Arc<[u64]>,
+) -> Result<ArrayRef> {
+    let offsets = list_array.value_offsets();
+    let values_start = offsets[0].as_usize();
+    let values_end = offsets[list_array.len()].as_usize();
+    if values_end < values_start {
+        return Err(Error::internal(format!(
+            "List field '{}' has invalid offsets while materializing blob v2 binary scan",
+            field.name
+        )));
+    }
+
+    let values_len = values_end - values_start;
+    let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1);
+    normalized_offsets.push(O::usize_as(0));
+    let mut child_row_addrs = Vec::with_capacity(values_len);
+    for row_idx in 0..list_array.len() {
+        let start = offsets[row_idx].as_usize();
+        let end = offsets[row_idx + 1].as_usize();
+        if end < start {
+            return Err(Error::internal(format!(
+                "List field '{}' has decreasing offsets while materializing blob v2 binary scan",
+                field.name
+            )));
+        }
+        let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| {
+            Error::internal(format!(
+                "List field '{}' row address count {} did not match row count {}",
+                field.name,
+                row_addrs.len(),
+                list_array.len()
+            ))
+        })?;
+        for _ in start..end {
+            child_row_addrs.push(row_addr);
+        }
+        normalized_offsets.push(O::usize_as(end - values_start));
+    }
+    let child_row_addrs: Arc<[u64]> = child_row_addrs.into();
+    let child = field.children.first().ok_or_else(|| {
+        Error::internal(format!(
+            "List field '{}' missing child while materializing blob v2 binary scan",
+            field.name
+        ))
+    })?;
+    let values = list_array.values().slice(values_start, values_len);
+    let values = materialize_blob_v2_binary_array(dataset, child, values, child_row_addrs).await?;
+    let child_field = public_blob_v2_binary_output_field(child.clone());
+    let list_array = GenericListArray::<O>::try_new(
+        Arc::new(ArrowField::from(&child_field)),
+        OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)),
+        values,
+        list_array.nulls().cloned(),
+    )?;
+    Ok(Arc::new(list_array))
+}
+
+async fn materialize_blob_v2_descriptors(
+    dataset: &Arc<Dataset>,
+    blob_field_id: u32,
+    descriptions: &StructArray,
+    row_addrs: &[u64],
+) -> Result<ArrayRef> {
+    if descriptions.len() != row_addrs.len() {
+        return Err(Error::internal(format!(
+            "blob v2 descriptor count {} did not match row address count {}",
+            descriptions.len(),
+            row_addrs.len()
+        )));
+    }
+    match blob_version_from_descriptions(descriptions)? {
+        BlobVersion::V1 => {
+            return Err(Error::not_supported(
+                "Blob v2 binary materialization received a legacy blob descriptor".to_string(),
+            ));
+        }
+        BlobVersion::V2 => {}
+    }
+
+    let columns = BlobV2DescriptorColumns::new(descriptions);
+    let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id);
+    let mut entries = Vec::with_capacity(descriptions.len());
+    let mut payloads = vec![None; descriptions.len()];
+
+    for (idx, row_addr) in row_addrs.iter().copied().enumerate() {
+        if descriptions.is_null(idx) || columns.kinds.is_null(idx) {
+            continue;
+        }
 
-        // Struct is non-nullable; null rows are encoded as inline with zero position/size and empty uri
-        if matches!(kind, BlobKind::Inline) && positions.value(idx) == 0 && sizes.value(idx) == 0 {
+        let kind = BlobKind::try_from(columns.kinds.value(idx))?;
+        if matches!(kind, BlobKind::Inline)
+            && columns.positions.value(idx) == 0
+            && columns.sizes.value(idx) == 0
+        {
+            payloads[idx] = Some(Bytes::new());
             continue;
         }
 
-        match kind {
+        let entry = read_context
+            .collect_entry(&columns, idx, idx, row_addr)
+            .await?
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 descriptor at index {idx} unexpectedly resolved to null"
+                ))
+            })?;
+        entries.push(entry);
+    }
+
+    let blobs = execute_blob_entries(entries, dataset.object_store.io_parallelism(), None).await?;
+    for blob in blobs {
+        let payload = payloads.get_mut(blob.selection_index).ok_or_else(|| {
+            Error::internal(format!(
+                "blob result selection index {} exceeded descriptor count {}",
+                blob.selection_index,
+                descriptions.len()
+            ))
+        })?;
+        if payload.replace(blob.data).is_some() {
+            return Err(Error::internal(format!(
+                "blob result selection index {} was produced more than once",
+                blob.selection_index
+            )));
+        }
+    }
+
+    let payload_capacity = payloads.iter().flatten().map(Bytes::len).sum::<usize>();
+    let mut builder = LargeBinaryBuilder::with_capacity(descriptions.len(), payload_capacity);
+    for (idx, payload) in payloads.into_iter().enumerate() {
+        if descriptions.is_null(idx) || columns.kinds.is_null(idx) {
+            builder.append_null();
+        } else {
+            let payload = payload.ok_or_else(|| {
+                Error::internal(format!(
+                    "blob v2 descriptor at index {idx} did not produce a payload"
+                ))
+            })?;
+            builder.append_value(payload);
+        }
+    }
+
+    Ok(Arc::new(builder.finish()))
+}
+
+struct BlobV2ReadContext<'a> {
+    dataset: &'a Arc<Dataset>,
+    blob_field_id: u32,
+    fragment_cache: HashMap<u32, BlobReadLocation>,
+    store_cache: HashMap<u32, Arc<ObjectStore>>,
+    external_base_path_cache: HashMap<u32, Path>,
+    source_cache: HashMap<BlobSourceKey, Arc<BlobSource>>,
+}
+
+impl<'a> BlobV2ReadContext<'a> {
+    fn new(dataset: &'a Arc<Dataset>, blob_field_id: u32) -> Self {
+        Self {
+            dataset,
+            blob_field_id,
+            fragment_cache: HashMap::new(),
+            store_cache: HashMap::new(),
+            external_base_path_cache: HashMap::new(),
+            source_cache: HashMap::new(),
+        }
+    }
+
+    async fn collect_entry(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<Option<BlobEntry>> {
+        if columns.is_null_blob(idx)? {
+            return Ok(None);
+        }
+
+        let kind = BlobKind::try_from(columns.kinds.value(idx))?;
+        let entry = match kind {
             BlobKind::Inline => {
-                let position = positions.value(idx);
-                let size = sizes.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let source = shared_blob_source(
-                    &mut source_cache,
-                    location.object_store,
-                    &location.data_file_path,
-                );
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, position, size, BlobKind::Inline, None),
-                });
+                self.collect_inline(columns, idx, selection_index, row_addr)
+                    .await?
             }
             BlobKind::Dedicated => {
-                let blob_id = blob_ids.value(idx);
-                let size = sizes.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None),
-                });
+                self.collect_dedicated(columns, idx, selection_index, row_addr)
+                    .await?
             }
             BlobKind::Packed => {
-                let blob_id = blob_ids.value(idx);
-                let size = sizes.value(idx);
-                let position = positions.value(idx);
-                let location = resolve_blob_read_location(
-                    dataset,
-                    blob_field_id,
-                    *row_addr,
-                    &mut fragment_cache,
-                    &mut store_cache,
-                )
-                .await?;
-                let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
-                let source = shared_blob_source(&mut source_cache, location.object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(source, position, size, BlobKind::Packed, None),
-                });
+                self.collect_packed(columns, idx, selection_index, row_addr)
+                    .await?
             }
             BlobKind::External => {
-                let uri_or_path = blob_uris.value(idx).to_string();
-                let position = positions.value(idx);
-                let size = sizes.value(idx);
-                let base_id = blob_ids.value(idx);
-                let (object_store, path) = if base_id == 0 {
-                    let registry = dataset.session.store_registry();
-                    let params = dataset
-                        .store_params
-                        .as_ref()
-                        .map(|p| Arc::new((**p).clone()))
-                        .unwrap_or_else(|| Arc::new(ObjectStoreParams::default()));
-                    ObjectStore::from_uri_and_params(registry, &uri_or_path, &params).await?
-                } else {
-                    let object_store = if let Some(store) = store_cache.get(&base_id) {
-                        store.clone()
-                    } else {
-                        let store = dataset.object_store(Some(base_id)).await?;
-                        store_cache.insert(base_id, store.clone());
-                        store
-                    };
-                    let base_root = if let Some(path) = external_base_path_cache.get(&base_id) {
-                        path.clone()
-                    } else {
-                        let base = dataset.manifest.base_paths.get(&base_id).ok_or_else(|| {
-                            Error::invalid_input(format!(
-                                "External blob references unknown base_id {}",
-                                base_id
-                            ))
-                        })?;
-                        let path = base.extract_path(dataset.session.store_registry())?;
-                        external_base_path_cache.insert(base_id, path.clone());
-                        path
-                    };
-                    let path = join_base_and_relative_path(&base_root, &uri_or_path)?;
-                    (object_store, path)
-                };
-                let size = if size > 0 {
-                    size
-                } else {
-                    object_store.size(&path).await?
-                };
-                let source = shared_blob_source(&mut source_cache, object_store, &path);
-                files.push(BlobEntry {
-                    selection_index,
-                    row_address: *row_addr,
-                    file: BlobFile::with_source(
-                        source,
-                        position,
-                        size,
-                        BlobKind::External,
-                        Some(uri_or_path),
-                    ),
-                });
+                self.collect_external(columns, idx, selection_index, row_addr)
+                    .await?
             }
-        }
+        };
+
+        Ok(Some(entry))
     }
 
-    Ok(files)
+    async fn blob_read_location(&mut self, row_addr: u64) -> Result<BlobReadLocation> {
+        resolve_blob_read_location(
+            self.dataset,
+            self.blob_field_id,
+            row_addr,
+            &mut self.fragment_cache,
+            &mut self.store_cache,
+        )
+        .await
+    }
+
+    async fn collect_inline(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let position = columns.positions.value(idx);
+        let size = columns.sizes.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let source = shared_blob_source(
+            &mut self.source_cache,
+            location.object_store,
+            &location.data_file_path,
+        );
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, position, size, BlobKind::Inline, None),
+        })
+    }
+
+    async fn collect_dedicated(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let blob_id = columns.blob_ids.value(idx);
+        let size = columns.sizes.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
+        let source = shared_blob_source(&mut self.source_cache, location.object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None),
+        })
+    }
+
+    async fn collect_packed(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let blob_id = columns.blob_ids.value(idx);
+        let size = columns.sizes.value(idx);
+        let position = columns.positions.value(idx);
+        let location = self.blob_read_location(row_addr).await?;
+        let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id);
+        let source = shared_blob_source(&mut self.source_cache, location.object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(source, position, size, BlobKind::Packed, None),
+        })
+    }
+
+    async fn collect_external(
+        &mut self,
+        columns: &BlobV2DescriptorColumns<'_>,
+        idx: usize,
+        selection_index: usize,
+        row_addr: u64,
+    ) -> Result<BlobEntry> {
+        let uri_or_path = columns.blob_uris.value(idx).to_string();
+        let position = columns.positions.value(idx);
+        let size = columns.sizes.value(idx);
+        let base_id = columns.blob_ids.value(idx);
+        let (object_store, path) = if base_id == 0 {
+            let registry = self.dataset.session.store_registry();
+            let params = self
+                .dataset
+                .store_params
+                .as_ref()
+                .map(|p| Arc::new((**p).clone()))
+                .unwrap_or_else(|| Arc::new(ObjectStoreParams::default()));
+            ObjectStore::from_uri_and_params(registry, &uri_or_path, &params).await?
+        } else {
+            let object_store = if let Some(store) = self.store_cache.get(&base_id) {
+                store.clone()
+            } else {
+                let store = self.dataset.object_store(Some(base_id)).await?;
+                self.store_cache.insert(base_id, store.clone());
+                store
+            };
+            let base_root = if let Some(path) = self.external_base_path_cache.get(&base_id) {
+                path.clone()
+            } else {
+                let base = self
+                    .dataset
+                    .manifest
+                    .base_paths
+                    .get(&base_id)
+                    .ok_or_else(|| {
+                        Error::invalid_input(format!(
+                            "External blob references unknown base_id {}",
+                            base_id
+                        ))
+                    })?;
+                let path = base.extract_path(self.dataset.session.store_registry())?;
+                self.external_base_path_cache.insert(base_id, path.clone());
+                path
+            };
+            let path = join_base_and_relative_path(&base_root, &uri_or_path)?;
+            (object_store, path)
+        };
+        let size = if size > 0 {
+            size
+        } else {
+            object_store.size(&path).await?
+        };
+        let source = shared_blob_source(&mut self.source_cache, object_store, &path);
+        Ok(BlobEntry {
+            selection_index,
+            row_address: row_addr,
+            file: BlobFile::with_source(
+                source,
+                position,
+                size,
+                BlobKind::External,
+                Some(uri_or_path),
+            ),
+        })
+    }
 }
 
 fn normalize_external_absolute_uri(uri: &str) -> Result<String> {
@@ -2103,17 +2880,23 @@ mod tests {
     };
     use arrow_array::RecordBatch;
     use arrow_array::{
-        ArrayRef, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array,
+        Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatchIterator, StringArray,
+        StructArray, UInt8Array, UInt32Array, UInt64Array,
     };
     use arrow_schema::{DataType, Field, Schema};
     use async_trait::async_trait;
     use bytes::Bytes;
     use chrono::Utc;
-    use futures::{StreamExt, TryStreamExt, future::try_join_all};
+    use futures::{StreamExt, TryStreamExt};
     use lance_arrow::{
-        ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, DataTypeExt,
+        ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+        BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+        BLOB_V2_EXT_NAME, DataTypeExt,
+    };
+    use lance_core::{
+        datatypes::{BlobHandling, BlobKind},
+        utils::blob::blob_path,
     };
-    use lance_core::datatypes::BlobKind;
     use lance_io::object_store::{
         ObjectStore, ObjectStoreParams, ObjectStoreRegistry, StorageOptionsAccessor,
     };
@@ -2128,21 +2911,29 @@ mod tests {
     use url::Url;
 
     use lance_core::{
-        Error, Result,
+        Error, ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION, Result,
         utils::tempfile::{TempDir, TempStrDir},
     };
     use lance_datagen::{BatchCount, RowCount, array};
-    use lance_file::version::LanceFileVersion;
+    use lance_file::{
+        version::LanceFileVersion,
+        writer::{FileWriter, FileWriterOptions},
+    };
+    use uuid::Uuid;
 
     use super::{
         BlobEntry, BlobFile, BlobSource, ExternalBaseCandidate, ExternalBaseResolver,
-        ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path,
+        ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path, execute_blob_entries,
         execute_blob_read_plan, plan_blob_read_plans,
     };
     use crate::{
         Dataset,
-        blob::{BlobArrayBuilder, blob_field},
-        dataset::{ExternalBlobMode, WriteParams},
+        blob::{BlobArrayBuilder, BlobDescriptorArrayBuilder, PackedBlobWriter, blob_field},
+        dataset::{
+            CommitBuilder, ExternalBlobMode, WriteMode, WriteParams,
+            scanner::MaterializationStyle,
+            transaction::{DataReplacementGroup, Operation, Transaction},
+        },
         utils::test::TestDatasetGenerator,
     };
 
@@ -2158,6 +2949,32 @@ mod tests {
         expected: Vec<u8>,
     }
 
+    fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc<Schema>, RecordBatch) {
+        let blob_field = blob_field("blob", true);
+        let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field];
+        let info_array: ArrayRef = Arc::new(
+            StructArray::try_new(
+                info_fields.clone().into(),
+                vec![
+                    Arc::new(StringArray::from_iter_values(
+                        (0..blob_array.len()).map(|idx| format!("name-{idx}")),
+                    )) as ArrayRef,
+                    blob_array,
+                ],
+                None,
+            )
+            .unwrap(),
+        );
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "info",
+            DataType::Struct(info_fields.into()),
+            true,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap();
+        (schema, batch)
+    }
+
     #[cfg(feature = "azure")]
     fn azure_store_params(account_name: &str) -> ObjectStoreParams {
         ObjectStoreParams {
@@ -2997,40 +3814,135 @@ mod tests {
         assert!(descriptors.column(0).data_type().is_struct());
     }
 
-    #[test]
-    fn test_data_file_key_from_path() {
-        assert_eq!(data_file_key_from_path("data/abc.lance"), "abc");
-        assert_eq!(data_file_key_from_path("abc.lance"), "abc");
-        assert_eq!(data_file_key_from_path("nested/path/xyz"), "xyz");
-    }
-
     #[tokio::test]
-    async fn test_write_and_take_blobs_with_blob_array_builder() {
+    async fn test_v2_0_legacy_blob_descriptor_projection_and_reads() {
         let test_dir = TempStrDir::default();
-
-        // Build a blob column with the new BlobArrayBuilder
-        let mut blob_builder = BlobArrayBuilder::new(2);
-        blob_builder.push_bytes(b"hello").unwrap();
-        blob_builder.push_bytes(b"world").unwrap();
-        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
-
-        let id_array: arrow_array::ArrayRef = Arc::new(UInt32Array::from(vec![0, 1]));
+        let blob_meta = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
         let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::UInt32, false),
-            blob_field("blob", true),
+            Field::new("id", DataType::Int32, false),
+            Field::new("blob", DataType::LargeBinary, true).with_metadata(blob_meta),
         ]));
-
-        let batch = RecordBatch::try_new(schema.clone(), vec![id_array, blob_array]).unwrap();
-        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
-
-        let params = WriteParams {
-            data_storage_version: Some(LanceFileVersion::V2_2),
-            ..Default::default()
-        };
+        let payloads = [
+            b"abc".as_slice(),
+            b"defgh".as_slice(),
+            b"ijklmnop".as_slice(),
+        ];
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1, 2])) as ArrayRef,
+                Arc::new(LargeBinaryArray::from_iter_values(payloads)) as ArrayRef,
+            ],
+        )
+        .unwrap();
         let dataset = Arc::new(
-            Dataset::write(reader, &test_dir, Some(params))
-                .await
-                .unwrap(),
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_0),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        for blob_handling in [
+            None,
+            Some(BlobHandling::BlobsDescriptions),
+            Some(BlobHandling::AllDescriptions),
+        ] {
+            let mut scanner = dataset.scan();
+            if let Some(blob_handling) = blob_handling {
+                scanner.blob_handling(blob_handling);
+            }
+            let descriptors = scanner
+                .project(&["blob"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let descriptor = descriptors.column(0).as_struct();
+            assert_eq!(descriptor.fields().len(), 2);
+            assert_eq!(descriptor.fields()[0].name(), "position");
+            assert_eq!(descriptor.fields()[1].name(), "size");
+            let sizes = descriptor
+                .column_by_name("size")
+                .unwrap()
+                .as_primitive::<UInt64Type>();
+            assert_eq!(sizes.values(), &[3, 5, 8]);
+        }
+
+        let mut scanner = dataset.scan();
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let bytes = scanner
+            .project(&["blob"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let bytes = bytes.column(0).as_binary::<i64>();
+        for (idx, expected) in payloads.iter().enumerate() {
+            assert_eq!(bytes.value(idx), *expected);
+        }
+
+        let blob_files = dataset
+            .take_blobs_by_indices(&[0, 1, 2], "blob")
+            .await
+            .unwrap();
+        assert_eq!(blob_files.len(), payloads.len());
+        for (blob_file, expected) in blob_files.iter().zip(payloads) {
+            assert_eq!(blob_file.read().await.unwrap().as_ref(), expected);
+        }
+
+        let read_blobs = dataset
+            .read_blobs("blob")
+            .unwrap()
+            .with_row_indices(vec![0, 1, 2])
+            .execute()
+            .await
+            .unwrap();
+        assert_eq!(read_blobs.len(), payloads.len());
+        for (read_blob, expected) in read_blobs.iter().zip(payloads) {
+            assert_eq!(read_blob.data.as_ref(), expected);
+        }
+    }
+
+    #[test]
+    fn test_data_file_key_from_path() {
+        assert_eq!(data_file_key_from_path("data/abc.lance"), "abc");
+        assert_eq!(data_file_key_from_path("abc.lance"), "abc");
+        assert_eq!(data_file_key_from_path("nested/path/xyz"), "xyz");
+    }
+
+    #[tokio::test]
+    async fn test_write_and_take_blobs_with_blob_descriptor_array_builder() {
+        let test_dir = TempStrDir::default();
+
+        // Build a blob column with the new BlobArrayBuilder
+        let mut blob_builder = BlobArrayBuilder::new(2);
+        blob_builder.push_bytes(b"hello").unwrap();
+        blob_builder.push_bytes(b"world").unwrap();
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+
+        let id_array: arrow_array::ArrayRef = Arc::new(UInt32Array::from(vec![0, 1]));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            blob_field("blob", true),
+        ]));
+
+        let batch = RecordBatch::try_new(schema.clone(), vec![id_array, blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone());
+
+        let params = WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_2),
+            ..Default::default()
+        };
+        let dataset = Arc::new(
+            Dataset::write(reader, &test_dir, Some(params))
+                .await
+                .unwrap(),
         );
 
         let blobs = dataset
@@ -3038,11 +3950,807 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(blobs.len(), 2);
-        let first = blobs[0].read().await.unwrap();
-        let second = blobs[1].read().await.unwrap();
-        assert_eq!(first.as_ref(), b"hello");
-        assert_eq!(second.as_ref(), b"world");
+        assert_eq!(blobs.len(), 2);
+        let first = blobs[0].read().await.unwrap();
+        let second = blobs[1].read().await.unwrap();
+        assert_eq!(first.as_ref(), b"hello");
+        assert_eq!(second.as_ref(), b"world");
+    }
+
+    #[tokio::test]
+    async fn test_write_prepared_blob_column_normalizes_schema() {
+        let test_dir = TempStrDir::default();
+        let mut prepared_writer = BlobDescriptorArrayBuilder::new("blob");
+        prepared_writer
+            .push_inline(Bytes::from_static(b"prepared-inline"))
+            .unwrap();
+        let (prepared_field, prepared_array) = prepared_writer.finish().unwrap().into_parts();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            prepared_field,
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![0])) as ArrayRef,
+                prepared_array,
+            ],
+        )
+        .unwrap();
+
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(batch)], schema),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let dataset_schema = Schema::from(dataset.schema());
+        let blob_field = dataset_schema.field_with_name("blob").unwrap();
+        let DataType::Struct(fields) = blob_field.data_type() else {
+            panic!("expected blob field to be a struct");
+        };
+        assert_eq!(fields.len(), 2);
+        assert_eq!(fields[0].name(), "data");
+        assert_eq!(fields[1].name(), "uri");
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-inline");
+    }
+
+    #[tokio::test]
+    async fn test_data_replacement_uses_blob_descriptor_array_builder_prepared_packed_blob_column()
+    {
+        let test_dir = TempStrDir::default();
+        let logical_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            blob_field("blob", true),
+        ]));
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(b"initial").unwrap();
+        let initial_batch = RecordBatch::try_new(
+            logical_schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![0])) as ArrayRef,
+                initial_builder.finish().unwrap(),
+            ],
+        )
+        .unwrap();
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(initial_batch)], logical_schema.clone()),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let file_id = Uuid::new_v4().to_string();
+        let data_file_name = format!("{file_id}.lance");
+        let data_file_path = dataset.data_dir().join(data_file_name.as_str());
+        let blob_id = 1;
+        let packed_path = blob_path(&dataset.data_dir(), &file_id, blob_id);
+        let mut blob_writer = BlobDescriptorArrayBuilder::new("blob");
+        let mut packed = PackedBlobWriter::try_new(
+            dataset.object_store.as_ref().clone(),
+            data_file_path.clone(),
+            blob_id,
+        )
+        .await
+        .unwrap();
+        assert_eq!(packed.path(), &packed_path);
+        packed.write_blob(b"prepared-packed").await.unwrap();
+        blob_writer.extend(packed.finish().await.unwrap()).unwrap();
+        let prepared_field = blob_writer.field().clone();
+        let prepared_array = blob_writer.finish().unwrap().into_parts().1;
+        let append_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            prepared_field,
+        ]));
+        let replacement_batch = RecordBatch::try_new(
+            append_schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![1])) as ArrayRef,
+                prepared_array,
+            ],
+        )
+        .unwrap();
+
+        let object_writer = dataset.object_store.create(&data_file_path).await.unwrap();
+        let mut file_writer = FileWriter::try_new(
+            object_writer,
+            crate::datatypes::Schema::try_from(append_schema.as_ref()).unwrap(),
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        file_writer.write_batch(&replacement_batch).await.unwrap();
+        file_writer.finish().await.unwrap();
+
+        let data_file = dataset
+            .create_data_file(&data_file_name, None)
+            .await
+            .unwrap();
+        let transaction = Transaction {
+            read_version: dataset.manifest.version,
+            uuid: Uuid::new_v4().hyphenated().to_string(),
+            operation: Operation::DataReplacement {
+                replacements: vec![DataReplacementGroup(0, data_file)],
+            },
+            tag: None,
+            transaction_properties: None,
+        };
+        let dataset = Arc::new(
+            CommitBuilder::new(dataset)
+                .execute(transaction)
+                .await
+                .unwrap(),
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-packed");
+        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+    }
+
+    #[tokio::test]
+    async fn test_append_uses_blob_descriptor_array_builder_prepared_blob_column() {
+        let test_dir = TempStrDir::default();
+        let logical_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            blob_field("blob", true),
+        ]));
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(b"initial").unwrap();
+        let initial_batch = RecordBatch::try_new(
+            logical_schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![0])) as ArrayRef,
+                initial_builder.finish().unwrap(),
+            ],
+        )
+        .unwrap();
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(initial_batch)], logical_schema.clone()),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let mut blob_writer = BlobDescriptorArrayBuilder::new("blob");
+        blob_writer
+            .push_inline(Bytes::from_static(b"append"))
+            .unwrap();
+        let prepared_column = blob_writer.finish().unwrap();
+        let (prepared_field, prepared_array) = prepared_column.into_parts();
+        let append_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            prepared_field,
+        ]));
+        let append_batch = RecordBatch::try_new(
+            append_schema.clone(),
+            vec![
+                Arc::new(UInt32Array::from(vec![1])) as ArrayRef,
+                prepared_array,
+            ],
+        )
+        .unwrap();
+
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(append_batch)], append_schema),
+                dataset,
+                Some(WriteParams {
+                    mode: WriteMode::Append,
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let ids = dataset
+            .scan()
+            .project(&["id"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let id_values = ids.column(0).as_primitive::<UInt32Type>();
+        assert_eq!(id_values.values(), &[0, 1]);
+
+        let blobs = dataset
+            .take_blobs_by_indices(&[0, 1], "blob")
+            .await
+            .unwrap();
+        assert_eq!(blobs.len(), 2);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"initial");
+        assert_eq!(blobs[1].read().await.unwrap().as_ref(), b"append");
+    }
+
+    #[tokio::test]
+    async fn test_data_replacement_uses_blob_descriptor_array_builder_prepared_nested_blob_column()
+    {
+        let test_dir = TempStrDir::default();
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(b"initial").unwrap();
+        let (logical_schema, initial_batch) =
+            nested_blob_v2_batch(initial_builder.finish().unwrap());
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(initial_batch)], logical_schema.clone()),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let file_id = Uuid::new_v4().to_string();
+        let data_file_name = format!("{file_id}.lance");
+        let data_file_path = dataset.data_dir().join(data_file_name.as_str());
+        let blob_id = 1;
+        let mut blob_writer = BlobDescriptorArrayBuilder::new("blob");
+        let mut packed = PackedBlobWriter::try_new(
+            dataset.object_store.as_ref().clone(),
+            data_file_path.clone(),
+            blob_id,
+        )
+        .await
+        .unwrap();
+        packed.write_blob(b"nested-replacement").await.unwrap();
+        blob_writer.extend(packed.finish().await.unwrap()).unwrap();
+        let prepared_column = blob_writer.finish().unwrap();
+        let (prepared_field, prepared_array) = prepared_column.into_parts();
+
+        let info_fields = vec![Field::new("name", DataType::Utf8, false), prepared_field];
+        let info_array = Arc::new(
+            StructArray::try_new(
+                info_fields.clone().into(),
+                vec![
+                    Arc::new(StringArray::from(vec!["replacement"])) as ArrayRef,
+                    prepared_array,
+                ],
+                None,
+            )
+            .unwrap(),
+        ) as ArrayRef;
+        let replacement_schema = Arc::new(Schema::new(vec![Field::new(
+            "info",
+            DataType::Struct(info_fields.into()),
+            true,
+        )]));
+        let replacement_batch =
+            RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap();
+
+        let object_writer = dataset.object_store.create(&data_file_path).await.unwrap();
+        let mut file_writer = FileWriter::try_new(
+            object_writer,
+            crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(),
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        file_writer.write_batch(&replacement_batch).await.unwrap();
+        file_writer.finish().await.unwrap();
+
+        let data_file = dataset
+            .create_data_file(&data_file_name, None)
+            .await
+            .unwrap();
+        let transaction = Transaction {
+            read_version: dataset.manifest.version,
+            uuid: Uuid::new_v4().hyphenated().to_string(),
+            operation: Operation::DataReplacement {
+                replacements: vec![DataReplacementGroup(0, data_file)],
+            },
+            tag: None,
+            transaction_properties: None,
+        };
+        let dataset = Arc::new(
+            CommitBuilder::new(dataset)
+                .execute(transaction)
+                .await
+                .unwrap(),
+        );
+
+        let blobs = dataset
+            .take_blobs_by_indices(&[0], "info.blob")
+            .await
+            .unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(
+            blobs[0].read().await.unwrap().as_ref(),
+            b"nested-replacement"
+        );
+        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+    }
+
+    #[tokio::test]
+    async fn test_create_data_file_skips_blob_orphans_with_top_level_name_collision() {
+        let test_dir = TempStrDir::default();
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(b"initial").unwrap();
+        let (_, initial_info_batch) = nested_blob_v2_batch(initial_builder.finish().unwrap());
+        let info_field = initial_info_batch.schema().field(0).as_ref().clone();
+        let logical_schema = Arc::new(Schema::new(vec![
+            info_field,
+            Field::new("kind", DataType::UInt8, false),
+        ]));
+        let initial_batch = RecordBatch::try_new(
+            logical_schema.clone(),
+            vec![
+                initial_info_batch.column(0).clone(),
+                Arc::new(UInt8Array::from(vec![7])) as ArrayRef,
+            ],
+        )
+        .unwrap();
+        let dataset = Arc::new(
+            Dataset::write(
+                RecordBatchIterator::new(vec![Ok(initial_batch)], logical_schema.clone()),
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let file_id = Uuid::new_v4().to_string();
+        let data_file_name = format!("{file_id}.lance");
+        let data_file_path = dataset.data_dir().join(data_file_name.as_str());
+        let mut blob_writer = BlobDescriptorArrayBuilder::new("blob");
+        blob_writer
+            .push_inline(Bytes::from_static(b"replacement"))
+            .unwrap();
+        let prepared_column = blob_writer.finish().unwrap();
+        let (prepared_field, prepared_array) = prepared_column.into_parts();
+
+        let info_fields = vec![Field::new("name", DataType::Utf8, false), prepared_field];
+        let info_array = Arc::new(
+            StructArray::try_new(
+                info_fields.clone().into(),
+                vec![
+                    Arc::new(StringArray::from(vec!["replacement"])) as ArrayRef,
+                    prepared_array,
+                ],
+                None,
+            )
+            .unwrap(),
+        ) as ArrayRef;
+        let replacement_schema = Arc::new(Schema::new(vec![Field::new(
+            "info",
+            DataType::Struct(info_fields.into()),
+            true,
+        )]));
+        let replacement_batch =
+            RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap();
+
+        let object_writer = dataset.object_store.create(&data_file_path).await.unwrap();
+        let mut file_writer = FileWriter::try_new(
+            object_writer,
+            crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(),
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        file_writer.write_batch(&replacement_batch).await.unwrap();
+        file_writer.finish().await.unwrap();
+
+        let data_file = dataset
+            .create_data_file(&data_file_name, None)
+            .await
+            .unwrap();
+
+        assert_eq!(data_file.fields.len(), 2);
+        assert_eq!(data_file.column_indices.as_ref(), &[0, 1]);
+    }
+
+    #[tokio::test]
+    async fn test_write_and_take_nested_blob_v2() {
+        let test_dir = TempStrDir::default();
+        let packed_payload = vec![0x4A; super::INLINE_MAX + 1024];
+
+        let mut blob_builder = BlobArrayBuilder::new(3);
+        blob_builder.push_bytes(b"hello").unwrap();
+        blob_builder.push_bytes(&packed_payload).unwrap();
+        blob_builder.push_null().unwrap();
+        let blob_array: ArrayRef = blob_builder.finish().unwrap();
+
+        let (schema, batch) = nested_blob_v2_batch(blob_array);
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let info_batch = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let blob_desc = info_batch
+            .column(0)
+            .as_struct()
+            .column_by_name("blob")
+            .unwrap()
+            .as_struct();
+        assert_eq!(
+            blob_desc
+                .column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Inline as u8
+        );
+        assert_eq!(
+            blob_desc
+                .column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(1),
+            BlobKind::Packed as u8
+        );
+
+        let blobs = dataset
+            .take_blobs_by_indices(&[0, 1], "info.blob")
+            .await
+            .unwrap();
+        assert_eq!(blobs.len(), 2);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello");
+        assert_eq!(
+            blobs[1].read().await.unwrap().as_ref(),
+            packed_payload.as_slice()
+        );
+
+        let null_blobs = dataset
+            .take_blobs_by_indices(&[2], "info.blob")
+            .await
+            .unwrap();
+        assert!(null_blobs.is_empty());
+
+        let filtered = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .filter("info.blob IS NOT NULL")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(filtered.num_rows(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_write_and_scan_list_blob_v2_descriptions() {
+        let test_dir = TempStrDir::default();
+        let packed_payload = vec![0x4B; super::INLINE_MAX + 1024];
+
+        let mut blob_builder = BlobArrayBuilder::new(4);
+        blob_builder.push_bytes(b"hello").unwrap();
+        blob_builder.push_null().unwrap();
+        blob_builder.push_bytes(&packed_payload).unwrap();
+        blob_builder.push_bytes(b"tail").unwrap();
+        let blob_values = blob_builder.finish().unwrap();
+
+        let item_field = Arc::new(blob_field("item", true));
+        let list_array: ArrayRef = Arc::new(
+            arrow_array::ListArray::try_new(
+                item_field.clone(),
+                arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![
+                    0i32, 3, 3, 3, 4,
+                ])),
+                blob_values,
+                Some(arrow_buffer::NullBuffer::from(vec![
+                    true, true, false, true,
+                ])),
+            )
+            .unwrap(),
+        );
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("blobs", DataType::List(item_field), true),
+            Field::new("id", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![list_array, Arc::new(Int32Array::from(vec![0, 1, 2, 3]))],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    enable_stable_row_ids: true,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let descriptions = dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let lists = descriptions.column(0).as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]);
+        assert!(lists.is_valid(0));
+        assert!(lists.is_valid(1));
+        assert!(lists.is_null(2));
+        assert!(lists.is_valid(3));
+
+        let DataType::List(descriptor_field) = lists.data_type() else {
+            panic!("unexpected list type: {}", lists.data_type());
+        };
+        assert!(matches!(descriptor_field.data_type(), DataType::Struct(_)));
+        assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let descriptors = lists.values().as_struct();
+        assert_eq!(descriptors.fields().len(), 5);
+        assert_eq!(descriptors.fields()[0].name(), "kind");
+        assert!(descriptors.is_valid(0));
+        assert!(descriptors.is_null(1));
+        assert!(descriptors.is_valid(2));
+        assert!(descriptors.is_valid(3));
+        let kinds = descriptors
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<UInt8Type>();
+        assert_eq!(kinds.value(0), BlobKind::Inline as u8);
+        assert_eq!(kinds.value(2), BlobKind::Packed as u8);
+        assert_eq!(kinds.value(3), BlobKind::Inline as u8);
+
+        let filtered = dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .filter("blobs IS NOT NULL")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(filtered.num_rows(), 3);
+
+        let mut scanner = dataset.scan();
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let bytes = scanner
+            .project(&["blobs"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let lists = bytes.column(0).as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]);
+        assert!(lists.is_valid(0));
+        assert!(lists.is_valid(1));
+        assert!(lists.is_null(2));
+        assert!(lists.is_valid(3));
+        let DataType::List(value_field) = lists.data_type() else {
+            panic!("unexpected list type: {}", lists.data_type());
+        };
+        assert_eq!(value_field.data_type(), &DataType::LargeBinary);
+        assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let values = lists.values().as_binary::<i64>();
+        assert_eq!(values.value(0), b"hello");
+        assert!(values.is_null(1));
+        assert_eq!(values.value(2), packed_payload.as_slice());
+        assert_eq!(values.value(3), b"tail");
+
+        for (filter, materialization_style) in [
+            (None, MaterializationStyle::Heuristic),
+            (Some("id >= 2"), MaterializationStyle::Heuristic),
+            (Some("id >= 2"), MaterializationStyle::AllEarly),
+        ] {
+            let mut scanner = dataset.scan();
+            scanner.blob_handling(BlobHandling::AllBinary);
+            scanner.materialization_style(materialization_style);
+            scanner
+                .project(&["blobs", ROW_LAST_UPDATED_AT_VERSION, ROW_CREATED_AT_VERSION])
+                .unwrap()
+                .with_row_id()
+                .with_row_address();
+            if let Some(filter) = filter {
+                scanner.filter(filter).unwrap();
+            }
+
+            let expected_schema = scanner.schema().await.unwrap();
+            let batch = scanner.try_into_batch().await.unwrap();
+            assert_eq!(batch.schema().as_ref(), expected_schema.as_ref());
+            assert_eq!(batch.num_rows(), if filter.is_some() { 2 } else { 4 });
+            for column in [
+                ROW_ID,
+                ROW_ADDR,
+                ROW_LAST_UPDATED_AT_VERSION,
+                ROW_CREATED_AT_VERSION,
+            ] {
+                assert!(
+                    batch.column_by_name(column).is_some(),
+                    "requested system column {column} was missing"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_and_scan_struct_nested_list_blob_v2() {
+        let test_dir = TempStrDir::default();
+
+        let mut blob_builder = BlobArrayBuilder::new(2);
+        blob_builder.push_bytes(b"nested").unwrap();
+        blob_builder.push_null().unwrap();
+        let blob_values = blob_builder.finish().unwrap();
+
+        let item_field = Arc::new(blob_field("item", true));
+        let list_field = Field::new("blobs", DataType::List(item_field.clone()), true);
+        let list_array: ArrayRef = Arc::new(
+            arrow_array::ListArray::try_new(
+                item_field,
+                arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![0i32, 2, 2])),
+                blob_values,
+                None,
+            )
+            .unwrap(),
+        );
+        let info_fields = vec![Field::new("name", DataType::Utf8, false), list_field];
+        let info_array: ArrayRef = Arc::new(
+            StructArray::try_new(
+                info_fields.clone().into(),
+                vec![
+                    Arc::new(StringArray::from(vec!["row-0", "row-1"])) as ArrayRef,
+                    list_array,
+                ],
+                None,
+            )
+            .unwrap(),
+        );
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "info",
+            DataType::Struct(info_fields.into()),
+            true,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &test_dir,
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let descriptions = dataset
+            .scan()
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let info = descriptions.column(0).as_struct();
+        assert_eq!(
+            info.column_by_name("name")
+                .unwrap()
+                .as_string::<i32>()
+                .value(0),
+            "row-0"
+        );
+        let lists = info.column_by_name("blobs").unwrap().as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]);
+        let DataType::List(descriptor_field) = lists.data_type() else {
+            panic!("unexpected nested list type: {}", lists.data_type());
+        };
+        assert!(matches!(descriptor_field.data_type(), DataType::Struct(_)));
+        assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let descriptors = lists.values().as_struct();
+        assert_eq!(descriptors.fields().len(), 5);
+        assert!(descriptors.is_valid(0));
+        assert!(descriptors.is_null(1));
+
+        let mut scanner = dataset.scan();
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let bytes = scanner
+            .project(&["info"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let info = bytes.column(0).as_struct();
+        let lists = info.column_by_name("blobs").unwrap().as_list::<i32>();
+        assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]);
+        let DataType::List(value_field) = lists.data_type() else {
+            panic!("unexpected nested list type: {}", lists.data_type());
+        };
+        assert_eq!(value_field.data_type(), &DataType::LargeBinary);
+        assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY));
+        let values = lists.values().as_binary::<i64>();
+        assert_eq!(values.value(0), b"nested");
+        assert!(values.is_null(1));
+    }
+
+    #[tokio::test]
+    async fn test_nested_blob_v2_requires_v2_2() {
+        let test_dir = TempStrDir::default();
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_bytes(b"hello").unwrap();
+        let blob_array: ArrayRef = blob_builder.finish().unwrap();
+
+        let (schema, batch) = nested_blob_v2_batch(blob_array);
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let result = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await;
+
+        assert!(
+            result.is_err(),
+            "Nested blob v2 should be rejected for file version 2.1"
+        );
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Blob v2 requires file version >= 2.2")
+        );
     }
 
     #[tokio::test]
@@ -3120,7 +4828,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_read_blobs_plan_preserves_order_and_coalesces() {
+    async fn test_execute_blob_entries_preserves_order_and_coalesces() {
         let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij"));
         let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin")));
         let entries = vec![
@@ -3135,15 +4843,7 @@ mod tests {
                 file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None),
             },
         ];
-        let execution = Arc::new(ReadBlobsExecution::new(None));
-        let blobs = try_join_all(
-            plan_blob_read_plans(entries)
-                .into_iter()
-                .map(|plan| execute_blob_read_plan(plan, execution.clone())),
-        )
-        .await
-        .unwrap();
-        let mut blobs = blobs.into_iter().flatten().collect::<Vec<_>>();
+        let mut blobs = execute_blob_entries(entries, 2, None).await.unwrap();
         blobs.sort_by_key(|blob| blob.selection_index);
 
         assert_eq!(blobs.len(), 2);
@@ -3621,6 +5321,50 @@ mod tests {
         assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
     }
 
+    #[tokio::test]
+    async fn test_blob_v2_external_ingest_respects_inline_threshold() {
+        let dataset_dir = TempDir::default();
+        let external_dir = TempDir::default();
+        let external_path = external_dir.std_path().join("external.bin");
+        let payload = vec![0x5A; 2048];
+        std::fs::write(&external_path, &payload).unwrap();
+        let external_uri = format!("file://{}", external_path.display());
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_uri(external_uri).unwrap();
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+
+        let mut field = blob_field("blob", true);
+        let mut metadata = field.metadata().clone();
+        metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "1024".to_string(),
+        );
+        field = field.with_metadata(metadata);
+        let schema = Arc::new(Schema::new(vec![field]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &dataset_dir.path_str(),
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    external_blob_mode: ExternalBlobMode::Ingest,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].kind(), BlobKind::Packed);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+    }
+
     #[tokio::test]
     async fn test_blob_v2_external_ingest_dedicated() {
         let dataset_dir = TempDir::default();
@@ -3637,83 +5381,481 @@ mod tests {
         let batch = RecordBatch::try_new(schema.clone(), vec![blob_array]).unwrap();
         let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
 
-        let dataset = Arc::new(
-            Dataset::write(
-                reader,
-                &dataset_dir.path_str(),
-                Some(WriteParams {
-                    data_storage_version: Some(LanceFileVersion::V2_2),
-                    external_blob_mode: ExternalBlobMode::Ingest,
-                    ..Default::default()
-                }),
-            )
-            .await
-            .unwrap(),
+        let dataset = Arc::new(
+            Dataset::write(
+                reader,
+                &dataset_dir.path_str(),
+                Some(WriteParams {
+                    data_storage_version: Some(LanceFileVersion::V2_2),
+                    external_blob_mode: ExternalBlobMode::Ingest,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let desc = dataset
+            .scan()
+            .project(&["blob"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .column(0)
+            .as_struct()
+            .to_owned();
+        assert_eq!(
+            desc.column_by_name("kind")
+                .unwrap()
+                .as_primitive::<UInt8Type>()
+                .value(0),
+            BlobKind::Dedicated as u8
+        );
+
+        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
+        assert_eq!(blobs.len(), 1);
+        assert_eq!(blobs[0].kind(), BlobKind::Dedicated);
+        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_requires_v2_2() {
+        let test_dir = TempStrDir::default();
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_bytes(b"hello").unwrap();
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+
+        let id_array: arrow_array::ArrayRef = Arc::new(UInt32Array::from(vec![0]));
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            blob_field("blob", true),
+        ]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![id_array, blob_array]).unwrap();
+        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+
+        let result = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await;
+
+        assert!(
+            result.is_err(),
+            "Blob v2 should be rejected for file version 2.1"
+        );
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Blob v2 requires file version >= 2.2")
+        );
+    }
+
+    async fn try_preprocess_blobs_with_blob_metadata(
+        metadata_entries: Vec<(&'static str, String)>,
+        pack_file_size_override: Option<usize>,
+        blob_len: usize,
+        num_blobs: usize,
+    ) -> Result<arrow_array::StructArray> {
+        let (object_store, base_path) = ObjectStore::from_uri_and_params(
+            Arc::new(ObjectStoreRegistry::default()),
+            "memory://blob_preprocessor",
+            &ObjectStoreParams::default(),
+        )
+        .await
+        .unwrap();
+        let object_store = object_store.as_ref().clone();
+        let data_dir = base_path.clone().join("data");
+
+        let mut field = blob_field("blob", true);
+        let mut metadata = field.metadata().clone();
+        for (key, value) in metadata_entries {
+            metadata.insert(key.to_string(), value);
+        }
+        field = field.with_metadata(metadata);
+
+        let writer_arrow_schema = Schema::new(vec![field.clone()]);
+        let writer_schema = lance_core::datatypes::Schema::try_from(&writer_arrow_schema).unwrap();
+
+        let mut preprocessor = super::BlobPreprocessor::new(
+            object_store.clone(),
+            data_dir,
+            "data_file_key".to_string(),
+            &writer_schema,
+            None,
+            false,
+            ExternalBlobMode::Reference,
+            Arc::new(ObjectStoreRegistry::default()),
+            ObjectStoreParams::default(),
+            pack_file_size_override,
+        )?;
+
+        let mut blob_builder = BlobArrayBuilder::new(num_blobs);
+        for _ in 0..num_blobs {
+            blob_builder.push_bytes(vec![0u8; blob_len]).unwrap();
+        }
+        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+
+        let field_without_metadata =
+            Field::new("blob", field.data_type().clone(), field.is_nullable());
+        let batch_schema = Arc::new(Schema::new(vec![field_without_metadata]));
+        let batch = RecordBatch::try_new(batch_schema, vec![blob_array]).unwrap();
+
+        let out = preprocessor.preprocess_batch(&batch).await?;
+        Ok(out
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow_array::StructArray>()
+            .unwrap()
+            .clone())
+    }
+
+    async fn try_preprocess_kind_with_blob_metadata(
+        metadata_entries: Vec<(&'static str, String)>,
+        data_len: usize,
+    ) -> Result<u8> {
+        let struct_arr =
+            try_preprocess_blobs_with_blob_metadata(metadata_entries, None, data_len, 1).await?;
+        Ok(struct_arr
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<arrow::datatypes::UInt8Type>()
+            .value(0))
+    }
+
+    async fn preprocess_kind_with_blob_metadata(
+        metadata_entries: Vec<(&'static str, String)>,
+        data_len: usize,
+    ) -> u8 {
+        try_preprocess_kind_with_blob_metadata(metadata_entries, data_len)
+            .await
+            .unwrap()
+    }
+
+    async fn packed_blobs_with_blob_metadata(
+        metadata_entries: Vec<(&'static str, String)>,
+        pack_file_size_override: Option<usize>,
+        blob_len: usize,
+        num_blobs: usize,
+    ) -> Vec<u32> {
+        let struct_arr = try_preprocess_blobs_with_blob_metadata(
+            metadata_entries,
+            pack_file_size_override,
+            blob_len,
+            num_blobs,
+        )
+        .await
+        .unwrap();
+        let blob_ids = struct_arr
+            .column_by_name("blob_id")
+            .unwrap()
+            .as_primitive::<arrow::datatypes::UInt32Type>();
+        (0..struct_arr.len()).map(|i| blob_ids.value(i)).collect()
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_dedicated_threshold_rejects_non_positive_metadata() {
+        let err = try_preprocess_kind_with_blob_metadata(
+            vec![(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "0".to_string())],
+            256 * 1024,
+        )
+        .await
+        .unwrap_err();
+        assert!(err.to_string().contains("expected a positive integer"));
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_inline_threshold_rejects_invalid_metadata() {
+        let err = try_preprocess_kind_with_blob_metadata(
+            vec![(
+                BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
+                "not-a-number".to_string(),
+            )],
+            256 * 1024,
+        )
+        .await
+        .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("expected a non-negative integer that fits in usize")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_write_rejects_invalid_inline_threshold_metadata() {
+        let dataset_dir = TempDir::default();
+        let mut field = blob_field("blob", true);
+        let mut metadata = field.metadata().clone();
+        metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "not-a-number".to_string(),
+        );
+        field = field.with_metadata(metadata);
+        let schema = Arc::new(Schema::new(vec![field]));
+
+        let mut blob_builder = BlobArrayBuilder::new(1);
+        blob_builder.push_bytes(vec![0u8; 256]).unwrap();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(blob_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let result = Dataset::write(
+            reader,
+            &dataset_dir.path_str(),
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await;
+        let Err(err) = result else {
+            panic!("write with invalid blob threshold metadata should fail");
+        };
+        assert!(
+            err.to_string()
+                .contains("expected a non-negative integer that fits in usize")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_dedicated_threshold_respects_smaller_metadata() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![(BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "131072".to_string())],
+            256 * 1024,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Dedicated as u8);
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_dedicated_threshold_respects_larger_metadata() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![(
+                BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+                "8388608".to_string(),
+            )],
+            super::DEDICATED_THRESHOLD + 1024,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Packed as u8);
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_pack_file_threshold_rolls_at_metadata_value() {
+        // Blobs must exceed the inline cutoff to be packed at all. With a pack-file
+        // threshold equal to a single blob's size, each of the three blobs rolls to its
+        // own pack file (a distinct blob_id).
+        let blob_len = super::INLINE_MAX + 1024;
+        let blob_ids = packed_blobs_with_blob_metadata(
+            vec![(BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, blob_len.to_string())],
+            None,
+            blob_len,
+            3,
+        )
+        .await;
+        let distinct: std::collections::HashSet<u32> = blob_ids.iter().copied().collect();
+        assert_eq!(
+            distinct.len(),
+            3,
+            "expected one pack file per blob: {blob_ids:?}"
         );
+    }
 
-        let desc = dataset
-            .scan()
-            .project(&["blob"])
-            .unwrap()
-            .try_into_batch()
-            .await
-            .unwrap()
-            .column(0)
-            .as_struct()
-            .to_owned();
+    #[tokio::test]
+    async fn test_blob_v2_pack_file_threshold_packs_within_metadata_value() {
+        // A pack-file threshold large enough for all three blobs keeps them in a single
+        // pack file (one shared blob_id).
+        let blob_len = super::INLINE_MAX + 1024;
+        let blob_ids = packed_blobs_with_blob_metadata(
+            vec![(
+                BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+                (blob_len * 3).to_string(),
+            )],
+            None,
+            blob_len,
+            3,
+        )
+        .await;
+        let distinct: std::collections::HashSet<u32> = blob_ids.iter().copied().collect();
         assert_eq!(
-            desc.column_by_name("kind")
-                .unwrap()
-                .as_primitive::<UInt8Type>()
-                .value(0),
-            BlobKind::Dedicated as u8
+            distinct.len(),
+            1,
+            "expected a single shared pack file: {blob_ids:?}"
         );
+    }
 
-        let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap();
-        assert_eq!(blobs.len(), 1);
-        assert_eq!(blobs[0].kind(), BlobKind::Dedicated);
-        assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice());
+    #[tokio::test]
+    async fn test_blob_v2_pack_file_threshold_rejects_non_positive_metadata() {
+        let err = try_preprocess_kind_with_blob_metadata(
+            vec![(BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, "0".to_string())],
+            1024,
+        )
+        .await
+        .expect_err("zero pack-file threshold should be rejected");
+        assert!(err.to_string().contains("expected a positive integer"));
     }
 
     #[tokio::test]
-    async fn test_blob_v2_requires_v2_2() {
-        let test_dir = TempStrDir::default();
+    async fn test_blob_v2_inline_threshold_respects_smaller_metadata() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![(BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "1024".to_string())],
+            2048,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Packed as u8);
+    }
 
-        let mut blob_builder = BlobArrayBuilder::new(1);
-        blob_builder.push_bytes(b"hello").unwrap();
-        let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
+    #[tokio::test]
+    async fn test_blob_v2_inline_threshold_respects_larger_metadata() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![(
+                BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
+                (super::INLINE_MAX + 8192).to_string(),
+            )],
+            super::INLINE_MAX + 4096,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Inline as u8);
+    }
 
-        let id_array: arrow_array::ArrayRef = Arc::new(UInt32Array::from(vec![0]));
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::UInt32, false),
-            blob_field("blob", true),
-        ]));
-        let batch = RecordBatch::try_new(schema.clone(), vec![id_array, blob_array]).unwrap();
-        let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    #[tokio::test]
+    async fn test_blob_v2_inline_threshold_uses_strict_greater_than() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![(BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "1024".to_string())],
+            1024,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Inline as u8);
+    }
 
-        let result = Dataset::write(
-            reader,
-            &test_dir,
-            Some(WriteParams {
-                data_storage_version: Some(LanceFileVersion::V2_1),
-                ..Default::default()
-            }),
+    #[tokio::test]
+    async fn test_blob_v2_dedicated_threshold_uses_strict_greater_than() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![
+                (BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "2048".to_string()),
+                (BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "1024".to_string()),
+            ],
+            1024,
         )
         .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Inline as u8);
+    }
 
-        assert!(
-            result.is_err(),
-            "Blob v2 should be rejected for file version 2.1"
+    #[tokio::test]
+    async fn test_blob_v2_inline_threshold_does_not_override_dedicated_threshold() {
+        let kind = preprocess_kind_with_blob_metadata(
+            vec![
+                (BLOB_INLINE_SIZE_THRESHOLD_META_KEY, "8192".to_string()),
+                (BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, "4096".to_string()),
+            ],
+            6144,
+        )
+        .await;
+        assert_eq!(kind, lance_core::datatypes::BlobKind::Dedicated as u8);
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_inline_threshold_is_per_column() {
+        let (object_store, base_path) = ObjectStore::from_uri_and_params(
+            Arc::new(ObjectStoreRegistry::default()),
+            "memory://blob_preprocessor",
+            &ObjectStoreParams::default(),
+        )
+        .await
+        .unwrap();
+        let object_store = object_store.as_ref().clone();
+        let data_dir = base_path.clone().join("data");
+
+        let mut inline_field = blob_field("inline_blob", true);
+        let mut inline_metadata = inline_field.metadata().clone();
+        inline_metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "4096".to_string(),
         );
-        assert!(
-            result
-                .unwrap_err()
-                .to_string()
-                .contains("Blob v2 requires file version >= 2.2")
+        inline_field = inline_field.with_metadata(inline_metadata);
+
+        let mut packed_field = blob_field("packed_blob", true);
+        let mut packed_metadata = packed_field.metadata().clone();
+        packed_metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "1024".to_string(),
         );
+        packed_field = packed_field.with_metadata(packed_metadata);
+
+        let writer_arrow_schema = Schema::new(vec![inline_field.clone(), packed_field.clone()]);
+        let writer_schema = lance_core::datatypes::Schema::try_from(&writer_arrow_schema).unwrap();
+
+        let mut preprocessor = super::BlobPreprocessor::new(
+            object_store.clone(),
+            data_dir,
+            "data_file_key".to_string(),
+            &writer_schema,
+            None,
+            false,
+            ExternalBlobMode::Reference,
+            Arc::new(ObjectStoreRegistry::default()),
+            ObjectStoreParams::default(),
+            None,
+        )
+        .unwrap();
+
+        let mut inline_builder = BlobArrayBuilder::new(1);
+        inline_builder.push_bytes(vec![0u8; 2048]).unwrap();
+        let inline_array: arrow_array::ArrayRef = inline_builder.finish().unwrap();
+
+        let mut packed_builder = BlobArrayBuilder::new(1);
+        packed_builder.push_bytes(vec![0u8; 2048]).unwrap();
+        let packed_array: arrow_array::ArrayRef = packed_builder.finish().unwrap();
+
+        let batch_schema = Arc::new(Schema::new(vec![
+            Field::new(
+                "inline_blob",
+                inline_field.data_type().clone(),
+                inline_field.is_nullable(),
+            ),
+            Field::new(
+                "packed_blob",
+                packed_field.data_type().clone(),
+                packed_field.is_nullable(),
+            ),
+        ]));
+        let batch = RecordBatch::try_new(batch_schema, vec![inline_array, packed_array]).unwrap();
+
+        let out = preprocessor.preprocess_batch(&batch).await.unwrap();
+        let inline_kind = out
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow_array::StructArray>()
+            .unwrap()
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<arrow::datatypes::UInt8Type>()
+            .value(0);
+        let packed_kind = out
+            .column(1)
+            .as_any()
+            .downcast_ref::<arrow_array::StructArray>()
+            .unwrap()
+            .column_by_name("kind")
+            .unwrap()
+            .as_primitive::<arrow::datatypes::UInt8Type>()
+            .value(0);
+
+        assert_eq!(inline_kind, lance_core::datatypes::BlobKind::Inline as u8);
+        assert_eq!(packed_kind, lance_core::datatypes::BlobKind::Packed as u8);
     }
 
-    async fn preprocess_kind_with_schema_metadata(metadata_value: &str, data_len: usize) -> u8 {
+    #[tokio::test]
+    async fn test_blob_v2_pack_file_threshold_starts_new_packed_blob() {
         let (object_store, base_path) = ObjectStore::from_uri_and_params(
             Arc::new(ObjectStoreRegistry::default()),
             "memory://blob_preprocessor",
@@ -3726,9 +5868,13 @@ mod tests {
 
         let mut field = blob_field("blob", true);
         let mut metadata = field.metadata().clone();
+        metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "0".to_string(),
+        );
         metadata.insert(
             BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY.to_string(),
-            metadata_value.to_string(),
+            "1024".to_string(),
         );
         field = field.with_metadata(metadata);
 
@@ -3745,48 +5891,249 @@ mod tests {
             ExternalBlobMode::Reference,
             Arc::new(ObjectStoreRegistry::default()),
             ObjectStoreParams::default(),
-            None,
-        );
+            Some(3),
+        )
+        .unwrap();
 
-        let mut blob_builder = BlobArrayBuilder::new(1);
-        blob_builder.push_bytes(vec![0u8; data_len]).unwrap();
+        let mut blob_builder = BlobArrayBuilder::new(3);
+        blob_builder.push_bytes(vec![0x11; 2]).unwrap();
+        blob_builder.push_bytes(vec![0x22; 2]).unwrap();
+        blob_builder.push_bytes(vec![0x33; 1]).unwrap();
         let blob_array: arrow_array::ArrayRef = blob_builder.finish().unwrap();
 
-        let field_without_metadata =
-            Field::new("blob", field.data_type().clone(), field.is_nullable());
-        let batch_schema = Arc::new(Schema::new(vec![field_without_metadata]));
+        let batch_schema = Arc::new(Schema::new(vec![Field::new(
+            "blob",
+            field.data_type().clone(),
+            field.is_nullable(),
+        )]));
         let batch = RecordBatch::try_new(batch_schema, vec![blob_array]).unwrap();
 
         let out = preprocessor.preprocess_batch(&batch).await.unwrap();
+        preprocessor.finish().await.unwrap();
+
         let struct_arr = out
             .column(0)
             .as_any()
             .downcast_ref::<arrow_array::StructArray>()
             .unwrap();
-        struct_arr
+        let kinds = struct_arr
             .column_by_name("kind")
             .unwrap()
-            .as_primitive::<arrow::datatypes::UInt8Type>()
-            .value(0)
+            .as_primitive::<UInt8Type>();
+        let blob_ids = struct_arr
+            .column_by_name("blob_id")
+            .unwrap()
+            .as_primitive::<UInt32Type>();
+        let sizes = struct_arr
+            .column_by_name("blob_size")
+            .unwrap()
+            .as_primitive::<UInt64Type>();
+        let positions = struct_arr
+            .column_by_name("position")
+            .unwrap()
+            .as_primitive::<UInt64Type>();
+
+        assert_eq!(kinds.values(), &[BlobKind::Packed as u8; 3]);
+        assert_eq!(blob_ids.values(), &[1, 2, 2]);
+        assert_eq!(sizes.values(), &[2, 2, 1]);
+        assert_eq!(positions.values(), &[0, 0, 2]);
     }
 
     #[tokio::test]
-    async fn test_blob_v2_dedicated_threshold_ignores_non_positive_metadata() {
-        let kind = preprocess_kind_with_schema_metadata("0", 256 * 1024).await;
-        assert_eq!(kind, lance_core::datatypes::BlobKind::Packed as u8);
+    async fn test_blob_v2_append_rejects_explicit_inline_threshold_mismatch() {
+        let dataset_dir = TempDir::default();
+        let payload = vec![0u8; 2048];
+
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(payload.clone()).unwrap();
+        let initial_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(initial_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let initial_reader = RecordBatchIterator::new(vec![Ok(initial_batch)], schema);
+        let dataset = Dataset::write(
+            initial_reader,
+            &dataset_dir.path_str(),
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let mut append_field = blob_field("blob", true);
+        let mut append_metadata = append_field.metadata().clone();
+        append_metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "1024".to_string(),
+        );
+        append_field = append_field.with_metadata(append_metadata);
+        let append_schema = Arc::new(Schema::new(vec![append_field]));
+        let mut append_builder = BlobArrayBuilder::new(1);
+        append_builder.push_bytes(payload).unwrap();
+        let append_batch = RecordBatch::try_new(
+            append_schema.clone(),
+            vec![Arc::new(append_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let append_reader = RecordBatchIterator::new(vec![Ok(append_batch)], append_schema);
+
+        let result = Dataset::write(
+            append_reader,
+            Arc::new(dataset),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await;
+        let Err(err) = result else {
+            panic!("append with explicit blob threshold mismatch should fail");
+        };
+        let message = err.to_string();
+        assert!(message.contains("Cannot append data with blob threshold metadata"));
+        assert!(message.contains(BLOB_INLINE_SIZE_THRESHOLD_META_KEY));
     }
 
     #[tokio::test]
-    async fn test_blob_v2_dedicated_threshold_respects_smaller_metadata() {
-        let kind = preprocess_kind_with_schema_metadata("131072", 256 * 1024).await;
-        assert_eq!(kind, lance_core::datatypes::BlobKind::Dedicated as u8);
+    async fn test_blob_v2_append_rejects_threshold_mismatch_with_non_blob_input_extension() {
+        let dataset_dir = TempDir::default();
+        let payload = vec![0u8; 2048];
+
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(payload.clone()).unwrap();
+        let initial_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(initial_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let initial_reader = RecordBatchIterator::new(vec![Ok(initial_batch)], schema);
+        let dataset = Dataset::write(
+            initial_reader,
+            &dataset_dir.path_str(),
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let mut append_field = blob_field("blob", true);
+        let mut append_metadata = append_field.metadata().clone();
+        append_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            "some.other.extension".to_string(),
+        );
+        append_metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            "1024".to_string(),
+        );
+        append_field = append_field.with_metadata(append_metadata);
+        let append_schema = Arc::new(Schema::new(vec![append_field]));
+        let mut append_builder = BlobArrayBuilder::new(1);
+        append_builder.push_bytes(payload).unwrap();
+        let append_batch = RecordBatch::try_new(
+            append_schema.clone(),
+            vec![Arc::new(append_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let append_reader = RecordBatchIterator::new(vec![Ok(append_batch)], append_schema);
+
+        let result = Dataset::write(
+            append_reader,
+            Arc::new(dataset),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await;
+        let Err(err) = result else {
+            panic!("append with ignored blob threshold metadata should fail");
+        };
+        let message = err.to_string();
+        assert!(message.contains("Cannot append data with blob threshold metadata"));
+        assert!(message.contains(BLOB_INLINE_SIZE_THRESHOLD_META_KEY));
     }
 
     #[tokio::test]
-    async fn test_blob_v2_dedicated_threshold_respects_larger_metadata() {
-        let kind =
-            preprocess_kind_with_schema_metadata("8388608", super::DEDICATED_THRESHOLD + 1024)
-                .await;
-        assert_eq!(kind, lance_core::datatypes::BlobKind::Packed as u8);
+    async fn test_blob_v2_append_accepts_explicit_default_inline_threshold() {
+        let dataset_dir = TempDir::default();
+        let payload = vec![0u8; 2048];
+
+        let schema = Arc::new(Schema::new(vec![blob_field("blob", true)]));
+        let mut initial_builder = BlobArrayBuilder::new(1);
+        initial_builder.push_bytes(payload.clone()).unwrap();
+        let initial_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(initial_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let initial_reader = RecordBatchIterator::new(vec![Ok(initial_batch)], schema);
+        let dataset = Dataset::write(
+            initial_reader,
+            &dataset_dir.path_str(),
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let mut append_field = blob_field("blob", true);
+        let mut append_metadata = append_field.metadata().clone();
+        append_metadata.insert(
+            BLOB_INLINE_SIZE_THRESHOLD_META_KEY.to_string(),
+            super::INLINE_MAX.to_string(),
+        );
+        append_field = append_field.with_metadata(append_metadata);
+        let append_schema = Arc::new(Schema::new(vec![append_field]));
+        let mut append_builder = BlobArrayBuilder::new(1);
+        append_builder.push_bytes(payload).unwrap();
+        let append_batch = RecordBatch::try_new(
+            append_schema.clone(),
+            vec![Arc::new(append_builder.finish().unwrap()) as ArrayRef],
+        )
+        .unwrap();
+        let append_reader = RecordBatchIterator::new(vec![Ok(append_batch)], append_schema);
+
+        let dataset = Dataset::write(
+            append_reader,
+            Arc::new(dataset),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 2);
+    }
+
+    #[tokio::test]
+    async fn test_blob_v2_pack_file_threshold_write_param_overrides_metadata() {
+        let blob_len = super::INLINE_MAX + 1024;
+        let blob_ids = packed_blobs_with_blob_metadata(
+            vec![(
+                BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+                (blob_len * 3).to_string(),
+            )],
+            Some(blob_len),
+            blob_len,
+            3,
+        )
+        .await;
+        let distinct: std::collections::HashSet<u32> = blob_ids.iter().copied().collect();
+        assert_eq!(
+            distinct.len(),
+            3,
+            "write-param override should force one pack file per blob: {blob_ids:?}"
+        );
     }
 }
diff --git a/rust/lance/src/dataset/branch_location.rs b/rust/lance/src/dataset/branch_location.rs
index 2dd9f3aa860..7ebce36ec86 100644
--- a/rust/lance/src/dataset/branch_location.rs
+++ b/rust/lance/src/dataset/branch_location.rs
@@ -31,14 +31,20 @@ impl BranchLocation {
     }
 
     fn get_root_path(path_str: &str, branch_name: &str) -> Result<String> {
+        // A uri may carry a query string (e.g. `s3+ddb://...?ddbTableName=t`);
+        // the branch suffix sits on the path part, before the query.
+        let (path_part, query) = match path_str.split_once('?') {
+            Some((path, query)) => (path, Some(query)),
+            None => (path_str, None),
+        };
         let branch_suffix = format!("{}/{}", BRANCH_DIR, branch_name);
         let branch_suffix = branch_suffix.as_str();
-        let root_path_str = path_str
+        let root_path_str = path_part
             .strip_suffix(branch_suffix)
             .or_else(|| {
                 if cfg!(windows) {
                     let windows_suffix = branch_suffix.replace('/', "\\");
-                    path_str.strip_suffix(&windows_suffix)
+                    path_part.strip_suffix(&windows_suffix)
                 } else {
                     None
                 }
@@ -59,7 +65,42 @@ impl BranchLocation {
                 root_path_str, path_str,
             )));
         };
-        Ok(root_path_str)
+        Ok(match query {
+            Some(query) => format!("{}?{}", root_path_str, query),
+            None => root_path_str,
+        })
+    }
+
+    /// The branch a location under `root` targets: the inverse of
+    /// [`Self::find_branch`]. `location` must be either `root` itself (main)
+    /// or `<root>/tree/<branch>`; anything else is rejected so a caller never
+    /// misattributes an unrelated location to a branch.
+    pub fn branch_of(root: &str, location: &str) -> Result<Option<String>> {
+        if location == root {
+            return Ok(None);
+        }
+        // Require the `/` component boundary after the root so a sibling path
+        // that merely shares the root as a string prefix is rejected.
+        let branch = location
+            .strip_prefix(root)
+            .and_then(|rel| {
+                if root.is_empty() {
+                    Some(rel)
+                } else {
+                    rel.strip_prefix('/')
+                }
+            })
+            .and_then(|rel| rel.strip_prefix(BRANCH_DIR))
+            .and_then(|rel| rel.strip_prefix('/'))
+            .filter(|name| !name.is_empty());
+
+        match branch {
+            Some(name) => Ok(Some(name.to_string())),
+            None => Err(Error::invalid_input(format!(
+                "cannot derive a branch for location '{}': expected the table root '{}' or a branch chain under '{}/{}'",
+                location, root, root, BRANCH_DIR
+            ))),
+        }
     }
 
     /// Find the target branch location
@@ -100,13 +141,23 @@ impl BranchLocation {
     }
 
     fn join_str(base: &str, segment: &str) -> Result<String> {
+        // A uri may carry a query string (e.g. `s3+ddb://...?ddbTableName=t`);
+        // path segments must be appended before it.
+        let (path_part, query) = match base.split_once('?') {
+            Some((path, query)) => (path, Some(query)),
+            None => (base, None),
+        };
         let normalized_segment = segment.trim_start_matches('/');
-        let is_base_dir = base.ends_with("/");
-        if is_base_dir {
-            Ok(format!("{}{}", base, normalized_segment))
+        let is_base_dir = path_part.ends_with("/");
+        let joined = if is_base_dir {
+            format!("{}{}", path_part, normalized_segment)
         } else {
-            Ok(format!("{}/{}", base, normalized_segment))
-        }
+            format!("{}/{}", path_part, normalized_segment)
+        };
+        Ok(match query {
+            Some(query) => format!("{}?{}", joined, query),
+            None => joined,
+        })
     }
 }
 
@@ -223,6 +274,59 @@ mod tests {
         assert!(fs::create_dir_all(std::path::Path::new(new_location.uri.as_str())).is_ok());
     }
 
+    #[test]
+    fn test_branch_location_with_query_uri() {
+        // Uris like `s3+ddb://...?ddbTableName=t` carry the commit handler
+        // config in the query string; branch path segments must be inserted
+        // before it and the query must survive the round trip.
+        let location = BranchLocation {
+            path: Path::parse("bucket/table.lance").unwrap(),
+            uri: "s3+ddb://bucket/table.lance?ddbTableName=t".to_string(),
+            branch: None,
+        };
+        let dev = location.find_branch(Some("dev")).unwrap();
+        assert_eq!(
+            dev.uri,
+            "s3+ddb://bucket/table.lance/tree/dev?ddbTableName=t"
+        );
+        assert_eq!(dev.path.as_ref(), "bucket/table.lance/tree/dev");
+        assert_eq!(dev.branch.as_deref(), Some("dev"));
+
+        let main = dev.find_main().unwrap();
+        assert_eq!(main.uri, "s3+ddb://bucket/table.lance?ddbTableName=t");
+        assert_eq!(main.path.as_ref(), "bucket/table.lance");
+        assert_eq!(main.branch, None);
+    }
+
+    #[test]
+    fn test_branch_of() {
+        let derive = |root: &str, location: &str| BranchLocation::branch_of(root, location);
+
+        // The table root targets main.
+        assert_eq!(derive("data/t.lance", "data/t.lance").unwrap(), None);
+
+        // Branch chains, including multi-segment branch names.
+        assert_eq!(
+            derive("data/t.lance", "data/t.lance/tree/exp").unwrap(),
+            Some("exp".to_string())
+        );
+        assert_eq!(
+            derive("data/t.lance", "data/t.lance/tree/bugfix/issue-123").unwrap(),
+            Some("bugfix/issue-123".to_string())
+        );
+
+        // A sibling path sharing the root as a string prefix is not a branch.
+        assert!(derive("data/t", "data/tx/tree/exp").is_err());
+        // Neither is a sub-path outside the branch directory.
+        assert!(derive("data/t.lance", "data/t.lance/other/exp").is_err());
+        // Nor a path missing the component boundary after the branch dir.
+        assert!(derive("data/t.lance", "data/t.lance/treex").is_err());
+        // An empty branch name is invalid.
+        assert!(derive("data/t.lance", "data/t.lance/tree").is_err());
+        // An unrelated location is invalid.
+        assert!(derive("data/t.lance", "elsewhere/u.lance").is_err());
+    }
+
     #[test]
     fn test_find_empty_branch() {
         let root_path = TempStdDir::default().to_owned();
diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs
index 393ff45c4ea..aecbf92f0d8 100644
--- a/rust/lance/src/dataset/builder.rs
+++ b/rust/lance/src/dataset/builder.rs
@@ -4,7 +4,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use lance_core::cache::CacheBackend;
 
-use super::refs::{Ref, Refs};
+use super::refs::{Branches, Ref, Refs, check_valid_branch, normalize_branch, standardize_branch};
 use super::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE, ReadParams, WriteParams};
 use crate::dataset::branch_location::BranchLocation;
 use crate::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
@@ -53,6 +53,12 @@ pub struct DatasetBuilder {
     storage_options_override: Option<HashMap<String, String>>,
     /// Runtime-only exact object store bindings keyed by base path URI.
     base_store_params: HashMap<String, ObjectStoreParams>,
+    /// Namespace-managed table info `(client, table_id)`, set by `from_namespace`
+    /// when the table uses managed versioning. The commit handler is built in
+    /// `build_object_store`, rooted at the resolved table path; the branch a
+    /// namespace request targets is derived per call from the base path the
+    /// handler is handed.
+    namespace_managed: Option<(Arc<dyn LanceNamespace>, Vec<String>)>,
 }
 
 impl std::fmt::Debug for DatasetBuilder {
@@ -71,6 +77,7 @@ impl std::fmt::Debug for DatasetBuilder {
                 &self.storage_options_override.is_some(),
             )
             .field("base_store_params", &!self.base_store_params.is_empty())
+            .field("namespace_managed", &self.namespace_managed.is_some())
             .finish()
     }
 }
@@ -90,6 +97,7 @@ impl DatasetBuilder {
             file_reader_options: None,
             storage_options_override: None,
             base_store_params: HashMap::new(),
+            namespace_managed: None,
         }
     }
 
@@ -149,16 +157,11 @@ impl DatasetBuilder {
 
         let mut builder = Self::from_uri(&table_uri);
 
-        // Check managed_versioning flag to determine if namespace-managed commits should be used
+        // Defer building the commit handler to load(): the manifest store is
+        // rooted at the resolved table path, which is only known once the
+        // object store is built.
         if response.managed_versioning == Some(true) {
-            let external_store = LanceNamespaceExternalManifestStore::new(
-                namespace_client.clone(),
-                table_id.clone(),
-            );
-            let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
-                external_manifest_store: Arc::new(external_store),
-            });
-            builder.commit_handler = Some(commit_handler);
+            builder.namespace_managed = Some((namespace_client.clone(), table_id.clone()));
         }
 
         // Use namespace storage options if available
@@ -302,6 +305,14 @@ impl DatasetBuilder {
     /// - [Azure options](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html#variants)
     /// - [S3 options](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html#variants)
     /// - [Google options](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html#variants)
+    ///
+    /// For datasets with additional registered base paths, a key of the form
+    /// `base_<id>.<key>` applies `<key>` only to the base path with that
+    /// manifest id, overriding the unscoped options that every base inherits.
+    /// For example `base_1.account_key = abc` makes base 1 use
+    /// `account_key = abc` while all other options are shared. Exact per-base
+    /// bindings set via [`Self::with_base_store_params`] take precedence over
+    /// base-scoped keys.
     pub fn with_storage_options(mut self, storage_options: HashMap<String, String>) -> Self {
         // Merge with existing options if accessor exists, otherwise create new static accessor
         if let Some(existing) = self.options.storage_options_accessor.take() {
@@ -456,8 +467,9 @@ impl DatasetBuilder {
     ///
     /// These params are not persisted in the manifest. They are used as-is
     /// whenever the dataset resolves an object store for the given
-    /// `BasePath.path`. Dataset-level store params remain the fallback for bases
-    /// without an explicit binding.
+    /// `BasePath.path`, taking precedence over `base_<id>.<key>` storage
+    /// options (see [`Self::with_storage_options`]). Dataset-level store params
+    /// remain the fallback for bases without an explicit binding.
     pub fn with_base_store_params(
         mut self,
         base_path: impl AsRef<str>,
@@ -524,13 +536,8 @@ impl DatasetBuilder {
 
     /// Build a lance object store for the given config
     pub async fn build_object_store(
-        self,
+        mut self,
     ) -> Result<(Arc<ObjectStore>, Path, Arc<dyn CommitHandler>)> {
-        let commit_handler = match self.commit_handler {
-            Some(commit_handler) => Ok(commit_handler),
-            None => commit_handler_from_url(&self.table_uri, &Some(self.options.clone())).await,
-        }?;
-
         let storage_options = self
             .options
             .storage_options()
@@ -546,13 +553,13 @@ impl DatasetBuilder {
             .unwrap_or_default();
 
         #[allow(deprecated)]
-        match &self.options.object_store {
-            Some(store) => Ok((
+        let (object_store, base_path) = match &self.options.object_store {
+            Some(store) => (
                 Arc::new(ObjectStore::new(
                     store.0.clone(),
                     store.1.clone(),
                     self.options.block_size,
-                    self.options.object_store_wrapper,
+                    self.options.object_store_wrapper.clone(),
                     self.options.use_constant_size_upload_parts,
                     store.1.scheme() != "file",
                     // If user supplied an object store then we just assume it's probably
@@ -562,18 +569,35 @@ impl DatasetBuilder {
                     None, // No storage_options available here
                 )),
                 Path::from(store.1.path()),
-                commit_handler,
-            )),
+            ),
             None => {
-                let (store, path) = ObjectStore::from_uri_and_params(
-                    store_registry,
-                    &self.table_uri,
-                    &self.options,
-                )
-                .await?;
-                Ok((store, path, commit_handler))
+                ObjectStore::from_uri_and_params(store_registry, &self.table_uri, &self.options)
+                    .await?
             }
-        }
+        };
+
+        // Resolve the commit handler: an explicitly set one wins; otherwise a
+        // namespace-managed table builds a manifest store rooted at the resolved
+        // table path (the branch a request targets is derived per call from the
+        // base path the handler is handed); otherwise fall back to the default
+        // for the uri. Resolving here (not in load) keeps this pub method
+        // consistent for every caller.
+        let commit_handler: Arc<dyn CommitHandler> =
+            if let Some(commit_handler) = self.commit_handler.take() {
+                commit_handler
+            } else if let Some((namespace_client, table_id)) = self.namespace_managed.take() {
+                Arc::new(ExternalManifestCommitHandler {
+                    external_manifest_store: Arc::new(LanceNamespaceExternalManifestStore::new(
+                        namespace_client,
+                        table_id,
+                        base_path.clone(),
+                    )),
+                })
+            } else {
+                commit_handler_from_url(&self.table_uri, &Some(self.options.clone())).await?
+            };
+
+        Ok((object_store, base_path, commit_handler))
     }
 
     #[instrument(skip_all)]
@@ -656,6 +680,14 @@ impl DatasetBuilder {
         let store_params = self.options.clone();
         let base_store_params = (!self.base_store_params.is_empty())
             .then(|| Arc::new(std::mem::take(&mut self.base_store_params)));
+
+        // A namespace-managed table is always addressed at its root uri, so the
+        // effective branch is resolvable before loading: the base path is
+        // qualified up front and the manifest store derives the branch from it.
+        // An explicitly supplied commit handler opts out of the managed flow.
+        let managed_store_active =
+            self.namespace_managed.is_some() && self.commit_handler.is_none();
+
         let (object_store, base_path, commit_handler) = self.build_object_store().await?;
 
         // Two cases that need to check out after loading the manifest:
@@ -667,7 +699,7 @@ impl DatasetBuilder {
         let mut need_delay_checkout = false;
         let (mut branch, mut version_number) = match target_ref.clone() {
             Some(Ref::Version(branch, version_number)) => {
-                if branch.is_some() {
+                if branch.is_some() && !managed_store_active {
                     need_delay_checkout = true;
                 }
                 (branch, version_number)
@@ -687,17 +719,57 @@ impl DatasetBuilder {
                         branch: None,
                     },
                 );
-                let tag_content = refs.tags().get(&tag_name).await;
-                if let Ok(tag_content) = tag_content {
-                    (tag_content.branch.clone(), Some(tag_content.version))
-                } else {
-                    need_delay_checkout = true;
-                    (None, None)
+                match refs.tags().get(&tag_name).await {
+                    Ok(tag_content) => {
+                        if tag_content.branch.is_some() && !managed_store_active {
+                            // The tag's chain lives under a different base path
+                            // and the unmanaged handler resolves versions by
+                            // base path only, so load the root's latest first
+                            // and check the tag's branch/version out from it.
+                            need_delay_checkout = true;
+                            (tag_content.branch, None)
+                        } else {
+                            (tag_content.branch.clone(), Some(tag_content.version))
+                        }
+                    }
+                    Err(e) => {
+                        // A managed table is always rooted at the namespace
+                        // location, so a tag missing here is missing.
+                        if managed_store_active {
+                            return Err(e);
+                        }
+                        need_delay_checkout = true;
+                        (None, None)
+                    }
                 }
             }
             None => (None, None),
         };
 
+        // Reject malformed branch names at the boundary (mirroring the branch
+        // CRUD paths) so they fail as InvalidRef instead of resolving oddly
+        if let Some(branch_name) = branch.as_deref()
+            && !Branches::is_main_branch(Some(branch_name))
+        {
+            check_valid_branch(branch_name)?;
+        }
+
+        // For a managed table the branch is known before loading; point the base
+        // path and uri at the branch chain so the loaded dataset is rooted there
+        // (data placement, refs and the path-derived store branch all follow the
+        // base path).
+        let (base_path, table_uri) = if managed_store_active && branch.is_some() {
+            let branch_location = BranchLocation {
+                path: base_path,
+                uri: table_uri,
+                branch: None,
+            }
+            .find_branch(branch.as_deref())?;
+            (branch_location.path, branch_location.uri)
+        } else {
+            (base_path, table_uri)
+        };
+
         let dataset = Self::load_by_uri(
             session,
             manifest,
@@ -712,6 +784,20 @@ impl DatasetBuilder {
         )
         .await?;
 
+        if managed_store_active {
+            // The base path was qualified above, so the loaded manifest must
+            // already be on the requested branch; a mismatch means the namespace
+            // resolved another chain.
+            let requested_branch = branch.as_deref().and_then(standardize_branch);
+            if dataset.manifest.branch.as_deref() != requested_branch.as_deref() {
+                return Err(Error::internal(format!(
+                    "open of branch '{}' resolved a manifest belonging to branch '{}'",
+                    normalize_branch(branch.as_deref()),
+                    normalize_branch(dataset.manifest.branch.as_deref()),
+                )));
+            }
+        }
+
         if need_delay_checkout {
             if let Some(Ref::Tag(tag_name)) = target_ref {
                 let tag_content = dataset.tags().get(tag_name.as_str()).await?;
@@ -756,7 +842,7 @@ impl DatasetBuilder {
                 let reader = object_store.open(&location.path).await?;
                 populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?;
             }
-            (manifest, location)
+            (Arc::new(manifest), location)
         } else {
             let manifest_location = match version_number {
                 Some(version) => {
@@ -789,7 +875,8 @@ impl DatasetBuilder {
                         _ => e,
                     })?,
             };
-            let manifest = Dataset::load_manifest(
+
+            let manifest = Dataset::get_manifest(
                 &object_store,
                 &manifest_location,
                 &table_uri,
@@ -803,7 +890,7 @@ impl DatasetBuilder {
             object_store,
             base_path,
             table_uri,
-            Arc::new(manifest),
+            manifest,
             location,
             session,
             commit_handler,
diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs
index b3ca60cfa0f..d8f69e7d36a 100644
--- a/rust/lance/src/dataset/cleanup.rs
+++ b/rust/lance/src/dataset/cleanup.rs
@@ -46,7 +46,8 @@ use lance_core::{
     Error, Result,
     utils::tracing::{
         AUDIT_MODE_DELETE, AUDIT_MODE_DELETE_UNVERIFIED, AUDIT_TYPE_DATA, AUDIT_TYPE_DELETION,
-        AUDIT_TYPE_INDEX, AUDIT_TYPE_MANIFEST, TRACE_FILE_AUDIT,
+        AUDIT_TYPE_INDEX, AUDIT_TYPE_MANIFEST, DATASET_CLEANING_EVENT, TRACE_DATASET_EVENTS,
+        TRACE_FILE_AUDIT,
     },
 };
 use lance_table::{
@@ -57,8 +58,8 @@ use lance_table::{
         manifest::{read_manifest, read_manifest_indexes},
     },
 };
+use object_store::ObjectMeta;
 use object_store::path::Path;
-use object_store::{Error as ObjectStoreError, ObjectMeta};
 use std::fmt::Debug;
 use std::{
     collections::{HashMap, HashSet},
@@ -78,7 +79,7 @@ struct ReferencedFiles {
     index_uuids: HashSet<String>,
 }
 
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct RemovalStats {
     pub bytes_removed: u64,
     pub old_versions: u64,
@@ -88,12 +89,194 @@ pub struct RemovalStats {
     pub deletion_files_removed: u64,
 }
 
-#[derive(Clone, Copy, Debug)]
-enum RemovedFileType {
+/// A read-only explanation of what a cleanup operation would remove.
+///
+/// This is an explanation, not a deletion plan.  Calling
+/// [`CleanupOperation::execute`] re-evaluates the current dataset and reference
+/// state before deleting files.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CleanupExplanation {
+    /// Dataset version observed when the explanation was produced.
+    pub read_version: u64,
+    /// Aggregate statistics for files that would be removed.
+    pub stats: RemovalStats,
+    /// Candidate files that would be removed, capped by `candidate_file_limit`.
+    pub candidate_files: Vec<CleanupCandidateFile>,
+    /// True if more candidate files were found than are included.
+    pub candidate_files_truncated: bool,
+    /// Maximum number of candidate files included in this explanation.
+    pub candidate_file_limit: usize,
+    /// Referenced child branches and whether cleanup would cascade into them.
+    pub referenced_branches: Vec<CleanupReferencedBranch>,
+    /// Non-fatal warnings about the explanation.
+    pub warnings: Vec<String>,
+}
+
+/// A file that cleanup identified as removable.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CleanupCandidateFile {
+    /// Dataset-relative or storage path for the candidate file.
+    pub path: String,
+    /// Kind of file identified by cleanup.
+    pub kind: CleanupFileKind,
+    /// True if the file is removable only because it aged past the unverified
+    /// retention threshold or `delete_unverified` is enabled.
+    pub unverified: bool,
+    /// Candidate file size in bytes.
+    pub size_bytes: u64,
+}
+
+/// A branch that references the current branch lineage.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct CleanupReferencedBranch {
+    /// Branch name.
+    pub name: String,
+    /// Version of the current lineage referenced by this branch.
+    pub referenced_version: u64,
+    /// True if this branch would be cleaned when cascading cleanup is enabled.
+    pub cleanup_candidate: bool,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CleanupFileKind {
+    Manifest,
     Data,
     Transaction,
     Index,
     Deletion,
+    /// A leftover `_versions/.tmp` manifest from a failed transaction.  These
+    /// are deleted but excluded from per-kind `RemovalStats` counts and audit
+    /// logs to match the long-standing cleanup behavior.  Their bytes
+    /// are still included in `bytes_removed`.
+    TemporaryManifest,
+}
+
+impl CleanupCandidateFile {
+    fn from_cleanup_file(file: &CleanupFile) -> Self {
+        Self {
+            path: file.path.to_string(),
+            kind: file.kind,
+            unverified: file.unverified,
+            size_bytes: file.size_bytes,
+        }
+    }
+}
+
+fn cleanup_file(
+    path: Path,
+    kind: CleanupFileKind,
+    unverified: bool,
+    size_bytes: u64,
+) -> Option<CleanupFile> {
+    Some(CleanupFile {
+        path,
+        kind,
+        unverified,
+        size_bytes,
+    })
+}
+
+#[derive(Clone, Debug)]
+struct CleanupFile {
+    path: Path,
+    kind: CleanupFileKind,
+    /// True when the file was kept on disk past its referenced lifetime
+    /// because we could not verify it was safe to remove (e.g. produced by an
+    /// unfinished commit) and is being deleted only because it has aged past
+    /// the unverified-retention threshold or `delete_unverified` is set.
+    unverified: bool,
+    size_bytes: u64,
+}
+
+impl RemovalStats {
+    fn record_file(&mut self, file: &CleanupFile) {
+        self.bytes_removed += file.size_bytes;
+        match file.kind {
+            CleanupFileKind::Manifest => self.old_versions += 1,
+            CleanupFileKind::Data => self.data_files_removed += 1,
+            CleanupFileKind::Transaction => self.transaction_files_removed += 1,
+            CleanupFileKind::Index => self.index_files_removed += 1,
+            CleanupFileKind::Deletion => self.deletion_files_removed += 1,
+            CleanupFileKind::TemporaryManifest => {}
+        }
+    }
+
+    fn merge(&mut self, other: &Self) {
+        self.bytes_removed += other.bytes_removed;
+        self.old_versions += other.old_versions;
+        self.data_files_removed += other.data_files_removed;
+        self.transaction_files_removed += other.transaction_files_removed;
+        self.index_files_removed += other.index_files_removed;
+        self.deletion_files_removed += other.deletion_files_removed;
+    }
+}
+
+#[derive(Debug, Default)]
+struct CleanupRunResult {
+    stats: RemovalStats,
+    removed_manifests: HashSet<Path>,
+    candidate_files: Vec<CleanupCandidateFile>,
+    candidate_files_truncated: bool,
+    referenced_branches: Vec<CleanupReferencedBranch>,
+}
+
+impl CleanupRunResult {
+    fn record_file(
+        &mut self,
+        file: &CleanupFile,
+        candidate_file_limit: Option<usize>,
+        track_removed_manifests: bool,
+    ) {
+        self.stats.record_file(file);
+        if track_removed_manifests && matches!(file.kind, CleanupFileKind::Manifest) {
+            self.removed_manifests.insert(file.path.clone());
+        }
+        if let Some(limit) = candidate_file_limit {
+            if self.candidate_files.len() < limit {
+                self.candidate_files
+                    .push(CleanupCandidateFile::from_cleanup_file(file));
+            } else {
+                self.candidate_files_truncated = true;
+            }
+        }
+    }
+
+    fn merge(&mut self, other: Self, candidate_file_limit: Option<usize>) {
+        self.stats.merge(&other.stats);
+        self.removed_manifests.extend(other.removed_manifests);
+        self.referenced_branches.extend(other.referenced_branches);
+        if let Some(limit) = candidate_file_limit {
+            for file in other.candidate_files {
+                if self.candidate_files.len() < limit {
+                    self.candidate_files.push(file);
+                } else {
+                    self.candidate_files_truncated = true;
+                }
+            }
+            self.candidate_files_truncated |= other.candidate_files_truncated;
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+enum CleanupAction {
+    Execute,
+    Explain { max_candidate_files: usize },
+}
+
+impl CleanupAction {
+    fn deletes_files(self) -> bool {
+        matches!(self, Self::Execute)
+    }
+
+    fn candidate_file_limit(self) -> Option<usize> {
+        match self {
+            Self::Execute => None,
+            Self::Explain {
+                max_candidate_files,
+            } => Some(max_candidate_files),
+        }
+    }
 }
 
 fn remove_prefix(path: &Path, prefix: &Path) -> Path {
@@ -108,6 +291,11 @@ fn remove_prefix(path: &Path, prefix: &Path) -> Path {
 struct CleanupTask<'a> {
     dataset: &'a Dataset,
     policy: CleanupPolicy,
+    action: CleanupAction,
+    read_version: u64,
+    ignored_manifests: HashSet<Path>,
+    track_removed_manifests: bool,
+    include_referenced_branches: bool,
 }
 
 /// Information about the dataset that we learn by inspecting all of the manifests
@@ -131,21 +319,131 @@ struct CleanupInspection {
 const UNVERIFIED_THRESHOLD_DAYS: i64 = 7;
 const S3_DELETE_STREAM_BATCH_SIZE: u64 = 1_000;
 const AZURE_DELETE_STREAM_BATCH_SIZE: u64 = 256;
+const DEFAULT_EXPLANATION_MAX_CANDIDATE_FILES: usize = 1_000;
+
+/// Builder-style cleanup operation.
+///
+/// Call [`Self::explain`] for a read-only explanation of what cleanup would
+/// remove, or [`Self::execute`] to re-evaluate the current dataset state and
+/// delete files.
+pub struct CleanupOperation<'a> {
+    dataset: &'a Dataset,
+    policy: CleanupPolicy,
+    max_candidate_files: usize,
+}
+
+impl<'a> CleanupOperation<'a> {
+    pub(crate) fn new(dataset: &'a Dataset, policy: CleanupPolicy) -> Self {
+        Self {
+            dataset,
+            policy,
+            max_candidate_files: DEFAULT_EXPLANATION_MAX_CANDIDATE_FILES,
+        }
+    }
+
+    /// Set the maximum number of candidate files included in explanations.
+    ///
+    /// The aggregate [`RemovalStats`] in [`CleanupExplanation`] still include
+    /// all files that would be removed.
+    pub fn with_max_candidate_files(mut self, max_candidate_files: usize) -> Self {
+        self.max_candidate_files = max_candidate_files;
+        self
+    }
+
+    /// Explain what cleanup would remove without deleting files.
+    pub async fn explain(&self) -> Result<CleanupExplanation> {
+        let cleanup = CleanupTask::new(
+            self.dataset,
+            self.policy.clone(),
+            CleanupAction::Explain {
+                max_candidate_files: self.max_candidate_files,
+            },
+        );
+        let read_version = cleanup.read_version;
+        let result = cleanup.run().await?;
+        let warnings = if result.candidate_files_truncated {
+            vec![format!(
+                "candidate_files truncated to {} entries",
+                self.max_candidate_files
+            )]
+        } else {
+            Vec::new()
+        };
+        Ok(CleanupExplanation {
+            read_version,
+            stats: result.stats,
+            candidate_files: result.candidate_files,
+            candidate_files_truncated: result.candidate_files_truncated,
+            candidate_file_limit: self.max_candidate_files,
+            referenced_branches: result.referenced_branches,
+            warnings,
+        })
+    }
+
+    /// Execute cleanup by re-evaluating the current dataset state.
+    pub async fn execute(&self) -> Result<RemovalStats> {
+        info!(target: TRACE_DATASET_EVENTS, event=DATASET_CLEANING_EVENT, uri=&self.dataset.uri);
+        let cleanup = CleanupTask::new(self.dataset, self.policy.clone(), CleanupAction::Execute);
+        Ok(cleanup.run().await?.stats)
+    }
+}
 
 impl<'a> CleanupTask<'a> {
-    fn new(dataset: &'a Dataset, policy: CleanupPolicy) -> Self {
-        Self { dataset, policy }
+    fn new(dataset: &'a Dataset, policy: CleanupPolicy, action: CleanupAction) -> Self {
+        let track_removed_manifests = policy.clean_referenced_branches;
+        let include_referenced_branches = action.candidate_file_limit().is_some();
+        Self::new_with_ignored_manifests(
+            dataset,
+            policy,
+            action,
+            HashSet::new(),
+            track_removed_manifests,
+            include_referenced_branches,
+        )
     }
 
-    async fn run(self) -> Result<RemovalStats> {
-        let mut final_stats = RemovalStats::default();
+    fn new_with_ignored_manifests(
+        dataset: &'a Dataset,
+        policy: CleanupPolicy,
+        action: CleanupAction,
+        ignored_manifests: HashSet<Path>,
+        track_removed_manifests: bool,
+        include_referenced_branches: bool,
+    ) -> Self {
+        Self {
+            dataset,
+            policy,
+            action,
+            read_version: dataset.version().version,
+            ignored_manifests,
+            track_removed_manifests,
+            include_referenced_branches,
+        }
+    }
+
+    async fn run(self) -> Result<CleanupRunResult> {
+        let mut final_result = CleanupRunResult::default();
+        let candidate_file_limit = self.action.candidate_file_limit();
         // First check if we need to clean referenced branches
         // For cases that referenced branches never clean and the current cleanup cannot clean anything
         // This must happen before cleaning the current branch if the setting is enabled.
 
         let referenced_branches: Vec<(String, u64)> = self.find_referenced_branches().await?;
+        if self.include_referenced_branches {
+            final_result.referenced_branches = referenced_branches
+                .iter()
+                .map(|(name, referenced_version)| CleanupReferencedBranch {
+                    name: name.clone(),
+                    referenced_version: *referenced_version,
+                    cleanup_candidate: self.policy.clean_referenced_branches,
+                })
+                .collect();
+        }
         if self.policy.clean_referenced_branches {
-            self.clean_referenced_branches(&referenced_branches).await?;
+            final_result.merge(
+                self.clean_referenced_branches(&referenced_branches).await?,
+                candidate_file_limit,
+            );
         }
 
         // we process all manifest files in parallel to figure
@@ -179,19 +477,21 @@ impl<'a> CleanupTask<'a> {
         }
 
         if !referenced_branches.is_empty() {
+            let ignored_manifests: HashSet<_> = final_result
+                .removed_manifests
+                .union(&self.ignored_manifests)
+                .cloned()
+                .collect();
             inspection = self
-                .retain_branch_lineage_files(inspection, &referenced_branches)
+                .retain_branch_lineage_files(inspection, &referenced_branches, &ignored_manifests)
                 .await?
         };
 
-        let stats = self.delete_unreferenced_files(inspection).await?;
-        final_stats.bytes_removed += stats.bytes_removed;
-        final_stats.old_versions += stats.old_versions;
-        final_stats.data_files_removed += stats.data_files_removed;
-        final_stats.transaction_files_removed += stats.transaction_files_removed;
-        final_stats.index_files_removed += stats.index_files_removed;
-        final_stats.deletion_files_removed += stats.deletion_files_removed;
-        Ok(final_stats)
+        final_result.merge(
+            self.delete_unreferenced_files(inspection).await?,
+            candidate_file_limit,
+        );
+        Ok(final_result)
     }
 
     #[instrument(level = "debug", skip_all)]
@@ -203,6 +503,7 @@ impl<'a> CleanupTask<'a> {
         self.dataset
             .commit_handler
             .list_manifest_locations(&self.dataset.base, &self.dataset.object_store, false)
+            .try_filter(|location| future::ready(!self.ignored_manifests.contains(&location.path)))
             .try_for_each_concurrent(self.dataset.object_store.io_parallelism(), |location| {
                 self.process_manifest_file(location, &inspection, tagged_versions)
             })
@@ -224,12 +525,10 @@ impl<'a> CleanupTask<'a> {
 
         let manifest =
             read_manifest(&self.dataset.object_store, &location.path, location.size).await?;
-        let dataset_version = self.dataset.version().version;
-
         // Don't delete the latest version, even if it is old. Don't delete tagged versions,
         // regardless of age. Don't delete manifests if their version is newer than the dataset
         // version.  These are either in-progress or newly added since we started.
-        let is_latest = dataset_version <= manifest.version;
+        let is_latest = self.read_version <= manifest.version;
         let is_tagged = tagged_versions.contains(&manifest.version);
         let in_working_set = is_latest || !self.policy.should_clean(&manifest) || is_tagged;
         let indexes =
@@ -319,25 +618,17 @@ impl<'a> CleanupTask<'a> {
     async fn delete_unreferenced_files(
         &self,
         inspection: CleanupInspection,
-    ) -> Result<RemovalStats> {
-        let removal_stats = Mutex::new(RemovalStats::default());
+    ) -> Result<CleanupRunResult> {
+        let cleanup_result = Mutex::new(CleanupRunResult::default());
+        let deletes_files = self.action.deletes_files();
+        let candidate_file_limit = self.action.candidate_file_limit();
         let verification_threshold = utc_now()
             - TimeDelta::try_days(UNVERIFIED_THRESHOLD_DAYS).expect("TimeDelta::try_days");
 
-        let is_not_found_err = |e: &Error| {
-            matches!(
-                e,
-                Error::IO { source,.. }
-                    if source
-                      .downcast_ref::<ObjectStoreError>()
-                      .map(|os_err| matches!(os_err, ObjectStoreError::NotFound {.. }))
-                      .unwrap_or(false)
-            )
-        };
+        let is_not_found_err = |e: &Error| matches!(e, Error::NotFound { .. });
         // Build stream for a managed subtree
-        let build_listing_stream = |dir: Path, file_type: Option<RemovedFileType>| {
+        let build_listing_stream = |dir: Path| {
             let inspection_ref = &inspection;
-            let removal_stats_ref = &removal_stats;
             self.dataset
                 .object_store
                 .read_dir_all(&dir, inspection.earliest_retained_manifest_time)
@@ -356,118 +647,133 @@ impl<'a> CleanupTask<'a> {
                     // delete it if we can verify it is part of an old version.
                     let maybe_in_progress = !self.policy.delete_unverified
                         && obj_meta.last_modified >= verification_threshold;
-                    let path_to_remove = self.path_if_not_referenced(
-                        obj_meta.location,
+                    let file_to_remove = self.cleanup_file_if_not_referenced(
+                        obj_meta,
                         maybe_in_progress,
                         inspection_ref,
                     );
-                    if matches!(path_to_remove, Ok(Some(..))) {
-                        let mut stats = removal_stats_ref.lock().unwrap();
-                        stats.bytes_removed += obj_meta.size;
-                        if let Some(file_type) = file_type {
-                            match file_type {
-                                RemovedFileType::Data => stats.data_files_removed += 1,
-                                RemovedFileType::Transaction => {
-                                    stats.transaction_files_removed += 1
-                                }
-                                RemovedFileType::Index => stats.index_files_removed += 1,
-                                RemovedFileType::Deletion => stats.deletion_files_removed += 1,
-                            }
-                        }
-                    }
-                    future::ready(path_to_remove)
+                    future::ready(file_to_remove)
                 })
                 .boxed()
         };
 
         // Restrict scanning to Lance-managed subtrees for safety and performance.
         let streams = vec![
-            build_listing_stream(self.dataset.versions_dir(), None),
-            build_listing_stream(
-                self.dataset.transactions_dir(),
-                Some(RemovedFileType::Transaction),
-            ),
-            build_listing_stream(self.dataset.data_dir(), Some(RemovedFileType::Data)),
-            build_listing_stream(self.dataset.indices_dir(), Some(RemovedFileType::Index)),
-            build_listing_stream(
-                self.dataset.deletions_dir(),
-                Some(RemovedFileType::Deletion),
-            ),
+            build_listing_stream(self.dataset.versions_dir()),
+            build_listing_stream(self.dataset.transactions_dir()),
+            build_listing_stream(self.dataset.data_dir()),
+            build_listing_stream(self.dataset.indices_dir()),
+            build_listing_stream(self.dataset.deletions_dir()),
         ];
-        let unreferenced_paths = stream::iter(streams).flatten().boxed();
+        let unreferenced_files = stream::iter(streams).flatten().boxed();
 
         let old_manifests = inspection.old_manifests.clone();
-        let num_old_manifests = old_manifests.len();
-
-        // Ideally this collect shouldn't be needed here but it seems necessary
-        // to avoid https://github.com/rust-lang/rust/issues/102211
-        let manifest_bytes_removed = stream::iter(old_manifests.keys())
-            .map(|path| self.dataset.object_store.size(path))
-            .collect::<Vec<_>>()
-            .await;
-        let manifest_bytes_removed = stream::iter(manifest_bytes_removed)
-            .buffer_unordered(self.dataset.object_store.io_parallelism())
-            .try_fold(0, |acc, size| async move { Ok(acc + (size)) })
-            .await;
-
-        let old_manifests_stream = stream::iter(old_manifests.into_keys())
-            .map(|path| {
-                info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = path.as_ref());
-                Ok(path)
+        let manifest_files = stream::iter(old_manifests)
+            .map(|(path, _version)| async move {
+                let size_bytes = self.dataset.object_store.size(&path).await?;
+                Ok::<CleanupFile, Error>(CleanupFile {
+                    path,
+                    kind: CleanupFileKind::Manifest,
+                    unverified: false,
+                    size_bytes,
+                })
             })
+            .buffer_unordered(self.dataset.object_store.io_parallelism())
             .boxed();
-        let all_paths_to_remove =
-            stream::iter(vec![unreferenced_paths, old_manifests_stream]).flatten();
-
-        let paths_to_delete: BoxStream<Result<Path>> = if let Some(rate) =
-            self.policy.delete_rate_limit
-        {
-            let duration = calculate_duration(self.dataset.object_store.scheme().to_string(), rate);
-            let mut ticker = interval(duration);
-            ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
-            IntervalStream::new(ticker)
-                .zip(all_paths_to_remove)
-                .map(|(_, path)| path)
-                .boxed()
-        } else {
-            all_paths_to_remove.boxed()
-        };
 
-        let delete_fut = self
-            .dataset
-            .object_store
-            .remove_stream(paths_to_delete)
-            .try_for_each(|_| future::ready(Ok(())));
+        let all_files = stream::iter(vec![unreferenced_files, manifest_files]).flatten();
+        let all_paths_to_remove = all_files.map(|file| {
+            let file = file?;
+            if deletes_files {
+                let mode = if file.unverified {
+                    AUDIT_MODE_DELETE_UNVERIFIED
+                } else {
+                    AUDIT_MODE_DELETE
+                };
+                let path_str = file.path.as_ref();
+                match file.kind {
+                    CleanupFileKind::Manifest => {
+                        info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = path_str);
+                    }
+                    CleanupFileKind::Data => {
+                        info!(target: TRACE_FILE_AUDIT, mode=mode, r#type=AUDIT_TYPE_DATA, path = path_str);
+                    }
+                    CleanupFileKind::Deletion => {
+                        info!(target: TRACE_FILE_AUDIT, mode=mode, r#type=AUDIT_TYPE_DELETION, path = path_str);
+                    }
+                    CleanupFileKind::Index => {
+                        info!(target: TRACE_FILE_AUDIT, mode=mode, r#type=AUDIT_TYPE_INDEX, path = path_str);
+                    }
+                    CleanupFileKind::Transaction | CleanupFileKind::TemporaryManifest => {}
+                }
+            }
+            cleanup_result
+                .lock()
+                .unwrap()
+                .record_file(&file, candidate_file_limit, self.track_removed_manifests);
+            Ok(file.path)
+        });
+
+        if deletes_files {
+            let paths_to_delete: BoxStream<Result<Path>> =
+                if let Some(rate) = self.policy.delete_rate_limit {
+                    let duration =
+                        calculate_duration(self.dataset.object_store.scheme().to_string(), rate);
+                    let mut ticker = interval(duration);
+                    ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
+                    IntervalStream::new(ticker)
+                        .zip(all_paths_to_remove)
+                        .map(|(_, path)| path)
+                        .boxed()
+                } else {
+                    all_paths_to_remove.boxed()
+                };
 
-        delete_fut.await?;
+            self.dataset
+                .object_store
+                .remove_stream(paths_to_delete)
+                .try_for_each(|_| future::ready(Ok(())))
+                .await?;
+        } else {
+            // Drain the stream to populate stats, but do not call remove_stream.
+            all_paths_to_remove
+                .try_for_each(|_| future::ready(Ok(())))
+                .await?;
+        }
 
-        let mut removal_stats = removal_stats.into_inner().unwrap();
-        removal_stats.old_versions = num_old_manifests as u64;
-        removal_stats.bytes_removed += manifest_bytes_removed?;
+        let cleanup_result = cleanup_result.into_inner().unwrap();
 
         let span = Span::current();
-        span.record("bytes_removed", removal_stats.bytes_removed);
-        span.record("data_files_removed", removal_stats.data_files_removed);
+        span.record("bytes_removed", cleanup_result.stats.bytes_removed);
+        span.record(
+            "data_files_removed",
+            cleanup_result.stats.data_files_removed,
+        );
         span.record(
             "transaction_files_removed",
-            removal_stats.transaction_files_removed,
+            cleanup_result.stats.transaction_files_removed,
+        );
+        span.record(
+            "index_files_removed",
+            cleanup_result.stats.index_files_removed,
         );
-        span.record("index_files_removed", removal_stats.index_files_removed);
         span.record(
             "deletion_files_removed",
-            removal_stats.deletion_files_removed,
+            cleanup_result.stats.deletion_files_removed,
         );
 
-        Ok(removal_stats)
+        Ok(cleanup_result)
     }
 
-    fn path_if_not_referenced(
+    fn cleanup_file_if_not_referenced(
         &self,
-        path: Path,
+        obj_meta: ObjectMeta,
         maybe_in_progress: bool,
         inspection: &CleanupInspection,
-    ) -> Result<Option<Path>> {
+    ) -> Result<Option<CleanupFile>> {
+        let path = obj_meta.location;
         let relative_path = remove_prefix(&path, &self.dataset.base);
+        let size_bytes = obj_meta.size;
         if relative_path.as_ref().starts_with("_versions/.tmp") {
             // This is a temporary manifest file.
             //
@@ -476,7 +782,12 @@ impl<'a> CleanupTask<'a> {
             if maybe_in_progress {
                 return Ok(None);
             } else {
-                return Ok(Some(path));
+                return Ok(cleanup_file(
+                    path,
+                    CleanupFileKind::TemporaryManifest,
+                    true,
+                    size_bytes,
+                ));
             }
         }
         if relative_path.as_ref().starts_with("_indices") {
@@ -490,15 +801,18 @@ impl<'a> CleanupTask<'a> {
                 {
                     return Ok(None);
                 } else if !maybe_in_progress {
-                    info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE_UNVERIFIED, r#type=AUDIT_TYPE_INDEX, path = path.to_string());
-                    return Ok(Some(path));
+                    return Ok(cleanup_file(path, CleanupFileKind::Index, true, size_bytes));
                 } else if inspection
                     .verified_files
                     .index_uuids
                     .contains(uuid.as_ref())
                 {
-                    info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_INDEX, path = path.to_string());
-                    return Ok(Some(path));
+                    return Ok(cleanup_file(
+                        path,
+                        CleanupFileKind::Index,
+                        false,
+                        size_bytes,
+                    ));
                 }
             } else {
                 return Ok(None);
@@ -514,15 +828,13 @@ impl<'a> CleanupTask<'a> {
                     {
                         Ok(None)
                     } else if !maybe_in_progress {
-                        info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE_UNVERIFIED, r#type=AUDIT_TYPE_DATA, path = path.to_string());
-                        Ok(Some(path))
+                        Ok(cleanup_file(path, CleanupFileKind::Data, true, size_bytes))
                     } else if inspection
                         .verified_files
                         .data_paths
                         .contains(&relative_path)
                     {
-                        info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_DATA, path = path.to_string());
-                        Ok(Some(path))
+                        Ok(cleanup_file(path, CleanupFileKind::Data, false, size_bytes))
                     } else {
                         Ok(None)
                     }
@@ -587,15 +899,13 @@ impl<'a> CleanupTask<'a> {
                 {
                     Ok(None)
                 } else if !maybe_in_progress {
-                    info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE_UNVERIFIED, r#type=AUDIT_TYPE_DATA, path = path.to_string());
-                    Ok(Some(path))
+                    Ok(cleanup_file(path, CleanupFileKind::Data, true, size_bytes))
                 } else if inspection
                     .verified_files
                     .data_paths
                     .contains(&parent_data_path)
                 {
-                    info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_DATA, path = path.to_string());
-                    Ok(Some(path))
+                    Ok(cleanup_file(path, CleanupFileKind::Data, false, size_bytes))
                 } else {
                     Ok(None)
                 }
@@ -613,15 +923,23 @@ impl<'a> CleanupTask<'a> {
                     {
                         Ok(None)
                     } else if !maybe_in_progress {
-                        info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE_UNVERIFIED, r#type=AUDIT_TYPE_DELETION, path = path.to_string());
-                        Ok(Some(path))
+                        Ok(cleanup_file(
+                            path,
+                            CleanupFileKind::Deletion,
+                            true,
+                            size_bytes,
+                        ))
                     } else if inspection
                         .verified_files
                         .delete_paths
                         .contains(&relative_path)
                     {
-                        info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_DELETION, path = path.to_string());
-                        Ok(Some(path))
+                        Ok(cleanup_file(
+                            path,
+                            CleanupFileKind::Deletion,
+                            false,
+                            size_bytes,
+                        ))
                     } else {
                         Ok(None)
                     }
@@ -640,7 +958,14 @@ impl<'a> CleanupTask<'a> {
                     } else if !maybe_in_progress
                         || inspection.verified_files.tx_paths.contains(&relative_path)
                     {
-                        Ok(Some(path))
+                        let unverified =
+                            !inspection.verified_files.tx_paths.contains(&relative_path);
+                        Ok(cleanup_file(
+                            path,
+                            CleanupFileKind::Transaction,
+                            unverified,
+                            size_bytes,
+                        ))
                     } else {
                         Ok(None)
                     }
@@ -709,8 +1034,8 @@ impl<'a> CleanupTask<'a> {
     async fn clean_referenced_branches(
         &self,
         referenced_branches: &[(String, u64)],
-    ) -> Result<RemovalStats> {
-        let final_stats = Mutex::new(RemovalStats::default());
+    ) -> Result<CleanupRunResult> {
+        let final_result = Mutex::new(CleanupRunResult::default());
 
         // Group branches by their lineage identifier (BranchIdentifier).
         // Branches with the same identifier share a lineage and must be cleaned sequentially
@@ -722,30 +1047,32 @@ impl<'a> CleanupTask<'a> {
                 .or_insert_with(Vec::new)
                 .push(branch.clone());
         }
+        let action = self.action;
+        let candidate_file_limit = self.action.candidate_file_limit();
         let tasks: Vec<_> = branches_chains
             .values()
             .map(|branch_chain| {
-                let final_stats = &final_stats;
+                let final_result = &final_result;
                 async move {
                     for branch in branch_chain {
                         let branch_dataset = self
                             .dataset
                             .checkout_version((branch.as_str(), None))
                             .await?;
-                        if let Some(stats) = cleanup_cascade_branch(
+                        let ignored_manifests =
+                            final_result.lock().unwrap().removed_manifests.clone();
+                        if let Some(result) = cleanup_cascade_branch_run(
                             &branch_dataset,
                             branch_dataset.manifest.as_ref(),
+                            action,
+                            ignored_manifests,
                         )
                         .await?
                         {
-                            let mut stats_guard = final_stats.lock().unwrap();
-                            stats_guard.bytes_removed += stats.bytes_removed;
-                            stats_guard.old_versions += stats.old_versions;
-                            stats_guard.data_files_removed += stats.data_files_removed;
-                            stats_guard.transaction_files_removed +=
-                                stats.transaction_files_removed;
-                            stats_guard.index_files_removed += stats.index_files_removed;
-                            stats_guard.deletion_files_removed += stats.deletion_files_removed;
+                            final_result
+                                .lock()
+                                .unwrap()
+                                .merge(result, candidate_file_limit);
                         }
                     }
                     Ok::<(), Error>(())
@@ -753,7 +1080,7 @@ impl<'a> CleanupTask<'a> {
             })
             .collect();
         try_join_all(tasks).await?;
-        Ok(final_stats.into_inner().unwrap())
+        Ok(final_result.into_inner().unwrap())
     }
 
     // Retain manifests containing files referenced by descendant branches.
@@ -762,6 +1089,7 @@ impl<'a> CleanupTask<'a> {
         &self,
         inspection: CleanupInspection,
         referenced_branches: &[(String, u64)],
+        removed_branch_manifests: &HashSet<Path>,
     ) -> Result<CleanupInspection> {
         let inspection = Mutex::new(inspection);
         for (branch, root_version_number) in referenced_branches {
@@ -772,6 +1100,9 @@ impl<'a> CleanupTask<'a> {
             self.dataset
                 .commit_handler
                 .list_manifest_locations(&branch_location.path, &self.dataset.object_store, false)
+                .try_filter(|location| {
+                    future::ready(!removed_branch_manifests.contains(&location.path))
+                })
                 .try_for_each_concurrent(self.dataset.object_store.io_parallelism(), |location| {
                     self.process_branch_referenced_manifests(
                         location,
@@ -1020,8 +1351,7 @@ pub async fn cleanup_old_versions(
     dataset: &Dataset,
     policy: CleanupPolicy,
 ) -> Result<RemovalStats> {
-    let cleanup = CleanupTask::new(dataset, policy);
-    cleanup.run().await
+    CleanupOperation::new(dataset, policy).execute().await
 }
 
 /// If the dataset config has `lance.auto_cleanup` parameters set,
@@ -1048,11 +1378,35 @@ pub async fn cleanup_cascade_branch(
     dataset: &Dataset,
     manifest: &Manifest,
 ) -> Result<Option<RemovalStats>> {
+    Ok(
+        cleanup_cascade_branch_run(dataset, manifest, CleanupAction::Execute, HashSet::new())
+            .await?
+            .map(|result| result.stats),
+    )
+}
+
+async fn cleanup_cascade_branch_run(
+    dataset: &Dataset,
+    manifest: &Manifest,
+    action: CleanupAction,
+    ignored_manifests: HashSet<Path>,
+) -> Result<Option<CleanupRunResult>> {
     let policy = build_cleanup_policy(dataset, manifest).await?;
     if let Some(mut policy) = policy {
         policy.clean_referenced_branches = false;
         policy.error_if_tagged_old_versions = false;
-        Ok(Some(dataset.cleanup_with_policy(policy).await?))
+        if action.deletes_files() {
+            info!(target: TRACE_DATASET_EVENTS, event=DATASET_CLEANING_EVENT, uri=&dataset.uri);
+        }
+        let cleanup = CleanupTask::new_with_ignored_manifests(
+            dataset,
+            policy,
+            action,
+            ignored_manifests,
+            true,
+            false,
+        );
+        Ok(Some(cleanup.run().await?))
     } else {
         Ok(None)
     }
@@ -1443,6 +1797,14 @@ mod tests {
             cleanup_old_versions(&db, policy).await
         }
 
+        async fn explain_cleanup_with_policy(
+            &self,
+            policy: CleanupPolicy,
+        ) -> Result<CleanupExplanation> {
+            let db = self.open().await?;
+            db.cleanup(policy).explain().await
+        }
+
         async fn run_cleanup_with_override(
             &self,
             before: DateTime<Utc>,
@@ -1670,6 +2032,51 @@ mod tests {
         assert_gt!(after_count.num_tx_files, 0);
     }
 
+    #[tokio::test]
+    async fn explain_cleanup_does_not_delete_files() {
+        let fixture = MockDatasetFixture::try_new().unwrap();
+        fixture.create_some_data().await.unwrap();
+        MockClock::set_system_time(TimeDelta::try_seconds(1).unwrap().to_std().unwrap());
+        fixture.overwrite_some_data().await.unwrap();
+
+        let before_count = fixture.count_files().await.unwrap();
+        let policy = CleanupPolicyBuilder::default()
+            .before_timestamp(utc_now())
+            .build();
+
+        let explanation = fixture
+            .explain_cleanup_with_policy(policy.clone())
+            .await
+            .unwrap();
+        let after_preview_count = fixture.count_files().await.unwrap();
+
+        // Files are not actually removed when explaining cleanup.
+        assert_eq!(before_count, after_preview_count);
+        assert_eq!(explanation.read_version, 2);
+        assert_eq!(explanation.stats.old_versions, 1);
+        assert_eq!(explanation.stats.data_files_removed, 1);
+        assert_eq!(explanation.stats.transaction_files_removed, 1);
+        assert_gt!(explanation.stats.bytes_removed, 0);
+        assert!(!explanation.candidate_files.is_empty());
+        assert!(!explanation.candidate_files_truncated);
+
+        // Running cleanup with the same policy should remove the same files the
+        // explanation reported for this unchanged dataset.
+        let removed = fixture.run_cleanup_with_policy(policy).await.unwrap();
+        let after_cleanup_count = fixture.count_files().await.unwrap();
+
+        assert_eq!(
+            removed.bytes_removed,
+            before_count.num_bytes - after_cleanup_count.num_bytes
+        );
+        assert_eq!(removed.old_versions, explanation.stats.old_versions);
+        assert_eq!(
+            removed.data_files_removed,
+            explanation.stats.data_files_removed
+        );
+        assert_eq!(removed.bytes_removed, explanation.stats.bytes_removed);
+    }
+
     #[tokio::test]
     async fn cleanup_blob_v2_sidecar_files() {
         let fixture = MockDatasetFixture::try_new().unwrap();
@@ -2571,10 +2978,8 @@ mod tests {
     async fn cleanup_before_ts_and_retain_n_recent_versions() {
         let fixture = MockDatasetFixture::try_new().unwrap();
         fixture.create_some_data().await.unwrap();
-        let mut time = 1i64;
-        for _ in 0..4 {
+        for time in (1i64..).take(4) {
             MockClock::set_system_time(TimeDelta::try_days(time).unwrap().to_std().unwrap());
-            time += 1i64;
             fixture.overwrite_some_data().await.unwrap();
         }
 
@@ -3073,6 +3478,17 @@ mod tests {
             self.run_cleanup_inner(policy).await
         }
 
+        async fn explain_cleanup_with_referenced_branches(&mut self) -> Result<CleanupExplanation> {
+            let policy = CleanupPolicyBuilder::default()
+                .error_if_tagged_old_versions(false)
+                .clean_referenced_branches(true)
+                .retain_n_versions(&self.dataset, 1)
+                .await?
+                .build();
+            self.dataset.checkout_latest().await?;
+            self.dataset.cleanup(policy).explain().await
+        }
+
         async fn run_cleanup_inner(&mut self, policy: CleanupPolicy) -> Result<RemovalStats> {
             let pre_count = self.count_data().await?;
             self.dataset.checkout_latest().await?;
@@ -3653,6 +4069,74 @@ mod tests {
         setup.assert_unchanged(&["branch4"]).await;
     }
 
+    #[tokio::test]
+    async fn explain_cleanup_with_referenced_branches_matches_cleanup() {
+        let mut setup = build_lineage_datasets().await.unwrap();
+
+        setup.enable_auto_cleanup().await.unwrap();
+        setup.main.write_data().await.unwrap();
+        setup.main.compact().await.unwrap();
+        setup.branch4.compact().await.unwrap();
+        setup.branch1.write_data().await.unwrap();
+        setup.branch1.compact().await.unwrap();
+        setup.branch2.write_data().await.unwrap();
+        setup.branch2.compact().await.unwrap();
+        setup.branch3.write_data().await.unwrap();
+        setup.branch3.compact().await.unwrap();
+
+        setup.main.refresh().await.unwrap();
+        setup.branch1.refresh().await.unwrap();
+        setup.branch2.refresh().await.unwrap();
+        setup.branch3.refresh().await.unwrap();
+        setup.branch4.refresh().await.unwrap();
+        let main_counts_before = setup.main.counts;
+        let branch1_counts_before = setup.branch1.counts;
+        let branch2_counts_before = setup.branch2.counts;
+        let branch3_counts_before = setup.branch3.counts;
+        let branch4_counts_before = setup.branch4.counts;
+
+        let explanation = setup
+            .main
+            .explain_cleanup_with_referenced_branches()
+            .await
+            .unwrap();
+
+        setup.main.refresh().await.unwrap();
+        setup.branch1.refresh().await.unwrap();
+        setup.branch2.refresh().await.unwrap();
+        setup.branch3.refresh().await.unwrap();
+        setup.branch4.refresh().await.unwrap();
+        assert_eq!(setup.main.counts, main_counts_before);
+        assert_eq!(setup.branch1.counts, branch1_counts_before);
+        assert_eq!(setup.branch2.counts, branch2_counts_before);
+        assert_eq!(setup.branch3.counts, branch3_counts_before);
+        assert_eq!(setup.branch4.counts, branch4_counts_before);
+
+        let removed = setup
+            .main
+            .run_cleanup_with_referenced_branches()
+            .await
+            .unwrap();
+
+        assert!(!explanation.referenced_branches.is_empty());
+        assert!(
+            explanation
+                .referenced_branches
+                .iter()
+                .any(|branch| branch.cleanup_candidate)
+        );
+        assert_eq!(explanation.stats, removed);
+        setup.branch1.refresh().await.unwrap();
+        setup.branch2.refresh().await.unwrap();
+        setup.branch3.refresh().await.unwrap();
+        setup.branch4.refresh().await.unwrap();
+        assert_eq!(setup.main.counts.num_manifest_files, 1);
+        assert_eq!(setup.branch1.counts.num_manifest_files, 1);
+        assert_eq!(setup.branch2.counts.num_manifest_files, 1);
+        assert_eq!(setup.branch3.counts.num_manifest_files, 1);
+        assert_eq!(setup.branch4.counts.num_manifest_files, 1);
+    }
+
     #[tokio::test]
     async fn auto_clean_referenced_branches_with_tags() {
         let mut setup = build_lineage_datasets().await.unwrap();
diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs
new file mode 100644
index 00000000000..2214822ed90
--- /dev/null
+++ b/rust/lance/src/dataset/files.rs
@@ -0,0 +1,1170 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Dataset file inspection APIs.
+
+use std::borrow::Cow;
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use arrow_array::RecordBatch;
+use arrow_array::builder::{
+    Int64Builder, StringBuilder, StringDictionaryBuilder, TimestampMicrosecondBuilder,
+};
+use arrow_array::types::Int32Type;
+use datafusion::execution::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use either::Either;
+use futures::stream::FuturesUnordered;
+use futures::{Future, StreamExt, TryStreamExt};
+use lance_table::format::IndexMetadata;
+use lance_table::utils::LanceIteratorExtension;
+use object_store::path::Path;
+use uuid::Uuid;
+
+use crate::Dataset;
+use crate::dataset::files::arrow::{TRACKED_FILES_SCHEMA, TrackedFileBatch};
+use crate::dataset::files::file_types::FileType;
+use crate::dataset::{DATA_DIR, INDICES_DIR, TRANSACTIONS_DIR};
+use lance_core::Result;
+use lance_table::io::deletion::relative_deletion_file_path;
+use lance_table::io::manifest::{read_manifest, read_manifest_indexes};
+
+mod arrow;
+mod file_types;
+
+const BATCH_SIZE: usize = 4096;
+/// Memory budget for in-flight manifests (estimated in-memory size).
+const MANIFEST_MEMORY_BUDGET: usize = 1024 * 1024 * 1024; // 1 GB
+/// Estimated ratio of in-memory size to on-disk size for manifests. Found
+/// empirically; manifests are protobuf with significant decompression and
+/// allocator overhead once parsed.
+const MANIFEST_DECOMPRESSION_RATIO: usize = 4;
+
+fn remove_prefix(path: &Path, prefix: &Path) -> Path {
+    match path.prefix_match(prefix) {
+        Some(parts) => Path::from_iter(parts),
+        None => path.clone(),
+    }
+}
+
+/// A single row destined for the `tracked_files` output.
+struct FileRow<'a> {
+    version: u64,
+    base_uri: Cow<'a, str>,
+    path: Cow<'a, str>,
+    file_type: FileType,
+}
+
+/// Resolve the base URI a file lives under. Files referenced from a shallow
+/// clone carry a `base_id` pointing into `manifest.base_paths`; otherwise they
+/// live under this dataset's own `base_uri`.
+fn resolve_base_uri<'a>(
+    manifest: &'a lance_table::format::Manifest,
+    base_id: Option<u32>,
+    base_uri: &'a str,
+) -> &'a str {
+    base_id
+        .and_then(|id| manifest.base_paths.get(&id).map(|bp| bp.path.as_str()))
+        .unwrap_or(base_uri)
+}
+
+fn manifest_file_rows<'a>(
+    manifest: &'a lance_table::format::Manifest,
+    base_uri: &'a str,
+    manifest_path: &'a str,
+) -> Box<dyn ExactSizeIterator<Item = FileRow<'a>> + Send + 'a> {
+    let mut files = 1;
+    let manifest_row = FileRow {
+        version: manifest.version,
+        base_uri: Cow::Borrowed(base_uri),
+        path: Cow::Borrowed(manifest_path),
+        file_type: FileType::Manifest,
+    };
+    let iter = std::iter::once(manifest_row);
+
+    let iter = if let Some(txn_file) = &manifest.transaction_file {
+        files += 1;
+        let txn_row = FileRow {
+            version: manifest.version,
+            base_uri: Cow::Borrowed(base_uri),
+            path: Cow::Owned(format!("{}/{}", TRANSACTIONS_DIR, txn_file)),
+            file_type: FileType::TransactionFile,
+        };
+        Either::Left(iter.chain(std::iter::once(txn_row)))
+    } else {
+        Either::Right(iter)
+    };
+
+    for fragment in manifest.fragments.iter() {
+        files += fragment.files.len();
+
+        if fragment.deletion_file.is_some() {
+            files += 1;
+        }
+    }
+
+    let data_files = manifest.fragments.iter().flat_map(move |fragment| {
+        fragment.files.iter().map(move |data_file| {
+            let effective_base_uri = resolve_base_uri(manifest, data_file.base_id, base_uri);
+            FileRow {
+                version: manifest.version,
+                base_uri: Cow::Borrowed(effective_base_uri),
+                path: Cow::Owned(format!("{}/{}", DATA_DIR, data_file.path)),
+                file_type: FileType::DataFile,
+            }
+        })
+    });
+
+    let deletion_files = manifest.fragments.iter().filter_map(|fragment| {
+        fragment.deletion_file.as_ref().map(|del_file| FileRow {
+            version: manifest.version,
+            base_uri: Cow::Borrowed(resolve_base_uri(manifest, del_file.base_id, base_uri)),
+            path: Cow::Owned(relative_deletion_file_path(fragment.id, del_file)),
+            file_type: FileType::DeletionFile,
+        })
+    });
+
+    Box::new(
+        iter.chain(data_files)
+            .chain(deletion_files)
+            .exact_size(files),
+    )
+}
+
+fn manifest_file_batches<'a>(
+    manifest: &'a lance_table::format::Manifest,
+    base_uri: &'a str,
+    manifest_path: &'a str,
+) -> Box<dyn ExactSizeIterator<Item = Result<RecordBatch>> + Send + 'a> {
+    let mut builder = TrackedFileBatch::with_capacity(BATCH_SIZE);
+
+    let mut iter = manifest_file_rows(manifest, base_uri, manifest_path);
+    let size = iter.len().div_ceil(BATCH_SIZE);
+
+    let mut flushed = false;
+    Box::new(
+        std::iter::from_fn(move || {
+            if flushed {
+                return None;
+            }
+            while let Some(row) = iter.next() {
+                builder.append(&row);
+                if builder.len() == BATCH_SIZE {
+                    let next_size = iter.len().div_ceil(BATCH_SIZE);
+                    let old_builder =
+                        std::mem::replace(&mut builder, TrackedFileBatch::with_capacity(next_size));
+                    return Some(old_builder.finish());
+                }
+            }
+            // Flush the remaining partial batch.
+            flushed = true;
+            if builder.len() != 0 {
+                let partial = std::mem::replace(&mut builder, TrackedFileBatch::with_capacity(0));
+                Some(partial.finish())
+            } else {
+                None
+            }
+        })
+        .exact_size(size),
+    )
+}
+
+async fn get_index_files(
+    uuids: impl IntoIterator<Item = Uuid>,
+    base: &Path,
+    object_store: &lance_io::object_store::ObjectStore,
+    cache: &mut HashMap<Uuid, Vec<object_store::ObjectMeta>>,
+) -> Result<Vec<Path>> {
+    let uuids: Vec<Uuid> = uuids.into_iter().collect();
+
+    // Phase 1: list uncached UUID directories concurrently.
+    let uncached: Vec<Uuid> = uuids
+        .iter()
+        .filter(|uuid| !cache.contains_key(*uuid))
+        .copied()
+        .collect();
+    if !uncached.is_empty() {
+        let parallelism = object_store.io_parallelism();
+        // Clone for use in async move closures (ObjectStore is Arc-backed).
+        let base_owned = base.clone();
+        let os = object_store.clone();
+        let new_entries: Vec<(Uuid, Vec<object_store::ObjectMeta>)> =
+            futures::stream::iter(uncached)
+                .map(|uuid| {
+                    let base = base_owned.clone();
+                    let os = os.clone();
+                    async move {
+                        let prefix = base.join(INDICES_DIR).join(uuid.to_string());
+                        let files: Vec<object_store::ObjectMeta> =
+                            os.list(Some(prefix)).try_collect().await?;
+                        lance_core::Result::Ok((uuid, files))
+                    }
+                })
+                .buffer_unordered(parallelism)
+                .try_collect()
+                .await?;
+
+        // Phase 2: insert results into cache (serial, no contention).
+        cache.extend(new_entries);
+    }
+
+    // Phase 3: collect paths for the requested UUIDs in order.
+    let mut paths = Vec::new();
+    for uuid in &uuids {
+        paths.extend(
+            cache[uuid]
+                .iter()
+                .map(|meta| remove_prefix(&meta.location, base)),
+        );
+    }
+    Ok(paths)
+}
+
+async fn index_file_batch(version: u64, base_uri: &str, paths: &[Path]) -> Result<RecordBatch> {
+    let mut builder = TrackedFileBatch::with_capacity(paths.len());
+    for path in paths {
+        builder.append(&FileRow {
+            version,
+            base_uri: Cow::Borrowed(base_uri),
+            path: Cow::Owned(path.to_string()),
+            file_type: FileType::IndexFile,
+        });
+    }
+    builder.finish()
+}
+
+/// Progress update for [`Dataset::tracked_files_with_options`].
+#[derive(Debug, Clone)]
+pub struct TrackedFilesProgress {
+    /// Number of manifests processed so far.
+    pub manifests_processed: usize,
+    /// Total number of manifests, if known. This becomes `Some` once the
+    /// listing stream is exhausted; until then it is `None`.
+    pub manifests_total: Option<usize>,
+}
+
+/// Options for [`Dataset::tracked_files_with_options`].
+#[derive(Default)]
+pub struct TrackedFilesOptions {
+    /// If set, only include manifests with `version >= min_version`.
+    pub min_version: Option<u64>,
+    /// If set, called each time a manifest has been fully processed. The
+    /// callback runs on a background tokio task, so it must not block (it
+    /// will stall the manifest reader pipeline). Order is the order in which
+    /// manifests finish processing, which is not the version order.
+    pub progress: Option<Box<dyn Fn(TrackedFilesProgress) + Send + Sync>>,
+}
+
+// A `ManifestLocation` is ~100 bytes, so a 50k-slot mpsc channel costs ~5 MB
+// in the worst case. That's enough headroom for the lister to run well ahead
+// of the reader on datasets with hundreds of thousands of manifests, while
+// still bounding memory.
+const MAX_BUFFERED_LOCATIONS: usize = 50_000;
+
+impl Dataset {
+    /// Returns one row per (version, file) for every file referenced in any manifest.
+    ///
+    /// Each row contains the manifest version, the storage root URI, the file path
+    /// relative to that URI, and the file type.
+    ///
+    /// # Schema
+    ///
+    /// | Column     | Type                              | Notes |
+    /// |------------|-----------------------------------|-------|
+    /// | `version`  | `Int64` (non-null)                | Manifest version number |
+    /// | `base_uri` | `Dictionary(Int32, Utf8)` (non-null) | Storage root for this file |
+    /// | `path`     | `Utf8` (non-null)                 | Relative to `base_uri` |
+    /// | `type`     | `Dictionary(Int8, Utf8)` (non-null)  | One of: `data file`, `manifest`, `deletion file`, `transaction file`, `index file` |
+    ///
+    /// Output order is non-deterministic.
+    pub async fn tracked_files(&self) -> SendableRecordBatchStream {
+        self.tracked_files_with_options(TrackedFilesOptions::default())
+            .await
+    }
+
+    /// Like [`Self::tracked_files`], but with additional options for filtering
+    /// and progress reporting.
+    pub async fn tracked_files_with_options(
+        &self,
+        options: TrackedFilesOptions,
+    ) -> SendableRecordBatchStream {
+        use lance_table::io::commit::ManifestLocation;
+
+        let base = self.base.clone();
+        let uri = self.uri().to_string();
+        let object_store = self.object_store.clone();
+        let commit_handler = self.commit_handler.clone();
+
+        // Pipeline architecture:
+        //
+        // Lister ──► tx_locations ──► Reader ──┬──► tx_manifest ──► Emitter ──► tx (output)
+        //                                      └──► tx_indexes  ──► IndexLister ──► tx (output)
+
+        // Output channel: Emitter and IndexLister both send batches here.
+        let (tx, rx) = tokio::sync::mpsc::channel::<datafusion::error::Result<RecordBatch>>(4);
+        // Location channel: Lister -> Reader. Large buffer since locations are
+        // small (~100 bytes each) and we want the lister to run ahead.
+        let (tx_locations, mut rx_locations) =
+            tokio::sync::mpsc::channel::<ManifestLocation>(MAX_BUFFERED_LOCATIONS);
+        // Manifest channel: Reader -> Emitter (small buffer for backpressure
+        // since manifests can be large).
+        let (tx_manifest, mut rx_manifest) =
+            tokio::sync::mpsc::channel::<(Arc<lance_table::format::Manifest>, String, usize)>(2);
+        // Index channel: Reader -> IndexLister.
+        let (tx_indexes, mut rx_indexes) =
+            tokio::sync::mpsc::channel::<(u64, Vec<IndexMetadata>)>(8);
+
+        // Tracks estimated in-memory size of in-flight manifests. Reader adds
+        // before sending; Emitter subtracts after processing.
+        let inflight_mem = Arc::new(AtomicUsize::new(0));
+        let mem_notify = Arc::new(tokio::sync::Notify::new());
+
+        // Progress: total is set by Lister once listing finishes, read by Emitter.
+        let total_manifests: Arc<std::sync::OnceLock<usize>> = Arc::new(std::sync::OnceLock::new());
+
+        // --- Lister task ---
+        // Lists manifest locations, applies min_version filter, and counts the
+        // total. Locations are lightweight so we buffer up to MAX_BUFFERED_LOCATIONS.
+        let tx_err_lister = tx.clone();
+        let os_lister = object_store.clone();
+        let base_lister = base.clone();
+        let total_manifests_lister = total_manifests.clone();
+        let min_version = options.min_version;
+        tokio::spawn(async move {
+            let result: lance_core::Result<()> = async {
+                let mut locations =
+                    commit_handler.list_manifest_locations(&base_lister, &os_lister, false);
+                let mut count = 0usize;
+                while let Some(loc) = locations.next().await {
+                    let loc = loc?;
+                    if let Some(min_v) = min_version
+                        && loc.version < min_v
+                    {
+                        continue;
+                    }
+                    count += 1;
+                    if tx_locations.send(loc).await.is_err() {
+                        return Ok(());
+                    }
+                }
+                let _ = total_manifests_lister.set(count);
+                Ok(())
+            }
+            .await;
+            if let Err(e) = result {
+                let _ = tx_err_lister
+                    .send(Err(datafusion::error::DataFusionError::from(e)))
+                    .await;
+            }
+        });
+
+        // --- Reader task ---
+        // Reads manifests with memory-aware parallelism and fans out to
+        // Emitter (file batches) and IndexLister (index metadata).
+        let tx_err_reader = tx.clone();
+        let os_reader = object_store.clone();
+        let base_reader = base.clone();
+        let inflight_mem_reader = inflight_mem.clone();
+        let mem_notify_reader = mem_notify.clone();
+        tokio::spawn(async move {
+            let result: lance_core::Result<()> = async {
+                let max_parallelism = os_reader.io_parallelism();
+
+                type ManifestResult = lance_core::Result<(
+                    Arc<lance_table::format::Manifest>,
+                    String,
+                    Vec<IndexMetadata>,
+                    usize,
+                )>;
+                let mut in_flight: FuturesUnordered<
+                    std::pin::Pin<Box<dyn Future<Output = ManifestResult> + Send>>,
+                > = FuturesUnordered::new();
+                let mut locations_exhausted = false;
+
+                loop {
+                    let can_launch = !locations_exhausted
+                        && in_flight.len() < max_parallelism
+                        && (in_flight.is_empty()
+                            || inflight_mem_reader.load(Ordering::Acquire)
+                                < MANIFEST_MEMORY_BUDGET);
+
+                    if in_flight.is_empty() && !can_launch {
+                        break;
+                    }
+
+                    tokio::select! {
+                        biased;
+                        // Always drain completed reads first.
+                        Some(item) = in_flight.next(), if !in_flight.is_empty() => {
+                            let (manifest, manifest_path, indexes, estimated) = item?;
+                            let version = manifest.version;
+                            if tx_manifest
+                                .send((manifest, manifest_path, estimated))
+                                .await
+                                .is_err()
+                            {
+                                return Ok(());
+                            }
+                            if !indexes.is_empty()
+                                && tx_indexes.send((version, indexes)).await.is_err()
+                            {
+                                return Ok(());
+                            }
+                        }
+                        // Receive next location and start a read.
+                        loc = rx_locations.recv(), if can_launch => {
+                            match loc {
+                                Some(loc) => {
+                                    let estimated =
+                                        loc.size.unwrap_or(0) as usize
+                                            * MANIFEST_DECOMPRESSION_RATIO;
+                                    inflight_mem_reader.fetch_add(estimated, Ordering::AcqRel);
+
+                                    let os = os_reader.clone();
+                                    let base = base_reader.clone();
+                                    in_flight.push(Box::pin(async move {
+                                        let manifest =
+                                            read_manifest(&os, &loc.path, loc.size).await?;
+                                        let indexes =
+                                            read_manifest_indexes(&os, &loc, &manifest).await?;
+                                        let manifest_path =
+                                            remove_prefix(&loc.path, &base).to_string();
+                                        lance_core::Result::Ok((
+                                            Arc::new(manifest),
+                                            manifest_path,
+                                            indexes,
+                                            estimated,
+                                        ))
+                                    }));
+                                }
+                                None => {
+                                    locations_exhausted = true;
+                                }
+                            }
+                        }
+                        // Wake up when Emitter frees memory.
+                        _ = mem_notify_reader.notified(),
+                            if !can_launch && !in_flight.is_empty() => {}
+                    }
+                }
+                Ok(())
+            }
+            .await;
+
+            if let Err(e) = result {
+                let _ = tx_err_reader
+                    .send(Err(datafusion::error::DataFusionError::from(e)))
+                    .await;
+            }
+        });
+
+        // --- Emitter task ---
+        // Converts manifests into file-row batches, releases memory budget,
+        // and reports progress.
+        let tx_emitter = tx.clone();
+        let uri_emitter = uri.clone();
+        let progress_cb = options.progress;
+        tokio::spawn(async move {
+            let mut processed = 0usize;
+            while let Some((manifest, manifest_path, estimated)) = rx_manifest.recv().await {
+                let batches = manifest_file_batches(&manifest, &uri_emitter, &manifest_path);
+                for batch_result in batches {
+                    let df_result = batch_result.map_err(datafusion::error::DataFusionError::from);
+                    if tx_emitter.send(df_result).await.is_err() {
+                        return;
+                    }
+                }
+                drop(manifest);
+                inflight_mem.fetch_sub(estimated, Ordering::AcqRel);
+                mem_notify.notify_one();
+
+                processed += 1;
+                if let Some(ref cb) = progress_cb {
+                    cb(TrackedFilesProgress {
+                        manifests_processed: processed,
+                        manifests_total: total_manifests.get().copied(),
+                    });
+                }
+            }
+        });
+
+        // --- IndexLister task ---
+        // Lists index directories and emits index file batches.
+        let tx_idx = tx;
+        let uri_idx = uri;
+        let os_idx = object_store;
+        let base_idx = base;
+        tokio::spawn(async move {
+            let mut uuid_cache: HashMap<Uuid, Vec<object_store::ObjectMeta>> = HashMap::new();
+            while let Some((version, indexes)) = rx_indexes.recv().await {
+                let uuids: Vec<Uuid> = indexes.iter().map(|idx| idx.uuid).collect();
+                match get_index_files(uuids, &base_idx, &os_idx, &mut uuid_cache).await {
+                    Ok(index_paths) if !index_paths.is_empty() => {
+                        match index_file_batch(version, &uri_idx, &index_paths).await {
+                            Ok(batch) => {
+                                if tx_idx.send(Ok(batch)).await.is_err() {
+                                    return;
+                                }
+                            }
+                            Err(e) => {
+                                let _ = tx_idx
+                                    .send(Err(datafusion::error::DataFusionError::from(e)))
+                                    .await;
+                                return;
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        let _ = tx_idx
+                            .send(Err(datafusion::error::DataFusionError::from(e)))
+                            .await;
+                        return;
+                    }
+                    _ => {}
+                }
+            }
+        });
+
+        let stream = tokio_stream::wrappers::ReceiverStream::new(rx);
+
+        Box::pin(RecordBatchStreamAdapter::new(
+            TRACKED_FILES_SCHEMA.clone(),
+            stream,
+        ))
+    }
+
+    /// Returns one row per file that physically exists at the dataset's base URI.
+    ///
+    /// This scans the primary object store root only. Additional `base_paths`
+    /// entries in the manifest (for externally-located data files) are not
+    /// scanned by this method.
+    ///
+    /// # Schema
+    ///
+    /// | Column          | Type                                       | Notes |
+    /// |-----------------|--------------------------------------------|-------|
+    /// | `base_uri`      | `Dictionary(Int32, Utf8)` (non-null)       | Storage root |
+    /// | `path`          | `Utf8` (non-null)                          | Relative to `base_uri` |
+    /// | `size_bytes`    | `Int64` (non-null)                         | File size in bytes |
+    /// | `last_modified` | `Timestamp(Microsecond, "UTC")` (non-null) | Last modification time |
+    pub async fn all_files(&self) -> SendableRecordBatchStream {
+        let base = self.base.clone();
+        let uri = self.uri().to_string();
+        let object_store = self.object_store.clone();
+
+        let stream = object_store
+            .list(Some(base.clone()))
+            .try_chunks(4000)
+            .map_err(|err| err.1)
+            .and_then(
+                move |chunk| match build_all_files_batch(&chunk, &base, &uri) {
+                    Ok(batch) => futures::future::ok(batch),
+                    Err(e) => futures::future::err(e),
+                },
+            )
+            .map_err(datafusion::error::DataFusionError::from);
+
+        Box::pin(RecordBatchStreamAdapter::new(
+            arrow::ALL_FILES_SCHEMA.clone(),
+            stream,
+        ))
+    }
+}
+
+fn build_all_files_batch(
+    chunk: &[object_store::ObjectMeta],
+    base: &Path,
+    uri: &str,
+) -> Result<RecordBatch> {
+    let n = chunk.len();
+    let mut base_uri_builder = StringDictionaryBuilder::<Int32Type>::with_capacity(n, 1, uri.len());
+    let path_capacity = chunk.iter().map(|m| m.location.as_ref().len()).sum();
+    let mut path_builder = StringBuilder::with_capacity(n, path_capacity);
+    let mut size_builder = Int64Builder::with_capacity(n);
+    let mut ts_builder = TimestampMicrosecondBuilder::with_capacity(n).with_timezone("UTC");
+
+    for meta in chunk {
+        let rel = remove_prefix(&meta.location, base);
+        base_uri_builder.append_value(uri);
+        path_builder.append_value(rel.as_ref());
+        size_builder.append_value(meta.size as i64);
+        ts_builder.append_value(meta.last_modified.timestamp_micros());
+    }
+
+    RecordBatch::try_new(
+        arrow::ALL_FILES_SCHEMA.clone(),
+        vec![
+            Arc::new(base_uri_builder.finish()),
+            Arc::new(path_builder.finish()),
+            Arc::new(size_builder.finish()),
+            Arc::new(ts_builder.finish()),
+        ],
+    )
+    .map_err(Into::into)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::Dataset;
+    use crate::index::DatasetIndexExt;
+    use crate::index::vector::VectorIndexParams;
+    use arrow_array::{Array, Int32Array, RecordBatchIterator, StringArray};
+    use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit};
+    use futures::TryStreamExt;
+    use lance_index::IndexType;
+    use lance_linalg::distance::MetricType;
+    use lance_testing::datagen::some_batch;
+    use std::collections::HashSet;
+
+    async fn collect_rows(stream: SendableRecordBatchStream) -> Vec<RecordBatch> {
+        stream.try_collect::<Vec<_>>().await.unwrap()
+    }
+
+    fn count_rows(batches: &[RecordBatch]) -> usize {
+        batches.iter().map(|b| b.num_rows()).sum()
+    }
+
+    fn dict_value_at(col: &dyn arrow_array::Array, i: usize) -> String {
+        if let Some(dict) = col
+            .as_any()
+            .downcast_ref::<arrow_array::DictionaryArray<arrow_array::types::Int8Type>>()
+        {
+            let values = dict
+                .values()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            values.value(dict.keys().value(i) as usize).to_string()
+        } else if let Some(dict) = col
+            .as_any()
+            .downcast_ref::<arrow_array::DictionaryArray<arrow_array::types::Int32Type>>()
+        {
+            let values = dict
+                .values()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            values.value(dict.keys().value(i) as usize).to_string()
+        } else {
+            panic!("expected a dictionary array with Int8 or Int32 keys");
+        }
+    }
+
+    fn collect_column_values(batches: &[RecordBatch], col: &str) -> Vec<String> {
+        batches
+            .iter()
+            .flat_map(|b| {
+                let col = b.column_by_name(col).unwrap();
+                (0..col.len()).map(|i| dict_value_at(col.as_ref(), i))
+            })
+            .collect()
+    }
+
+    fn make_simple_batch() -> impl arrow_array::RecordBatchReader {
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        RecordBatchIterator::new(vec![Ok(batch)], schema)
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_basic() {
+        let uri = "memory://test_tracked_files_basic";
+
+        // Create then append twice to get 3 manifest versions.
+        let mut ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+
+        let stream = ds.tracked_files().await;
+        let schema = stream.schema();
+        let batches = collect_rows(stream).await;
+
+        // Schema is correct.
+        assert_eq!(schema.field(0).name(), "version");
+        assert_eq!(schema.field(1).name(), "base_uri");
+        assert_eq!(schema.field(2).name(), "path");
+        assert_eq!(schema.field(3).name(), "type");
+
+        let n = count_rows(&batches);
+        // At minimum: 3 manifests + 3 data files = 6 rows
+        assert!(n >= 6, "expected at least 6 rows, got {n}");
+
+        let types: HashSet<String> = collect_column_values(&batches, "type")
+            .into_iter()
+            .collect();
+        assert!(types.contains("manifest"), "missing 'manifest' rows");
+        assert!(types.contains("data file"), "missing 'data file' rows");
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_deletion() {
+        let uri = "memory://test_tracked_files_deletion";
+
+        let mut ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+        ds.delete("id = 2").await.unwrap();
+
+        let stream = ds.tracked_files().await;
+        let batches = collect_rows(stream).await;
+
+        let types: HashSet<String> = collect_column_values(&batches, "type")
+            .into_iter()
+            .collect();
+        assert!(
+            types.contains("deletion file"),
+            "missing 'deletion file' rows after delete; got types: {:?}",
+            types
+        );
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_transaction() {
+        let uri = "memory://test_tracked_files_transaction";
+
+        // Normal writes record transaction files by default.
+        let mut ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+
+        let stream = ds.tracked_files().await;
+        let batches = collect_rows(stream).await;
+
+        let types: HashSet<String> = collect_column_values(&batches, "type")
+            .into_iter()
+            .collect();
+        assert!(
+            types.contains("transaction file"),
+            "expected 'transaction file' rows; got types: {:?}",
+            types
+        );
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_index() {
+        let uri = "memory://test_tracked_files_index";
+
+        let mut ds = Dataset::write(some_batch(), uri, None).await.unwrap();
+        let params = VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 5);
+        ds.create_index(&["indexable"], IndexType::Vector, None, &params, true)
+            .await
+            .unwrap();
+
+        let stream = ds.tracked_files().await;
+        let batches = collect_rows(stream).await;
+
+        let types: HashSet<String> = collect_column_values(&batches, "type")
+            .into_iter()
+            .collect();
+        assert!(
+            types.contains("index file"),
+            "expected 'index file' rows after vector index creation; got types: {:?}",
+            types
+        );
+    }
+
+    fn collect_versions(batches: &[RecordBatch]) -> Vec<i64> {
+        batches
+            .iter()
+            .flat_map(|b| {
+                let col = b
+                    .column_by_name("version")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<arrow_array::Int64Array>()
+                    .unwrap();
+                (0..col.len()).map(|i| col.value(i)).collect::<Vec<_>>()
+            })
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_min_version() {
+        let uri = "memory://test_tracked_files_min_version";
+
+        // Create 3 versions.
+        let mut ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+
+        // Without filter: should have rows from versions 1, 2, 3.
+        let stream = ds.tracked_files().await;
+        let all_batches = collect_rows(stream).await;
+        let all_versions: HashSet<i64> = collect_versions(&all_batches).into_iter().collect();
+        assert!(all_versions.contains(&1));
+        assert!(all_versions.contains(&2));
+        assert!(all_versions.contains(&3));
+
+        // With min_version=3: should only have version 3.
+        let stream = ds
+            .tracked_files_with_options(TrackedFilesOptions {
+                min_version: Some(3),
+                ..Default::default()
+            })
+            .await;
+        let filtered_batches = collect_rows(stream).await;
+        let filtered_versions: HashSet<i64> =
+            collect_versions(&filtered_batches).into_iter().collect();
+        assert_eq!(filtered_versions, HashSet::from([3]));
+
+        // With min_version=2: should have versions 2 and 3.
+        let stream = ds
+            .tracked_files_with_options(TrackedFilesOptions {
+                min_version: Some(2),
+                ..Default::default()
+            })
+            .await;
+        let filtered_batches = collect_rows(stream).await;
+        let filtered_versions: HashSet<i64> =
+            collect_versions(&filtered_batches).into_iter().collect();
+        assert_eq!(filtered_versions, HashSet::from([2, 3]));
+    }
+
+    #[tokio::test]
+    async fn test_tracked_files_progress() {
+        let uri = "memory://test_tracked_files_progress";
+
+        let mut ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+        ds.append(make_simple_batch(), None).await.unwrap();
+
+        let updates = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let updates_clone = updates.clone();
+
+        let stream = ds
+            .tracked_files_with_options(TrackedFilesOptions {
+                progress: Some(Box::new(move |p| {
+                    updates_clone.lock().unwrap().push(p);
+                })),
+                ..Default::default()
+            })
+            .await;
+        // Consume the full stream to drive all tasks to completion.
+        let _batches = collect_rows(stream).await;
+
+        let updates = updates.lock().unwrap();
+        // Should have exactly 3 progress updates (one per manifest).
+        assert_eq!(updates.len(), 3, "expected 3 progress updates");
+        // Processed counts should be monotonically increasing.
+        for (i, u) in updates.iter().enumerate() {
+            assert_eq!(u.manifests_processed, i + 1);
+        }
+        // The last update should know the total.
+        let last = updates.last().unwrap();
+        assert_eq!(last.manifests_total, Some(3));
+    }
+
+    fn make_multi_row_batch(rows: usize) -> impl arrow_array::RecordBatchReader {
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..rows as i32))],
+        )
+        .unwrap();
+        RecordBatchIterator::new(vec![Ok(batch)], schema)
+    }
+
+    /// Multi-fragment scenario: write 6 rows split across 3 fragments, delete
+    /// one row to produce a deletion file, then assert that every path
+    /// `tracked_files` emits for the latest version actually exists in the
+    /// `all_files` listing of the dataset directory.
+    #[tokio::test]
+    async fn test_tracked_files_paths_match_disk() {
+        use crate::dataset::WriteParams;
+
+        let uri = "memory://test_tracked_files_paths_match_disk";
+
+        let write_params = WriteParams {
+            max_rows_per_file: 2,
+            ..Default::default()
+        };
+        let mut ds = Dataset::write(make_multi_row_batch(6), uri, Some(write_params))
+            .await
+            .unwrap();
+        // Triggers a deletion file on one of the fragments.
+        ds.delete("id = 1").await.unwrap();
+        let latest_version = ds.version().version as i64;
+
+        // Sanity-check the multi-fragment setup: 3 data files in the latest manifest.
+        assert_eq!(
+            ds.get_fragments().len(),
+            3,
+            "expected 3 fragments from max_rows_per_file=2 over 6 rows"
+        );
+
+        let tracked = collect_rows(ds.tracked_files().await).await;
+        let all = collect_rows(ds.all_files().await).await;
+
+        let all_paths: HashSet<String> = all
+            .iter()
+            .flat_map(|b| {
+                let col = b
+                    .column_by_name("path")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                (0..col.len()).map(|i| col.value(i).to_string())
+            })
+            .collect();
+
+        // Collect tracked paths grouped by type for the latest version only.
+        let mut tracked_at_latest: HashMap<String, Vec<String>> = HashMap::new();
+        for batch in &tracked {
+            let versions = batch
+                .column_by_name("version")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<arrow_array::Int64Array>()
+                .unwrap();
+            let paths = batch
+                .column_by_name("path")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let types = batch.column_by_name("type").unwrap();
+            for i in 0..batch.num_rows() {
+                if versions.value(i) == latest_version {
+                    tracked_at_latest
+                        .entry(dict_value_at(types.as_ref(), i))
+                        .or_default()
+                        .push(paths.value(i).to_string());
+                }
+            }
+        }
+
+        // Every file tracked at the latest version must exist on disk.
+        for (file_type, paths) in &tracked_at_latest {
+            for p in paths {
+                assert!(
+                    all_paths.contains(p),
+                    "tracked {file_type} path {p:?} not present in all_files (got {all_paths:?})"
+                );
+            }
+        }
+
+        // The latest manifest references one manifest, 3 data files, and 1 deletion file.
+        assert_eq!(
+            tracked_at_latest.get("manifest").map(Vec::len),
+            Some(1),
+            "expected 1 manifest row at latest version"
+        );
+        assert_eq!(
+            tracked_at_latest.get("data file").map(Vec::len),
+            Some(3),
+            "expected 3 data files at latest version"
+        );
+        assert_eq!(
+            tracked_at_latest.get("deletion file").map(Vec::len),
+            Some(1),
+            "expected 1 deletion file at latest version"
+        );
+
+        // Path shapes are as documented (relative to base_uri, no leading slash).
+        for p in tracked_at_latest.get("data file").unwrap() {
+            assert!(
+                p.starts_with("data/"),
+                "data file path {p:?} should start with data/"
+            );
+        }
+        let manifest_path = &tracked_at_latest.get("manifest").unwrap()[0];
+        assert!(
+            manifest_path.starts_with("_versions/") && manifest_path.ends_with(".manifest"),
+            "manifest path {manifest_path:?} should match _versions/<n>.manifest"
+        );
+        let deletion_path = &tracked_at_latest.get("deletion file").unwrap()[0];
+        assert!(
+            deletion_path.starts_with("_deletions/"),
+            "deletion path {deletion_path:?} should start with _deletions/"
+        );
+    }
+
+    /// Each `DataFile` inside a fragment carries its own `base_id`; the
+    /// emitted `base_uri` must be looked up per file, not per fragment.
+    #[test]
+    fn test_manifest_file_rows_per_file_base_id() {
+        use lance_core::datatypes::{Field as LanceField, Schema as LanceSchema};
+        use lance_io::utils::CachedFileSize;
+        use lance_table::format::{
+            BasePath, DataFile, DataStorageFormat, DeletionFile, DeletionFileType, Fragment,
+            Manifest,
+        };
+
+        let schema = LanceSchema {
+            fields: vec![LanceField::try_from(&Field::new("id", DataType::Int32, false)).unwrap()],
+            metadata: Default::default(),
+        };
+
+        let mk_file = |path: &str, base_id: Option<u32>| DataFile {
+            path: path.to_string(),
+            fields: Arc::from(vec![0]),
+            column_indices: Arc::from(Vec::<i32>::new()),
+            file_major_version: 2,
+            file_minor_version: 0,
+            file_size_bytes: CachedFileSize::unknown(),
+            base_id,
+        };
+
+        let fragment = Fragment {
+            id: 0,
+            files: vec![
+                mk_file("a.lance", Some(1)),
+                mk_file("b.lance", Some(2)),
+                // No base_id -> falls back to the dataset base_uri.
+                mk_file("c.lance", None),
+            ],
+            overlays: vec![],
+            // Deletion files also carry a base_id when they originate from a
+            // shallow clone, and must resolve against base_paths too.
+            deletion_file: Some(DeletionFile {
+                read_version: 1,
+                id: 7,
+                file_type: DeletionFileType::Bitmap,
+                num_deleted_rows: Some(1),
+                base_id: Some(2),
+            }),
+            row_id_meta: None,
+            physical_rows: Some(3),
+            last_updated_at_version_meta: None,
+            created_at_version_meta: None,
+        };
+
+        let mut base_paths = HashMap::new();
+        base_paths.insert(
+            1,
+            BasePath::new(1, "s3://bucket-a/root".to_string(), None, false),
+        );
+        base_paths.insert(
+            2,
+            BasePath::new(2, "s3://bucket-b/root".to_string(), None, false),
+        );
+
+        let manifest = Manifest::new(
+            schema,
+            Arc::new(vec![fragment]),
+            DataStorageFormat::default(),
+            base_paths,
+        );
+
+        let rows: Vec<_> =
+            manifest_file_rows(&manifest, "memory://main", "_versions/1.manifest").collect();
+        let by_path: HashMap<&str, &str> = rows
+            .iter()
+            .filter(|r| matches!(r.file_type, FileType::DataFile))
+            .map(|r| (r.path.as_ref(), r.base_uri.as_ref()))
+            .collect();
+
+        assert_eq!(by_path.get("data/a.lance"), Some(&"s3://bucket-a/root"));
+        assert_eq!(by_path.get("data/b.lance"), Some(&"s3://bucket-b/root"));
+        assert_eq!(by_path.get("data/c.lance"), Some(&"memory://main"));
+
+        let deletion = rows
+            .iter()
+            .find(|r| matches!(r.file_type, FileType::DeletionFile))
+            .expect("deletion file row");
+        assert_eq!(deletion.path.as_ref(), "_deletions/0-1-7.bin");
+        assert_eq!(deletion.base_uri.as_ref(), "s3://bucket-b/root");
+    }
+
+    #[tokio::test]
+    async fn test_all_files_basic() {
+        let uri = "memory://test_all_files_basic";
+        let ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+
+        let stream = ds.all_files().await;
+        let schema = stream.schema();
+        let batches = collect_rows(stream).await;
+
+        assert_eq!(schema.field(0).name(), "base_uri");
+        assert_eq!(schema.field(1).name(), "path");
+        assert_eq!(schema.field(2).name(), "size_bytes");
+        assert_eq!(schema.field(3).name(), "last_modified");
+
+        let n = count_rows(&batches);
+        // A dataset always has at least a manifest and a data file.
+        assert!(n >= 2, "expected at least 2 physical files, got {n}");
+
+        // Verify sizes and timestamps are populated (non-zero).
+        for batch in &batches {
+            let sizes = batch
+                .column_by_name("size_bytes")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<arrow_array::Int64Array>()
+                .unwrap();
+            for i in 0..sizes.len() {
+                assert!(
+                    sizes.value(i) > 0,
+                    "size_bytes should be positive, got {}",
+                    sizes.value(i)
+                );
+            }
+
+            let ts = batch
+                .column_by_name("last_modified")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<arrow_array::TimestampMicrosecondArray>()
+                .unwrap();
+            for i in 0..ts.len() {
+                assert!(
+                    ts.value(i) > 0,
+                    "last_modified should be positive, got {}",
+                    ts.value(i)
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_all_files_schema() {
+        let uri = "memory://test_all_files_schema";
+        let ds = Dataset::write(make_simple_batch(), uri, None)
+            .await
+            .unwrap();
+
+        let stream = ds.all_files().await;
+        let schema = stream.schema();
+
+        assert_eq!(schema.fields().len(), 4);
+        assert_eq!(schema.field(0).name(), "base_uri");
+        assert!(matches!(
+            schema.field(0).data_type(),
+            DataType::Dictionary(_, _)
+        ));
+        assert_eq!(schema.field(1).name(), "path");
+        assert_eq!(schema.field(1).data_type(), &DataType::Utf8);
+        assert_eq!(schema.field(2).name(), "size_bytes");
+        assert_eq!(schema.field(2).data_type(), &DataType::Int64);
+        assert_eq!(schema.field(3).name(), "last_modified");
+        assert!(matches!(
+            schema.field(3).data_type(),
+            DataType::Timestamp(TimeUnit::Microsecond, _)
+        ));
+    }
+}
diff --git a/rust/lance/src/dataset/files/arrow.rs b/rust/lance/src/dataset/files/arrow.rs
new file mode 100644
index 00000000000..22d767fe13b
--- /dev/null
+++ b/rust/lance/src/dataset/files/arrow.rs
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::{Arc, LazyLock};
+
+use arrow::array::{ArrayBuilder, Int8Builder};
+use arrow::datatypes::Int8Type;
+use arrow_array::builder::{Int64Builder, StringBuilder, StringDictionaryBuilder};
+use arrow_array::types::Int32Type;
+use arrow_array::{ArrayRef, DictionaryArray, RecordBatch};
+use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
+use lance_core::Result;
+
+use super::FileRow;
+use super::file_types::FileType;
+
+pub static FILE_TYPE_DICT_ARRAY: LazyLock<ArrayRef> = LazyLock::new(|| {
+    let mut builder = StringBuilder::with_capacity(5, 20);
+    builder.append_value(FileType::Manifest.to_string());
+    builder.append_value(FileType::DataFile.to_string());
+    builder.append_value(FileType::DeletionFile.to_string());
+    builder.append_value(FileType::TransactionFile.to_string());
+    builder.append_value(FileType::IndexFile.to_string());
+    Arc::new(builder.finish())
+});
+
+pub struct FileTypeArrayBuilder {
+    builder: Int8Builder,
+}
+
+impl FileTypeArrayBuilder {
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            builder: Int8Builder::with_capacity(capacity),
+        }
+    }
+
+    pub fn append_value(&mut self, file_type: FileType) {
+        let value = file_type.into();
+        self.builder.append_value(value);
+    }
+
+    pub fn finish(mut self) -> DictionaryArray<Int8Type> {
+        let indices = self.builder.finish();
+        DictionaryArray::new(indices, FILE_TYPE_DICT_ARRAY.clone())
+    }
+}
+
+pub(super) static TRACKED_FILES_SCHEMA: LazyLock<SchemaRef> = LazyLock::new(|| {
+    Arc::new(Schema::new(vec![
+        Field::new("version", DataType::Int64, false),
+        Field::new(
+            "base_uri",
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+            false,
+        ),
+        Field::new("path", DataType::Utf8, false),
+        Field::new(
+            "type",
+            DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)),
+            false,
+        ),
+    ]))
+});
+
+pub(super) static ALL_FILES_SCHEMA: LazyLock<SchemaRef> = LazyLock::new(|| {
+    Arc::new(Schema::new(vec![
+        Field::new(
+            "base_uri",
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+            false,
+        ),
+        Field::new("path", DataType::Utf8, false),
+        Field::new("size_bytes", DataType::Int64, false),
+        Field::new(
+            "last_modified",
+            DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
+            false,
+        ),
+    ]))
+});
+
+/// Arrow batch builder for the `tracked_files` schema.
+///
+/// Construct with [`with_capacity`](Self::with_capacity) to pre-size the
+/// underlying buffers, then call [`extend`](Self::extend) to fill rows in bulk.
+pub(super) struct TrackedFileBatch {
+    version: Int64Builder,
+    base_uri: StringDictionaryBuilder<Int32Type>,
+    path: StringBuilder,
+    file_type: FileTypeArrayBuilder,
+}
+
+impl TrackedFileBatch {
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            version: Int64Builder::with_capacity(capacity),
+            // Most of the time, there is only 1 base_uri
+            base_uri: StringDictionaryBuilder::with_capacity(capacity, 1, 20),
+            path: StringBuilder::with_capacity(capacity, capacity * 50),
+            file_type: FileTypeArrayBuilder::with_capacity(capacity),
+        }
+    }
+
+    pub fn append(&mut self, row: &FileRow) {
+        self.version.append_value(row.version as i64);
+        self.base_uri.append_value(&row.base_uri);
+        self.path.append_value(&row.path);
+        self.file_type.append_value(row.file_type);
+    }
+
+    pub fn len(&self) -> usize {
+        self.version.len()
+    }
+
+    pub fn finish(mut self) -> Result<RecordBatch> {
+        RecordBatch::try_new(
+            TRACKED_FILES_SCHEMA.clone(),
+            vec![
+                Arc::new(self.version.finish()),
+                Arc::new(self.base_uri.finish()),
+                Arc::new(self.path.finish()),
+                Arc::new(self.file_type.finish()),
+            ],
+        )
+        .map_err(Into::into)
+    }
+}
diff --git a/rust/lance/src/dataset/files/file_types.rs b/rust/lance/src/dataset/files/file_types.rs
new file mode 100644
index 00000000000..7c20a81eae5
--- /dev/null
+++ b/rust/lance/src/dataset/files/file_types.rs
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+// Discriminants are the dictionary keys used in the `tracked_files` output
+// schema; they must stay in sync with `FILE_TYPE_DICT_ARRAY` in `arrow.rs`.
+#[repr(i8)]
+#[derive(Debug, Clone, Copy)]
+pub enum FileType {
+    Manifest = 0,
+    DataFile = 1,
+    DeletionFile = 2,
+    TransactionFile = 3,
+    IndexFile = 4,
+}
+
+impl std::fmt::Display for FileType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = match self {
+            Self::Manifest => "manifest",
+            Self::DataFile => "data file",
+            Self::DeletionFile => "deletion file",
+            Self::TransactionFile => "transaction file",
+            Self::IndexFile => "index file",
+        };
+        write!(f, "{s}")
+    }
+}
+
+impl From<FileType> for i8 {
+    fn from(file_type: FileType) -> Self {
+        file_type as Self
+    }
+}
diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs
index 11851e8846e..b365dcb4d26 100644
--- a/rust/lance/src/dataset/fragment.rs
+++ b/rust/lance/src/dataset/fragment.rs
@@ -22,6 +22,7 @@ use datafusion::logical_expr::Expr;
 use datafusion::scalar::ScalarValue;
 use futures::future::{BoxFuture, try_join_all};
 use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, join, stream};
+use lance_arrow::json::{convert_json_columns, has_json_fields, is_arrow_json_field};
 use lance_arrow::{RecordBatchExt, SchemaExt};
 use lance_core::datatypes::{OnMissing, OnTypeMismatch, SchemaCompareOptions};
 use lance_core::utils::address::RowAddress;
@@ -37,7 +38,9 @@ use lance_encoding::decoder::DecoderPlugins;
 use lance_file::previous::reader::{
     FileReader as PreviousFileReader, read_batch as previous_read_batch,
 };
-use lance_file::reader::{CachedFileMetadata, FileReaderOptions, ReaderProjection};
+use lance_file::reader::{
+    CachedFileMetadata, FileMetadataIndex, FileReaderOptions, ProjectedFileReader, ReaderProjection,
+};
 use lance_file::version::LanceFileVersion;
 use lance_file::{LanceEncodingsIo, determine_file_version};
 use lance_io::ReadBatchParams;
@@ -62,6 +65,9 @@ use super::updater::Updater;
 use super::{NewColumnTransform, WriteParams, schema_evolution};
 use crate::dataset::Dataset;
 use crate::dataset::fragment::session::FragmentSession;
+use crate::dataset::overlay::{
+    OverlayReadPlanner, merge_overlay_batch, plan_overlays, resolve_overlays,
+};
 use crate::io::deletion::read_dataset_deletion_file;
 
 /// Result of [`FileFragment::update_columns_with_offsets`]: updated fragment metadata, modified field ids,
@@ -133,7 +139,7 @@ pub trait GenericFileReader: std::fmt::Debug + Send + Sync {
     fn projection(&self) -> &Arc<Schema>;
 
     /// Get storage statistics for this file (ignored by v1 reader)
-    fn storage_stats(&self) -> Vec<(u32, u64)>;
+    fn storage_stats(&self) -> Result<Vec<(u32, u64)>>;
 
     // Helper functions to fallback to the legacy implementation while we
     // slowly migrate functionality over to the generic reader
@@ -299,9 +305,9 @@ impl GenericFileReader for V1Reader {
         self.reader.len() as u32
     }
 
-    fn storage_stats(&self) -> Vec<(u32, u64)> {
+    fn storage_stats(&self) -> Result<Vec<(u32, u64)>> {
         // No-op for v1 files
-        Vec::new()
+        Ok(Vec::new())
     }
 
     fn clone_box(&self) -> Box<dyn GenericFileReader> {
@@ -328,7 +334,7 @@ mod v2_adapter {
 
     #[derive(Debug, Clone)]
     pub struct Reader {
-        reader: Arc<lance_file::reader::FileReader>,
+        reader: Arc<ProjectedFileReader>,
         projection: Arc<Schema>,
         field_id_to_column_idx: Arc<BTreeMap<u32, u32>>,
         default_priority: u32,
@@ -337,7 +343,7 @@ mod v2_adapter {
 
     impl Reader {
         pub fn new(
-            reader: Arc<lance_file::reader::FileReader>,
+            reader: Arc<ProjectedFileReader>,
             projection: Arc<Schema>,
             field_id_to_column_idx: Arc<BTreeMap<u32, u32>>,
             default_priority: u32,
@@ -363,7 +369,7 @@ mod v2_adapter {
         ) -> BoxFuture<'_, Result<ReadBatchTaskStream>> {
             async move {
                 let projection = ReaderProjection::from_field_ids(
-                    self.reader.metadata().version(),
+                    self.reader.version(),
                     projection.as_ref(),
                     self.field_id_to_column_idx.as_ref(),
                 )?;
@@ -393,7 +399,7 @@ mod v2_adapter {
         ) -> BoxFuture<'_, Result<ReadBatchTaskStream>> {
             async move {
                 let projection = ReaderProjection::from_field_ids(
-                    self.reader.metadata().version(),
+                    self.reader.version(),
                     projection.as_ref(),
                     self.field_id_to_column_idx.as_ref(),
                 )?;
@@ -422,7 +428,7 @@ mod v2_adapter {
         ) -> BoxFuture<'_, Result<ReadBatchTaskStream>> {
             async move {
                 let projection = ReaderProjection::from_field_ids(
-                    self.reader.metadata().version(),
+                    self.reader.version(),
                     projection.as_ref(),
                     self.field_id_to_column_idx.as_ref(),
                 )?;
@@ -454,7 +460,7 @@ mod v2_adapter {
             let indices = UInt32Array::from(indices.to_vec());
             async move {
                 let projection = ReaderProjection::from_field_ids(
-                    self.reader.metadata().version(),
+                    self.reader.version(),
                     projection.as_ref(),
                     self.field_id_to_column_idx.as_ref(),
                 )?;
@@ -487,8 +493,10 @@ mod v2_adapter {
             .boxed()
         }
 
-        fn storage_stats(&self) -> Vec<(u32, u64)> {
-            let file_statistics = self.reader.file_statistics();
+        fn storage_stats(&self) -> Result<Vec<(u32, u64)>> {
+            let file_statistics = self.reader.file_statistics().ok_or_else(|| {
+                Error::internal("storage_stats requires full file metadata".to_string())
+            })?;
             let column_idx_to_field_id = self
                 .field_id_to_column_idx
                 .iter()
@@ -505,7 +513,7 @@ mod v2_adapter {
                 }
                 stats.push((current_field_id, col_stats.size_bytes));
             }
-            stats
+            Ok(stats)
         }
 
         fn projection(&self) -> &Arc<Schema> {
@@ -514,7 +522,7 @@ mod v2_adapter {
 
         /// Return the number of rows in the file
         fn len(&self) -> u32 {
-            self.reader.metadata().num_rows as u32
+            self.reader.num_rows() as u32
         }
 
         fn clone_box(&self) -> Box<dyn GenericFileReader> {
@@ -614,9 +622,9 @@ impl GenericFileReader for NullReader {
         self.read_ranges_tasks(vec![0..num_rows].into(), batch_size, projection)
     }
 
-    fn storage_stats(&self) -> Vec<(u32, u64)> {
+    fn storage_stats(&self) -> Result<Vec<(u32, u64)>> {
         // No-op for null reader
-        Vec::new()
+        Ok(Vec::new())
     }
 
     fn projection(&self) -> &Arc<Schema> {
@@ -644,7 +652,7 @@ impl GenericFileReader for NullReader {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct FragReadConfig {
     // Add the row id column
     pub with_row_id: bool,
@@ -715,6 +723,12 @@ impl FragReadConfig {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum MetadataMode {
+    LazyAllowed,
+    Full,
+}
+
 impl FileFragment {
     /// Creates a new FileFragment.
     pub fn new(dataset: Arc<Dataset>, metadata: Fragment) -> Self {
@@ -839,13 +853,13 @@ impl FileFragment {
     ) -> Result<Vec<(u32, u64)>> {
         let mut stats = Vec::new();
         for reader in self
-            .open_readers(
+            .open_readers_with_full_metadata(
                 dataset_schema,
                 &FragReadConfig::default().with_scan_scheduler(scan_scheduler),
             )
             .await?
         {
-            stats.extend(reader.storage_stats());
+            stats.extend(reader.storage_stats()?);
         }
         Ok(stats)
     }
@@ -938,6 +952,19 @@ impl FileFragment {
             Arc::new(self.metadata.clone()),
         )?;
 
+        // Plan overlay resolution from coverage metadata (no files opened here); the
+        // readers are opened lazily on read, pruned to the rows each read touches.
+        if !self.metadata.overlays.is_empty() {
+            let planner = plan_overlays(self, projection)?;
+            if !planner.is_empty() {
+                reader.overlay = Some(OverlayReadState {
+                    planner: Arc::new(planner),
+                    fragment: Arc::new(self.clone()),
+                    read_config: Arc::new(read_config.clone()),
+                });
+            }
+        }
+
         if read_config.with_row_id {
             reader.with_row_id();
         }
@@ -958,123 +985,225 @@ impl FileFragment {
         data_file.fields.first().copied().unwrap_or(0) as u32
     }
 
-    async fn open_reader(
+    fn should_try_indexed_metadata(
+        data_file: &DataFile,
+        projection: &ReaderProjection,
+        file_version: LanceFileVersion,
+    ) -> bool {
+        if !ProjectedFileReader::supports_projection(projection, file_version) {
+            return false;
+        }
+        let total_columns = data_file
+            .column_indices
+            .iter()
+            .filter(|column_index| **column_index >= 0)
+            .count();
+        let selected_columns = projection.column_indices.len();
+        selected_columns.saturating_mul(4) < total_columns
+    }
+
+    pub(super) async fn open_reader(
+        &self,
+        data_file: &DataFile,
+        projection: Option<&Schema>,
+        read_config: &FragReadConfig,
+    ) -> Result<Option<Box<dyn GenericFileReader>>> {
+        self.open_reader_impl(
+            data_file,
+            projection,
+            read_config,
+            MetadataMode::LazyAllowed,
+        )
+        .await
+    }
+
+    async fn open_reader_with_full_metadata(
         &self,
         data_file: &DataFile,
         projection: Option<&Schema>,
         read_config: &FragReadConfig,
     ) -> Result<Option<Box<dyn GenericFileReader>>> {
-        let full_schema = self.dataset.schema();
-        // The data file may contain fields that are not part of the dataset any longer, remove those
-        let data_file_schema = data_file.schema(full_schema);
-        let projection = projection.unwrap_or(full_schema);
-        // Also remove any fields that are not part of the user's provided projection
-        let schema_per_file = Arc::new(projection.intersection_ignore_types(&data_file_schema)?);
-
-        if data_file.is_legacy_file() {
-            let max_field_id = data_file.fields.iter().max().unwrap();
-            if !schema_per_file.fields.is_empty() {
+        self.open_reader_impl(data_file, projection, read_config, MetadataMode::Full)
+            .await
+    }
+
+    fn open_reader_impl<'a>(
+        &'a self,
+        data_file: &'a DataFile,
+        projection: Option<&'a Schema>,
+        read_config: &'a FragReadConfig,
+        metadata_mode: MetadataMode,
+    ) -> BoxFuture<'a, Result<Option<Box<dyn GenericFileReader>>>> {
+        async move {
+            let full_schema = self.dataset.schema();
+            // The data file may contain fields that are not part of the dataset any longer, remove those
+            let data_file_schema = Arc::new(data_file.schema(full_schema));
+            let projection = projection.unwrap_or(full_schema);
+            // Also remove any fields that are not part of the user's provided projection
+            let schema_per_file =
+                Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?);
+
+            if data_file.is_legacy_file() {
+                let max_field_id = data_file.fields.iter().max().unwrap();
+                if !schema_per_file.fields.is_empty() {
+                    let path = self
+                        .dataset
+                        .data_file_dir(data_file)?
+                        .join(data_file.path.as_str());
+                    let object_store = self.dataset.object_store_for_data_file(data_file).await?;
+                    let field_id_offset = Self::get_field_id_offset(data_file);
+                    let reader = PreviousFileReader::try_new_with_fragment_id(
+                        &object_store,
+                        &path,
+                        self.schema().clone(),
+                        self.id() as u32,
+                        field_id_offset as i32,
+                        *max_field_id,
+                        Some(&self.dataset.metadata_cache.file_metadata_cache(&path)),
+                    )
+                    .await?;
+                    let initialized_schema = reader.schema().project_by_schema(
+                        schema_per_file.as_ref(),
+                        OnMissing::Error,
+                        OnTypeMismatch::Error,
+                    )?;
+                    let reader = V1Reader::new(reader, Arc::new(initialized_schema));
+                    let reader: Box<dyn GenericFileReader> = Box::new(reader);
+                    Ok(Some(reader))
+                } else {
+                    Ok(None)
+                }
+            } else if schema_per_file.fields.is_empty() {
+                Ok(None)
+            } else {
                 let path = self
                     .dataset
                     .data_file_dir(data_file)?
                     .join(data_file.path.as_str());
-                let object_store = self.dataset.object_store_for_data_file(data_file).await?;
-                let field_id_offset = Self::get_field_id_offset(data_file);
-                let reader = PreviousFileReader::try_new_with_fragment_id(
-                    &object_store,
-                    &path,
-                    self.schema().clone(),
-                    self.id() as u32,
-                    field_id_offset as i32,
-                    *max_field_id,
-                    Some(&self.dataset.metadata_cache.file_metadata_cache(&path)),
-                )
-                .await?;
-                let initialized_schema = reader.schema().project_by_schema(
+                let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id {
+                    // TODO: make object stores for non-default bases reuse the same scan scheduler
+                    //  currently we always create a new one
+                    let object_store = self.dataset.object_store(Some(base_id)).await?;
+                    let config = SchedulerConfig::max_bandwidth(&object_store);
+                    (
+                        ScanScheduler::new(object_store, config),
+                        read_config.reader_priority.unwrap_or(0),
+                    )
+                } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() {
+                    (
+                        scan_scheduler.clone(),
+                        read_config.reader_priority.unwrap_or(0),
+                    )
+                } else {
+                    (
+                        ScanScheduler::new(
+                            self.dataset.object_store.clone(),
+                            SchedulerConfig::max_bandwidth(&self.dataset.object_store),
+                        ),
+                        0,
+                    )
+                };
+                let file_scheduler = store_scheduler
+                    .open_file_with_priority(
+                        &path,
+                        reader_priority as u64,
+                        &data_file.file_size_bytes,
+                    )
+                    .await?;
+                let path = file_scheduler.reader().path().clone();
+                let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path);
+                let field_id_to_column_idx = Arc::new(BTreeMap::from_iter(
+                    data_file
+                        .fields
+                        .iter()
+                        .copied()
+                        .zip(data_file.column_indices.iter().copied())
+                        .filter_map(|(field_id, column_index)| {
+                            if column_index < 0 {
+                                None
+                            } else {
+                                Some((field_id as u32, column_index as u32))
+                            }
+                        }),
+                ));
+                let file_version = LanceFileVersion::try_from_major_minor(
+                    data_file.file_major_version,
+                    data_file.file_minor_version,
+                )?;
+                let reader_projection = ReaderProjection::from_field_ids(
+                    file_version,
                     schema_per_file.as_ref(),
-                    OnMissing::Error,
-                    OnTypeMismatch::Error,
+                    field_id_to_column_idx.as_ref(),
                 )?;
-                let reader = V1Reader::new(reader, Arc::new(initialized_schema));
-                Ok(Some(Box::new(reader)))
-            } else {
-                Ok(None)
+                let file_reader_options = read_config
+                    .file_reader_options
+                    .clone()
+                    .or_else(|| self.dataset.file_reader_options.clone())
+                    .unwrap_or_default();
+                let metadata_index = if metadata_mode == MetadataMode::LazyAllowed
+                    && Self::should_try_indexed_metadata(
+                        data_file,
+                        &reader_projection,
+                        file_version,
+                    ) {
+                    let known_schema = self
+                        .metadata
+                        .physical_rows
+                        .map(|num_rows| (data_file_schema.clone(), num_rows as u64));
+                    let metadata_index = self
+                        .get_file_metadata_index(&file_scheduler, known_schema)
+                        .await?;
+                    if (reader_projection.column_indices.len() as u32).saturating_mul(4)
+                        < metadata_index.num_columns()
+                    {
+                        Some(metadata_index)
+                    } else {
+                        None
+                    }
+                } else {
+                    None
+                };
+
+                let encodings_io = Arc::new(
+                    LanceEncodingsIo::new(file_scheduler.clone())
+                        .with_read_chunk_size(file_reader_options.read_chunk_size),
+                );
+                let reader = if let Some(metadata_index) = metadata_index {
+                    ProjectedFileReader::try_open_with_metadata_index(
+                        encodings_io.clone(),
+                        path.clone(),
+                        Some(reader_projection.clone()),
+                        Arc::<DecoderPlugins>::default(),
+                        metadata_index,
+                        &metadata_cache,
+                        file_reader_options.clone(),
+                    )
+                    .await?
+                } else {
+                    let file_metadata = self.get_file_metadata(&file_scheduler).await?;
+                    ProjectedFileReader::try_open_with_file_metadata(
+                        encodings_io,
+                        path.clone(),
+                        None,
+                        Arc::<DecoderPlugins>::default(),
+                        file_metadata,
+                        &metadata_cache,
+                        file_reader_options,
+                    )
+                    .await?
+                };
+                let reader = v2_adapter::Reader::new(
+                    Arc::new(reader),
+                    schema_per_file,
+                    field_id_to_column_idx,
+                    reader_priority,
+                    file_scheduler,
+                );
+                let reader: Box<dyn GenericFileReader> = Box::new(reader);
+                Ok(Some(reader))
             }
-        } else if schema_per_file.fields.is_empty() {
-            Ok(None)
-        } else {
-            let path = self
-                .dataset
-                .data_file_dir(data_file)?
-                .join(data_file.path.as_str());
-            let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id {
-                // TODO: make object stores for non-default bases reuse the same scan scheduler
-                //  currently we always create a new one
-                let object_store = self.dataset.object_store(Some(base_id)).await?;
-                let config = SchedulerConfig::max_bandwidth(&object_store);
-                (
-                    ScanScheduler::new(object_store, config),
-                    read_config.reader_priority.unwrap_or(0),
-                )
-            } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() {
-                (
-                    scan_scheduler.clone(),
-                    read_config.reader_priority.unwrap_or(0),
-                )
-            } else {
-                (
-                    ScanScheduler::new(
-                        self.dataset.object_store.clone(),
-                        SchedulerConfig::max_bandwidth(&self.dataset.object_store),
-                    ),
-                    0,
-                )
-            };
-            let file_scheduler = store_scheduler
-                .open_file_with_priority(&path, reader_priority as u64, &data_file.file_size_bytes)
-                .await?;
-            let file_metadata = self.get_file_metadata(&file_scheduler).await?;
-            let path = file_scheduler.reader().path().clone();
-            let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path);
-            let reader = Arc::new(
-                lance_file::reader::FileReader::try_open_with_file_metadata(
-                    Arc::new(LanceEncodingsIo::new(file_scheduler.clone())),
-                    path,
-                    None,
-                    Arc::<DecoderPlugins>::default(),
-                    file_metadata,
-                    &metadata_cache,
-                    read_config
-                        .file_reader_options
-                        .clone()
-                        .or_else(|| self.dataset.file_reader_options.clone())
-                        .unwrap_or_default(),
-                )
-                .await?,
-            );
-            let field_id_to_column_idx = Arc::new(BTreeMap::from_iter(
-                data_file
-                    .fields
-                    .iter()
-                    .copied()
-                    .zip(data_file.column_indices.iter().copied())
-                    .filter_map(|(field_id, column_index)| {
-                        if column_index < 0 {
-                            None
-                        } else {
-                            Some((field_id as u32, column_index as u32))
-                        }
-                    }),
-            ));
-            let reader = v2_adapter::Reader::new(
-                reader,
-                schema_per_file,
-                field_id_to_column_idx,
-                reader_priority,
-                file_scheduler,
-            );
-            Ok(Some(Box::new(reader)))
         }
+        .boxed()
     }
 
     async fn open_readers(
@@ -1082,35 +1211,68 @@ impl FileFragment {
         projection: &Schema,
         read_config: &FragReadConfig,
     ) -> Result<Vec<Box<dyn GenericFileReader>>> {
-        let mut opened_files = vec![];
-        for data_file in &self.metadata.files {
-            if let Some(reader) = self
-                .open_reader(data_file, Some(projection), read_config)
-                .await?
-            {
-                opened_files.push(reader);
+        self.open_readers_impl(projection, read_config, MetadataMode::LazyAllowed)
+            .await
+    }
+
+    async fn open_readers_with_full_metadata(
+        &self,
+        projection: &Schema,
+        read_config: &FragReadConfig,
+    ) -> Result<Vec<Box<dyn GenericFileReader>>> {
+        self.open_readers_impl(projection, read_config, MetadataMode::Full)
+            .await
+    }
+
+    fn open_readers_impl<'a>(
+        &'a self,
+        projection: &'a Schema,
+        read_config: &'a FragReadConfig,
+        metadata_mode: MetadataMode,
+    ) -> BoxFuture<'a, Result<Vec<Box<dyn GenericFileReader>>>> {
+        async move {
+            let mut opened_files = vec![];
+            for data_file in &self.metadata.files {
+                let reader = match metadata_mode {
+                    MetadataMode::LazyAllowed => {
+                        self.open_reader(data_file, Some(projection), read_config)
+                            .await?
+                    }
+                    MetadataMode::Full => {
+                        self.open_reader_with_full_metadata(
+                            data_file,
+                            Some(projection),
+                            read_config,
+                        )
+                        .await?
+                    }
+                };
+                if let Some(reader) = reader {
+                    opened_files.push(reader);
+                }
             }
-        }
 
-        // This should return immediately on modern datasets.  Need to use physical_rows because
-        // deletions will be applied later
-        let num_rows = self.physical_rows().await?;
+            // This should return immediately on modern datasets.  Need to use physical_rows because
+            // deletions will be applied later
+            let num_rows = self.physical_rows().await?;
 
-        // Check if there are any fields that are not in any data files
-        let field_ids_in_files = opened_files
-            .iter()
-            .flat_map(|r| r.projection().fields_pre_order().map(|f| f.id))
-            .filter(|id| *id >= 0)
-            .collect::<HashSet<_>>();
-        let mut missing_fields = projection.field_ids();
-        missing_fields.retain(|f| !field_ids_in_files.contains(f) && *f >= 0);
-        if !missing_fields.is_empty() {
-            let missing_projection = projection.project_by_ids(&missing_fields, true);
-            let null_reader = NullReader::new(Arc::new(missing_projection), num_rows as u32);
-            opened_files.push(Box::new(null_reader));
-        }
+            // Check if there are any fields that are not in any data files
+            let field_ids_in_files = opened_files
+                .iter()
+                .flat_map(|r| r.projection().fields_pre_order().map(|f| f.id))
+                .filter(|id| *id >= 0)
+                .collect::<HashSet<_>>();
+            let mut missing_fields = projection.field_ids();
+            missing_fields.retain(|f| !field_ids_in_files.contains(f) && *f >= 0);
+            if !missing_fields.is_empty() {
+                let missing_projection = projection.project_by_ids(&missing_fields, true);
+                let null_reader = NullReader::new(Arc::new(missing_projection), num_rows as u32);
+                opened_files.push(Box::new(null_reader));
+            }
 
-        Ok(opened_files)
+            Ok(opened_files)
+        }
+        .boxed()
     }
 
     /// Count the rows in this fragment.
@@ -1427,8 +1589,22 @@ impl FileFragment {
         };
 
         // Then call take rows
-        self.take_rows(&row_ids, projection, false, false, false, false)
-            .await
+        let batch = self
+            .take_rows(&row_ids, projection, false, false, false, false)
+            .await?;
+
+        // Convert Lance JSON columns (LargeBinary/JSONB) back to Arrow JSON (Utf8)
+        // for user-facing output.
+        if batch
+            .schema()
+            .fields()
+            .iter()
+            .any(|f| lance_arrow::json::is_json_field(f) || lance_arrow::json::has_json_fields(f))
+        {
+            Ok(lance_arrow::json::convert_lance_json_to_arrow(&batch)?)
+        } else {
+            Ok(batch)
+        }
     }
 
     /// Get the deletion vector for this fragment, using the cache if available.
@@ -1444,7 +1620,7 @@ impl FileFragment {
     }
 
     /// Get the file metadata for this fragment, using the cache if available.
-    async fn get_file_metadata(
+    pub async fn get_file_metadata(
         &self,
         file_scheduler: &FileScheduler,
     ) -> Result<Arc<CachedFileMetadata>> {
@@ -1461,6 +1637,32 @@ impl FileFragment {
         Ok(file_metadata)
     }
 
+    pub async fn get_file_metadata_index(
+        &self,
+        file_scheduler: &FileScheduler,
+        known_schema: Option<(Arc<Schema>, u64)>,
+    ) -> Result<Arc<FileMetadataIndex>> {
+        let path = file_scheduler.reader().path();
+        let cache = self.dataset.metadata_cache.file_metadata_cache(path);
+
+        let metadata_index = cache
+            .get_or_insert_with_key(FileMetadataIndexCacheKey, || async {
+                let metadata_index = if let Some((file_schema, num_rows)) = known_schema {
+                    lance_file::reader::FileReader::read_metadata_index_with_schema(
+                        file_scheduler,
+                        file_schema,
+                        num_rows,
+                    )
+                    .await?
+                } else {
+                    lance_file::reader::FileReader::read_metadata_index(file_scheduler).await?
+                };
+                Ok(metadata_index)
+            })
+            .await?;
+        Ok(metadata_index)
+    }
+
     /// Take rows based on internal local row offsets
     ///
     /// If the row offsets are out-of-bounds, this will return an error. But if the
@@ -1721,6 +1923,17 @@ impl FileFragment {
             )
             .await?;
         // Hash join: rows matched on the right-hand stream rewrite columns; track physical offsets via `_rowaddr`.
+        // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) in the right stream
+        // so they match the physical storage format read from the fragment's left batch.
+        let right_stream: Box<dyn RecordBatchReader + Send> = if right_schema
+            .fields()
+            .iter()
+            .any(|f| is_arrow_json_field(f) || has_json_fields(f))
+        {
+            Box::new(JsonConvertingReader::new(right_stream))
+        } else {
+            right_stream
+        };
         let joiner = Arc::new(HashJoiner::try_new(right_stream, right_on).await?);
         let mut matched_offsets = RoaringBitmap::new();
         let frag_id_u32 = u32::try_from(self.metadata.id).map_err(|_| {
@@ -1792,7 +2005,7 @@ impl FileFragment {
         read_columns: Option<Vec<String>>,
         batch_size: Option<u32>,
     ) -> Result<(Fragment, Schema)> {
-        let (fragments, schema) = schema_evolution::add_columns_to_fragments(
+        let (fragments, schema, _) = schema_evolution::add_columns_to_fragments(
             self.dataset.as_ref(),
             transforms,
             read_columns,
@@ -1983,6 +2196,21 @@ impl CacheKey for FileMetadataCacheKey {
     }
 }
 
+#[derive(Debug, Clone)]
+struct FileMetadataIndexCacheKey;
+
+impl CacheKey for FileMetadataIndexCacheKey {
+    type ValueType = FileMetadataIndex;
+
+    fn key(&self) -> std::borrow::Cow<'_, str> {
+        "metadata_index".into()
+    }
+
+    fn type_name() -> &'static str {
+        "FileMetadataIndex"
+    }
+}
+
 impl From<FileFragment> for Fragment {
     fn from(fragment: FileFragment) -> Self {
         fragment.metadata
@@ -2042,6 +2270,23 @@ pub struct FragmentReader {
 
     // total number of physical rows in the fragment (all rows, ignoring deletions)
     num_physical_rows: usize,
+
+    /// Read-time state for resolving data overlay files: the coverage plan plus
+    /// what is needed to open overlay readers. `None` when the fragment has no
+    /// overlays. Overlays are merged into base batches (by `offset_in_frag`) before
+    /// deletion filtering, opening only the files each read's rows touch.
+    overlay: Option<OverlayReadState>,
+}
+
+/// What [`FragmentReader`] needs to resolve overlays at read time: the coverage
+/// plan (from metadata, cheap to build), and the fragment + config needed to open
+/// overlay readers once the read's rows — and therefore which files it touches —
+/// are known. All `Arc` so cloning a reader stays cheap.
+#[derive(Clone, Debug)]
+struct OverlayReadState {
+    planner: Arc<OverlayReadPlanner>,
+    fragment: Arc<FileFragment>,
+    read_config: Arc<FragReadConfig>,
 }
 
 // Custom clone impl needed because it is not easy to clone Box<dyn GenericFileReader>
@@ -2070,6 +2315,7 @@ impl Clone for FragmentReader {
             created_at_sequence: self.created_at_sequence.clone(),
             num_rows: self.num_rows,
             num_physical_rows: self.num_physical_rows,
+            overlay: self.overlay.clone(),
         }
     }
 }
@@ -2137,6 +2383,7 @@ impl FragmentReader {
             created_at_sequence: None,
             num_rows,
             num_physical_rows,
+            overlay: None,
         })
     }
 
@@ -2394,6 +2641,77 @@ impl FragmentReader {
         Ok(result.project_by_schema(&output_schema)?)
     }
 
+    /// Merge data overlay values onto a stream of base batches.
+    ///
+    /// Runs on physical rows in read order, *before* deletion filtering, so each
+    /// row can be addressed by its position in the fragment (its `offset_in_frag`,
+    /// derived from `params`) and deletions take precedence naturally: an overlay
+    /// value for a deleted row is dropped along with the row downstream. A no-op
+    /// when the fragment has no overlays.
+    ///
+    /// The read's `offset_in_frag` values are known from `params` up front, so
+    /// overlays are resolved here to just the files this read's rows touch — an
+    /// overlay whose cells fall outside the read is not opened at all. Within each
+    /// batch, the overlay reads (only the values that batch needs) are then issued
+    /// concurrently with the base read rather than after it.
+    async fn merge_overlays(
+        &self,
+        merged: ReadBatchTaskStream,
+        params: &ReadBatchParams,
+        total_num_rows: u32,
+    ) -> Result<ReadBatchTaskStream> {
+        let Some(overlay) = &self.overlay else {
+            return Ok(merged);
+        };
+        // The offset_in_frag of every row this read will return, materialized once.
+        // Cost is one u32 per output row (a whole-fragment scan is 4 bytes/row), and
+        // it lets us both prune overlays to the read and slice each batch's offsets
+        // below without reading any data. Only paid when the fragment has overlays.
+        //
+        // TODO(overlay perf): this could be avoided by teaching `ReadBatchParams` to
+        // yield a coverage bitmap directly (for pruning) and to slice per batch (for
+        // the routing below), or by moving `ReadBatchParams` to a roaring bitmap
+        // wholesale — a larger refactor tracked separately.
+        let offsets_in_frag: Arc<Vec<u32>> =
+            Arc::new(params.to_offsets_total(total_num_rows).values().to_vec());
+
+        // Open only the overlay readers this read touches (pruned by row selection).
+        let plans = resolve_overlays(
+            &overlay.planner,
+            &offsets_in_frag,
+            &overlay.fragment,
+            &overlay.read_config,
+        )
+        .await?;
+        if plans.is_empty() {
+            return Ok(merged);
+        }
+        let plans = Arc::new(plans);
+
+        // Batches arrive in physical read order, so a running total of the rows seen
+        // so far gives each batch its starting offset_in_batch into `offsets_in_frag`.
+        let mut rows_seen = 0usize;
+        let stream = merged
+            .map(move |task| {
+                let num_rows = task.num_rows;
+                let start = rows_seen;
+                rows_seen += num_rows as usize;
+                let offsets_in_frag = offsets_in_frag.clone();
+                let plans = plans.clone();
+                let inner = task.task;
+                ReadBatchTask {
+                    num_rows,
+                    task: async move {
+                        let batch_offsets = &offsets_in_frag[start..start + num_rows as usize];
+                        merge_overlay_batch(inner, batch_offsets, &plans).await
+                    }
+                    .boxed(),
+                }
+            })
+            .boxed();
+        Ok(stream)
+    }
+
     async fn new_read_impl<'a, F>(
         &'a self,
         params: ReadBatchParams,
@@ -2461,6 +2779,8 @@ impl FragmentReader {
             lance_table::utils::stream::merge_streams(read_streams)
         };
 
+        let merged = self.merge_overlays(merged, &params, total_num_rows).await?;
+
         // Add the row id column (if needed) and delete rows (if a deletion
         // vector is present).
         let config = RowIdAndDeletesConfig {
@@ -2631,6 +2951,11 @@ impl FragmentReader {
             lance_table::utils::stream::merge_streams(read_streams)
         };
 
+        let params = ReadBatchParams::Ranges(ranges);
+        let merged_stream = self
+            .merge_overlays(merged_stream, &params, total_num_rows)
+            .await?;
+
         // Add the row id column (if needed) and delete rows (if a deletion
         // vector is present).
         let config = RowIdAndDeletesConfig {
@@ -2643,7 +2968,7 @@ impl FragmentReader {
             with_row_created_at_version: self.with_row_created_at_version,
             last_updated_at_sequence: self.last_updated_at_sequence.clone(),
             created_at_sequence: self.created_at_sequence.clone(),
-            params: ReadBatchParams::Ranges(ranges),
+            params,
             total_num_rows,
         };
         let output_schema = Arc::new(self.output_schema.clone());
@@ -2746,6 +3071,59 @@ impl FragmentReader {
     }
 }
 
+/// A wrapper around a `RecordBatchReader` that converts Arrow JSON columns
+/// (Utf8/LargeUtf8 with `arrow.json` extension) to Lance JSON columns
+/// (LargeBinary with `lance.json` extension / JSONB format).
+///
+/// This is needed when user-provided data contains Arrow JSON fields but the
+/// dataset stores them in Lance's JSONB binary format.
+struct JsonConvertingReader {
+    inner: Box<dyn RecordBatchReader + Send>,
+    schema: arrow_schema::SchemaRef,
+}
+
+impl JsonConvertingReader {
+    fn new(inner: Box<dyn RecordBatchReader + Send>) -> Self {
+        use lance_arrow::json::arrow_json_to_lance_json;
+
+        // Build the converted schema (Arrow JSON fields → Lance JSON fields)
+        let orig_schema = inner.schema();
+        let new_fields: Vec<arrow_schema::FieldRef> = orig_schema
+            .fields()
+            .iter()
+            .map(|f| {
+                if is_arrow_json_field(f) || has_json_fields(f) {
+                    Arc::new(arrow_json_to_lance_json(f))
+                } else {
+                    Arc::clone(f)
+                }
+            })
+            .collect();
+        let schema = Arc::new(arrow_schema::Schema::new_with_metadata(
+            new_fields,
+            orig_schema.metadata().clone(),
+        ));
+
+        Self { inner, schema }
+    }
+}
+
+impl Iterator for JsonConvertingReader {
+    type Item = std::result::Result<RecordBatch, arrow_schema::ArrowError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner
+            .next()
+            .map(|result| result.and_then(|batch| convert_json_columns(&batch)))
+    }
+}
+
+impl RecordBatchReader for JsonConvertingReader {
+    fn schema(&self) -> arrow_schema::SchemaRef {
+        self.schema.clone()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use arrow_arith::numeric::mul;
@@ -2839,72 +3217,1585 @@ mod tests {
         Dataset::open(test_uri).await.unwrap()
     }
 
-    #[rstest]
-    #[tokio::test]
-    async fn test_fragment_scan(
-        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
-        data_storage_version: LanceFileVersion,
-    ) {
-        let test_dir = TempStrDir::default();
-        let test_uri = &test_dir;
-        let dataset = create_dataset(test_uri, data_storage_version).await;
-        let fragment = &dataset.get_fragments()[2];
-        let mut scanner = fragment.scan();
-        let batches = scanner
-            .with_row_id()
-            .filter(" i < 105")
-            .unwrap()
-            .try_into_stream()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
+    /// End-to-end tests for reading data overlay files (OSS-1324): overlays are
+    /// written, committed via the `DataOverlay` transaction, and then resolved on
+    /// the `take` and scan read paths.
+    mod overlay_read {
+        use std::sync::Arc;
 
-        if data_storage_version == LanceFileVersion::Legacy {
-            assert_eq!(batches.len(), 3);
+        use arrow_array::{
+            Array, ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StructArray, UInt64Array,
+        };
+        use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
+        use lance_core::datatypes::Schema;
+        use lance_file::version::LanceFileVersion;
+        use lance_file::writer::{FileWriter, FileWriterOptions};
+        use lance_io::utils::CachedFileSize;
+        use lance_table::format::DataFile;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use object_store::path::Path;
+        use roaring::RoaringBitmap;
+        use rstest::rstest;
+
+        use crate::dataset::transaction::{DataOverlayGroup, Operation};
+        use crate::dataset::{Dataset, WriteDestination, WriteParams};
+
+        fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+            RoaringBitmap::from_iter(offsets)
+        }
 
-            assert_eq!(
-                batches[0].column_by_name("i").unwrap().as_ref(),
-                &Int32Array::from_iter_values(80..90)
-            );
-            assert_eq!(
-                batches[1].column_by_name("i").unwrap().as_ref(),
-                &Int32Array::from_iter_values(90..100)
-            );
-            assert_eq!(
-                batches[2].column_by_name("i").unwrap().as_ref(),
-                &Int32Array::from_iter_values(100..105)
-            );
-        } else {
-            assert_eq!(batches.len(), 1);
+        fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+            Arc::new(Int32Array::from_iter(values))
+        }
 
-            assert_eq!(
-                batches[0].column_by_name("i").unwrap().as_ref(),
-                &Int32Array::from_iter_values(80..105)
+        /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1)
+        /// = id * 10, written 6 rows per file (fragments 0 and 1).
+        ///
+        /// Uses an in-memory store so the test can write overlay files with a
+        /// store-relative `data/<name>.lance` path and commit against the returned
+        /// dataset directly.
+        async fn create_base_dataset(version: LanceFileVersion) -> Dataset {
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..12)),
+                    Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))),
+                ],
             )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap()
         }
-    }
 
-    #[tokio::test]
-    async fn test_fragment_scan_v2() {
-        let test_dir = TempStrDir::default();
-        let test_uri = &test_dir;
-        let dataset = create_dataset_v2(test_uri).await;
-        let fragment = &dataset.get_fragments()[2];
-        let mut scanner = fragment.scan();
-        let batches = scanner
-            .with_row_id()
-            .try_into_stream()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
+        /// Write an overlay file covering `fields` (dataset field ids) of
+        /// `fragment_id` with the given coverage and per-field value columns, then
+        /// commit it as a `DataOverlay` transaction. `name` makes the file unique.
+        #[allow(clippy::too_many_arguments)]
+        async fn commit_overlay(
+            dataset: Dataset,
+            name: &str,
+            fragment_id: u64,
+            fields: &[i32],
+            coverage: OverlayCoverage,
+            columns: Vec<ArrayRef>,
+            version: LanceFileVersion,
+        ) -> Dataset {
+            let read_version = dataset.version().version;
+            let overlay_schema = dataset.schema().project_by_ids(fields, true);
+
+            let filename = format!("{name}.lance");
+            let path = Path::from(format!("data/{filename}"));
+            let obj_writer = dataset.object_store.create(&path).await.unwrap();
+            let mut writer = FileWriter::try_new(
+                obj_writer,
+                overlay_schema,
+                FileWriterOptions {
+                    format_version: Some(version),
+                    ..Default::default()
+                },
+            )
             .unwrap();
+            let (major, minor) = writer.version().to_numbers();
+            for (column_index, array) in columns.into_iter().enumerate() {
+                writer.write_column(column_index, array).await.unwrap();
+            }
+            let summary = writer.finish().await.unwrap();
 
-        assert_eq!(batches.len(), 1);
-
-        assert_eq!(
+            let mut data_file = DataFile::new_unstarted(filename, major, minor);
+            data_file.fields = writer
+                .field_id_to_column_indices()
+                .iter()
+                .map(|(field_id, _)| *field_id as i32)
+                .collect::<Vec<_>>()
+                .into();
+            data_file.column_indices = writer
+                .field_id_to_column_indices()
+                .iter()
+                .map(|(_, column_index)| *column_index as i32)
+                .collect::<Vec<_>>()
+                .into();
+            data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+            let overlay = DataOverlayFile {
+                data_file,
+                coverage,
+                committed_version: 0,
+            };
+            Dataset::commit(
+                WriteDestination::Dataset(Arc::new(dataset)),
+                Operation::DataOverlay {
+                    groups: vec![DataOverlayGroup {
+                        fragment_id,
+                        overlays: vec![overlay],
+                    }],
+                },
+                Some(read_version),
+                None,
+                None,
+                Arc::new(Default::default()),
+                false,
+            )
+            .await
+            .unwrap()
+        }
+
+        fn full_schema(dataset: &Dataset) -> Schema {
+            dataset.schema().clone()
+        }
+
+        fn col(batch: &RecordBatch, name: &str) -> Int32Array {
+            let idx = batch.schema().index_of(name).unwrap();
+            batch
+                .column(idx)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .clone()
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_covered_and_uncovered(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay fragment 0's `val` at physical offsets {1, 4}.
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag
+                .take(&[0, 1, 2, 4], &full_schema(&dataset))
+                .await
+                .unwrap();
+            // Offsets 1 and 4 take overlay values; 0 and 2 fall through to base.
+            assert_eq!(col(&batch, "val").values(), &[0, 111, 20, 444]);
+            // The unrelated `id` column is untouched.
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 4]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_newest_overlay_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+            // A newer overlay (later commit -> higher committed_version) re-covers
+            // offset 1.
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 4], &full_schema(&dataset)).await.unwrap();
+            // Offset 1 -> newest overlay (999); offset 4 -> only older covers it.
+            assert_eq!(col(&batch, "val").values(), &[999, 444]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_per_field_coverage(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Sparse overlay: `id` covers {2}, `val` covers {2, 3} — different
+            // offset sets and therefore unequal-length value columns.
+            let dataset = commit_overlay(
+                dataset,
+                "sparse",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([2]), bitmap([2, 3])]),
+                vec![i32_array([Some(777)]), i32_array([Some(220), Some(330)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap();
+            // id: offset 2 covered (777), offset 3 falls through (3).
+            assert_eq!(col(&batch, "id").values(), &[777, 3]);
+            // val: both offsets covered (220, 330).
+            assert_eq!(col(&batch, "val").values(), &[220, 330]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_null_override(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "nullov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([None])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[0, 1], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            // Offset 0 is covered with a NULL value -> resolves to NULL; offset 1
+            // falls through to the base value.
+            assert!(val.is_null(0));
+            assert_eq!(val.value(1), 10);
+        }
+
+        /// Overlays interact correctly with NULL *base* cells (distinct from a NULL
+        /// overlay value): a covered row whose base value is NULL is overridden to the
+        /// overlay's non-null value, while an uncovered NULL base cell falls through
+        /// and stays NULL.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_null_base_cell(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    // `val` is NULL at offsets 1 and 3.
+                    Arc::new(Int32Array::from_iter([
+                        Some(0),
+                        None,
+                        Some(20),
+                        None,
+                        Some(40),
+                        Some(50),
+                    ])),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Cover offset 1 (NULL base) and offset 4 (non-null base); leave offset
+            // 3's NULL base uncovered.
+            let dataset = commit_overlay(
+                dataset,
+                "nullbase",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 3, 4], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            // Offset 1: NULL base overridden to 111. Offset 3: uncovered NULL base
+            // stays NULL. Offset 4: non-null base overridden to 444.
+            assert_eq!(val.value(0), 111);
+            assert!(val.is_null(1));
+            assert_eq!(val.value(2), 444);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_on_deleted_row_is_inert(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let mut dataset = create_base_dataset(version).await;
+            // Delete global row 1 (fragment 0, physical offset 1).
+            dataset.delete("id = 1").await.unwrap();
+            // Overlay covers the deleted offset 1 and the live offset 4.
+            let dataset = commit_overlay(
+                dataset,
+                "delov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            // Scan fragment 0: row 1 is gone, and offset 4's overlay value survives
+            // even though the deletion shifts logical positions — coverage is keyed
+            // by physical offset.
+            let frag = dataset.get_fragment(0).unwrap();
+            let mut scanner = frag.scan();
+            let batch = scanner
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 2, 3, 4, 5]);
+            assert_eq!(col(&batch, "val").values(), &[0, 20, 30, 444, 50]);
+        }
+
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_multi_fragment_overlays(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay fragment 0 at offset 0 and fragment 1 at offset 0 (global
+            // row 6). Each fragment's coverage is independent.
+            let dataset = commit_overlay(
+                dataset,
+                "frag0",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag1",
+                1,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(6000)])],
+                version,
+            )
+            .await;
+
+            let batch = dataset
+                .scan()
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(batch.num_rows(), 12);
+            let expected: Vec<i32> = (0..12)
+                .map(|i| match i {
+                    0 => 1000,
+                    6 => 6000,
+                    other => other * 10,
+                })
+                .collect();
+            assert_eq!(col(&batch, "val").values(), &expected);
+        }
+
+        /// A `take` of a few rows must read only the overlay values those rows
+        /// touch — not the whole column. Uses v2.1 (which slices pages on read) and
+        /// an incompressible, all-covering overlay, so reading the full column would
+        /// be far more bytes than reading a couple of values. This is the regression
+        /// guard for the lazy, value-pushdown overlay read.
+        #[tokio::test]
+        async fn test_take_reads_only_needed_overlay_values() {
+            let version = LanceFileVersion::V2_1;
+            const N: usize = 100_000;
+
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let base = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..N as i32)),
+                    Arc::new(Int32Array::from_iter_values((0..N as i32).map(|v| v * 10))),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: N,
+                max_rows_per_group: N,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(base)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `val` over ALL N offsets with incompressible values, so the
+            // value column is ~N*4 bytes on disk.
+            let values: Vec<i32> = (0..N as u64)
+                .map(|i| {
+                    let mut x = i;
+                    x ^= x >> 33;
+                    x = x.wrapping_mul(0xff51_afd7_ed55_8ccd);
+                    x ^= x >> 33;
+                    x as i32
+                })
+                .collect();
+            let dataset = commit_overlay(
+                dataset,
+                "big",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap(0..N as u32)),
+                vec![Arc::new(Int32Array::from(values.clone())) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // Measure only the reads that resolve the take.
+            dataset.object_store.io_stats_incremental();
+            let batch = frag.take(&[0, 1], &val_only).await.unwrap();
+            let io = dataset.object_store.io_stats_incremental();
+
+            // The overlay's `val` column alone is N*4 bytes; resolving two adjacent
+            // offsets must read only a small fraction of it.
+            let full_column_bytes = (N * std::mem::size_of::<i32>()) as u64;
+            assert!(
+                io.read_bytes > 0 && io.read_bytes < full_column_bytes / 4,
+                "take read {} bytes; expected far less than the {}-byte overlay \
+                 column (a take must not read the whole value column)",
+                io.read_bytes,
+                full_column_bytes,
+            );
+
+            // ...and it still resolves correctly.
+            let val = col(&batch, "val");
+            assert_eq!(val.value(0), values[0]);
+            assert_eq!(val.value(1), values[1]);
+        }
+
+        /// Row-selection pruning: an overlay whose coverage is disjoint from the
+        /// requested rows must not be opened at all. Proven by deleting the overlay's
+        /// data file — a `take` that misses its coverage still succeeds (the file is
+        /// never touched), while a `take` that hits it then fails because the file is
+        /// genuinely needed.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_prunes_overlays_outside_row_selection(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay on fragment 0 (offsets 0..6) covering only offset_in_frag 5.
+            let dataset = commit_overlay(
+                dataset,
+                "miss",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([5])),
+                vec![i32_array([Some(5000)])],
+                version,
+            )
+            .await;
+
+            // Delete the overlay's data file: opening it now fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/miss.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // A take that misses the overlay's coverage must not open it, so it
+            // succeeds and returns base values (val = offset * 10).
+            let batch = frag.take(&[0, 1], &val_only).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[0, 10]);
+
+            // A take that hits the coverage does need the file, so it now fails with
+            // a not-found error naming the missing overlay file.
+            let err = frag.take(&[5], &val_only).await.unwrap_err();
+            let err = format!("{err:?}");
+            assert!(
+                err.contains("miss.lance") && err.to_lowercase().contains("not found"),
+                "take hitting the overlay's coverage should fail with a not-found error \
+                 for its missing file, got: {err}",
+            );
+        }
+
+        /// The overlay merge runs before `wrap_with_row_id_and_delete`, so the
+        /// `_rowid` system column must coexist with overlay-resolved data columns:
+        /// the row ids are unaffected by the merge and the overlay value still wins.
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_with_row_id_alongside_overlay(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "rowidov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag
+                .scan()
+                .with_row_id()
+                .project(&["id", "val"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            // Overlay value resolves...
+            assert_eq!(col(&batch, "val").values()[0], 1000);
+            assert_eq!(&col(&batch, "val").values()[1..], &[10, 20, 30, 40, 50]);
+            // ...and the row ids for fragment 0 are the untouched physical offsets.
+            let row_ids = batch
+                .column(batch.schema().index_of("_rowid").unwrap())
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .unwrap();
+            assert_eq!(row_ids.values(), &[0, 1, 2, 3, 4, 5]);
+        }
+
+        /// When the newest overlay covers every requested offset, an older overlay
+        /// in the same plan needs zero values and its value column must not be read
+        /// (the empty-input branch of `fetch_overlay_values`). The result still
+        /// resolves to the newest overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_older_overlay_contributes_no_values(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Older covers {1, 4}; newer re-covers {1}. A take of only offset 1
+            // routes entirely to the newer overlay, leaving the older one with no
+            // values to fetch even though it is part of the field's plan.
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[999]);
+        }
+
+        /// A newest overlay whose value is NULL must shadow an older overlay's
+        /// non-null value at the same offset — the merge resolves to NULL, it does
+        /// not fall back to the older overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_newest_null_shadows_older(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([Some(111)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "newer_null",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1])),
+                vec![i32_array([None])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap();
+            let val = col(&batch, "val");
+            assert!(val.is_null(0), "newest NULL must win over older 111");
+        }
+
+        /// Newest-wins is resolved independently per field across multiple sparse
+        /// overlays: for the same offset, `id` can resolve to one overlay while
+        /// `val` resolves to the other, depending on which overlay newly covers
+        /// that field at that offset.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_multi_sparse_per_field_newest_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Older: id covers {3}, val covers {2}.
+            let dataset = commit_overlay(
+                dataset,
+                "older",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([3]), bitmap([2])]),
+                vec![i32_array([Some(7773)]), i32_array([Some(2772)])],
+                version,
+            )
+            .await;
+            // Newer: id covers {2}, val covers {3} — the mirror image.
+            let dataset = commit_overlay(
+                dataset,
+                "newer",
+                0,
+                &[0, 1],
+                OverlayCoverage::sparse(vec![bitmap([2]), bitmap([3])]),
+                vec![i32_array([Some(9992)]), i32_array([Some(9993)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap();
+            // id: offset 2 -> newer (9992), offset 3 -> older (7773).
+            assert_eq!(col(&batch, "id").values(), &[9992, 7773]);
+            // val: offset 2 -> older (2772), offset 3 -> newer (9993).
+            assert_eq!(col(&batch, "val").values(), &[2772, 9993]);
+        }
+
+        /// A fragment with an overlay plan, but a take that touches only uncovered
+        /// offsets, must fall entirely through to the base values (the
+        /// `!routing.any_overlay` early-return with a plan present).
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_plan_present_all_offsets_uncovered(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            // None of {0, 2, 5} are covered: the plan exists but contributes nothing.
+            let batch = frag.take(&[0, 2, 5], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(col(&batch, "val").values(), &[0, 20, 50]);
+            assert_eq!(col(&batch, "id").values(), &[0, 2, 5]);
+        }
+
+        /// A dataset-level `take` spanning multiple fragments, each with its own
+        /// overlay, routes every global row index to the right fragment's overlay.
+        #[rstest]
+        #[tokio::test]
+        async fn test_dataset_take_multi_fragment_overlays(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag0",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(1000)])],
+                version,
+            )
+            .await;
+            let dataset = commit_overlay(
+                dataset,
+                "frag1",
+                1,
+                &[1],
+                OverlayCoverage::dense(bitmap([0])),
+                vec![i32_array([Some(6000)])],
+                version,
+            )
+            .await;
+
+            // Global rows 0 and 6 are the overlaid offset-0 rows of fragments 0 and
+            // 1; rows 1 and 7 fall through to base.
+            let batch = dataset
+                .take(&[0, 1, 6, 7], full_schema(&dataset))
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 6, 7]);
+            assert_eq!(col(&batch, "val").values(), &[1000, 10, 6000, 70]);
+        }
+
+        /// A scan whose read splits into multiple batches must slice
+        /// `offsets_in_frag` per batch correctly — the running `rows_seen`
+        /// accumulator in `merge_overlays` gives each batch its start. Every other
+        /// scan test uses single-batch fragments, so this is the only guard for the
+        /// cross-batch (`start > 0`) path.
+        #[rstest]
+        #[tokio::test]
+        async fn test_scan_multi_batch_overlay_slicing(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use futures::TryStreamExt;
+
+            // One fragment of 10 rows so the read can be chunked below.
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("val", DataType::Int32, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..10)),
+                    Arc::new(Int32Array::from_iter_values((0..10).map(|v| v * 10))),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 100,
+                max_rows_per_group: 100,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay one offset in each batch that batch_size 4 produces (batches
+            // [0,4), [4,8), [8,10)): offsets 1, 5, 9 with distinct values. A wrong
+            // per-batch slice would misalign these.
+            let dataset = commit_overlay(
+                dataset,
+                "multibatch",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 5, 9])),
+                vec![i32_array([Some(111), Some(555), Some(999)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let mut scanner = frag.scan();
+            scanner.batch_size(4).project(&["val"]).unwrap();
+            let batches: Vec<RecordBatch> = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect()
+                .await
+                .unwrap();
+            // Guard the guard: the read must actually span multiple batches, else
+            // this would not exercise the cross-batch slice at all.
+            assert!(
+                batches.len() > 1,
+                "expected a multi-batch scan, got {} batch(es)",
+                batches.len()
+            );
+
+            let merged =
+                arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap();
+            let expected: Vec<i32> = (0..10)
+                .map(|i| match i {
+                    1 => 111,
+                    5 => 555,
+                    9 => 999,
+                    other => other * 10,
+                })
+                .collect();
+            assert_eq!(col(&merged, "val").values(), &expected);
+        }
+
+        /// An empty selection must not trip over the overlay path: the plan exists
+        /// but there are no offsets to route, so the result is an empty batch.
+        #[rstest]
+        #[tokio::test]
+        async fn test_take_empty_selection(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            let dataset = commit_overlay(
+                dataset,
+                "ov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![i32_array([Some(111), Some(444)])],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[], &full_schema(&dataset)).await.unwrap();
+            assert_eq!(batch.num_rows(), 0);
+        }
+
+        /// Overlays resolve variable-width columns end-to-end, not just fixed-width
+        /// ones: the value column is fetched through the real file reader (a
+        /// different value-pushdown path than the fixed-width case) and assembled.
+        #[rstest]
+        #[tokio::test]
+        async fn test_string_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use arrow_array::StringArray;
+
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("name", DataType::Utf8, true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e", "f"])),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `name` at offsets {1, 4}, one of the values NULL.
+            let dataset = commit_overlay(
+                dataset,
+                "strov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([1, 4])),
+                vec![Arc::new(StringArray::from(vec![Some("B"), None])) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[0, 1, 4], &full_schema(&dataset)).await.unwrap();
+            let name = batch
+                .column(batch.schema().index_of("name").unwrap())
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            assert_eq!(name.value(0), "a"); // falls through to base
+            assert_eq!(name.value(1), "B"); // overlay value
+            assert!(name.is_null(2)); // overlay NULL wins
+        }
+
+        /// Projection pruning must do NO IO to overlay files whose fields are not
+        /// projected. Proven the same way as row-selection pruning: delete the
+        /// overlay's data file, then read projecting only the *unrelated* `id`
+        /// column — it must succeed (the `val` overlay file is never opened), while
+        /// projecting the overlaid `val` column then fails because its file is gone.
+        #[rstest]
+        #[tokio::test]
+        async fn test_projection_prunes_overlay_files_no_io(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let dataset = create_base_dataset(version).await;
+            // Overlay covers `val` (field 1) only.
+            let dataset = commit_overlay(
+                dataset,
+                "valov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([0, 1])),
+                vec![i32_array([Some(1000), Some(1010)])],
+                version,
+            )
+            .await;
+
+            // Delete the overlay's data file: opening it now fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/valov.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let id_only = dataset.schema().project_by_ids(&[0], true);
+            let val_only = dataset.schema().project_by_ids(&[1], true);
+
+            // Projecting only `id` must not open the `val` overlay file, so it
+            // succeeds and returns untouched base values.
+            let batch = frag.take(&[0, 1], &id_only).await.unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1]);
+            // A scan projecting only `id` must likewise never touch the file.
+            let batch = frag
+                .scan()
+                .project(&["id"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 3, 4, 5]);
+
+            // Projecting the overlaid `val` column does need the file, so it fails
+            // with a not-found error naming the missing overlay file.
+            let err = frag.take(&[0], &val_only).await.unwrap_err();
+            let err = format!("{err:?}");
+            assert!(
+                err.contains("valov.lance") && err.to_lowercase().contains("not found"),
+                "projecting the overlaid column should fail with a not-found error \
+                 for its missing file, got: {err}",
+            );
+        }
+
+        /// A top-level struct column resolves through overlays: the overlay stores
+        /// the struct's leaf columns (under V2_1 those are the only ids in
+        /// `data_file.fields`), and `plan_overlays` maps them back to the top-level
+        /// struct so the whole value is fetched and replaced as a unit.
+        #[rstest]
+        #[tokio::test]
+        async fn test_struct_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let struct_fields = Fields::from(vec![
+                ArrowField::new("x", DataType::Int32, true),
+                ArrowField::new("y", DataType::Int32, true),
+            ]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("info", DataType::Struct(struct_fields.clone()), true),
+            ]));
+            let info = Arc::new(StructArray::new(
+                struct_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), info],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay the whole `info` struct (top-level field id 1) at offset 2.
+            let overlay_info = Arc::new(StructArray::new(
+                struct_fields,
+                vec![
+                    Arc::new(Int32Array::from(vec![777])),
+                    Arc::new(Int32Array::from(vec![888])),
+                ],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "structov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay_info],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let info = batch
+                .column(batch.schema().index_of("info").unwrap())
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            let x = info
+                .column(0)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            let y = info
+                .column(1)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            // Offset 1 falls through to base {1, 100}; offset 2 takes the overlay.
+            assert_eq!(x.values(), &[1, 777]);
+            assert_eq!(y.values(), &[100, 888]);
+        }
+
+        /// A top-level list column resolves through overlays the same way — the
+        /// overlay's leaf (item) id maps back to the top-level list, and the whole
+        /// list value at a covered offset is replaced.
+        #[rstest]
+        #[tokio::test]
+        async fn test_list_overlay_end_to_end(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            use arrow_array::ListArray;
+            use arrow_array::types::Int32Type;
+
+            let item = Arc::new(ArrowField::new("item", DataType::Int32, true));
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("tags", DataType::List(item.clone()), true),
+            ]));
+            let base_tags = ListArray::from_iter_primitive::<Int32Type, _, _>(
+                (0..6i32).map(|i| Some(vec![Some(i), Some(i * 10)])),
+            );
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(base_tags),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `tags` (top-level field id 1) at offset 2 with a new list.
+            let overlay_tags =
+                ListArray::from_iter_primitive::<Int32Type, _, _>(std::iter::once(Some(vec![
+                    Some(77),
+                    Some(88),
+                    Some(99),
+                ])));
+            let dataset = commit_overlay(
+                dataset,
+                "listov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(overlay_tags) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let tags = batch
+                .column(batch.schema().index_of("tags").unwrap())
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap();
+            let row1 = tags.value(0);
+            let row1 = row1.as_any().downcast_ref::<Int32Array>().unwrap();
+            let row2 = tags.value(1);
+            let row2 = row2.as_any().downcast_ref::<Int32Array>().unwrap();
+            // Offset 1 falls through to base [1, 10]; offset 2 takes the overlay.
+            assert_eq!(row1.values(), &[1, 10]);
+            assert_eq!(row2.values(), &[77, 88, 99]);
+        }
+
+        /// A top-level Map column resolves as a single atomic field even though its
+        /// value spans two leaves (key and value): both leaf ids map back to the one
+        /// Map atomic field, and the whole map value at a covered offset is replaced.
+        /// Maps require
+        /// the 2.2+ file format, so this runs only at V2_2 (unlike the V2_0/V2_1
+        /// parametrized tests).
+        #[tokio::test]
+        async fn test_map_overlay_end_to_end() {
+            use arrow_array::MapArray;
+            use arrow_array::builder::{Int32Builder, MapBuilder};
+
+            let version = LanceFileVersion::V2_2;
+
+            // Base row i holds the single entry {i: i * 10}.
+            let mut builder = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new());
+            for i in 0..6i32 {
+                builder.keys().append_value(i);
+                builder.values().append_value(i * 10);
+                builder.append(true).unwrap();
+            }
+            let base_attrs = builder.finish();
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("attrs", base_attrs.data_type().clone(), true),
+            ]));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(base_attrs),
+                ],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay `attrs` (top-level field id 1) at offset 2 with a two-entry map.
+            let mut ov = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new());
+            ov.keys().append_value(7);
+            ov.values().append_value(77);
+            ov.keys().append_value(8);
+            ov.values().append_value(88);
+            ov.append(true).unwrap();
+            let overlay_attrs = ov.finish();
+            let dataset = commit_overlay(
+                dataset,
+                "mapov",
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(overlay_attrs) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let attrs = batch
+                .column(batch.schema().index_of("attrs").unwrap())
+                .as_any()
+                .downcast_ref::<MapArray>()
+                .unwrap();
+
+            let entries = |i: usize| -> (Vec<i32>, Vec<i32>) {
+                let row = attrs.value(i);
+                let keys = row.column(0).as_any().downcast_ref::<Int32Array>().unwrap();
+                let vals = row.column(1).as_any().downcast_ref::<Int32Array>().unwrap();
+                (keys.values().to_vec(), vals.values().to_vec())
+            };
+            // Offset 1 falls through to the base entry {1: 10}; offset 2 takes the
+            // overlay map {7: 77, 8: 88}.
+            assert_eq!(entries(0), (vec![1], vec![10]));
+            assert_eq!(entries(1), (vec![7, 8], vec![77, 88]));
+        }
+
+        /// Base `id` + a struct `s { a, b }` (6 rows). Field ids: s=1, a=2, b=3.
+        async fn create_struct_dataset(version: LanceFileVersion) -> (Dataset, Fields) {
+            let s_fields = Fields::from(vec![
+                ArrowField::new("a", DataType::Int32, true),
+                ArrowField::new("b", DataType::Int32, true),
+            ]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("s", DataType::Struct(s_fields.clone()), true),
+            ]));
+            let s = Arc::new(StructArray::new(
+                s_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), s],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+            (dataset, s_fields)
+        }
+
+        fn struct_col<'a>(batch: &'a RecordBatch, name: &str) -> &'a StructArray {
+            batch
+                .column(batch.schema().index_of(name).unwrap())
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap()
+        }
+
+        fn i32_child(s: &StructArray, i: usize) -> Int32Array {
+            s.column(i)
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .clone()
+        }
+
+        /// The reviewer's core case (r3553495147): an overlay stores only sub-field
+        /// `s.a`, but the read projects the whole struct `s`. The overlay must splice
+        /// into `a` and leave `b` untouched (previously this panicked because the merge
+        /// fetched the whole `s` from an overlay file holding only `a`).
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_subfield_projecting_parent_struct(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            // Overlay ONLY `s.a` (field id 2) at offset 2.
+            let a_only = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let overlay = Arc::new(StructArray::new(
+                a_only,
+                vec![Arc::new(Int32Array::from(vec![777]))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "aov",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let s = struct_col(&batch, "s");
+            // a: offset 1 base (1), offset 2 overlaid (777).
+            assert_eq!(i32_child(s, 0).values(), &[1, 777]);
+            // b: untouched base (100, 200).
+            assert_eq!(i32_child(s, 1).values(), &[100, 200]);
+        }
+
+        /// An overlay on a non-projected sibling leaf must be skipped and its file
+        /// never opened: overlay covers `s.b`, but the read projects only `s.a`.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_nonprojected_sibling_skipped(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            let b_only = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]);
+            let overlay = Arc::new(StructArray::new(
+                b_only,
+                vec![Arc::new(Int32Array::from(vec![888]))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "bov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+            // Delete the overlay file: if projecting only `s.a` opened it, this fails.
+            dataset
+                .object_store
+                .delete(&Path::from("data/bov.lance"))
+                .await
+                .unwrap();
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let a_only = dataset.schema().project_by_ids(&[2], true);
+            let batch = frag.take(&[1, 2], &a_only).await.unwrap();
+            let s = struct_col(&batch, "s");
+            // Only `a` is projected, unchanged base values.
+            assert_eq!(i32_child(s, 0).values(), &[1, 2]);
+        }
+
+        /// Two overlays target different sub-fields of the same struct, and a third
+        /// re-overlays `s.a`. Each leaf resolves independently and newest wins on `a`.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_multiple_subfields_newest_wins(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let (dataset, _) = create_struct_dataset(version).await;
+            let a_field = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let b_field = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]);
+            // Older: a := 700 at offset 2.
+            let dataset = commit_overlay(
+                dataset,
+                "a_old",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    a_field.clone(),
+                    vec![Arc::new(Int32Array::from(vec![700]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+            // b := 800 at offset 2.
+            let dataset = commit_overlay(
+                dataset,
+                "b_ov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    b_field,
+                    vec![Arc::new(Int32Array::from(vec![800]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+            // Newest: a := 999 at offset 2 (shadows the older `a` overlay).
+            let dataset = commit_overlay(
+                dataset,
+                "a_new",
+                0,
+                &[2],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![Arc::new(StructArray::new(
+                    a_field,
+                    vec![Arc::new(Int32Array::from(vec![999]))],
+                    None,
+                )) as ArrayRef],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[2], &full_schema(&dataset)).await.unwrap();
+            let s = struct_col(&batch, "s");
+            assert_eq!(i32_child(s, 0).values(), &[999]); // newest `a` wins
+            assert_eq!(i32_child(s, 1).values(), &[800]); // `b` from its own overlay
+        }
+
+        /// Three levels of nesting: `outer { middle { a, b } }`. An overlay on the
+        /// deep leaf `outer.middle.a` splices correctly when the whole `outer` is read.
+        #[rstest]
+        #[tokio::test]
+        async fn test_overlay_deeply_nested_subfield(
+            #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion,
+        ) {
+            let mid_fields = Fields::from(vec![
+                ArrowField::new("a", DataType::Int32, true),
+                ArrowField::new("b", DataType::Int32, true),
+            ]);
+            let outer_fields = Fields::from(vec![ArrowField::new(
+                "middle",
+                DataType::Struct(mid_fields.clone()),
+                true,
+            )]);
+            let schema = Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("id", DataType::Int32, true),
+                ArrowField::new("outer", DataType::Struct(outer_fields.clone()), true),
+            ]));
+            // Field ids: outer=1, middle=2, a=3, b=4.
+            let middle = Arc::new(StructArray::new(
+                mid_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from_iter_values(0..6)),
+                    Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))),
+                ],
+                None,
+            ));
+            let outer = Arc::new(StructArray::new(outer_fields, vec![middle], None));
+            let batch = RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(0..6)), outer],
+            )
+            .unwrap();
+            let write_params = WriteParams {
+                max_rows_per_file: 6,
+                max_rows_per_group: 6,
+                data_storage_version: Some(version),
+                ..Default::default()
+            };
+            let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+            let dataset = Dataset::write(reader, "memory://", Some(write_params))
+                .await
+                .unwrap();
+
+            // Overlay the deep leaf `outer.middle.a` (field id 3) at offset 2.
+            let a_leaf = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]);
+            let mid_a = Fields::from(vec![ArrowField::new(
+                "middle",
+                DataType::Struct(a_leaf.clone()),
+                true,
+            )]);
+            let overlay = Arc::new(StructArray::new(
+                mid_a,
+                vec![Arc::new(StructArray::new(
+                    a_leaf,
+                    vec![Arc::new(Int32Array::from(vec![777]))],
+                    None,
+                ))],
+                None,
+            )) as ArrayRef;
+            let dataset = commit_overlay(
+                dataset,
+                "deepov",
+                0,
+                &[3],
+                OverlayCoverage::dense(bitmap([2])),
+                vec![overlay],
+                version,
+            )
+            .await;
+
+            let frag = dataset.get_fragment(0).unwrap();
+            let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap();
+            let outer = struct_col(&batch, "outer");
+            let middle = outer
+                .column(0)
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            // a: offset 1 base (1), offset 2 overlaid (777); b untouched.
+            assert_eq!(i32_child(middle, 0).values(), &[1, 777]);
+            assert_eq!(i32_child(middle, 1).values(), &[100, 200]);
+
+            // Projecting the *intermediate* struct `outer.middle` (field id 2) while
+            // the overlay targets a deeper field (id 3) must still apply: the
+            // overlay's leaf id falls inside the projected subtree, so it maps to a
+            // projected atomic field. (This is the case wjones127/westonpace flagged where a
+            // top-level-only mapping would miss the overlay.)
+            let middle_only = dataset.schema().project_by_ids(&[2], true);
+            let batch = frag.take(&[2], &middle_only).await.unwrap();
+            let middle = struct_col(&batch, "outer")
+                .column(0)
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            assert_eq!(i32_child(middle, 0).values(), &[777]);
+        }
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_fragment_scan(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+        let dataset = create_dataset(test_uri, data_storage_version).await;
+        let fragment = &dataset.get_fragments()[2];
+        let mut scanner = fragment.scan();
+        let batches = scanner
+            .with_row_id()
+            .filter(" i < 105")
+            .unwrap()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        if data_storage_version == LanceFileVersion::Legacy {
+            assert_eq!(batches.len(), 3);
+
+            assert_eq!(
+                batches[0].column_by_name("i").unwrap().as_ref(),
+                &Int32Array::from_iter_values(80..90)
+            );
+            assert_eq!(
+                batches[1].column_by_name("i").unwrap().as_ref(),
+                &Int32Array::from_iter_values(90..100)
+            );
+            assert_eq!(
+                batches[2].column_by_name("i").unwrap().as_ref(),
+                &Int32Array::from_iter_values(100..105)
+            );
+        } else {
+            assert_eq!(batches.len(), 1);
+
+            assert_eq!(
+                batches[0].column_by_name("i").unwrap().as_ref(),
+                &Int32Array::from_iter_values(80..105)
+            )
+        }
+    }
+
+    #[tokio::test]
+    async fn test_fragment_scan_v2() {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+        let dataset = create_dataset_v2(test_uri).await;
+        let fragment = &dataset.get_fragments()[2];
+        let mut scanner = fragment.scan();
+        let batches = scanner
+            .with_row_id()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(batches.len(), 1);
+
+        assert_eq!(
             batches[0].column_by_name("i").unwrap().as_ref(),
             &Int32Array::from_iter_values(80..120)
         );
@@ -4060,6 +5951,153 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_lazy_column_metadata_scan_reads_less_than_full_projection() {
+        let num_columns = 512;
+        let rows_per_batch = 100;
+        let num_batches = 10;
+        let schema = Arc::new(ArrowSchema::new(
+            (0..num_columns)
+                .map(|i| ArrowField::new(format!("col_{i}"), DataType::Int32, true))
+                .collect::<Vec<_>>(),
+        ));
+        let batches = (0..num_batches)
+            .map(|batch_idx| {
+                let columns = (0..num_columns)
+                    .map(|column_idx| {
+                        Arc::new(Int32Array::from_iter_values((0..rows_per_batch).map(
+                            |row_idx| (batch_idx * rows_per_batch + row_idx) as i32 + column_idx,
+                        ))) as ArrayRef
+                    })
+                    .collect::<Vec<_>>();
+                RecordBatch::try_new(schema.clone(), columns).unwrap()
+            })
+            .collect::<Vec<_>>();
+
+        let test_dir = TempStrDir::default();
+        let write_params = WriteParams {
+            max_rows_per_file: rows_per_batch * num_batches,
+            max_rows_per_group: rows_per_batch,
+            data_storage_version: Some(LanceFileVersion::V2_1),
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, &test_dir, Some(write_params))
+            .await
+            .unwrap();
+
+        let projection = dataset.schema().project(&["col_0"]).unwrap();
+        let fragment = dataset.get_fragment(0).unwrap();
+
+        dataset.object_store.as_ref().io_stats_incremental();
+        let narrow_reader = fragment
+            .open(&projection, FragReadConfig::default())
+            .await
+            .unwrap();
+        let mut empty_narrow_stream = narrow_reader.take_range(0..0, 1024).await.unwrap();
+        assert!(empty_narrow_stream.next().await.is_none());
+        let narrow_metadata_stats = dataset.object_store.as_ref().io_stats_incremental();
+        assert!(
+            narrow_metadata_stats.read_iops <= 3,
+            "expected lazy metadata open to skip the schema buffer read, iops={}, bytes={}",
+            narrow_metadata_stats.read_iops,
+            narrow_metadata_stats.read_bytes
+        );
+
+        let full_projection = dataset.schema().clone();
+        let full_reader = fragment
+            .open(&full_projection, FragReadConfig::default())
+            .await
+            .unwrap();
+        let mut empty_full_stream = full_reader.take_range(0..0, 1024).await.unwrap();
+        assert!(empty_full_stream.next().await.is_none());
+        let full_metadata_stats = dataset.object_store.as_ref().io_stats_incremental();
+
+        assert!(
+            full_metadata_stats.read_bytes > narrow_metadata_stats.read_bytes * 4,
+            "expected narrow lazy metadata read to fetch much less than full metadata, narrow={} bytes, full={} bytes",
+            narrow_metadata_stats.read_bytes,
+            full_metadata_stats.read_bytes
+        );
+
+        let mut narrow_scan = dataset.scan();
+        let narrow_batch = narrow_scan
+            .project(&["col_0"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(narrow_batch.num_columns(), 1);
+        assert_eq!(narrow_batch.num_rows(), rows_per_batch * num_batches);
+
+        let taken = fragment.take(&[0, 777, 999], &projection).await.unwrap();
+        let taken_values = taken
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(taken_values.values(), &[0, 777, 999]);
+
+        let projected_readers = fragment
+            .open_readers(&projection, &FragReadConfig::default())
+            .await
+            .unwrap();
+        let err = projected_readers[0].storage_stats().unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("storage_stats requires full file metadata"),
+            "expected storage_stats to reject projected metadata, got {err:?}"
+        );
+    }
+
+    #[test]
+    fn test_indexed_metadata_heuristic_counts_selected_physical_columns() {
+        let schema = Schema::try_from(&ArrowSchema::new(vec![
+            ArrowField::new(
+                "s",
+                DataType::Struct(
+                    vec![
+                        ArrowField::new("x", DataType::Int32, true),
+                        ArrowField::new("y", DataType::Int32, true),
+                    ]
+                    .into(),
+                ),
+                true,
+            ),
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+            ArrowField::new("c", DataType::Int32, true),
+        ]))
+        .unwrap();
+        let data_file = DataFile {
+            path: "wide.lance".to_string(),
+            fields: Arc::from([0, 1, 2, 3, 4, 5]),
+            column_indices: Arc::from([-1, 0, 1, 2, 3, 4]),
+            file_major_version: 2,
+            file_minor_version: 1,
+            file_size_bytes: CachedFileSize::unknown(),
+            base_id: None,
+        };
+
+        let full_struct =
+            ReaderProjection::from_column_names(LanceFileVersion::V2_1, &schema, &["s"]).unwrap();
+        assert_eq!(full_struct.column_indices.len(), 2);
+        assert!(!FileFragment::should_try_indexed_metadata(
+            &data_file,
+            &full_struct,
+            LanceFileVersion::V2_1
+        ));
+
+        let partial_struct =
+            ReaderProjection::from_column_names(LanceFileVersion::V2_1, &schema, &["s.x"]).unwrap();
+        assert_eq!(partial_struct.column_indices.len(), 1);
+        assert!(FileFragment::should_try_indexed_metadata(
+            &data_file,
+            &partial_struct,
+            LanceFileVersion::V2_1
+        ));
+    }
+
     #[tokio::test]
     async fn test_iops_read_small() {
         // Create a file that has 8 columns.
@@ -4121,4 +6159,80 @@ mod tests {
         assert_io_eq!(stats, read_iops, 1);
         assert_io_lt!(stats, read_bytes, 4096);
     }
+
+    #[tokio::test]
+    async fn test_update_columns_with_json_extension_type() {
+        use arrow_array::UInt64Array;
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::ARROW_JSON_EXT_NAME;
+        use lance_core::ROW_ID;
+        use std::collections::HashMap;
+
+        // Create a dataset with an Arrow JSON extension column
+        let test_dir = TempStrDir::default();
+        let mut json_metadata = HashMap::new();
+        json_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            ARROW_JSON_EXT_NAME.to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int64, false),
+            ArrowField::new("name", DataType::Utf8, true),
+            ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])),
+                Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"x":1}"#,
+                    r#"{"x":2}"#,
+                    r#"{"x":3}"#,
+                    r#"{"x":4}"#,
+                    r#"{"x":5}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let dataset = Dataset::write(reader, test_dir.as_ref(), None)
+            .await
+            .unwrap();
+
+        // Build the right stream with Arrow JSON column (Utf8 + arrow.json extension)
+        // Only update rows with row_id 1 and 3
+        let update_schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new(ROW_ID, DataType::UInt64, false),
+            ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata),
+        ]));
+        let update_batch = RecordBatch::try_new(
+            update_schema.clone(),
+            vec![
+                Arc::new(UInt64Array::from(vec![1, 3])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true,"id":2}"#,
+                    r#"{"updated":true,"id":4}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let right_stream: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(update_batch)],
+            update_schema,
+        ));
+
+        // Perform update_columns - this should NOT fail with type mismatch
+        // Previously this would error with:
+        //   "It is not possible to interleave arrays of different data types (Utf8 and LargeBinary)"
+        let mut fragment = dataset.get_fragment(0).unwrap();
+        let (updated_fragment, fields_modified) = fragment
+            .update_columns(right_stream, ROW_ID, ROW_ID)
+            .await
+            .unwrap();
+
+        // Verify the operation produced valid results
+        assert!(!fields_modified.is_empty());
+        assert!(!updated_fragment.files.is_empty());
+    }
 }
diff --git a/rust/lance/src/dataset/fragment/session.rs b/rust/lance/src/dataset/fragment/session.rs
index 64fa427580a..de50255bb72 100644
--- a/rust/lance/src/dataset/fragment/session.rs
+++ b/rust/lance/src/dataset/fragment/session.rs
@@ -54,7 +54,20 @@ impl FragmentSession {
         };
 
         // Then call take rows
-        self.take_rows(&row_ids).await
+        let batch = self.take_rows(&row_ids).await?;
+
+        // Convert Lance JSON columns (LargeBinary/JSONB) back to Arrow JSON (Utf8)
+        // for user-facing output, mirroring FileFragment::take.
+        if batch
+            .schema()
+            .fields()
+            .iter()
+            .any(|f| lance_arrow::json::is_json_field(f) || lance_arrow::json::has_json_fields(f))
+        {
+            Ok(lance_arrow::json::convert_lance_json_to_arrow(&batch)?)
+        } else {
+            Ok(batch)
+        }
     }
 
     pub(crate) async fn take_rows(&self, row_offsets: &[u32]) -> Result<RecordBatch> {
@@ -204,6 +217,78 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_fragment_session_take_json() {
+        use arrow_array::LargeBinaryArray;
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::{ARROW_JSON_EXT_NAME, JsonArray, json_field};
+
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        // Build a schema with a Lance JSON column (LargeBinary + lance.json metadata).
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            json_field("j", true),
+        ]));
+
+        let json_strings = (0..10)
+            .map(|v| Some(format!("{{\"v\":{}}}", v)))
+            .collect::<Vec<_>>();
+        let jsonb = JsonArray::try_from_iter(json_strings).unwrap().into_inner();
+
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..10)),
+                Arc::new(jsonb),
+            ],
+        )
+        .unwrap();
+
+        let write_params = WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_2),
+            ..Default::default()
+        };
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let dataset = Dataset::write(batches, test_uri, Some(write_params))
+            .await
+            .unwrap();
+
+        let fragment = dataset.get_fragments().into_iter().next().unwrap();
+        let take_session = fragment
+            .open_session(dataset.schema(), false)
+            .await
+            .unwrap();
+        let result = take_session.take(&[0, 3, 7]).await.unwrap();
+
+        // The JSON column must be returned as Arrow JSON (Utf8 + arrow.json), not raw JSONB.
+        let field = result.schema().field_with_name("j").unwrap().clone();
+        assert_eq!(field.data_type(), &DataType::Utf8);
+        assert_eq!(
+            field.metadata().get(ARROW_EXT_NAME_KEY).map(|s| s.as_str()),
+            Some(ARROW_JSON_EXT_NAME)
+        );
+        assert!(
+            result
+                .column_by_name("j")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<LargeBinaryArray>()
+                .is_none(),
+            "JSON column should not be raw LargeBinary JSONB"
+        );
+        let json_col = result
+            .column_by_name("j")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("JSON column should be Utf8 strings");
+        assert_eq!(json_col.value(0), "{\"v\":0}");
+        assert_eq!(json_col.value(1), "{\"v\":3}");
+        assert_eq!(json_col.value(2), "{\"v\":7}");
+    }
+
     async fn create_dataset(test_uri: &str, data_storage_version: LanceFileVersion) -> Dataset {
         let schema = Arc::new(ArrowSchema::new(vec![
             ArrowField::new("i", DataType::Int32, true),
diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs
index b10158224f9..42b05051124 100644
--- a/rust/lance/src/dataset/fragment/write.rs
+++ b/rust/lance/src/dataset/fragment/write.rs
@@ -12,6 +12,7 @@ use lance_file::previous::writer::FileWriter as PreviousFileWriter;
 use lance_file::version::LanceFileVersion;
 use lance_file::writer::FileWriterOptions;
 use lance_io::object_store::ObjectStore;
+use lance_io::utils::CachedFileSize;
 use lance_table::format::{DataFile, Fragment};
 use lance_table::io::manifest::ManifestDescribing;
 use std::borrow::Cow;
@@ -20,7 +21,8 @@ use uuid::Uuid;
 
 use crate::Result;
 use crate::dataset::builder::DatasetBuilder;
-use crate::dataset::write::{do_write_fragments, validate_and_resolve_target_bases};
+use crate::dataset::utils::SchemaAdapter;
+use crate::dataset::write::{do_write_fragments, validate_and_resolve_target_bases_with_primary};
 use crate::dataset::{DATA_DIR, Dataset, ReadParams, WriteMode, WriteParams};
 
 /// Generates a filename optimized for S3 throughput using a UUID-based approach.
@@ -105,6 +107,12 @@ impl<'a> FragmentCreateBuilder<'a> {
         id: Option<u64>,
     ) -> Result<Fragment> {
         let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?;
+        // Convert Arrow JSON columns (`arrow.json`, stored as Utf8) into Lance JSON
+        // (`lance.json`, stored as JSONB-encoded LargeBinary) before writing. The
+        // multi-fragment and dataset write paths perform this through `do_write_fragments`;
+        // the single-fragment create path must do the same or the raw UTF-8 string bytes
+        // would be written into a column whose schema declares JSONB, corrupting reads.
+        let stream = SchemaAdapter::new(stream.schema()).to_physical_stream(stream);
         self.write_impl(stream, schema, id).await
     }
 
@@ -165,7 +173,8 @@ impl<'a> FragmentCreateBuilder<'a> {
             writer.write_batches(batch_chunk.iter()).await?;
         }
 
-        fragment.physical_rows = Some(writer.finish().await? as usize);
+        let write_summary = writer.finish().await?;
+        fragment.physical_rows = Some(write_summary.num_rows as usize);
 
         if matches!(fragment.physical_rows, Some(0)) {
             return Err(Error::invalid_input("Input data was empty."));
@@ -186,6 +195,7 @@ impl<'a> FragmentCreateBuilder<'a> {
 
         fragment.files[0].fields = field_ids;
         fragment.files[0].column_indices = column_indices;
+        fragment.files[0].file_size_bytes = CachedFileSize::new(write_summary.size_bytes);
 
         progress.complete(&fragment).await?;
 
@@ -203,6 +213,7 @@ impl<'a> FragmentCreateBuilder<'a> {
         let version = params.data_storage_version.unwrap_or_default();
         let needs_existing_dataset = params.target_base_names_or_paths.is_some()
             || params.target_bases.is_some()
+            || params.target_all_bases.is_some()
             || params.initial_bases.is_some();
         let existing_dataset = if needs_existing_dataset {
             self.existing_dataset(&params).await?
@@ -212,17 +223,24 @@ impl<'a> FragmentCreateBuilder<'a> {
         let existing_base_paths = existing_dataset
             .as_ref()
             .map(|dataset| &dataset.manifest.base_paths);
-        let target_bases_info = if needs_existing_dataset {
-            validate_and_resolve_target_bases(&mut params, existing_base_paths).await?
-        } else {
-            None
-        };
         let (object_store, base_path) = ObjectStore::from_uri_and_params(
             params.store_registry(),
             self.dataset_uri,
             &params.store_params.clone().unwrap_or_default(),
         )
         .await?;
+        let target_bases_info = if needs_existing_dataset {
+            validate_and_resolve_target_bases_with_primary(
+                &mut params,
+                existing_base_paths,
+                &object_store,
+                &base_path,
+                self.dataset_uri,
+            )
+            .await?
+        } else {
+            None
+        };
         do_write_fragments(
             existing_dataset.as_ref(),
             object_store,
@@ -284,7 +302,7 @@ impl<'a> FragmentCreateBuilder<'a> {
             return Err(Error::invalid_input("Input data was empty."));
         }
 
-        fragment.physical_rows = Some(writer.finish().await?);
+        fragment.physical_rows = Some(writer.finish().await?.num_rows as usize);
 
         progress.complete(&fragment).await?;
 
@@ -404,7 +422,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Cannot write with an empty schema.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing empty reader produces an error
@@ -418,7 +436,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Input data was empty.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing with incorrect schema produces an error.
@@ -436,7 +454,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
             if difference.contains("fields did not match")),
             "{:?}",
-            &result
+            result
         );
     }
 
@@ -495,7 +513,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. }
             if source.to_string().contains("Cannot write with an empty schema.")),
             "{:?}",
-            &result
+            result
         );
 
         // Writing empty reader produces an error
@@ -522,7 +540,7 @@ mod tests {
             matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. }
             if difference.contains("fields did not match")),
             "{:?}",
-            &result
+            result
         );
     }
 
@@ -603,6 +621,51 @@ mod tests {
         assert_eq!(fragments[0].files[0].base_id, Some(2));
     }
 
+    #[tokio::test]
+    async fn test_write_fragments_with_target_all_bases() {
+        let primary = TempStrDir::default();
+        let base1 = TempStrDir::default();
+        let base2 = TempStrDir::default();
+        let create_params = WriteParams::default().with_initial_bases(vec![
+            BasePath::new(0, base1.to_string(), Some("base1".to_string()), false),
+            BasePath::new(0, base2.to_string(), Some("base2".to_string()), false),
+        ]);
+
+        let dataset = InsertBuilder::new(primary.as_str())
+            .with_params(&create_params)
+            .execute_stream(test_data())
+            .await
+            .unwrap();
+
+        // Without primary, the first slot is the lowest registered base id.
+        let append_params = WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }
+        .with_target_all_bases(false);
+        let fragments = FragmentCreateBuilder::new(dataset.uri.as_str())
+            .write_params(&append_params)
+            .write_fragments(test_data())
+            .await
+            .unwrap();
+        assert_eq!(fragments.len(), 1);
+        assert_eq!(fragments[0].files[0].base_id, Some(1));
+
+        // With primary included, the first slot is primary storage.
+        let append_params = WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }
+        .with_target_all_bases(true);
+        let fragments = FragmentCreateBuilder::new(dataset.uri.as_str())
+            .write_params(&append_params)
+            .write_fragments(test_data())
+            .await
+            .unwrap();
+        assert_eq!(fragments.len(), 1);
+        assert_eq!(fragments[0].files[0].base_id, None);
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_write_with_format_version(
diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs
index 354cdaf7f86..7cac7815125 100644
--- a/rust/lance/src/dataset/index.rs
+++ b/rust/lance/src/dataset/index.rs
@@ -3,7 +3,8 @@
 
 pub mod frag_reuse;
 
-use std::collections::{HashMap, HashSet};
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use crate::Dataset;
@@ -21,6 +22,7 @@ use lance_index::pb::VectorIndexDetails;
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_table::format::IndexMetadata;
 use serde::{Deserialize, Serialize};
+use uuid::Uuid;
 
 use super::optimize::{IndexRemapper, IndexRemapperOptions};
 
@@ -46,7 +48,7 @@ impl DatasetIndexRemapper {
     async fn remap_index(
         &self,
         index: &IndexMetadata,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
     ) -> Result<RemapResult> {
         remap_index(&self.dataset, &index.uuid, mapping).await
     }
@@ -56,7 +58,7 @@ impl DatasetIndexRemapper {
 impl IndexRemapper for DatasetIndexRemapper {
     async fn remap_indices(
         &self,
-        mapping: HashMap<u64, Option<u64>>,
+        mapping: RowAddrRemap,
         affected_fragment_ids: &[u64],
     ) -> Result<Vec<RemappedIndex>> {
         let affected_frag_ids = HashSet::<u64>::from_iter(affected_fragment_ids.iter().copied());
@@ -121,7 +123,7 @@ impl IndexRemapper for DatasetIndexRemapper {
 #[async_trait]
 pub trait LanceIndexStoreExt {
     /// Create an index store for a new index (will always be absolute with no base id)
-    fn from_dataset_for_new(dataset: &Dataset, uuid: &str) -> Result<Self>
+    fn from_dataset_for_new(dataset: &Dataset, uuid: &Uuid) -> Result<Self>
     where
         Self: Sized;
 
@@ -147,8 +149,8 @@ pub(crate) fn dataset_format_version(dataset: &Dataset) -> LanceFileVersion {
 
 #[async_trait]
 impl LanceIndexStoreExt for LanceIndexStore {
-    fn from_dataset_for_new(dataset: &Dataset, uuid: &str) -> Result<Self> {
-        let index_dir = dataset.indices_dir().join(uuid);
+    fn from_dataset_for_new(dataset: &Dataset, uuid: &Uuid) -> Result<Self> {
+        let index_dir = dataset.indices_dir().join(uuid.to_string());
         let cache = dataset.metadata_cache.file_metadata_cache(&index_dir);
         let format_version = dataset_format_version(dataset);
         Ok(Self::with_format_version(
@@ -174,8 +176,6 @@ impl LanceIndexStoreExt for LanceIndexStore {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
-
     use super::*;
     use crate::dataset::WriteParams;
     use crate::index::DatasetIndexExt;
@@ -223,7 +223,7 @@ mod tests {
         let built_index = dataset
             .create_index_builder(&["vector"], IndexType::Vector, &params)
             .name("vector_idx".to_string())
-            .index_uuid(first_segment_uuid.to_string())
+            .index_uuid(first_segment_uuid)
             .execute_uncommitted()
             .await
             .unwrap();
@@ -298,7 +298,7 @@ mod tests {
             .create_remapper(&dataset)
             .unwrap();
         let remapped = remapper
-            .remap_indices(HashMap::new(), &[target_fragments[0].id() as u64])
+            .remap_indices(RowAddrRemap::empty(), &[target_fragments[0].id() as u64])
             .await
             .unwrap();
 
diff --git a/rust/lance/src/dataset/index/frag_reuse.rs b/rust/lance/src/dataset/index/frag_reuse.rs
index 4fbefcd4725..ceebe456bbf 100644
--- a/rust/lance/src/dataset/index/frag_reuse.rs
+++ b/rust/lance/src/dataset/index/frag_reuse.rs
@@ -243,4 +243,198 @@ mod tests {
             Err(Error::RetryableCommitConflict { .. })
         ));
     }
+
+    /// With more than one index on the table, remapping every index must catch
+    /// all of them up so the reuse index can be trimmed.
+    ///
+    /// Regression: `remap_column_index` used to decide whether to remap an
+    /// index's data from the presence of the old fragments in its fragment
+    /// bitmap. But `load_indices` coverage-remaps the bitmap onto the new
+    /// fragments in memory, and remapping the *first* index commits a manifest
+    /// that persists that cleaned bitmap for the others — so remapping the
+    /// remaining indexes became a silent no-op (their data was never remapped
+    /// and their `dataset_version` never advanced), and the reuse index could
+    /// never be trimmed.
+    #[tokio::test]
+    async fn test_cleanup_frag_reuse_index_multiple_indices() {
+        let mut dataset = lance_datagen::gen_batch()
+            .col("i", lance_datagen::array::step::<Int32Type>())
+            .col("j", lance_datagen::array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        for col in ["i", "j"] {
+            dataset
+                .create_index(
+                    &[col],
+                    IndexType::Scalar,
+                    Some(format!("{col}_idx")),
+                    &ScalarIndexParams::default(),
+                    false,
+                )
+                .await
+                .unwrap();
+        }
+
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 2_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        let frag_reuse_index_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("Fragment reuse index must be available");
+        let frag_reuse_details = load_frag_reuse_index_details(&dataset, &frag_reuse_index_meta)
+            .await
+            .unwrap();
+        assert_eq!(frag_reuse_details.versions.len(), 1);
+
+        for col in ["i", "j"] {
+            remapping::remap_column_index(&mut dataset, &[col], Some(format!("{col}_idx")))
+                .await
+                .unwrap();
+        }
+
+        // Every index must now be caught up (data remapped, version advanced).
+        let indices = dataset.load_indices().await.unwrap();
+        for col in ["i", "j"] {
+            let index = indices
+                .iter()
+                .find(|idx| idx.name == format!("{col}_idx"))
+                .unwrap();
+            assert!(
+                is_index_remap_caught_up(&frag_reuse_details.versions[0], index).unwrap(),
+                "index {col}_idx was not caught up after remap"
+            );
+        }
+
+        // ... so the reuse index trims down to zero versions.
+        cleanup_frag_reuse_index(&mut dataset).await.unwrap();
+        let frag_reuse_index_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("Fragment reuse index must be available");
+        let frag_reuse_details = load_frag_reuse_index_details(&dataset, &frag_reuse_index_meta)
+            .await
+            .unwrap();
+        assert_eq!(frag_reuse_details.versions.len(), 0);
+
+        // Data correctness, not just version bookkeeping: with the reuse index
+        // trimmed there is no auto-remap safety net, so each index must resolve
+        // to LIVE rows. An index whose data was not actually remapped (e.g. one
+        // whose bitmap was coverage-remapped by a sibling's commit before its
+        // own data remap) points at compacted-away fragments and errors on take.
+        use futures::TryStreamExt;
+        for col in ["i", "j"] {
+            let rows: usize = dataset
+                .scan()
+                .filter(&format!("{col} >= 2000 AND {col} < 3000"))
+                .unwrap()
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap()
+                .iter()
+                .map(|b| b.num_rows())
+                .sum();
+            assert_eq!(
+                rows, 1000,
+                "index {col}_idx must resolve to live rows after remap+trim"
+            );
+        }
+    }
+
+    /// When the reuse index has accumulated several versions, a single remap
+    /// must compose them and rebuild + commit the index exactly ONCE, not once
+    /// per version.
+    #[tokio::test]
+    async fn test_remap_index_batches_multiple_reuse_versions() {
+        let mut dataset = lance_datagen::gen_batch()
+            .col("i", lance_datagen::array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(8), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["i"],
+                IndexType::Scalar,
+                Some("i_idx".into()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Accumulate multiple reuse versions: each round deletes a prefix, which
+        // shrinks fragments below target and forces another deferred compaction.
+        let options = CompactionOptions {
+            target_rows_per_fragment: 4_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+        for round in 0..4 {
+            dataset
+                .delete(&format!("i < {}", 1_000 * (round + 1)))
+                .await
+                .unwrap();
+            compact_files(&mut dataset, options.clone(), None)
+                .await
+                .unwrap();
+        }
+
+        let frag_reuse_index_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("Fragment reuse index must be available");
+        let num_versions = load_frag_reuse_index_details(&dataset, &frag_reuse_index_meta)
+            .await
+            .unwrap()
+            .versions
+            .len();
+        assert!(
+            num_versions >= 2,
+            "test needs multiple reuse versions to exercise batching, got {num_versions}"
+        );
+
+        // A single remap must commit exactly once, regardless of version count.
+        let version_before = dataset.manifest.version;
+        remapping::remap_column_index(&mut dataset, &["i"], Some("i_idx".into()))
+            .await
+            .unwrap();
+        let commits = dataset.manifest.version - version_before;
+        assert_eq!(
+            commits, 1,
+            "batched remap must commit once, not once per reuse version ({num_versions})"
+        );
+
+        // ... and the reuse index then trims to zero.
+        cleanup_frag_reuse_index(&mut dataset).await.unwrap();
+        let frag_reuse_index_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("Fragment reuse index must be available");
+        assert_eq!(
+            load_frag_reuse_index_details(&dataset, &frag_reuse_index_meta)
+                .await
+                .unwrap()
+                .versions
+                .len(),
+            0
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs
index 5f3bc2ed483..f5b89d06ff4 100644
--- a/rust/lance/src/dataset/mem_wal.rs
+++ b/rust/lance/src/dataset/mem_wal.rs
@@ -39,10 +39,59 @@ mod manifest;
 pub mod memtable;
 pub mod scanner;
 pub mod sharding;
+#[cfg(test)]
+pub(crate) mod test_util;
 pub mod util;
 mod wal;
 pub mod write;
 
+use std::sync::Arc;
+
+use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+
+/// Column name for the mem_wal tombstone (delete sentinel) marker.
+///
+/// `_tombstone` is a *physical* column present only in mem_wal memtables and
+/// flushed generations — it is deliberately kept out of the base table (hard
+/// delete), so it is **not** a virtual [`is_system_column`](lance_core::is_system_column).
+/// A row with `_tombstone = true` is a delete sentinel: the newest value for
+/// its primary key, carrying null in every non-PK column, that wins
+/// newest-per-PK resolution and is then silently dropped from query results.
+///
+/// The column is owned end-to-end by lance: callers pass the base schema and
+/// lance injects the column on the write path ([`write::ShardWriter::put`] /
+/// [`write::ShardWriter::delete`]), so no caller ever constructs or names it.
+pub const TOMBSTONE: &str = "_tombstone";
+
+/// The mem_wal tombstone field appended to the base schema to form the
+/// memtable/generation schema.
+///
+/// Non-nullable: the write path always populates it (`false` for normal rows,
+/// `true` for tombstones). Non-nullability also lets the point-lookup base arm
+/// synthesize a matching `Literal(false)` column for the `CoalesceFirstExec`
+/// exact-schema check.
+pub fn tombstone_field() -> ArrowField {
+    ArrowField::new(TOMBSTONE, DataType::Boolean, false)
+}
+
+/// Extend a base schema with the trailing `_tombstone` column to form the
+/// mem_wal memtable/generation schema.
+///
+/// Idempotent: a schema that already carries `_tombstone` (a reopen/replay
+/// path) is returned unchanged. Schema-level metadata and per-field metadata
+/// (e.g. the `lance-schema:unenforced-primary-key` marker) are preserved.
+pub fn schema_with_tombstone(base: &ArrowSchema) -> Arc<ArrowSchema> {
+    if base.column_with_name(TOMBSTONE).is_some() {
+        return Arc::new(base.clone());
+    }
+    let mut fields: Vec<ArrowField> = base.fields().iter().map(|f| f.as_ref().clone()).collect();
+    fields.push(tombstone_field());
+    Arc::new(ArrowSchema::new_with_metadata(
+        fields,
+        base.metadata().clone(),
+    ))
+}
+
 pub use api::{DatasetMemWalExt, InitializeMemWalBuilder};
 pub use manifest::ShardManifestStore;
 pub use memtable::scanner::MemTableScanner;
@@ -51,6 +100,7 @@ pub use sharding::{
     evaluate_sharding_spec, evaluate_sharding_spec_with_embedded_columns,
     evaluate_sharding_spec_with_source_columns,
 };
-pub use wal::{WalAppendResult, WalAppender, WalReadEntry, WalTailer};
+pub use wal::{BatchDurableWatcher, WalAppendResult, WalAppender, WalReadEntry, WalTailer};
 pub use write::ShardWriter;
 pub use write::ShardWriterConfig;
+pub use write::WriteResult;
diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs
index ec78ff89c72..1bd95092e7c 100644
--- a/rust/lance/src/dataset/mem_wal/api.rs
+++ b/rust/lance/src/dataset/mem_wal/api.rs
@@ -14,7 +14,6 @@ use async_trait::async_trait;
 use lance_core::{Error, Result};
 use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, ShardingField, ShardingSpec};
 use lance_index::vector::hnsw::builder::HnswBuildParams;
-use lance_io::object_store::ObjectStore;
 use uuid::Uuid;
 
 use crate::Dataset;
@@ -26,7 +25,8 @@ use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta};
 
 use super::ShardWriterConfig;
 use super::scanner::flushed_cache::open_flushed_dataset;
-use super::scanner::{FlushedMemTableCache, ShardSnapshot};
+use super::scanner::{DatasetCache, ShardSnapshot};
+use super::util::derived_store_params;
 use super::write::MemIndexConfig;
 use super::write::ShardWriter;
 
@@ -400,6 +400,14 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap<String, Stri
             "max_wal_buffer_size".to_string(),
             config.max_wal_buffer_size.to_string(),
         ),
+        (
+            "max_wal_persist_retries".to_string(),
+            config.max_wal_persist_retries.to_string(),
+        ),
+        (
+            "wal_persist_retry_base_delay_ms".to_string(),
+            config.wal_persist_retry_base_delay.as_millis().to_string(),
+        ),
         (
             "max_memtable_size".to_string(),
             config.max_memtable_size.to_string(),
@@ -500,7 +508,7 @@ pub trait DatasetMemWalExt {
     async fn prewarm_mem_wal(
         &self,
         _snapshots: &[ShardSnapshot],
-        _cache: Option<&Arc<FlushedMemTableCache>>,
+        _cache: Option<&Arc<dyn DatasetCache>>,
     ) -> Result<()> {
         Ok(())
     }
@@ -586,9 +594,11 @@ impl DatasetMemWalExt for Dataset {
     async fn prewarm_mem_wal(
         &self,
         snapshots: &[ShardSnapshot],
-        cache: Option<&Arc<FlushedMemTableCache>>,
+        cache: Option<&Arc<dyn DatasetCache>>,
     ) -> Result<()> {
         let session = self.session();
+        // Every open below targets a generation URI, never the base's own.
+        let store_params = self.store_params().map(derived_store_params);
         // Resolve flushed paths exactly as the LSM collector does, so the
         // session/cache entries we warm key-match the paths later lookups open.
         let base_path = self.uri().trim_end_matches('/').to_string();
@@ -598,10 +608,18 @@ impl DatasetMemWalExt for Dataset {
                 let shard_id = snapshot.shard_id;
                 let base_path = &base_path;
                 let session = &session;
+                let store_params = &store_params;
                 snapshot.flushed_generations.iter().map(move |flushed| {
                     let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, flushed.path);
                     async move {
-                        let dataset = open_flushed_dataset(&path, Some(session), cache).await?;
+                        let dataset = open_flushed_dataset(
+                            &path,
+                            Some(session),
+                            store_params.as_ref(),
+                            cache,
+                            None,
+                        )
+                        .await?;
                         prewarm_all_indexes(&dataset).await
                     }
                 })
@@ -691,9 +709,17 @@ impl DatasetMemWalExt for Dataset {
         // Set shard_id in config
         config.shard_id = shard_id;
 
-        // Get object store and base path
+        // Inject the dataset's store params + session so the flusher opens the
+        // base + generations with the same store the base was resolved with.
+        config.store_params = self.store_params().cloned();
+        config.session = Some(self.session());
+
+        // Reuse the dataset's own object store + base path; `ObjectStore::from_uri`
+        // would discard the store params the dataset was opened with, signing WAL
+        // writes with the ambient identity. Mirrors `list_mem_wal_latest_shard_ids`.
         let base_uri = self.uri();
-        let (store, base_path) = ObjectStore::from_uri(base_uri).await?;
+        let store = self.object_store(None).await?;
+        let base_path = self.branch_location().path;
 
         // Create ShardWriter
         ShardWriter::open(
@@ -738,7 +764,7 @@ async fn load_vector_index_config(
     // bake this metric into their on-disk metadata, so a wrong default would
     // be durable corruption.
     let distance_type = dataset
-        .open_vector_index(&column, &index_meta.uuid.to_string(), &NoOpMetricsCollector)
+        .open_vector_index(&column, &index_meta.uuid, &NoOpMetricsCollector)
         .await
         .map_err(|e| {
             Error::invalid_input(format!(
@@ -762,6 +788,7 @@ async fn load_vector_index_config(
 
 #[cfg(test)]
 mod tests {
+    use super::super::scanner::FlushedMemTableCache;
     use super::*;
 
     use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
@@ -831,7 +858,7 @@ mod tests {
             .with_current_generation(2)
             .with_flushed_generation(1, folder.to_string());
 
-        let cache = Arc::new(FlushedMemTableCache::new(4));
+        let cache: Arc<dyn DatasetCache> = Arc::new(FlushedMemTableCache::new(4));
         base.prewarm_mem_wal(std::slice::from_ref(&snapshot), Some(&cache))
             .await
             .expect("prewarm must open the generation and warm its index");
@@ -839,7 +866,7 @@ mod tests {
         // The generation is resident in the cache (same session), with its
         // index loadable — a later lookup that opens this path is a pure hit.
         let warmed = cache
-            .get_or_open(&gen_uri, Some(base.session()))
+            .get_or_open(&gen_uri, Some(base.session()), base.store_params().cloned())
             .await
             .unwrap();
         assert_eq!(warmed.load_indices().await.unwrap().len(), 1);
diff --git a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
index 6dfa09840e9..bbeb57a5fe2 100644
--- a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
+++ b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs
@@ -8,7 +8,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 
 use arrow_array::cast::AsArray;
 use arrow_array::types::Float32Type;
-use arrow_array::{Array, ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt64Array};
+use arrow_array::{
+    Array, ArrayRef, BooleanArray, FixedSizeListArray, Float32Array, RecordBatch, UInt64Array,
+};
 use arrow_schema::{DataType, Field, Schema as ArrowSchema, SchemaRef};
 use lance_core::{Error, ROW_ID, Result};
 use lance_linalg::distance::{DistanceType, cosine_distance, dot_f32, l2_f32};
@@ -205,6 +207,12 @@ impl ArrowFixedSizeListVectorStore {
     }
 
     /// Append one Arrow `FixedSizeList<Float32>` batch by reference.
+    ///
+    /// Null-vector rows (e.g. tombstones, or rows written without an embedding)
+    /// are skipped — they get no graph node — matching base-table vector index
+    /// semantics. Surviving non-null rows are compacted into a contiguous node
+    /// range, but each node's `row_ids` entry preserves its *original* offset in
+    /// the input batch so search still resolves a hit back to the correct row.
     pub fn append_batch(
         &self,
         vectors: Arc<FixedSizeListArray>,
@@ -221,13 +229,39 @@ impl ArrowFixedSizeListVectorStore {
                 vectors.value_length()
             )));
         }
-        if vectors.null_count() > 0 {
-            return Err(Error::invalid_input(format!(
-                "null vectors are not supported, got {} null row(s)",
-                vectors.null_count()
-            )));
+
+        // Compact away null-vector rows, remembering each survivor's original
+        // offset. The all-non-null case keeps the input array as-is (offsets are
+        // the identity) to avoid a copy on the hot path.
+        let (stored_array, original_offsets): (Arc<FixedSizeListArray>, Option<Vec<u32>>) =
+            if vectors.null_count() == 0 {
+                (vectors, None)
+            } else {
+                let valid = vectors
+                    .nulls()
+                    .ok_or_else(|| Error::internal("null_count > 0 but no null buffer"))?;
+                let mask = BooleanArray::new(valid.inner().clone(), None);
+                let filtered = arrow_select::filter::filter(vectors.as_ref(), &mask)?;
+                let fsl = filtered
+                    .as_fixed_size_list_opt()
+                    .ok_or_else(|| {
+                        Error::internal("filtered vector column is not a FixedSizeList")
+                    })?
+                    .clone();
+                let offsets: Vec<u32> = (0..vectors.len() as u32)
+                    .filter(|&i| valid.is_valid(i as usize))
+                    .collect();
+                (Arc::new(fsl), Some(offsets))
+            };
+
+        let num_rows = stored_array.len();
+        if num_rows == 0 {
+            // All rows were null (e.g. an all-tombstone memtable): no graph node.
+            let start = self.committed_len.load(Ordering::Relaxed) as u32;
+            return Ok(start..start);
         }
-        let values = vectors.values();
+
+        let values = stored_array.values();
         let Some(values_f32) = values.as_primitive_opt::<Float32Type>() else {
             return Err(Error::invalid_input(format!(
                 "vector values must be Float32, got {:?}",
@@ -236,11 +270,10 @@ impl ArrowFixedSizeListVectorStore {
         };
 
         let start = self.committed_len.load(Ordering::Relaxed);
-        let end = start.checked_add(vectors.len()).ok_or_else(|| {
+        let end = start.checked_add(num_rows).ok_or_else(|| {
             Error::invalid_input(format!(
                 "vector count overflow: start={}, batch_len={}",
-                start,
-                vectors.len()
+                start, num_rows
             ))
         })?;
         if end > self.capacity {
@@ -264,19 +297,25 @@ impl ArrowFixedSizeListVectorStore {
             self.batches
                 .add(batch_idx)
                 .write(MaybeUninit::new(StoredArrowBatch {
-                    _array: vectors.clone(),
+                    _array: stored_array.clone(),
                     values_ptr: values_f32.values().as_ptr(),
                 }));
-            for offset in 0..vectors.len() {
+            for offset in 0..num_rows {
                 self.row_to_batch
                     .add(start + offset)
                     .write(MaybeUninit::new(RowLookup {
                         batch_idx: batch_idx as u32,
                         offset: offset as u32,
                     }));
+                // The node maps back to the survivor's *original* row position,
+                // so a compacted node still resolves to the right row.
+                let original = match &original_offsets {
+                    Some(offsets) => offsets[offset] as u64,
+                    None => offset as u64,
+                };
                 self.row_ids
                     .add(start + offset)
-                    .write(MaybeUninit::new(row_id_start + offset as u64));
+                    .write(MaybeUninit::new(row_id_start + original));
             }
         }
 
@@ -459,4 +498,88 @@ mod tests {
             8.0
         );
     }
+
+    /// Build a `FixedSizeList<Float32>` where `None` rows are null at the list
+    /// level (the representation a tombstone / embedding-less row produces).
+    fn fsl_opt(rows: &[Option<Vec<f32>>], dim: usize) -> Arc<FixedSizeListArray> {
+        let mut values: Vec<f32> = Vec::new();
+        let mut validity: Vec<bool> = Vec::new();
+        for r in rows {
+            match r {
+                Some(v) => {
+                    assert_eq!(v.len(), dim);
+                    values.extend_from_slice(v);
+                    validity.push(true);
+                }
+                None => {
+                    values.extend(std::iter::repeat_n(0.0f32, dim));
+                    validity.push(false);
+                }
+            }
+        }
+        let values = Arc::new(Float32Array::from(values)) as ArrayRef;
+        let nulls = arrow_buffer::NullBuffer::new(arrow_buffer::BooleanBuffer::from(validity));
+        Arc::new(
+            FixedSizeListArray::try_new(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                dim as i32,
+                values,
+                Some(nulls),
+            )
+            .unwrap(),
+        )
+    }
+
+    #[test]
+    fn test_append_skips_null_vectors_preserves_mapping() {
+        // Mid-batch null: rows [(1,1), null, (3,3)] → two nodes that map back to
+        // the *original* offsets 0 and 2 (no off-by-one after the skip).
+        let store =
+            Arc::new(ArrowFixedSizeListVectorStore::try_new(8, 2, 2, DistanceType::L2).unwrap());
+        let batch = fsl_opt(&[Some(vec![1.0, 1.0]), None, Some(vec![3.0, 3.0])], 2);
+        assert_eq!(
+            store.append_batch(batch, 0).unwrap(),
+            0..2,
+            "two non-null survivors → two contiguous nodes"
+        );
+        assert_eq!(store.committed_len(), 2);
+
+        // `to_record_batch` exposes each node's row id = its original offset.
+        let rb = store.to_record_batch(None).unwrap();
+        let row_ids = rb.column(0).as_any().downcast_ref::<UInt64Array>().unwrap();
+        let got: Vec<u64> = (0..row_ids.len()).map(|i| row_ids.value(i)).collect();
+        assert_eq!(got, vec![0, 2], "node→row mapping skips the null row");
+
+        let snapshot = store.snapshot();
+        assert_eq!(snapshot.vector(0), &[1.0, 1.0]);
+        assert_eq!(snapshot.vector(1), &[3.0, 3.0]);
+        assert_eq!(snapshot.row_id(0), 0);
+        assert_eq!(snapshot.row_id(1), 2);
+    }
+
+    #[test]
+    fn test_append_skips_null_vectors_offsets_with_row_id_start() {
+        // The survivor's original offset is added to `row_id_start`.
+        let store =
+            Arc::new(ArrowFixedSizeListVectorStore::try_new(8, 2, 2, DistanceType::L2).unwrap());
+        // rows at row_id_start=100: [null, (2,2)] → one node, row id 100+1.
+        let batch = fsl_opt(&[None, Some(vec![2.0, 2.0])], 2);
+        assert_eq!(store.append_batch(batch, 100).unwrap(), 0..1);
+        let snapshot = store.snapshot();
+        assert_eq!(snapshot.row_id(0), 101);
+        assert_eq!(snapshot.vector(0), &[2.0, 2.0]);
+    }
+
+    #[test]
+    fn test_append_all_null_batch_is_empty_range() {
+        // All-null batch (e.g. an all-tombstone memtable) adds no node.
+        let store =
+            Arc::new(ArrowFixedSizeListVectorStore::try_new(8, 2, 2, DistanceType::L2).unwrap());
+        let batch = fsl_opt(&[None, None], 2);
+        assert!(
+            store.append_batch(batch, 0).unwrap().is_empty(),
+            "all-null batch yields an empty node range"
+        );
+        assert_eq!(store.committed_len(), 0);
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs
index 116ea6c60ce..8077f06149e 100644
--- a/rust/lance/src/dataset/mem_wal/index.rs
+++ b/rust/lance/src/dataset/mem_wal/index.rs
@@ -18,9 +18,13 @@ mod arena_skiplist;
 mod btree;
 mod fts;
 mod hnsw;
+mod pk_key;
 
 use std::collections::HashMap;
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use datafusion::common::ScalarValue;
 
 use super::memtable::batch_store::StoredBatch;
 use arrow_array::RecordBatch;
@@ -28,6 +32,7 @@ use lance_core::datatypes::Schema as LanceSchema;
 use lance_core::{Error, Result};
 use lance_index::pbold;
 use lance_index::scalar::InvertedIndexParams;
+use lance_index::scalar::inverted::InvertedListFormatVersion;
 use lance_index::vector::hnsw::builder::HnswBuildParams;
 use lance_linalg::distance::DistanceType;
 use lance_table::format::IndexMetadata;
@@ -44,6 +49,32 @@ pub type RowPosition = u64;
 pub use btree::{BTreeIndexConfig, BTreeMemIndex};
 pub use fts::{FtsIndexConfig, FtsMemIndex, FtsQueryExpr, SearchOptions};
 pub use hnsw::{HnswIndexConfig, HnswMemIndex};
+pub use pk_key::encode_pk_tuple;
+
+use pk_key::encode_pk_batch;
+
+/// Synthetic column the composite PK index is keyed on: the order-preserving
+/// encoded tuple (see [`encode_pk_tuple`]), stored as `Binary` so a
+/// [`BTreeMemIndex`]'s byte backend indexes it directly.
+const PK_KEY_COLUMN: &str = "__pk_key__";
+
+/// The memtable's primary-key index, used to answer "newest visible version of
+/// this key" for dedup. Single-column PKs reuse the column's compact typed
+/// [`BTreeMemIndex`] (no second copy); composite PKs key a `BTreeMemIndex` on
+/// the order-preserving encoded tuple ([`encode_pk_tuple`]) instead. Either way
+/// the lookup is a single seek on one `BTreeMemIndex`.
+enum PkIndex {
+    /// Arity 1: aliases a `btree_indexes` entry, so the insert loop maintains it.
+    Single(Arc<BTreeMemIndex>),
+    /// Arity >= 2: a `BTreeMemIndex` over the encoded-tuple `Binary` key,
+    /// maintained explicitly in the insert paths (the original batch lacks the
+    /// synthetic key column). `columns` are the PK columns in order, resolved
+    /// against each batch's schema at insert time.
+    Composite {
+        index: Arc<BTreeMemIndex>,
+        columns: Vec<String>,
+    },
+}
 
 // ============================================================================
 // Index Store
@@ -116,13 +147,14 @@ impl MemIndexConfig {
         } else {
             InvertedIndexParams::default()
         };
+        let params = params.format_version(Self::fts_format_version_from_metadata(index_meta)?);
 
-        Ok(Self::Fts(FtsIndexConfig::with_params(
+        Ok(Self::Fts(FtsIndexConfig::try_with_params(
             index_meta.name.clone(),
             field_id,
             column,
             params,
-        )))
+        )?))
     }
 
     /// Create an HNSW vector index config.
@@ -165,6 +197,22 @@ impl MemIndexConfig {
         }
     }
 
+    fn fts_format_version_from_metadata(
+        index_meta: &IndexMetadata,
+    ) -> Result<InvertedListFormatVersion> {
+        match index_meta.index_version {
+            // Legacy Arrow FTS indexes did not use the v1/v2 metadata values, but
+            // the maintained-index path can only write the modern format.
+            0 | 1 => Ok(InvertedListFormatVersion::V1),
+            2 => Ok(InvertedListFormatVersion::V2),
+            3 => Ok(InvertedListFormatVersion::V3),
+            version => Err(Error::invalid_input(format!(
+                "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3",
+                index_meta.name, version
+            ))),
+        }
+    }
+
     /// Extract field ID and column name from index metadata.
     fn extract_field_info(
         index_meta: &IndexMetadata,
@@ -195,16 +243,26 @@ impl MemIndexConfig {
 /// therefore safe for scanners to read. Scanners snapshot this at plan
 /// construction time so every plan keys on a stable MVCC cursor.
 pub struct IndexStore {
-    /// BTree indexes keyed by index name.
-    btree_indexes: HashMap<String, BTreeMemIndex>,
+    /// BTree indexes keyed by index name. `Arc` so the primary-key BTrees can be
+    /// shared into [`Self::pk_btrees`] without a second copy or a second insert.
+    btree_indexes: HashMap<String, Arc<BTreeMemIndex>>,
     /// HNSW vector indexes keyed by index name.
     hnsw_indexes: HashMap<String, HnswMemIndex>,
     /// FTS indexes keyed by index name.
     fts_indexes: HashMap<String, FtsMemIndex>,
+    /// The primary-key index (single-column or composite), or `None` without a
+    /// primary key. Queried via [`Self::pk_newest_visible`] (see
+    /// [`Self::enable_pk_index`]).
+    pk_index: Option<PkIndex>,
     /// Maximum batch position that is durable in the WAL and therefore
     /// visible to scanners. Advanced unconditionally after a WAL append
     /// succeeds; not gated on whether any indexes are configured.
     max_visible_batch_position: AtomicUsize,
+    /// Conservative flag set once this memtable has observed any primary-key
+    /// rewrite while maintaining a search index. Search planners can push top-k
+    /// into HNSW/FTS for append-only PK data, but must switch to
+    /// newest-before-top-k search after an overwrite.
+    pk_has_overrides: AtomicBool,
 }
 
 impl Default for IndexStore {
@@ -213,7 +271,9 @@ impl Default for IndexStore {
             btree_indexes: HashMap::new(),
             hnsw_indexes: HashMap::new(),
             fts_indexes: HashMap::new(),
+            pk_index: None,
             max_visible_batch_position: AtomicUsize::new(0),
+            pk_has_overrides: AtomicBool::new(false),
         }
     }
 }
@@ -230,10 +290,24 @@ impl std::fmt::Debug for IndexStore {
                 &self.hnsw_indexes.keys().collect::<Vec<_>>(),
             )
             .field("fts_indexes", &self.fts_indexes.keys().collect::<Vec<_>>())
+            .field(
+                "pk_index",
+                &match &self.pk_index {
+                    None => "none".to_string(),
+                    Some(PkIndex::Single(b)) => format!("single({})", b.column_name()),
+                    Some(PkIndex::Composite { columns, .. }) => {
+                        format!("composite({})", columns.join(", "))
+                    }
+                },
+            )
             .field(
                 "max_visible_batch_position",
                 &self.max_visible_batch_position.load(Ordering::Acquire),
             )
+            .field(
+                "pk_has_overrides",
+                &self.pk_has_overrides.load(Ordering::Acquire),
+            )
             .finish()
     }
 }
@@ -264,7 +338,7 @@ impl IndexStore {
         for config in configs {
             match config {
                 MemIndexConfig::BTree(c) => {
-                    let index = BTreeMemIndex::new(c.field_id, c.column.clone());
+                    let index = Arc::new(BTreeMemIndex::new(c.field_id, c.column.clone()));
                     registry.btree_indexes.insert(c.name.clone(), index);
                 }
                 MemIndexConfig::Hnsw(c) => {
@@ -279,8 +353,11 @@ impl IndexStore {
                     registry.hnsw_indexes.insert(c.name.clone(), index);
                 }
                 MemIndexConfig::Fts(c) => {
-                    let index =
-                        FtsMemIndex::with_params(c.field_id, c.column.clone(), c.params.clone());
+                    let index = FtsMemIndex::try_with_params(
+                        c.field_id,
+                        c.column.clone(),
+                        c.params.clone(),
+                    )?;
                     registry.fts_indexes.insert(c.name.clone(), index);
                 }
             }
@@ -293,10 +370,14 @@ impl IndexStore {
     /// the production memtable path goes through [`Self::from_configs`].
     pub fn add_btree(&mut self, name: String, field_id: i32, column: String) {
         self.btree_indexes
-            .insert(name, BTreeMemIndex::new(field_id, column));
+            .insert(name, Arc::new(BTreeMemIndex::new(field_id, column)));
     }
 
     /// Add an HNSW vector index with default build parameters.
+    ///
+    /// HNSW indexes must be configured before rows are inserted into a
+    /// PK-indexed memtable. The vector planner's append-only fast path relies
+    /// on `pk_has_overrides` being maintained for every row visible to HNSW.
     pub fn add_hnsw(
         &mut self,
         name: String,
@@ -306,6 +387,10 @@ impl IndexStore {
         capacity: usize,
         max_batches: usize,
     ) {
+        assert!(
+            self.pk_index.is_none() || self.pk_is_empty(),
+            "HNSW indexes must be configured before inserting rows into a PK memtable"
+        );
         self.hnsw_indexes.insert(
             name,
             HnswMemIndex::with_capacity(
@@ -320,6 +405,8 @@ impl IndexStore {
     }
 
     /// Add an HNSW vector index with explicit build parameters.
+    ///
+    /// See [`Self::add_hnsw`] for the PK-indexed memtable lifecycle invariant.
     #[allow(clippy::too_many_arguments)]
     pub fn add_hnsw_with_params(
         &mut self,
@@ -331,6 +418,10 @@ impl IndexStore {
         capacity: usize,
         max_batches: usize,
     ) {
+        assert!(
+            self.pk_index.is_none() || self.pk_is_empty(),
+            "HNSW indexes must be configured before inserting rows into a PK memtable"
+        );
         self.hnsw_indexes.insert(
             name,
             HnswMemIndex::with_capacity(
@@ -345,7 +436,15 @@ impl IndexStore {
     }
 
     /// Add an FTS index with default tokenizer parameters.
+    ///
+    /// FTS indexes must be configured before rows are inserted into a
+    /// PK-indexed memtable. FTS top-k pushdown relies on `pk_has_overrides`
+    /// being maintained for every row visible to the index.
     pub fn add_fts(&mut self, name: String, field_id: i32, column: String) {
+        assert!(
+            self.pk_index.is_none() || self.pk_is_empty(),
+            "FTS indexes must be configured before inserting rows into a PK memtable"
+        );
         self.fts_indexes
             .insert(name, FtsMemIndex::new(field_id, column));
     }
@@ -357,9 +456,210 @@ impl IndexStore {
         field_id: i32,
         column: String,
         params: InvertedIndexParams,
-    ) {
-        self.fts_indexes
-            .insert(name, FtsMemIndex::with_params(field_id, column, params));
+    ) -> Result<()> {
+        assert!(
+            self.pk_index.is_none() || self.pk_is_empty(),
+            "FTS indexes must be configured before inserting rows into a PK memtable"
+        );
+        self.fts_indexes.insert(
+            name,
+            FtsMemIndex::try_with_params(field_id, column, params)?,
+        );
+        Ok(())
+    }
+
+    /// Maintain a primary-key index so the memtable can answer "newest visible
+    /// version of this key" (see [`Self::pk_newest_visible`]).
+    ///
+    /// Single-column PKs reuse an existing BTree on the field, else auto-create
+    /// one under a `__pk__*` name so the normal insert loop maintains it (no
+    /// second copy). Composite (arity >= 2) PKs key a `BTreeMemIndex` on the
+    /// order-preserving encoded tuple (synthetic `PK_KEY_COLUMN`), maintained
+    /// explicitly in the insert paths. Call once at construction, after
+    /// [`Self::from_configs`] and before any inserts; a no-op when `pk_columns`
+    /// is empty. Search indexes (HNSW/FTS) must also still be empty so every
+    /// search-visible row participates in PK override tracking.
+    pub fn enable_pk_index(&mut self, pk_columns: &[(String, i32)]) {
+        if !pk_columns.is_empty() {
+            assert!(
+                self.hnsw_indexes.values().all(|idx| idx.is_empty())
+                    && self.fts_indexes.values().all(|idx| idx.is_empty()),
+                "Primary-key indexes must be configured before inserting rows into a search-indexed memtable"
+            );
+        }
+        self.pk_index = match pk_columns {
+            [] => None,
+            [(column, field_id)] => {
+                let btree = match self
+                    .btree_indexes
+                    .values()
+                    .find(|b| b.field_id() == *field_id)
+                {
+                    Some(existing) => existing.clone(),
+                    None => {
+                        let btree = Arc::new(BTreeMemIndex::new(*field_id, column.clone()));
+                        self.btree_indexes
+                            .insert(format!("__pk__{column}"), btree.clone());
+                        btree
+                    }
+                };
+                Some(PkIndex::Single(btree))
+            }
+            multi => Some(PkIndex::Composite {
+                // Synthetic field id (-1): the composite index is held directly,
+                // never resolved by field id.
+                index: Arc::new(BTreeMemIndex::new(-1, PK_KEY_COLUMN.to_string())),
+                columns: multi.iter().map(|(c, _)| c.clone()).collect(),
+            }),
+        };
+    }
+
+    /// Whether the memtable has a primary-key index.
+    pub fn has_pk_index(&self) -> bool {
+        self.pk_index.is_some()
+    }
+
+    /// Sorted `(value, row_id)` training batches for the flushed on-disk PK
+    /// BTree (the sidecar dedup index). Single-column emits the typed PK value;
+    /// composite emits the order-preserving `Binary` encoded tuple. Empty when
+    /// there is no primary key. Row positions line up 1:1 with the forward-
+    /// written data file, so they are the flushed row ids directly.
+    pub fn pk_training_batches(&self, batch_size: usize) -> Result<Vec<RecordBatch>> {
+        match &self.pk_index {
+            None => Ok(Vec::new()),
+            Some(PkIndex::Single(btree)) => btree.to_training_batches(batch_size),
+            Some(PkIndex::Composite { index, .. }) => index.to_training_batches(batch_size),
+        }
+    }
+
+    /// Resolve the PK columns' positions in `batch` (composite insert helper).
+    fn pk_batch_indices(batch: &RecordBatch, columns: &[String]) -> Result<Vec<usize>> {
+        columns
+            .iter()
+            .map(|c| {
+                batch
+                    .schema()
+                    .column_with_name(c)
+                    .map(|(i, _)| i)
+                    .ok_or_else(|| {
+                        Error::invalid_input(format!("PK column '{c}' not found in batch"))
+                    })
+            })
+            .collect()
+    }
+
+    /// Maintain the composite PK index for `batch` (no-op for single/no PK):
+    /// encode the PK columns into the synthetic `PK_KEY_COLUMN` `Binary` column
+    /// and feed that to the keyed `BTreeMemIndex`.
+    fn insert_composite_pk(
+        &self,
+        batch: &RecordBatch,
+        row_offset: u64,
+        report_existing: bool,
+    ) -> Result<bool> {
+        if let Some(PkIndex::Composite { index, columns }) = &self.pk_index {
+            let pk_indices = Self::pk_batch_indices(batch, columns)?;
+            let encoded = encode_pk_batch(batch, &pk_indices)?;
+            let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+                PK_KEY_COLUMN,
+                arrow_schema::DataType::Binary,
+                false,
+            )]));
+            let key_batch = RecordBatch::try_new(schema, vec![Arc::new(encoded)])
+                .map_err(|e| Error::invalid_input(e.to_string()))?;
+            if report_existing {
+                return index.insert_and_report_existing(&key_batch, row_offset);
+            }
+            index.insert(&key_batch, row_offset)?;
+        }
+        Ok(false)
+    }
+
+    /// The newest row position of the primary-key tuple `values` (in PK order)
+    /// visible at `max_visible_row`, or `None`. A single seek either way:
+    /// single-column probes the typed BTree; composite probes the encoded-tuple
+    /// index. Collision-free, since `position` is the row identity.
+    pub fn pk_newest_visible(
+        &self,
+        values: &[ScalarValue],
+        max_visible_row: RowPosition,
+    ) -> Option<RowPosition> {
+        match &self.pk_index {
+            None => None,
+            Some(PkIndex::Single(btree)) => btree.get_newest_visible(&values[0], max_visible_row),
+            Some(PkIndex::Composite { index, .. }) => {
+                // An unsupported PK type would have failed at insert, so the
+                // index can't hold a tuple this fails to encode. The probe key is
+                // the same `Binary`-encoded tuple the insert path indexed.
+                let key = encode_pk_tuple(values).ok()?;
+                index.get_newest_visible(&ScalarValue::Binary(Some(key)), max_visible_row)
+            }
+        }
+    }
+
+    /// Whether `position` is the newest visible row of `values` — the recency
+    /// check the active index-search arms apply to drop predicate-crossing
+    /// stale hits. Callers gate on [`Self::has_pk_index`] first, since this is
+    /// `false` (drop) when the memtable has no primary-key index.
+    pub fn pk_is_newest(
+        &self,
+        values: &[ScalarValue],
+        position: RowPosition,
+        max_visible_row: RowPosition,
+    ) -> bool {
+        self.pk_newest_visible(values, max_visible_row) == Some(position)
+    }
+
+    /// Whether `key` has any version visible at `max_visible_row` — the
+    /// cross-source block-list's existence query, snapshot-bounded so a
+    /// not-yet-visible write can't shadow an older visible copy.
+    ///
+    /// `key` is already in the index's key space: the typed PK value for a
+    /// single-column key, the `Binary`-encoded tuple for a composite one (built
+    /// by `block_list::on_disk_pk_key`, the same key the flushed on-disk index is
+    /// probed with). Both arities forward it straight to the keyed BTree.
+    pub fn pk_contains_key(&self, key: &ScalarValue, max_visible_row: RowPosition) -> bool {
+        match &self.pk_index {
+            None => false,
+            Some(PkIndex::Single(btree)) | Some(PkIndex::Composite { index: btree, .. }) => {
+                btree.get_newest_visible(key, max_visible_row).is_some()
+            }
+        }
+    }
+
+    /// Whether the primary-key index holds no rows (or doesn't exist).
+    pub fn pk_is_empty(&self) -> bool {
+        match &self.pk_index {
+            None => true,
+            Some(PkIndex::Single(btree)) => btree.is_empty(),
+            Some(PkIndex::Composite { index, .. }) => index.is_empty(),
+        }
+    }
+
+    /// Whether this memtable has observed at least one PK rewrite.
+    ///
+    /// This is intentionally conservative: once true, it never resets for the
+    /// lifetime of the memtable. That is enough for query planning because a
+    /// memtable is flushed as a unit, and any rewrite means search-index top-k
+    /// pushdown can be polluted by stale entries that must be removed before
+    /// top-k. Scalar-only PK tables skip tracking because no search index uses
+    /// the flag.
+    pub fn pk_has_overrides(&self) -> bool {
+        self.pk_has_overrides.load(Ordering::Acquire)
+    }
+
+    fn should_track_pk_overrides(&self) -> bool {
+        (!self.hnsw_indexes.is_empty() || !self.fts_indexes.is_empty()) && !self.pk_has_overrides()
+    }
+
+    fn is_single_pk_btree(&self, index: &Arc<BTreeMemIndex>) -> bool {
+        matches!(&self.pk_index, Some(PkIndex::Single(pk)) if Arc::ptr_eq(pk, index))
+    }
+
+    fn mark_pk_overrides_if_needed(&self, had_existing_pk: bool) {
+        if had_existing_pk {
+            self.pk_has_overrides.store(true, Ordering::Release);
+        }
     }
 
     /// Insert a batch into all indexes.
@@ -375,8 +675,14 @@ impl IndexStore {
         row_offset: u64,
         batch_position: Option<usize>,
     ) -> Result<()> {
+        let track_pk_overrides = self.should_track_pk_overrides();
         for index in self.btree_indexes.values() {
-            index.insert(batch, row_offset)?;
+            if track_pk_overrides && self.is_single_pk_btree(index) {
+                let had_existing = index.insert_and_report_existing(batch, row_offset)?;
+                self.mark_pk_overrides_if_needed(had_existing);
+            } else {
+                index.insert(batch, row_offset)?;
+            }
         }
         for index in self.hnsw_indexes.values() {
             index.insert(batch, row_offset)?;
@@ -384,6 +690,10 @@ impl IndexStore {
         for index in self.fts_indexes.values() {
             index.insert(batch, row_offset)?;
         }
+        // Single-column PK aliases a `btree_indexes` entry (maintained above);
+        // a composite PK has its own index, maintained here.
+        let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?;
+        self.mark_pk_overrides_if_needed(had_existing);
 
         // Update global watermark after all indexes have been updated
         if let Some(bp) = batch_position {
@@ -395,11 +705,11 @@ impl IndexStore {
 
     /// Advance the visibility watermark to at least `batch_pos`.
     ///
-    /// The watermark only ever moves forward (idempotent max). Public so the
-    /// WAL flush handler can advance it after a successful WAL append, which
-    /// is the authoritative durability signal regardless of whether any
-    /// indexes are configured.
-    pub fn advance_max_visible_batch_position(&self, batch_pos: usize) {
+    /// The watermark only ever moves forward (idempotent max). The vector
+    /// planner relies on the insert paths setting `pk_has_overrides` before
+    /// calling this method, so any snapshot that can see a PK rewrite also
+    /// observes `pk_has_overrides == true`.
+    pub(crate) fn advance_max_visible_batch_position(&self, batch_pos: usize) {
         let mut current = self.max_visible_batch_position.load(Ordering::Acquire);
         while batch_pos > current {
             match self.max_visible_batch_position.compare_exchange_weak(
@@ -421,11 +731,20 @@ impl IndexStore {
             return Ok(());
         }
 
+        let track_pk_overrides = self.should_track_pk_overrides();
         // BTree indexes: iterate batches (no cross-batch optimization benefit)
         for index in self.btree_indexes.values() {
+            let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
+            let mut had_existing = false;
             for stored in batches {
-                index.insert(&stored.data, stored.row_offset)?;
+                if track_this_index {
+                    had_existing |=
+                        index.insert_and_report_existing(&stored.data, stored.row_offset)?;
+                } else {
+                    index.insert(&stored.data, stored.row_offset)?;
+                }
             }
+            self.mark_pk_overrides_if_needed(had_existing);
         }
 
         // HNSW indexes: use batched insert
@@ -440,6 +759,15 @@ impl IndexStore {
             }
         }
 
+        // Single-column PK aliases a `btree_indexes` entry (maintained above);
+        // a composite PK has its own index, maintained here.
+        let mut had_existing = false;
+        for stored in batches {
+            had_existing |=
+                self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?;
+        }
+        self.mark_pk_overrides_if_needed(had_existing);
+
         // Update global watermark to the max batch position
         let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
         self.advance_max_visible_batch_position(max_bp);
@@ -468,24 +796,32 @@ impl IndexStore {
             return Ok(std::collections::HashMap::new());
         }
 
+        let track_pk_overrides = self.should_track_pk_overrides();
         // Use std::thread::scope for parallel CPU-bound work
         std::thread::scope(|scope| {
             // Each handle returns (index_name, index_type, duration, Result)
             let mut handles: Vec<(
                 &str,
                 &str,
-                std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result<()>)>,
+                std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result<bool>)>,
             )> = Vec::new();
 
             // Spawn a thread for each BTree index
             for (name, index) in &self.btree_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<()>) {
+                let track_this_index = track_pk_overrides && self.is_single_pk_btree(index);
+                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
                     let start = Instant::now();
                     let result = (|| {
+                        let mut had_existing = false;
                         for stored in batches {
-                            index.insert(&stored.data, stored.row_offset)?;
+                            if track_this_index {
+                                had_existing |= index
+                                    .insert_and_report_existing(&stored.data, stored.row_offset)?;
+                            } else {
+                                index.insert(&stored.data, stored.row_offset)?;
+                            }
                         }
-                        Ok(())
+                        Ok(had_existing)
                     })();
                     (start.elapsed(), result)
                 });
@@ -494,9 +830,9 @@ impl IndexStore {
 
             // Spawn a thread for each HNSW index
             for (name, index) in &self.hnsw_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<()>) {
+                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
                     let start = Instant::now();
-                    let result = index.insert_batches(batches);
+                    let result = index.insert_batches(batches).map(|_| false);
                     (start.elapsed(), result)
                 });
                 handles.push((name.as_str(), "hnsw", handle));
@@ -504,13 +840,13 @@ impl IndexStore {
 
             // Spawn a thread for each FTS index
             for (name, index) in &self.fts_indexes {
-                let handle = scope.spawn(move || -> (std::time::Duration, Result<()>) {
+                let handle = scope.spawn(move || -> (std::time::Duration, Result<bool>) {
                     let start = Instant::now();
                     let result = (|| {
                         for stored in batches {
                             index.insert(&stored.data, stored.row_offset)?;
                         }
-                        Ok(())
+                        Ok(false)
                     })();
                     (start.elapsed(), result)
                 });
@@ -522,11 +858,13 @@ impl IndexStore {
             // BTree updates) are preserved instead of getting truncated to 0.
             let mut first_error: Option<Error> = None;
             let mut timings: Vec<(&str, &str, std::time::Duration)> = Vec::new();
+            let mut had_existing_pk = false;
 
             for (name, idx_type, handle) in handles {
                 match handle.join() {
-                    Ok((duration, Ok(()))) => {
+                    Ok((duration, Ok(had_existing))) => {
                         timings.push((name, idx_type, duration));
+                        had_existing_pk |= had_existing;
                     }
                     Ok((duration, Err(e))) => {
                         timings.push((name, idx_type, duration));
@@ -546,12 +884,24 @@ impl IndexStore {
             if let Some(e) = first_error {
                 return Err(e);
             }
+            self.mark_pk_overrides_if_needed(had_existing_pk);
 
             let duration_map: std::collections::HashMap<String, std::time::Duration> = timings
                 .into_iter()
                 .map(|(name, _idx_type, duration)| (name.to_string(), duration))
                 .collect();
 
+            // Single-column PK aliases a `btree_indexes` entry — its thread above
+            // already maintained it (and joined). A composite PK has its own
+            // index; maintain it here before the watermark advances so the
+            // visible prefix is fully indexed.
+            let mut had_existing = false;
+            for stored in batches {
+                had_existing |=
+                    self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?;
+            }
+            self.mark_pk_overrides_if_needed(had_existing);
+
             // Update global watermark to the max batch position
             let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap();
             self.advance_max_visible_batch_position(max_bp);
@@ -562,7 +912,7 @@ impl IndexStore {
 
     /// Get a BTree index by name.
     pub fn get_btree(&self, name: &str) -> Option<&BTreeMemIndex> {
-        self.btree_indexes.get(name)
+        self.btree_indexes.get(name).map(Arc::as_ref)
     }
 
     /// Get an HNSW vector index by name.
@@ -583,6 +933,7 @@ impl IndexStore {
         self.btree_indexes
             .values()
             .find(|idx| idx.field_id() == field_id)
+            .map(Arc::as_ref)
     }
 
     /// Get an HNSW vector index by field ID.
@@ -607,6 +958,7 @@ impl IndexStore {
         self.btree_indexes
             .values()
             .find(|idx| idx.column_name() == column)
+            .map(Arc::as_ref)
     }
 
     /// Get an HNSW vector index by column name.
@@ -652,6 +1004,7 @@ mod tests {
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use log::warn;
     use std::sync::Arc;
+    use uuid::Uuid;
 
     /// Check if an index type is supported and log warning if not.
     fn check_index_type_supported(index_type: &str) -> bool {
@@ -694,6 +1047,298 @@ mod tests {
         .unwrap()
     }
 
+    fn fts_index_metadata(index_version: i32) -> IndexMetadata {
+        let details =
+            pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap();
+        fts_index_metadata_with_details(index_version, Some(details))
+    }
+
+    fn fts_index_metadata_with_details(
+        index_version: i32,
+        details: Option<pbold::InvertedIndexDetails>,
+    ) -> IndexMetadata {
+        let index_details = details.map(|details| {
+            let mut value = Vec::new();
+            details.encode(&mut value).unwrap();
+            Arc::new(prost_types::Any {
+                type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(),
+                value,
+            })
+        });
+
+        IndexMetadata {
+            uuid: Uuid::new_v4(),
+            fields: vec![2],
+            name: "desc_idx".to_string(),
+            dataset_version: 1,
+            fragment_bitmap: None,
+            index_details,
+            index_version,
+            created_at: None,
+            base_id: None,
+            files: None,
+        }
+    }
+
+    /// Single-column `id` batch for primary-key lookup tests.
+    fn id_batch(ids: &[i32]) -> RecordBatch {
+        RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "id",
+                DataType::Int32,
+                false,
+            )])),
+            vec![Arc::new(Int32Array::from(ids.to_vec()))],
+        )
+        .unwrap()
+    }
+
+    fn id_vector_batch(ids: &[i32]) -> RecordBatch {
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                false,
+            ),
+        ]));
+        let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), 2);
+        for id in ids {
+            vectors.values().append_value(*id as f32);
+            vectors.values().append_value(*id as f32 + 0.5);
+            vectors.append(true);
+        }
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(ids.to_vec())),
+                Arc::new(vectors.finish()),
+            ],
+        )
+        .unwrap()
+    }
+
+    fn id_name_vector_batch(rows: &[(i32, &str)]) -> RecordBatch {
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                false,
+            ),
+        ]));
+        let mut ids = Vec::with_capacity(rows.len());
+        let mut names = Vec::with_capacity(rows.len());
+        let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), 2);
+        for (id, name) in rows {
+            ids.push(*id);
+            names.push(*name);
+            vectors.values().append_value(*id as f32);
+            vectors.values().append_value(name.len() as f32);
+            vectors.append(true);
+        }
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+                Arc::new(vectors.finish()),
+            ],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn pk_newest_visible_single_column() {
+        let mut store = IndexStore::new();
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+        // id=1 at positions 0 and 2 (an update), id=2 at position 1.
+        store.insert(&id_batch(&[1, 2]), 0).unwrap();
+        store.insert(&id_batch(&[1]), 2).unwrap();
+
+        let one = [ScalarValue::Int32(Some(1))];
+        // Watermark above the update sees the newest position; below it, the older.
+        assert_eq!(store.pk_newest_visible(&one, 5), Some(2));
+        assert_eq!(store.pk_newest_visible(&one, 1), Some(0));
+        assert!(store.pk_is_newest(&one, 2, 5));
+        assert!(!store.pk_is_newest(&one, 0, 5));
+        // Absent key (probed by the typed value, as the block-list does).
+        assert!(!store.pk_contains_key(&ScalarValue::Int32(Some(9)), 5));
+    }
+
+    #[test]
+    fn pk_has_overrides_tracks_single_column_rewrites() {
+        let mut store = IndexStore::new();
+        store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+
+        store.insert(&id_vector_batch(&[1, 2]), 0).unwrap();
+        assert!(
+            !store.pk_has_overrides(),
+            "append-only PK inserts should keep HNSW eligible"
+        );
+
+        store.insert(&id_vector_batch(&[3, 3]), 2).unwrap();
+        assert!(
+            store.pk_has_overrides(),
+            "duplicate PKs within one insert must disable HNSW"
+        );
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "Primary-key indexes must be configured before inserting rows into a search-indexed memtable"
+    )]
+    fn enable_pk_index_after_search_rows_panics() {
+        let mut store = IndexStore::new();
+        store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        store.insert(&id_vector_batch(&[1, 2]), 0).unwrap();
+
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+    }
+
+    #[test]
+    fn pk_has_overrides_tracks_single_column_rewrites_across_inserts() {
+        let mut store = IndexStore::new();
+        store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+
+        store.insert(&id_vector_batch(&[1, 2]), 0).unwrap();
+        assert!(
+            !store.pk_has_overrides(),
+            "append-only PK inserts should keep HNSW eligible"
+        );
+
+        store.insert(&id_vector_batch(&[1]), 2).unwrap();
+        assert!(
+            store.pk_has_overrides(),
+            "single-column PK rewrites across inserts must disable HNSW"
+        );
+    }
+
+    #[test]
+    fn pk_has_overrides_skips_scalar_only_tables() {
+        let mut store = IndexStore::new();
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+
+        store.insert(&id_batch(&[1, 1]), 0).unwrap();
+        assert!(
+            !store.pk_has_overrides(),
+            "scalar-only PK tables should not pay override tracking cost"
+        );
+    }
+
+    #[test]
+    fn pk_has_overrides_tracks_fts_rewrites() {
+        let mut store = IndexStore::new();
+        store.enable_pk_index(&[("id".to_string(), 0)]);
+        store.add_fts("text_fts".to_string(), 1, "text".to_string());
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1])),
+                Arc::new(StringArray::from(vec!["alpha", "beta"])),
+            ],
+        )
+        .unwrap();
+        store.insert(&batch, 0).unwrap();
+        assert!(
+            store.pk_has_overrides(),
+            "FTS PK rewrites must disable index-level FTS limit/WAND pushdown"
+        );
+    }
+
+    #[test]
+    fn pk_newest_visible_composite_seeks_encoded_tuple() {
+        let mut store = IndexStore::new();
+        store.enable_pk_index(&[("id".to_string(), 0), ("name".to_string(), 1)]);
+        // Rows: (1,"a")@0, (1,"b")@1, (1,"a")@2 — an update of (1,"a").
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1, 1])),
+                Arc::new(StringArray::from(vec!["a", "b", "a"])),
+            ],
+        )
+        .unwrap();
+        store.insert(&batch, 0).unwrap();
+
+        let tuple_1a = [ScalarValue::Int32(Some(1)), ScalarValue::from("a")];
+        let tuple_1b = [ScalarValue::Int32(Some(1)), ScalarValue::from("b")];
+        // (1,"a")'s newest visible row is its re-write at position 2.
+        assert_eq!(store.pk_newest_visible(&tuple_1a, 5), Some(2));
+        assert!(store.pk_is_newest(&tuple_1a, 2, 5));
+        assert!(!store.pk_is_newest(&tuple_1a, 0, 5));
+        // (1,"b") only exists at position 1.
+        assert_eq!(store.pk_newest_visible(&tuple_1b, 5), Some(1));
+        // Watermark below the re-write: the older (1,"a")@0 is the newest visible.
+        assert_eq!(store.pk_newest_visible(&tuple_1a, 1), Some(0));
+        // An absent tuple (probed by its Binary-encoded key, as the block-list
+        // does).
+        let tuple_2a = [ScalarValue::Int32(Some(2)), ScalarValue::from("a")];
+        let key_2a = ScalarValue::Binary(Some(encode_pk_tuple(&tuple_2a).unwrap()));
+        assert!(!store.pk_contains_key(&key_2a, 5));
+    }
+
+    #[test]
+    fn pk_has_overrides_tracks_composite_rewrites() {
+        let mut store = IndexStore::new();
+        store.add_hnsw(
+            "vector_hnsw".to_string(),
+            2,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        store.enable_pk_index(&[("id".to_string(), 0), ("name".to_string(), 1)]);
+        let first = id_name_vector_batch(&[(1, "a"), (1, "b")]);
+        store.insert(&first, 0).unwrap();
+        assert!(!store.pk_has_overrides());
+
+        let rewrite = id_name_vector_batch(&[(1, "a")]);
+        store.insert(&rewrite, 2).unwrap();
+        assert!(
+            store.pk_has_overrides(),
+            "repeated composite PK must disable HNSW"
+        );
+    }
+
     #[test]
     fn test_index_registry() {
         let schema = create_test_schema();
@@ -727,6 +1372,102 @@ mod tests {
         assert!(!check_index_type_supported("unknown"));
     }
 
+    #[test]
+    fn fts_from_metadata_preserves_format_version() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        for (index_version, expected_format_version) in [
+            (0, InvertedListFormatVersion::V1),
+            (1, InvertedListFormatVersion::V1),
+            (2, InvertedListFormatVersion::V2),
+        ] {
+            let config =
+                MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema)
+                    .unwrap();
+
+            match config {
+                MemIndexConfig::Fts(config) => {
+                    assert_eq!(
+                        config.params.resolved_format_version(),
+                        expected_format_version
+                    );
+                }
+                _ => unreachable!("fts metadata should create an FTS config"),
+            }
+        }
+    }
+
+    #[test]
+    fn fts_from_metadata_rejects_unsupported_format_version() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err();
+        assert!(
+            err.to_string().contains("unsupported index_version 4"),
+            "{err}"
+        );
+    }
+
+    #[test]
+    fn fts_from_metadata_rejects_v3_without_256_block_size() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let missing_details =
+            MemIndexConfig::fts_from_metadata(&fts_index_metadata_with_details(3, None), &schema)
+                .unwrap_err();
+        assert!(
+            missing_details
+                .to_string()
+                .contains("requires block_size=256"),
+            "{missing_details}"
+        );
+        assert!(
+            missing_details.to_string().contains("got 128"),
+            "{missing_details}"
+        );
+
+        let default_details =
+            MemIndexConfig::fts_from_metadata(&fts_index_metadata(3), &schema).unwrap_err();
+        assert!(
+            default_details
+                .to_string()
+                .contains("requires block_size=256"),
+            "{default_details}"
+        );
+        assert!(
+            default_details.to_string().contains("got 128"),
+            "{default_details}"
+        );
+    }
+
+    #[test]
+    fn fts_from_metadata_accepts_v3_with_256_block_size() {
+        let arrow_schema = create_test_schema();
+        let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap();
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let details = pbold::InvertedIndexDetails::try_from(&params).unwrap();
+
+        let config = MemIndexConfig::fts_from_metadata(
+            &fts_index_metadata_with_details(3, Some(details)),
+            &schema,
+        )
+        .unwrap();
+
+        match config {
+            MemIndexConfig::Fts(config) => {
+                assert_eq!(
+                    config.params.resolved_format_version(),
+                    InvertedListFormatVersion::V3
+                );
+                assert_eq!(config.params.posting_block_size(), 256);
+            }
+            _ => unreachable!("fts metadata should create an FTS config"),
+        }
+    }
+
     #[test]
     fn test_from_configs() {
         let configs = vec![
diff --git a/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs b/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs
index b600f6296de..6b7361e9f1b 100644
--- a/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs
+++ b/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs
@@ -250,9 +250,22 @@ impl<K: Ord> SkipListWriter<K> {
         height
     }
 
-    /// Insert `key`. Keys must be unique (the MemTable key carries a row
-    /// position, making every entry distinct); equal keys are not expected.
+    #[cfg(test)]
     pub fn insert(&mut self, key: K) {
+        self.insert_and_check_neighbors(key, |_, _| false);
+    }
+
+    /// Insert `key` and let the caller inspect its immediate level-0 neighbors.
+    ///
+    /// Keys must be unique (the MemTable key carries a row position, making
+    /// every entry distinct). Backends that index `(value, row_position)` use
+    /// the neighbor check to detect whether the same value already existed,
+    /// reusing the insertion traversal instead of issuing a second lookup.
+    pub fn insert_and_check_neighbors(
+        &mut self,
+        key: K,
+        check: impl FnOnce(Option<&K>, Option<&K>) -> bool,
+    ) -> bool {
         let cur_height = self.core.height.load(Ordering::Relaxed);
 
         // Find the predecessor at every level. For levels at/above the current
@@ -274,6 +287,15 @@ impl<K: Ord> SkipListWriter<K> {
             preds[level] = pred;
         }
 
+        let succ = self.core.next_slot(preds[0], 0).load(Ordering::Acquire);
+        // SAFETY: `preds[0]` and `succ` are either null or point to nodes owned
+        // by this skiplist's arena. The single writer never removes nodes, so
+        // any non-null neighbor remains alive for the duration of this call.
+        let had_neighbor_match =
+            check(unsafe { preds[0].as_ref().map(|node| &node.key) }, unsafe {
+                succ.as_ref().map(|node| &node.key)
+            });
+
         let height = self.random_height();
 
         // Allocate the node in one block and initialize it fully (key + tower)
@@ -309,6 +331,7 @@ impl<K: Ord> SkipListWriter<K> {
         }
 
         self.core.len.fetch_add(1, Ordering::Release);
+        had_neighbor_match
     }
 }
 
diff --git a/rust/lance/src/dataset/mem_wal/index/btree.rs b/rust/lance/src/dataset/mem_wal/index/btree.rs
index c2f89b9932d..ca4dd178548 100644
--- a/rust/lance/src/dataset/mem_wal/index/btree.rs
+++ b/rust/lance/src/dataset/mem_wal/index/btree.rs
@@ -305,7 +305,8 @@ impl FixedIntBackend {
         }
     }
 
-    fn insert_array(&self, array: &dyn Array, row_offset: u64) -> Result<()> {
+    fn insert_array_and_report_existing(&self, array: &dyn Array, row_offset: u64) -> Result<bool> {
+        let mut had_existing = false;
         macro_rules! insert_int {
             ($array_type:ty, $to_i64:expr) => {{
                 let typed = array
@@ -314,14 +315,26 @@ impl FixedIntBackend {
                     .unwrap();
                 let mut writer = self.writer.lock().unwrap();
                 let mut nulls: Vec<RowPosition> = Vec::new();
+                let had_existing_nulls = !self.null_positions.lock().unwrap().is_empty();
+                let mut saw_null = false;
                 for (row_idx, value) in typed.iter().enumerate() {
                     let position = row_offset + row_idx as u64;
                     match value {
-                        Some(v) => writer.insert(FixedKey {
-                            enc: $to_i64(v),
-                            position,
-                        }),
-                        None => nulls.push(position),
+                        Some(v) => {
+                            let enc = $to_i64(v);
+                            let key = FixedKey { enc, position };
+                            had_existing |= writer.insert_and_check_neighbors(key, |prev, next| {
+                                prev.is_some_and(|key| key.enc == enc)
+                                    || next.is_some_and(|key| key.enc == enc)
+                            });
+                        }
+                        None => {
+                            if had_existing_nulls || saw_null {
+                                had_existing = true;
+                            }
+                            saw_null = true;
+                            nulls.push(position);
+                        }
                     }
                 }
                 drop(writer);
@@ -348,7 +361,7 @@ impl FixedIntBackend {
                 )));
             }
         }
-        Ok(())
+        Ok(had_existing)
     }
 
     fn get_newest_visible(
@@ -458,7 +471,8 @@ impl BytesBackend {
         }
     }
 
-    fn insert_array(&self, array: &dyn Array, row_offset: u64) -> Result<()> {
+    fn insert_array_and_report_existing(&self, array: &dyn Array, row_offset: u64) -> Result<bool> {
+        let mut had_existing = false;
         // Append (position, key bytes) for each row; nulls go to the side list.
         // `$v => $to_bytes` extracts the key bytes from each non-null value
         // inline (no closure, so the borrow ties directly to the row value).
@@ -467,16 +481,26 @@ impl BytesBackend {
                 let typed = array.as_any().downcast_ref::<$array_type>().unwrap();
                 let mut writer = self.writer.lock().unwrap();
                 let mut nulls: Vec<RowPosition> = Vec::new();
+                let had_existing_nulls = !self.null_positions.lock().unwrap().is_empty();
+                let mut saw_null = false;
                 for row_idx in 0..typed.len() {
                     let position = row_offset + row_idx as u64;
                     if typed.is_null(row_idx) {
+                        if had_existing_nulls || saw_null {
+                            had_existing = true;
+                        }
+                        saw_null = true;
                         nulls.push(position);
                     } else {
                         let $v = typed.value(row_idx);
                         let bytes: &[u8] = $to_bytes;
-                        writer.insert(BytesKey {
+                        let key = BytesKey {
                             bytes: InlineBytes::new(bytes),
                             position,
+                        };
+                        had_existing |= writer.insert_and_check_neighbors(key, |prev, next| {
+                            prev.is_some_and(|key| key.bytes.as_slice() == bytes)
+                                || next.is_some_and(|key| key.bytes.as_slice() == bytes)
                         });
                     }
                 }
@@ -501,7 +525,7 @@ impl BytesBackend {
                 )));
             }
         }
-        Ok(())
+        Ok(had_existing)
     }
 
     fn get_newest_visible(
@@ -609,14 +633,22 @@ impl ScalarBackend {
         }
     }
 
-    fn add(&self, value: OrderableScalarValue, row_position: RowPosition) {
-        self.writer.lock().unwrap().insert(IndexKey {
-            value,
-            row_position,
-        });
+    fn add(&self, value: OrderableScalarValue, row_position: RowPosition) -> bool {
+        let probe = value.clone();
+        self.writer.lock().unwrap().insert_and_check_neighbors(
+            IndexKey {
+                value,
+                row_position,
+            },
+            |prev, next| {
+                prev.is_some_and(|key| key.value == probe)
+                    || next.is_some_and(|key| key.value == probe)
+            },
+        )
     }
 
-    fn insert_array(&self, array: &dyn Array, row_offset: u64) -> Result<()> {
+    fn insert_array_and_report_existing(&self, array: &dyn Array, row_offset: u64) -> Result<bool> {
+        let mut had_existing = false;
         macro_rules! insert_primitive {
             ($array_type:ty, $scalar_variant:ident) => {{
                 let typed_array = array
@@ -625,7 +657,7 @@ impl ScalarBackend {
                     .unwrap();
                 for (row_idx, value) in typed_array.iter().enumerate() {
                     let row_position = row_offset + row_idx as u64;
-                    self.add(
+                    had_existing |= self.add(
                         OrderableScalarValue(ScalarValue::$scalar_variant(value)),
                         row_position,
                     );
@@ -653,7 +685,7 @@ impl ScalarBackend {
                     .unwrap();
                 for (row_idx, value) in typed_array.iter().enumerate() {
                     let row_position = row_offset + row_idx as u64;
-                    self.add(
+                    had_existing |= self.add(
                         OrderableScalarValue(ScalarValue::Utf8(value.map(|s| s.to_string()))),
                         row_position,
                     );
@@ -666,7 +698,7 @@ impl ScalarBackend {
                     .unwrap();
                 for (row_idx, value) in typed_array.iter().enumerate() {
                     let row_position = row_offset + row_idx as u64;
-                    self.add(
+                    had_existing |= self.add(
                         OrderableScalarValue(ScalarValue::LargeUtf8(value.map(|s| s.to_string()))),
                         row_position,
                     );
@@ -679,7 +711,7 @@ impl ScalarBackend {
                     .unwrap();
                 for (row_idx, value) in typed_array.iter().enumerate() {
                     let row_position = row_offset + row_idx as u64;
-                    self.add(
+                    had_existing |= self.add(
                         OrderableScalarValue(ScalarValue::Boolean(value)),
                         row_position,
                     );
@@ -690,11 +722,11 @@ impl ScalarBackend {
                 for row_idx in 0..array.len() {
                     let value = ScalarValue::try_from_array(array, row_idx)?;
                     let row_position = row_offset + row_idx as u64;
-                    self.add(OrderableScalarValue(value), row_position);
+                    had_existing |= self.add(OrderableScalarValue(value), row_position);
                 }
             }
         }
-        Ok(())
+        Ok(had_existing)
     }
 
     fn get_newest_visible(
@@ -769,11 +801,11 @@ impl Backend {
         }
     }
 
-    fn insert_array(&self, array: &dyn Array, row_offset: u64) -> Result<()> {
+    fn insert_array_and_report_existing(&self, array: &dyn Array, row_offset: u64) -> Result<bool> {
         match self {
-            Self::FixedInt(b) => b.insert_array(array, row_offset),
-            Self::Bytes(b) => b.insert_array(array, row_offset),
-            Self::Scalar(b) => b.insert_array(array, row_offset),
+            Self::FixedInt(b) => b.insert_array_and_report_existing(array, row_offset),
+            Self::Bytes(b) => b.insert_array_and_report_existing(array, row_offset),
+            Self::Scalar(b) => b.insert_array_and_report_existing(array, row_offset),
         }
     }
 
@@ -873,6 +905,13 @@ impl BTreeMemIndex {
 
     /// Insert rows from a batch into the index.
     pub fn insert(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> {
+        self.insert_and_report_existing(batch, row_offset)
+            .map(|_| ())
+    }
+
+    /// Insert rows and report whether any inserted key already existed in the
+    /// index or repeated earlier in this batch.
+    pub fn insert_and_report_existing(&self, batch: &RecordBatch, row_offset: u64) -> Result<bool> {
         let col_idx = batch
             .schema()
             .column_with_name(&self.column_name)
@@ -885,7 +924,7 @@ impl BTreeMemIndex {
         let backend = self
             .backend
             .get_or_init(|| Backend::for_type(column.data_type()));
-        backend.insert_array(column.as_ref(), row_offset)
+        backend.insert_array_and_report_existing(column.as_ref(), row_offset)
     }
 
     /// Look up row positions for an exact value.
diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs
index 8c6c9d85de7..6e0d12a276e 100644
--- a/rust/lance/src/dataset/mem_wal/index/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/index/fts.rs
@@ -54,9 +54,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use arc_swap::ArcSwap;
 use arrow_array::{Array, LargeStringArray, RecordBatch, StringArray, StringViewArray};
 use arrow_schema::DataType;
-use bitpacking::{BitPacker, BitPacker4x};
 use crossbeam_skiplist::SkipMap;
 use fst::{Map, Streamer};
+use lance_bitpacking::{BitPacker, BitPacker4x};
 use lance_core::{Error, Result};
 use lance_index::scalar::InvertedIndexParams;
 use lance_index::scalar::inverted::query::Operator;
@@ -88,6 +88,8 @@ pub enum FtsQueryExpr {
     Match {
         /// The search query string.
         query: String,
+        /// The operator used to combine tokenized query terms.
+        operator: Operator,
         /// Boost factor applied to the score (default 1.0).
         boost: f32,
     },
@@ -107,6 +109,8 @@ pub enum FtsQueryExpr {
         /// Maximum edit distance (Levenshtein distance).
         /// `None` means auto-fuzziness based on token length.
         fuzziness: Option<u32>,
+        /// Number of initial characters that must match exactly.
+        prefix_length: u32,
         /// Maximum number of terms to expand to.
         max_expansions: usize,
         /// Boost factor applied to the score (default 1.0).
@@ -204,8 +208,13 @@ impl SearchOptions {
 
 impl FtsQueryExpr {
     pub fn match_query(query: impl Into<String>) -> Self {
+        Self::match_query_with_operator(query, Operator::Or)
+    }
+
+    pub fn match_query_with_operator(query: impl Into<String>, operator: Operator) -> Self {
         Self::Match {
             query: query.into(),
+            operator,
             boost: 1.0,
         }
     }
@@ -230,6 +239,7 @@ impl FtsQueryExpr {
         Self::Fuzzy {
             query: query.into(),
             fuzziness: None,
+            prefix_length: 0,
             max_expansions: DEFAULT_MAX_EXPANSIONS,
             boost: 1.0,
         }
@@ -239,6 +249,7 @@ impl FtsQueryExpr {
         Self::Fuzzy {
             query: query.into(),
             fuzziness: Some(fuzziness),
+            prefix_length: 0,
             max_expansions: DEFAULT_MAX_EXPANSIONS,
             boost: 1.0,
         }
@@ -247,11 +258,13 @@ impl FtsQueryExpr {
     pub fn fuzzy_with_options(
         query: impl Into<String>,
         fuzziness: Option<u32>,
+        prefix_length: u32,
         max_expansions: usize,
     ) -> Self {
         Self::Fuzzy {
             query: query.into(),
             fuzziness,
+            prefix_length,
             max_expansions,
             boost: 1.0,
         }
@@ -279,16 +292,24 @@ impl FtsQueryExpr {
 
     pub fn with_boost(self, boost: f32) -> Self {
         match self {
-            Self::Match { query, .. } => Self::Match { query, boost },
+            Self::Match {
+                query, operator, ..
+            } => Self::Match {
+                query,
+                operator,
+                boost,
+            },
             Self::Phrase { query, slop, .. } => Self::Phrase { query, slop, boost },
             Self::Fuzzy {
                 query,
                 fuzziness,
+                prefix_length,
                 max_expansions,
                 ..
             } => Self::Fuzzy {
                 query,
                 fuzziness,
+                prefix_length,
                 max_expansions,
                 boost,
             },
@@ -340,6 +361,16 @@ pub fn levenshtein_distance(a: &str, b: &str) -> u32 {
     prev_row[n]
 }
 
+fn char_prefix(term: &str, prefix_length: u32) -> &str {
+    if prefix_length == 0 {
+        return "";
+    }
+    term.char_indices()
+        .nth(prefix_length as usize)
+        .map(|(idx, _)| &term[..idx])
+        .unwrap_or(term)
+}
+
 /// Builder for constructing Boolean queries.
 #[derive(Debug, Clone, Default)]
 pub struct BooleanQueryBuilder {
@@ -929,10 +960,20 @@ impl FtsMemIndex {
 
     /// Create a new FTS index with custom tokenizer parameters.
     pub fn with_params(field_id: i32, column_name: String, params: InvertedIndexParams) -> Self {
-        let pool = TokenizerPool::new(&params, Self::DEFAULT_TOKENIZER_POOL_CAP)
-            .expect("Failed to build tokenizer");
+        Self::try_with_params(field_id, column_name, params)
+            .expect("invalid MemWAL FTS index parameters")
+    }
+
+    /// Try to create a new FTS index with custom tokenizer parameters.
+    pub fn try_with_params(
+        field_id: i32,
+        column_name: String,
+        params: InvertedIndexParams,
+    ) -> Result<Self> {
+        params.validate_format_version()?;
+        let pool = TokenizerPool::new(&params, Self::DEFAULT_TOKENIZER_POOL_CAP)?;
         let writer_tokenizer = pool.template.box_clone();
-        Self {
+        Ok(Self {
             field_id,
             column_name,
             params,
@@ -941,7 +982,7 @@ impl FtsMemIndex {
             state: ArcSwap::from(IndexState::empty()),
             freeze_threshold_rows: Self::DEFAULT_FREEZE_THRESHOLD_ROWS,
             merge: Arc::new(Mutex::new(None)),
-        }
+        })
     }
 
     /// Override the tail freeze threshold (docs) — the analogue of Lucene's
@@ -1237,7 +1278,7 @@ impl FtsMemIndex {
     pub fn search(&self, term: &str) -> Vec<FtsEntry> {
         let st = self.state.load_full();
         let tokens = self.tokenize_for_search(term);
-        self.search_match(&st, &tokens, None, true, true)
+        self.search_match(&st, &tokens, Operator::Or, None, true, true)
     }
 
     /// Search for documents containing an exact phrase, optionally allowing
@@ -1269,7 +1310,7 @@ impl FtsMemIndex {
         max_expansions: usize,
     ) -> Vec<(String, u32)> {
         let st = self.state.load_full();
-        self.expand_fuzzy_term(&st, term, max_distance, max_expansions, true)
+        self.expand_fuzzy_term(&st, term, max_distance, 0, max_expansions, true)
     }
 
     /// Search for documents using fuzzy matching on each query token.
@@ -1281,7 +1322,7 @@ impl FtsMemIndex {
     ) -> Vec<FtsEntry> {
         let st = self.state.load_full();
         let tokens = self.tokenize_for_search(query);
-        self.search_fuzzy_tokens(&st, &tokens, fuzziness, max_expansions, true)
+        self.search_fuzzy_tokens(&st, &tokens, fuzziness, 0, max_expansions, true)
     }
 
     /// BM25 OR-search over the query tokens, scored with one corpus-wide
@@ -1294,6 +1335,7 @@ impl FtsMemIndex {
         &self,
         st: &IndexState,
         tokens: &[String],
+        operator: Operator,
         limit: Option<usize>,
         include_tail: bool,
         tail_skip: bool,
@@ -1309,8 +1351,8 @@ impl FtsMemIndex {
         if scorer.num_docs() == 0 {
             return Vec::new();
         }
-        match limit {
-            Some(k) if k > 0 => {
+        match (operator, limit) {
+            (Operator::Or, Some(k)) if k > 0 => {
                 let mut topk = TopK::new(k);
                 // Scan the block-max partitions first to warm the shared
                 // threshold, then the (un-skippable) tail last — so the tail
@@ -1325,7 +1367,14 @@ impl FtsMemIndex {
                     } else {
                         f32::NEG_INFINITY
                     };
-                    for e in score_terms(&tail_snap, &st.tail.terms, tokens, &scorer, theta) {
+                    for e in score_terms(
+                        &tail_snap,
+                        &st.tail.terms,
+                        tokens,
+                        Operator::Or,
+                        &scorer,
+                        theta,
+                    ) {
                         topk.offer(e.score, e.row_position);
                     }
                 }
@@ -1334,13 +1383,14 @@ impl FtsMemIndex {
             _ => {
                 let mut results = Vec::new();
                 for p in st.partitions.iter() {
-                    results.extend(p.search_match(tokens, Operator::Or, &scorer));
+                    results.extend(p.search_match(tokens, operator, &scorer));
                 }
                 if scan_tail {
                     results.extend(score_terms(
                         &tail_snap,
                         &st.tail.terms,
                         tokens,
+                        operator,
                         &scorer,
                         f32::NEG_INFINITY,
                     ));
@@ -1362,7 +1412,7 @@ impl FtsMemIndex {
         }
         if tokens.len() == 1 {
             // A single-token phrase reduces to a regular term search.
-            return self.search_match(st, tokens, None, include_tail, true);
+            return self.search_match(st, tokens, Operator::Or, None, include_tail, true);
         }
         // A multi-token phrase needs token positions; without them (the index
         // was built `with_position = false`) phrase search is unsupported, as
@@ -1397,6 +1447,7 @@ impl FtsMemIndex {
         st: &IndexState,
         tokens: &[String],
         fuzziness: Option<u32>,
+        prefix_length: u32,
         max_expansions: usize,
         include_tail: bool,
     ) -> Vec<FtsEntry> {
@@ -1407,9 +1458,14 @@ impl FtsMemIndex {
         let mut seen: HashSet<String> = HashSet::new();
         for tok in tokens {
             let max_dist = fuzziness.unwrap_or_else(|| auto_fuzziness(tok));
-            for (matched, _) in
-                self.expand_fuzzy_term(st, tok, max_dist, max_expansions, include_tail)
-            {
+            for (matched, _) in self.expand_fuzzy_term(
+                st,
+                tok,
+                max_dist,
+                prefix_length,
+                max_expansions,
+                include_tail,
+            ) {
                 if seen.insert(matched.clone()) {
                     expanded.push(matched);
                 }
@@ -1418,7 +1474,7 @@ impl FtsMemIndex {
         if expanded.is_empty() {
             return Vec::new();
         }
-        self.search_match(st, &expanded, None, include_tail, true)
+        self.search_match(st, &expanded, Operator::Or, None, include_tail, true)
     }
 
     /// Expand `term` against the term dictionaries of every partition (and the
@@ -1428,6 +1484,7 @@ impl FtsMemIndex {
         st: &IndexState,
         term: &str,
         max_distance: u32,
+        prefix_length: u32,
         max_expansions: usize,
         include_tail: bool,
     ) -> Vec<(String, u32)> {
@@ -1449,11 +1506,15 @@ impl FtsMemIndex {
         }
         let mut matches: Vec<(String, u32)> = Vec::new();
         let mut seen: HashSet<String> = HashSet::new();
+        let prefix = char_prefix(term, prefix_length);
         for entry in st.tail.terms.iter() {
             if !include_tail || !has_visible_chunk(&entry.value().load(), tail_snap.visible_count) {
                 continue;
             }
             let key: &Arc<str> = entry.key();
+            if !key.starts_with(prefix) {
+                continue;
+            }
             let dist = levenshtein_distance(term, key);
             if dist <= max_distance && seen.insert(key.to_string()) {
                 matches.push((key.to_string(), dist));
@@ -1461,6 +1522,9 @@ impl FtsMemIndex {
         }
         for p in st.partitions.iter() {
             for key in p.collect_terms() {
+                if !key.starts_with(prefix) {
+                    continue;
+                }
                 let dist = levenshtein_distance(term, key.as_ref());
                 if dist <= max_distance && seen.insert(key.to_string()) {
                     matches.push((key.to_string(), dist));
@@ -1497,9 +1561,14 @@ impl FtsMemIndex {
         tail_skip: bool,
     ) -> Vec<FtsEntry> {
         match query {
-            FtsQueryExpr::Match { query, boost } => {
+            FtsQueryExpr::Match {
+                query,
+                operator,
+                boost,
+            } => {
                 let tokens = self.tokenize_for_search(query);
-                let mut results = self.search_match(st, &tokens, limit, include_tail, tail_skip);
+                let mut results =
+                    self.search_match(st, &tokens, *operator, limit, include_tail, tail_skip);
                 apply_boost(&mut results, *boost);
                 results
             }
@@ -1512,6 +1581,7 @@ impl FtsMemIndex {
             FtsQueryExpr::Fuzzy {
                 query,
                 fuzziness,
+                prefix_length,
                 max_expansions,
                 boost,
             } => {
@@ -1520,6 +1590,7 @@ impl FtsMemIndex {
                     st,
                     &tokens,
                     *fuzziness,
+                    *prefix_length,
                     *max_expansions,
                     include_tail,
                 );
@@ -1551,6 +1622,10 @@ impl FtsMemIndex {
         query: &FtsQueryExpr,
         options: SearchOptions,
     ) -> Vec<FtsEntry> {
+        if options.limit == Some(0) {
+            return Vec::new();
+        }
+
         let st = self.state.load_full();
         let mut results = self.search_query_with_state(
             query,
@@ -1699,6 +1774,9 @@ impl FtsMemIndex {
 
         let st = self.state.load_full();
         let with_position = self.params.has_positions();
+        let block_size = self.params.posting_block_size();
+        let format_version = self.params.resolved_format_version();
+        let posting_tail_codec = format_version.posting_tail_codec();
         let total_rows_u64 = total_rows as u64;
 
         // Step 1: collect (original_pos, num_tokens) for every doc across all
@@ -1716,10 +1794,12 @@ impl FtsMemIndex {
             }
         }
         if all_docs.is_empty() {
-            return Ok(InnerBuilder::new(
+            return Ok(InnerBuilder::new_with_format_version_and_block_size(
                 partition_id,
                 with_position,
                 Default::default(),
+                format_version,
+                block_size,
             ));
         }
 
@@ -1805,7 +1885,13 @@ impl FtsMemIndex {
             docs_for_term.sort_by_key(|(doc_id, _, _)| *doc_id);
             let token_id = tokens.add(token) as usize;
             debug_assert_eq!(token_id, posting_lists.len());
-            posting_lists.push(PostingListBuilder::new(with_position));
+            posting_lists.push(
+                PostingListBuilder::new_with_posting_tail_codec_and_block_size(
+                    with_position,
+                    posting_tail_codec,
+                    block_size,
+                ),
+            );
             let plb = &mut posting_lists[token_id];
             for (doc_id, freq, pos) in docs_for_term {
                 let recorder = if with_position {
@@ -1817,7 +1903,13 @@ impl FtsMemIndex {
             }
         }
 
-        let mut builder = InnerBuilder::new(partition_id, with_position, Default::default());
+        let mut builder = InnerBuilder::new_with_format_version_and_block_size(
+            partition_id,
+            with_position,
+            Default::default(),
+            format_version,
+            block_size,
+        );
         builder.set_tokens(tokens);
         builder.set_docs(docs);
         builder.set_posting_lists(posting_lists);
@@ -2021,12 +2113,13 @@ fn tail_token_df(
     }
 }
 
-/// OR-score `tokens` against the visible tail, summing each token's BM25
+/// Score `tokens` against the visible tail, summing each token's BM25
 /// contribution per document. Uses the shared corpus-wide `scorer`.
 fn score_terms(
     snap: &Snapshot,
     terms: &SkipMap<Arc<str>, Arc<ArcSwap<TermSlice>>>,
     tokens: &[String],
+    operator: Operator,
     scorer: &MemBM25Scorer,
     theta: f32,
 ) -> Vec<FtsEntry> {
@@ -2038,6 +2131,9 @@ fn score_terms(
     let mut tail_terms: Vec<(f32, Arc<TermSlice>)> = Vec::with_capacity(tokens.len());
     for token in tokens {
         let Some(entry) = terms.get(token.as_str()) else {
+            if operator == Operator::And {
+                return Vec::new();
+            }
             continue;
         };
         let qw = scorer.query_weight(token);
@@ -2058,6 +2154,8 @@ fn score_terms(
         return Vec::new();
     }
     let mut doc_scores: HashMap<RowPosition, f32> = HashMap::new();
+    let mut doc_hits: Option<HashMap<RowPosition, u32>> =
+        (operator == Operator::And).then(HashMap::new);
     for (qw, slice) in tail_terms {
         for chunk in slice.chunks() {
             if chunk.batch_position >= snap.visible_count {
@@ -2070,11 +2168,21 @@ fn score_terms(
                 let dl = meta.dl(row_position).unwrap_or(1);
                 let score = qw * scorer.doc_weight(chunk.frequencies[i], dl);
                 *doc_scores.entry(row_position).or_default() += score;
+                if let Some(doc_hits) = &mut doc_hits {
+                    *doc_hits.entry(row_position).or_default() += 1;
+                }
             }
         }
     }
     doc_scores
         .into_iter()
+        .filter(|(row_position, _)| {
+            operator == Operator::Or
+                || doc_hits
+                    .as_ref()
+                    .and_then(|doc_hits| doc_hits.get(row_position))
+                    .is_some_and(|hits| *hits >= tokens.len() as u32)
+        })
         .map(|(row_position, score)| FtsEntry {
             row_position,
             score,
@@ -2240,12 +2348,23 @@ impl FtsIndexConfig {
         column: String,
         params: InvertedIndexParams,
     ) -> Self {
-        Self {
+        Self::try_with_params(name, field_id, column, params)
+            .expect("invalid MemWAL FTS index config parameters")
+    }
+
+    pub fn try_with_params(
+        name: String,
+        field_id: i32,
+        column: String,
+        params: InvertedIndexParams,
+    ) -> Result<Self> {
+        params.validate_format_version()?;
+        Ok(Self {
             name,
             field_id,
             column,
             params,
-        }
+        })
     }
 }
 
@@ -4057,6 +4176,47 @@ mod tests {
         assert!(!entries.is_empty());
     }
 
+    #[test]
+    fn test_search_query_fuzzy_respects_prefix_length() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::new(1, "description".to_string());
+
+        let batch = create_fuzzy_test_batch(&schema);
+        index.insert(&batch, 0).unwrap();
+
+        let query = FtsQueryExpr::fuzzy_with_options("alpha", Some(5), 2, 50);
+        let entries = index.search_query(&query);
+        assert!(!entries.is_empty());
+        assert!(
+            entries.iter().all(|entry| entry.row_position != 3),
+            "the omega row is within edit distance but does not share the required prefix"
+        );
+    }
+
+    #[test]
+    fn test_search_query_fuzzy_prefix_length_uses_char_boundaries() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::new(1, "description".to_string());
+
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![0, 1])),
+                Arc::new(StringArray::from(vec!["éclair", "xclair"])),
+            ],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+
+        let query = FtsQueryExpr::fuzzy_with_options("éclair", Some(2), 1, 50);
+        let entries = index.search_query(&query);
+        assert!(!entries.is_empty());
+        assert!(
+            entries.iter().all(|entry| entry.row_position != 1),
+            "the ASCII-prefixed row is within edit distance but not the character prefix"
+        );
+    }
+
     #[test]
     fn test_search_query_fuzzy_with_boost() {
         let schema = create_test_schema();
@@ -4332,6 +4492,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_search_with_options_zero_limit_with_wand_factor() {
+        let schema = create_test_schema();
+        let index = FtsMemIndex::new(1, "description".to_string());
+
+        let batch = create_wand_test_batch(&schema);
+        index.insert(&batch, 0).unwrap();
+
+        let query = FtsQueryExpr::match_query("alpha");
+        let options = SearchOptions::new().with_limit(0).with_wand_factor(0.5);
+        let results = index.search_with_options(&query, options);
+        assert!(results.is_empty());
+    }
+
     #[test]
     fn test_search_with_options_empty_results() {
         let schema = create_test_schema();
@@ -4519,6 +4693,18 @@ mod tests {
         assert!(builder.id() > 0 || builder.id() == 42);
     }
 
+    #[test]
+    fn test_to_index_builder_supports_block_size_256() {
+        let schema = create_test_schema();
+        let params = InvertedIndexParams::default().block_size(256).unwrap();
+        let index = FtsMemIndex::try_with_params(1, "description".to_string(), params).unwrap();
+        let batch = create_test_batch(&schema);
+        index.insert(&batch, 0).unwrap();
+
+        let builder = index.to_index_builder(42, 3).unwrap();
+        assert_eq!(builder.id(), 42);
+    }
+
     #[test]
     fn test_unsupported_column_type_errors() {
         let schema = Arc::new(ArrowSchema::new(vec![
diff --git a/rust/lance/src/dataset/mem_wal/index/hnsw.rs b/rust/lance/src/dataset/mem_wal/index/hnsw.rs
index 8ece292871f..7c7993bb298 100644
--- a/rust/lance/src/dataset/mem_wal/index/hnsw.rs
+++ b/rust/lance/src/dataset/mem_wal/index/hnsw.rs
@@ -237,14 +237,8 @@ impl HnswMemIndex {
                 fsl_ref.values().data_type()
             )));
         }
-        if fsl_ref.null_count() > 0 {
-            return Err(Error::invalid_input(format!(
-                "HNSW index column '{}' has {} null row(s); null vectors are not supported",
-                self.column,
-                fsl_ref.null_count()
-            )));
-        }
-
+        // Null-vector rows (e.g. tombstones) are skipped inside `append_batch`,
+        // which yields an empty range for an all-null batch — handled below.
         let dim = fsl_ref.value_length() as usize;
         let state = self.ensure_state(dim)?;
         let vectors = Arc::new(fsl_ref.clone());
@@ -290,14 +284,9 @@ impl HnswMemIndex {
                     fsl_ref.values().data_type()
                 )));
             }
-            if fsl_ref.null_count() > 0 {
-                return Err(Error::invalid_input(format!(
-                    "HNSW index column '{}' has {} null row(s); null vectors are not supported",
-                    self.column,
-                    fsl_ref.null_count()
-                )));
-            }
-
+            // Null-vector rows (e.g. tombstones) are skipped inside
+            // `append_batch`; an all-null batch yields an empty range, handled
+            // by the `id_range.is_empty()` continue below.
             let dim = fsl_ref.value_length() as usize;
             let current_state = match state {
                 Some(state) => {
@@ -538,13 +527,11 @@ mod tests {
 
         let results = index.search(&query, 5, Some(32), u64::MAX).unwrap();
         assert!(!results.is_empty());
-        let (best_dist, best_pos) = results[0];
         assert!(
-            best_dist < 1e-4,
-            "expected near-zero distance, got {}",
-            best_dist
+            results.iter().any(|&(dist, pos)| pos == 82 && dist < 1e-4),
+            "expected exact row position 82 in top-5 candidates, got {:?}",
+            results
         );
-        assert_eq!(best_pos, 82);
     }
 
     #[test]
@@ -676,4 +663,117 @@ mod tests {
         assert!(index.len() > 1);
         assert!(total_reader_iters > 0);
     }
+
+    /// Build a 3-row batch whose middle vector row is null at the list level.
+    fn batch_with_null_middle(dim: usize) -> RecordBatch {
+        let mut values: Vec<f32> = Vec::new();
+        values.extend(std::iter::repeat_n(1.0f32, dim)); // row 0
+        values.extend(std::iter::repeat_n(0.0f32, dim)); // row 1 (null placeholder)
+        values.extend(std::iter::repeat_n(3.0f32, dim)); // row 2
+        let inner = Arc::new(Float32Array::from(values)) as arrow_array::ArrayRef;
+        let nulls = arrow_buffer::NullBuffer::new(arrow_buffer::BooleanBuffer::from(vec![
+            true, false, true,
+        ]));
+        let fsl = FixedSizeListArray::try_new(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            dim as i32,
+            inner,
+            Some(nulls),
+        )
+        .unwrap();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dim as i32,
+                ),
+                true,
+            ),
+        ]));
+        RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from(vec![0, 1, 2])), Arc::new(fsl)],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn test_index_skips_null_vector_row() {
+        // A null vector row gets no graph node; the surviving rows keep their
+        // original positions (the tombstone-enabling fix).
+        let dim = 4;
+        let index = HnswMemIndex::with_capacity(
+            1,
+            "vector".to_string(),
+            DistanceType::L2,
+            HnswBuildParams::default().num_edges(16).ef_construction(64),
+            16,
+            16,
+        );
+        index.insert(&batch_with_null_middle(dim), 0).unwrap();
+        assert_eq!(index.len(), 2, "the null row is skipped");
+
+        let query = FixedSizeListArray::try_new_from_values(
+            Float32Array::from(vec![3.0f32; dim]),
+            dim as i32,
+        )
+        .unwrap();
+        let results = index.search(&query, 2, Some(16), u64::MAX).unwrap();
+        assert!(!results.is_empty());
+        let (best_dist, best_pos) = results[0];
+        assert!(best_dist < 1e-4, "got {}", best_dist);
+        assert_eq!(
+            best_pos, 2,
+            "row 2 resolves to its original offset, not 1, after the skip"
+        );
+        assert!(
+            results.iter().all(|(_, pos)| *pos != 1),
+            "the skipped null row must never be returned"
+        );
+    }
+
+    #[test]
+    fn test_index_all_null_batch_adds_no_nodes() {
+        // An all-null batch (e.g. an all-tombstone memtable) inserts cleanly and
+        // adds no nodes.
+        let dim = 4;
+        let index = HnswMemIndex::with_capacity(
+            1,
+            "vector".to_string(),
+            DistanceType::L2,
+            HnswBuildParams::default(),
+            8,
+            8,
+        );
+        let inner = Arc::new(Float32Array::from(vec![0.0f32; dim * 2])) as arrow_array::ArrayRef;
+        let nulls =
+            arrow_buffer::NullBuffer::new(arrow_buffer::BooleanBuffer::from(vec![false, false]));
+        let fsl = FixedSizeListArray::try_new(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            dim as i32,
+            inner,
+            Some(nulls),
+        )
+        .unwrap();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dim as i32,
+                ),
+                true,
+            ),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(fsl)],
+        )
+        .unwrap();
+        index.insert(&batch, 0).unwrap();
+        assert_eq!(index.len(), 0, "no nodes for an all-null batch");
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/index/pk_key.rs b/rust/lance/src/dataset/mem_wal/index/pk_key.rs
new file mode 100644
index 00000000000..b31fe42c995
--- /dev/null
+++ b/rust/lance/src/dataset/mem_wal/index/pk_key.rs
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Composite primary-key encoding for MemWAL dedup.
+//!
+//! A multi-column primary key is reduced to a single order-preserving byte
+//! string ([`encode_pk_tuple`]) so the whole tuple is one comparable key:
+//! lexicographic byte order equals tuple order, and distinct tuples never
+//! collide. Encoded as a `Binary` value, the tuple is indexed directly by a
+//! [`super::BTreeMemIndex`] (its byte backend) — both in memory and, after
+//! flush, as the on-disk BTree's `Binary` value column — so a probe builds
+//! `ScalarValue::Binary(key)` and every layer agrees.
+//!
+//! Single-column primary keys do **not** use this — they key the typed
+//! `BTreeMemIndex` on the column value directly.
+
+use arrow_array::{BinaryArray, RecordBatch};
+use datafusion::common::ScalarValue;
+use lance_core::{Error, Result};
+
+/// Sign-flip a signed integer to an order-preserving unsigned key (matches the
+/// fixed-int BTree backend). Big-endian bytes of the result sort like the value.
+#[inline]
+fn encode_signed(v: i64) -> u64 {
+    (v as u64) ^ (1u64 << 63)
+}
+
+/// Append an order-preserving encoding of one non-null byte string: each `0x00`
+/// is escaped to `0x00 0xFF`, then a `0x00 0x00` terminator is appended. The
+/// terminator sorts before any escaped content, so a prefix orders before its
+/// extensions and no value can forge a column boundary.
+fn encode_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
+    for &b in bytes {
+        out.push(b);
+        if b == 0x00 {
+            out.push(0xFF);
+        }
+    }
+    out.extend_from_slice(&[0x00, 0x00]);
+}
+
+/// Append the order-preserving encoding of a single PK column value. A leading
+/// tag (`0x00` null / `0x01` non-null) makes nulls sort first and keeps the
+/// per-column encoding self-delimiting (fixed-width for ints, terminated for
+/// bytes), so concatenating columns stays injective and order-preserving.
+fn encode_value(out: &mut Vec<u8>, value: &ScalarValue) -> Result<()> {
+    if value.is_null() {
+        out.push(0x00);
+        return Ok(());
+    }
+    out.push(0x01);
+    macro_rules! be_signed {
+        ($v:expr) => {
+            out.extend_from_slice(&encode_signed($v as i64).to_be_bytes())
+        };
+    }
+    match value {
+        ScalarValue::Int8(Some(v)) => be_signed!(*v),
+        ScalarValue::Int16(Some(v)) => be_signed!(*v),
+        ScalarValue::Int32(Some(v)) => be_signed!(*v),
+        ScalarValue::Int64(Some(v)) => be_signed!(*v),
+        ScalarValue::Date32(Some(v)) => be_signed!(*v),
+        ScalarValue::Date64(Some(v)) => be_signed!(*v),
+        ScalarValue::UInt8(Some(v)) => out.extend_from_slice(&(*v as u64).to_be_bytes()),
+        ScalarValue::UInt16(Some(v)) => out.extend_from_slice(&(*v as u64).to_be_bytes()),
+        ScalarValue::UInt32(Some(v)) => out.extend_from_slice(&(*v as u64).to_be_bytes()),
+        ScalarValue::UInt64(Some(v)) => out.extend_from_slice(&v.to_be_bytes()),
+        ScalarValue::Boolean(Some(b)) => out.push(*b as u8),
+        ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)) => {
+            encode_bytes(out, s.as_bytes())
+        }
+        ScalarValue::Binary(Some(b))
+        | ScalarValue::LargeBinary(Some(b))
+        | ScalarValue::FixedSizeBinary(_, Some(b)) => encode_bytes(out, b),
+        other => {
+            return Err(Error::invalid_input(format!(
+                "Unsupported primary-key column type for composite key: {other:?}"
+            )));
+        }
+    }
+    Ok(())
+}
+
+/// Encode a PK tuple (values in PK column order) to one order-preserving key.
+pub fn encode_pk_tuple(values: &[ScalarValue]) -> Result<Vec<u8>> {
+    let mut out = Vec::with_capacity(values.len() * 9);
+    for value in values {
+        encode_value(&mut out, value)?;
+    }
+    Ok(out)
+}
+
+/// Encode row `row` of `batch`'s PK columns (at `pk_indices`) to one key.
+fn encode_pk_row(batch: &RecordBatch, pk_indices: &[usize], row: usize) -> Result<Vec<u8>> {
+    let mut out = Vec::with_capacity(pk_indices.len() * 9);
+    for &col in pk_indices {
+        let value = ScalarValue::try_from_array(batch.column(col), row)?;
+        encode_value(&mut out, &value)?;
+    }
+    Ok(out)
+}
+
+/// Encode every row of `batch`'s PK columns (at `pk_indices`) into a `Binary`
+/// column of order-preserving composite keys — the form a [`super::BTreeMemIndex`]
+/// indexes directly (its byte backend), so the composite PK reuses the same
+/// index as a single-column one.
+pub fn encode_pk_batch(batch: &RecordBatch, pk_indices: &[usize]) -> Result<BinaryArray> {
+    let mut keys: Vec<Vec<u8>> = Vec::with_capacity(batch.num_rows());
+    for row in 0..batch.num_rows() {
+        keys.push(encode_pk_row(batch, pk_indices, row)?);
+    }
+    Ok(BinaryArray::from_iter_values(keys.iter()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::{Int32Array, StringArray};
+    use arrow_schema::{DataType, Field, Schema};
+    use std::sync::Arc;
+
+    fn tuple(a: i32, b: &str) -> Vec<ScalarValue> {
+        vec![ScalarValue::Int32(Some(a)), ScalarValue::from(b)]
+    }
+
+    #[test]
+    fn encoding_is_order_preserving_and_injective() {
+        // Sorting tuples by their encoding must match tuple order, and distinct
+        // tuples must produce distinct bytes.
+        let tuples = [
+            tuple(1, "a"),
+            tuple(1, "ab"),
+            tuple(1, "b"),
+            tuple(2, "a"),
+            tuple(-1, "z"),
+        ];
+        let mut encoded: Vec<(Vec<u8>, &Vec<ScalarValue>)> = tuples
+            .iter()
+            .map(|t| (encode_pk_tuple(t).unwrap(), t))
+            .collect();
+        encoded.sort_by(|x, y| x.0.cmp(&y.0));
+        let order: Vec<_> = encoded.iter().map(|(_, t)| (*t).clone()).collect();
+        // -1 < 1 < 2; within id=1, "a" < "ab" < "b".
+        assert_eq!(
+            order,
+            vec![
+                tuple(-1, "z"),
+                tuple(1, "a"),
+                tuple(1, "ab"),
+                tuple(1, "b"),
+                tuple(2, "a"),
+            ]
+        );
+        // Injective: 5 distinct tuples → 5 distinct keys.
+        let mut keys: Vec<Vec<u8>> = tuples.iter().map(|t| encode_pk_tuple(t).unwrap()).collect();
+        keys.sort();
+        keys.dedup();
+        assert_eq!(keys.len(), 5);
+    }
+
+    #[test]
+    fn null_sorts_first_and_is_distinct() {
+        let null_a = vec![ScalarValue::Int32(None), ScalarValue::from("a")];
+        let one_a = tuple(1, "a");
+        assert!(encode_pk_tuple(&null_a).unwrap() < encode_pk_tuple(&one_a).unwrap());
+        assert_ne!(
+            encode_pk_tuple(&null_a).unwrap(),
+            encode_pk_tuple(&one_a).unwrap()
+        );
+    }
+
+    #[test]
+    fn prefix_safety_with_embedded_zero() {
+        // A string containing 0x00 must not collide with or sort incorrectly
+        // against a shorter one (escaping + terminator).
+        let with_zero = vec![ScalarValue::Binary(Some(vec![0x00]))];
+        let empty = vec![ScalarValue::Binary(Some(vec![]))];
+        assert!(encode_pk_tuple(&empty).unwrap() < encode_pk_tuple(&with_zero).unwrap());
+    }
+
+    #[test]
+    fn encode_pk_batch_matches_per_tuple_encoding() {
+        // Each row of the encoded `Binary` column equals `encode_pk_tuple` of
+        // that row's PK values — so the column a BTreeMemIndex indexes is exactly
+        // what a probe builds.
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int32Array::from(vec![2, 1])),
+                Arc::new(StringArray::from(vec!["a", "b"])),
+            ],
+        )
+        .unwrap();
+        let encoded = encode_pk_batch(&batch, &[0, 1]).unwrap();
+        assert_eq!(encoded.value(0), encode_pk_tuple(&tuple(2, "a")).unwrap());
+        assert_eq!(encoded.value(1), encode_pk_tuple(&tuple(1, "b")).unwrap());
+        // (1,"b") encodes below (2,"a").
+        assert!(encoded.value(1) < encoded.value(0));
+    }
+}
diff --git a/rust/lance/src/dataset/mem_wal/manifest.rs b/rust/lance/src/dataset/mem_wal/manifest.rs
index 48df22dca88..9c2a3aa2163 100644
--- a/rust/lance/src/dataset/mem_wal/manifest.rs
+++ b/rust/lance/src/dataset/mem_wal/manifest.rs
@@ -35,7 +35,7 @@ use bytes::Bytes;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
 use lance_core::{Error, Result};
-use lance_index::mem_wal::ShardManifest;
+use lance_index::mem_wal::{ShardManifest, ShardStatus};
 use lance_io::object_store::ObjectStore;
 use lance_table::format::pb;
 use log::{info, warn};
@@ -147,6 +147,7 @@ impl ShardManifestStore {
             wal_entry_position_last_seen: 0,
             current_generation: 1,
             flushed_generations: vec![],
+            status: ShardStatus::Active,
         };
 
         match self.write(&manifest).await {
@@ -425,6 +426,21 @@ impl ShardManifestStore {
         for _ in 0..MAX_CLAIM_RETRIES {
             let current = self.read_latest().await?;
 
+            // A sealed shard is mid-drop (drop-table 2PC). Refuse the claim
+            // with a distinguishable error rather than minting a new epoch,
+            // so a caller that skips its own status check still cannot
+            // resurrect a shard being dropped. Sophon's reconcile keys on
+            // the "sealed" marker in this message to tell it apart from an
+            // ordinary epoch fence.
+            if let Some(m) = &current
+                && m.status == ShardStatus::Sealed
+            {
+                return Err(Error::invalid_input(format!(
+                    "shard {} is sealed; refusing claim (drop in flight)",
+                    self.shard_id
+                )));
+            }
+
             let (next_version, next_epoch, base_manifest) = match current {
                 Some(m) => (m.version + 1, m.writer_epoch + 1, Some(m)),
                 None => (1, 1, None),
@@ -447,6 +463,7 @@ impl ShardManifestStore {
                     wal_entry_position_last_seen: 0,
                     current_generation: 1,
                     flushed_generations: vec![],
+                    status: ShardStatus::Active,
                 }
             };
 
@@ -502,8 +519,8 @@ impl ShardManifestStore {
         shard_id: Uuid,
     ) -> Result<()> {
         match manifest {
-            Some(m) if m.writer_epoch > local_epoch => Err(Error::io(format!(
-                "Writer fenced: local epoch {} < stored epoch {} for shard {}",
+            Some(m) if m.writer_epoch > local_epoch => Err(Error::fenced_by_peer(format!(
+                "local epoch {} < stored epoch {} for shard {}",
                 local_epoch, m.writer_epoch, shard_id
             ))),
             _ => Ok(()),
@@ -603,6 +620,7 @@ mod tests {
             wal_entry_position_last_seen: 0,
             current_generation: 1,
             flushed_generations: vec![],
+            status: ShardStatus::Active,
         }
     }
 
@@ -752,6 +770,51 @@ mod tests {
         assert_eq!(second.wal_entry_position_last_seen, 42);
     }
 
+    #[tokio::test]
+    async fn test_claim_epoch_refuses_sealed_manifest() {
+        // A `Sealed` manifest is the drop-table 2PC in-doubt marker:
+        // `claim_epoch` must refuse it with a distinguishable error rather
+        // than mint a new epoch, so a sealed shard can't be resurrected even
+        // by a caller that skips its own status check. Rolling the status
+        // back to `Active` makes the shard claimable again (reversible).
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2);
+
+        let (epoch, claimed) = manifest_store.claim_epoch(0).await.unwrap();
+        assert_eq!(claimed.status, ShardStatus::Active);
+
+        // Seal it (drop-table prepare).
+        let sealed = ShardManifest {
+            version: claimed.version + 1,
+            status: ShardStatus::Sealed,
+            ..claimed
+        };
+        manifest_store.write(&sealed).await.unwrap();
+
+        // The claim is refused with the distinguishable "sealed" error —
+        // and the manifest is left untouched (no new epoch minted).
+        let err = manifest_store.claim_epoch(0).await.unwrap_err();
+        assert!(
+            err.to_string().contains("sealed"),
+            "expected a distinguishable sealed-refusal error, got: {err}"
+        );
+        let after = manifest_store.read_latest().await.unwrap().unwrap();
+        assert_eq!(after.writer_epoch, sealed.writer_epoch, "no epoch minted");
+        assert_eq!(after.status, ShardStatus::Sealed);
+
+        // Roll back to Active (drop-table abort) → re-claimable.
+        let active = ShardManifest {
+            version: sealed.version + 1,
+            status: ShardStatus::Active,
+            ..sealed
+        };
+        manifest_store.write(&active).await.unwrap();
+        let (next_epoch, reclaimed) = manifest_store.claim_epoch(0).await.unwrap();
+        assert!(next_epoch > epoch, "rolled-back shard mints the next epoch");
+        assert_eq!(reclaimed.status, ShardStatus::Active);
+    }
+
     #[tokio::test]
     async fn test_initialize_shard_rejects_conflict_with_mismatch() {
         let (store, base_path, _temp_dir) = create_local_store().await;
diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs
index fd23fd6d577..77611fd7c47 100644
--- a/rust/lance/src/dataset/mem_wal/memtable.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable.rs
@@ -14,14 +14,15 @@ use std::time::{Duration, Instant};
 use arrow_array::{Array, RecordBatch, RecordBatchIterator};
 use arrow_schema::Schema as ArrowSchema;
 use lance_core::datatypes::Schema;
+use lance_core::utils::bloomfilter::sbbf::Sbbf;
 use lance_core::{Error, Result};
-use lance_index::scalar::bloomfilter::sbbf::Sbbf;
 use tokio::sync::RwLock;
 use tracing::instrument;
 use uuid::Uuid;
 
 use super::index::IndexStore;
 use super::util::{WatchableOnceCell, WatchableOnceCellReader};
+use super::wal::WalFlushFailure;
 use super::write::{DurabilityResult, WalFlushResult};
 use crate::Dataset;
 use batch_store::BatchStore;
@@ -103,9 +104,10 @@ pub struct MemTable {
     /// Set when the memtable is frozen and a WAL flush request is sent.
     /// The reader can be awaited to know when WAL flush is complete.
     /// Uses Mutex for interior mutability since the MemTable is wrapped in Arc when frozen.
-    /// Uses Result<WalFlushResult, String> since lance_core::Error doesn't implement Clone.
+    /// Uses `WalFlushFailure` (not `Error`) since `lance_core::Error` doesn't
+    /// implement Clone; the carrier preserves the fence reason for the waiter.
     wal_flush_completion: std::sync::Mutex<
-        Option<WatchableOnceCellReader<std::result::Result<WalFlushResult, String>>>,
+        Option<WatchableOnceCellReader<std::result::Result<WalFlushResult, WalFlushFailure>>>,
     >,
 
     /// Cell for memtable flush completion notification.
@@ -266,7 +268,7 @@ impl MemTable {
     /// the WAL flush is complete.
     pub fn set_wal_flush_completion(
         &self,
-        reader: WatchableOnceCellReader<std::result::Result<WalFlushResult, String>>,
+        reader: WatchableOnceCellReader<std::result::Result<WalFlushResult, WalFlushFailure>>,
     ) {
         *self.wal_flush_completion.lock().unwrap() = Some(reader);
     }
@@ -278,7 +280,7 @@ impl MemTable {
     /// Thread-safe via interior mutability.
     pub fn take_wal_flush_completion(
         &self,
-    ) -> Option<WatchableOnceCellReader<std::result::Result<WalFlushResult, String>>> {
+    ) -> Option<WatchableOnceCellReader<std::result::Result<WalFlushResult, WalFlushFailure>>> {
         self.wal_flush_completion.lock().unwrap().take()
     }
 
diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
index f4d4d797acc..30e47e1a9bb 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs
@@ -43,7 +43,9 @@ use std::cell::UnsafeCell;
 use std::mem::MaybeUninit;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
+use arrow::array::ArrayData;
 use arrow_array::RecordBatch;
+use arrow_schema::DataType;
 
 /// A batch stored in the lock-free store.
 #[derive(Clone)]
@@ -75,14 +77,56 @@ impl StoredBatch {
     }
 
     /// Estimate the memory size of a RecordBatch.
+    ///
+    /// Sums each column's slice-aware buffer size (see
+    /// [`Self::estimate_array_size`]) plus the struct overhead, so a column that
+    /// is a zero-copy slice of a larger parent contributes only its own window
+    /// rather than the whole shared buffer.
     fn estimate_batch_size(batch: &RecordBatch) -> usize {
         batch
             .columns()
             .iter()
-            .map(|col| col.get_array_memory_size())
+            .map(|col| Self::estimate_array_size(&col.to_data()))
             .sum::<usize>()
             + std::mem::size_of::<RecordBatch>()
     }
+
+    /// Slice-aware buffer size of a single array.
+    ///
+    /// [`ArrayData::get_slice_memory_size`] reports each buffer's own window
+    /// (not the whole shared buffer), but omits the variadic data buffers of
+    /// `Utf8View`/`BinaryView` (values > 12 bytes) while still returning `Ok`, so
+    /// [`Self::view_data_buffers_size`] adds them. Those buffers are shared across
+    /// zero-copy slices and are counted at full capacity for each slice — an
+    /// over-count in the safe direction.
+    fn estimate_array_size(data: &ArrayData) -> usize {
+        match data.get_slice_memory_size() {
+            Ok(size) => size + Self::view_data_buffers_size(data),
+            // Fall back to the full-buffer sum for layouts the slice-aware call
+            // cannot handle.
+            Err(_) => data.get_array_memory_size(),
+        }
+    }
+
+    /// Capacity of the variadic `Utf8View`/`BinaryView` data buffers that
+    /// [`ArrayData::get_slice_memory_size`] omits, summed recursively over children.
+    fn view_data_buffers_size(data: &ArrayData) -> usize {
+        let mut size = 0;
+        if matches!(data.data_type(), DataType::Utf8View | DataType::BinaryView) {
+            // buffers()[0] is the 16-byte view array that get_slice_memory_size
+            // already counts; [1..] are the data buffers it skips.
+            size += data
+                .buffers()
+                .iter()
+                .skip(1)
+                .map(|b| b.capacity())
+                .sum::<usize>();
+        }
+        for child in data.child_data() {
+            size += Self::view_data_buffers_size(child);
+        }
+        size
+    }
 }
 
 /// Snapshot of the active batches that have not yet been flushed to WAL.
@@ -615,6 +659,22 @@ impl BatchStore {
         (0..end).collect()
     }
 
+    /// The inclusive maximum visible *row* position at `max_visible_batch_position`,
+    /// or `None` when no rows are visible. The visible batches are the committed
+    /// prefix `[0, last_visible_idx]`; each batch carries its cumulative
+    /// `row_offset`, so this is the end of the last visible batch minus one.
+    /// Used to bound MVCC seeks against the maintained PK-position index.
+    pub fn max_visible_row(&self, max_visible_batch_position: usize) -> Option<u64> {
+        let len = self.committed_len.load(Ordering::Acquire);
+        if len == 0 {
+            return None;
+        }
+        let last_visible_idx = max_visible_batch_position.min(len - 1);
+        let last = self.get(last_visible_idx)?;
+        let visible_end = last.row_offset + last.num_rows as u64; // exclusive
+        visible_end.checked_sub(1)
+    }
+
     /// Check if a specific batch is visible at a given visibility position.
     #[inline]
     pub fn is_batch_visible(
@@ -910,6 +970,37 @@ mod tests {
         assert!(!store.is_batch_visible(3, 10));
     }
 
+    #[test]
+    fn test_max_visible_row() {
+        // (1) Empty store: no rows are visible at any position.
+        let store = BatchStore::with_capacity(10);
+        assert_eq!(store.max_visible_row(0), None);
+        assert_eq!(store.max_visible_row(100), None);
+
+        // Three batches → rows [0,10) [10,30) [30,60); row_offsets 0, 10, 30.
+        store.append(create_test_batch(10)).unwrap(); // position 0
+        store.append(create_test_batch(20)).unwrap(); // position 1
+        store.append(create_test_batch(30)).unwrap(); // position 2
+
+        // (2) A position within range yields the inclusive end of that prefix.
+        assert_eq!(store.max_visible_row(0), Some(9)); // batch 0: 0..10
+        assert_eq!(store.max_visible_row(1), Some(29)); // batch 1: 10..30
+        assert_eq!(store.max_visible_row(2), Some(59)); // batch 2: 30..60
+
+        // (3) A position beyond the committed range clamps to the last batch,
+        // i.e. the inclusive max over all rows.
+        assert_eq!(store.max_visible_row(100), Some(59));
+
+        // (4) An empty leading batch contributes no rows: at its own position
+        // the inclusive end underflows to None, while a later non-empty batch
+        // is reported correctly.
+        let store = BatchStore::with_capacity(10);
+        store.append(create_test_batch(0)).unwrap(); // position 0: rows [0,0)
+        store.append(create_test_batch(5)).unwrap(); // position 1: rows [0,5)
+        assert_eq!(store.max_visible_row(0), None); // empty prefix → no rows
+        assert_eq!(store.max_visible_row(1), Some(4)); // through batch 1
+    }
+
     #[test]
     fn test_recommended_capacity() {
         // 64MB memtable, 64KB avg batch = 1024 batches * 1.2 = ~1228
@@ -925,6 +1016,122 @@ mod tests {
         assert_eq!(cap, 16); // minimum
     }
 
+    #[test]
+    fn test_estimated_size_is_slice_aware() {
+        // A batch that is a zero-copy slice of a larger parent must contribute
+        // only its own window to the estimate, not the whole shared buffer.
+        // `get_array_memory_size` counts every buffer's full capacity regardless
+        // of offset/length, so N slices tiling one parent each report the
+        // parent's size and inflate the memtable estimate ~N×, tripping the
+        // flush threshold far below the configured size.
+        let chunk = 1_000;
+        let num_slices = 100;
+        let parent = create_test_batch(chunk * num_slices);
+
+        // One window vs an equivalently-sized owned batch should track each
+        // other; the buggy per-slice estimate would be ~num_slices× larger.
+        let slice_est = StoredBatch::estimate_batch_size(&parent.slice(0, chunk));
+        let owned_est = StoredBatch::estimate_batch_size(&create_test_batch(chunk));
+        assert!(
+            slice_est <= owned_est * 2,
+            "slice estimate {slice_est} should track its own window (~{owned_est}), not the parent"
+        );
+
+        // End-to-end: tiling the parent with zero-copy slices must not multiply
+        // the store's running estimate. Track what the old full-buffer behavior
+        // would have summed to for contrast.
+        let store = BatchStore::with_capacity(num_slices);
+        let mut over_counting_sum = 0usize;
+        for k in 0..num_slices {
+            let s = parent.slice(k * chunk, chunk);
+            over_counting_sum += s
+                .columns()
+                .iter()
+                .map(|col| col.get_array_memory_size())
+                .sum::<usize>()
+                + std::mem::size_of::<RecordBatch>();
+            store.append(s).unwrap();
+        }
+
+        // Two non-nullable Int32 columns → exactly 4 bytes/row/col of payload.
+        let payload_bytes = num_slices * chunk * 2 * std::mem::size_of::<i32>();
+        let estimated = store.estimated_bytes();
+        assert!(
+            estimated >= payload_bytes,
+            "estimate {estimated} should cover the actual payload {payload_bytes}"
+        );
+        // The old behavior over-counts by ~num_slices×; the fix must be far
+        // below it (generous 10× margin against struct/alignment overhead).
+        assert!(
+            estimated * 10 < over_counting_sum,
+            "estimate {estimated} should be far below the over-counting sum {over_counting_sum}"
+        );
+    }
+
+    #[test]
+    fn test_estimated_size_counts_view_data_buffers() {
+        // Long Utf8View/BinaryView values live in variadic data buffers that
+        // `get_slice_memory_size` ignores (returning ~16 * rows). The estimate
+        // must include them, both for a top-level view column and for a view
+        // array nested in a container, which is only reached via child_data
+        // recursion.
+        use arrow_array::{Array, ArrayRef, StringViewArray, StructArray};
+
+        let num_rows = 1_000;
+        // Each value exceeds the 12-byte inline limit, so it spills to a data buffer.
+        let long_value = "x".repeat(64);
+        let payload_bytes = num_rows * long_value.len();
+        // What the slice-aware call alone reports: just the 16-byte view entries.
+        let view_entries_only = num_rows * 16;
+
+        let make_views = || {
+            StringViewArray::from(
+                (0..num_rows)
+                    .map(|_| Some(long_value.as_str()))
+                    .collect::<Vec<_>>(),
+            )
+        };
+        let assert_covers = |batch: &RecordBatch| {
+            let estimated = StoredBatch::estimate_batch_size(batch);
+            assert!(
+                estimated >= payload_bytes,
+                "estimate {estimated} should cover the view data-buffer payload {payload_bytes}"
+            );
+            assert!(
+                estimated > view_entries_only * 2,
+                "estimate {estimated} must exceed the ~{view_entries_only}-byte view-entry-only undercount"
+            );
+        };
+
+        // Top-level view column.
+        let flat = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "s",
+                DataType::Utf8View,
+                false,
+            )])),
+            vec![Arc::new(make_views())],
+        )
+        .unwrap();
+        assert_covers(&flat);
+
+        // View nested inside a struct — reachable only through child_data recursion.
+        let nested = StructArray::from(vec![(
+            Arc::new(Field::new("s", DataType::Utf8View, false)),
+            Arc::new(make_views()) as ArrayRef,
+        )]);
+        let nested = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "st",
+                nested.data_type().clone(),
+                false,
+            )])),
+            vec![Arc::new(nested)],
+        )
+        .unwrap();
+        assert_covers(&nested);
+    }
+
     #[test]
     fn test_to_vec() {
         let store = BatchStore::with_capacity(10);
diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
index 8f0e34db5ec..e88be539e30 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs
@@ -14,11 +14,11 @@ use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_index::mem_wal::{FlushedGeneration, ShardManifest};
 use lance_index::scalar::{IndexStore, ScalarIndexParams};
-use lance_io::object_store::ObjectStore;
+use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use lance_table::format::IndexMetadata;
 use lance_table::io::commit::write_manifest_file_to_path;
 use lance_table::io::deletion::write_deletion_file;
-use log::info;
+use log::{info, warn};
 use object_store::ObjectStoreExt;
 use object_store::path::Path;
 use roaring::RoaringBitmap;
@@ -28,9 +28,14 @@ use uuid::Uuid;
 use super::super::index::MemIndexConfig;
 use super::super::memtable::MemTable;
 use crate::Dataset;
+use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::mem_wal::manifest::ShardManifestStore;
+use crate::dataset::mem_wal::scanner::GenerationWarmer;
 use crate::dataset::mem_wal::scanner::exec::{compute_pk_hash, validate_pk_types};
-use crate::dataset::mem_wal::util::{flushed_memtable_path, generate_random_hash};
+use crate::dataset::mem_wal::util::{
+    derived_store_params, flushed_memtable_path, generate_random_hash,
+};
+use crate::session::Session;
 
 #[derive(Debug, Clone)]
 pub struct FlushResult {
@@ -68,6 +73,16 @@ pub struct MemTableFlusher {
     base_uri: String,
     shard_id: Uuid,
     manifest_store: Arc<ShardManifestStore>,
+    /// When present, each new generation is warmed before it is committed, so
+    /// the first query sees zero cold reads. `None` => no warming.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Store params the base dataset was opened with, reused for the flusher's
+    /// own opens + writes. Used verbatim only for the base's own URI; generation
+    /// URIs go through [`derived_store_params`]. `None` opens by URI alone.
+    store_params: Option<ObjectStoreParams>,
+    /// Session for those opens, sharing the base's store registry. `None` opens
+    /// with a fresh session.
+    session: Option<Arc<Session>>,
 }
 
 impl MemTableFlusher {
@@ -84,6 +99,73 @@ impl MemTableFlusher {
             base_uri: base_uri.into(),
             shard_id,
             manifest_store,
+            warmer: None,
+            store_params: None,
+            session: None,
+        }
+    }
+
+    /// Attach the warmer fired pre-commit for each new generation.
+    pub fn with_warmer(mut self, warmer: Option<Arc<dyn GenerationWarmer>>) -> Self {
+        self.warmer = warmer;
+        self
+    }
+
+    /// Set the store params + session used for derived-URI opens. Injected by
+    /// `mem_wal_writer` from the base `Dataset`.
+    pub fn with_storage_context(
+        mut self,
+        store_params: Option<ObjectStoreParams>,
+        session: Option<Arc<Session>>,
+    ) -> Self {
+        self.store_params = store_params;
+        self.session = session;
+        self
+    }
+
+    /// Open the base table, reusing the injected store params verbatim — they
+    /// were resolved for exactly this URI, so a path-bound `object_store`
+    /// binding still points where it should.
+    async fn open_base(&self) -> Result<Dataset> {
+        self.open_uri(&self.base_uri, self.store_params.clone())
+            .await
+    }
+
+    /// Open a flushed generation under `_mem_wal/`. The params must be adapted
+    /// first: a path-bound store binding would redirect the open at the base
+    /// table (see [`derived_store_params`]).
+    async fn open_generation(&self, uri: &str) -> Result<Dataset> {
+        self.open_uri(uri, self.store_params.as_ref().map(derived_store_params))
+            .await
+    }
+
+    /// Open `uri` with the injected session, or by URI alone when nothing was
+    /// injected.
+    async fn open_uri(
+        &self,
+        uri: &str,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Dataset> {
+        let mut builder = DatasetBuilder::from_uri(uri);
+        if let Some(params) = store_params {
+            builder = builder.with_store_params(params);
+        }
+        if let Some(session) = &self.session {
+            builder = builder.with_session(session.clone());
+        }
+        builder.load().await
+    }
+
+    /// Warm a just-written generation before it is committed. Best-effort: a
+    /// failure is logged and the flush proceeds — warming is never a commit
+    /// gate. No-op without a warmer. `uri` must be the resolved reader path
+    /// (`path_to_uri(gen_path)`) so warmed entries key-match later queries.
+    async fn warm_generation(&self, uri: &str) {
+        let Some(warmer) = &self.warmer else {
+            return;
+        };
+        if let Err(e) = warmer.warm(uri).await {
+            warn!("pre-commit warm failed for generation {uri}; committing cold: {e}");
         }
     }
 
@@ -115,7 +197,7 @@ impl MemTableFlusher {
     /// In production MemWAL is always initialized on a real dataset, so the base
     /// version is inherited; other open errors are propagated.
     async fn base_storage_version(&self) -> Result<lance_file::version::LanceFileVersion> {
-        match Dataset::open(&self.base_uri).await {
+        match self.open_base().await {
             Ok(dataset) => dataset.manifest().data_storage_format.lance_file_version(),
             Err(Error::DatasetNotFound { .. }) => {
                 Ok(lance_file::version::LanceFileVersion::default())
@@ -170,7 +252,7 @@ impl MemTableFlusher {
         // generation exposes newest-per-PK on every read path.
         if !deleted.is_empty() {
             let uri = self.path_to_uri(&gen_path);
-            let dataset = Dataset::open(&uri).await?;
+            let dataset = self.open_generation(&uri).await?;
             self.finalize_generation(&dataset, &deleted, None).await?;
         }
 
@@ -178,6 +260,16 @@ impl MemTableFlusher {
         self.write_bloom_filter(&bloom_path, memtable.bloom_filter())
             .await?;
 
+        // Write the standalone primary-key dedup sidecar. A primary key needs
+        // no secondary index, so this is required on the plain-flush path too —
+        // the LSM scanner opens it to dedup the generation. (`flush_with_indexes`
+        // writes it on the indexed path.) No-op when the memtable has no PK.
+        self.create_pk_index(&gen_path, memtable.indexes()).await?;
+
+        // Warm before commit (zero cold window); no-op without a warmer.
+        let warm_uri = self.path_to_uri(&gen_path);
+        self.warm_generation(&warm_uri).await;
+
         let new_manifest = self
             .update_manifest(
                 epoch,
@@ -269,6 +361,12 @@ impl MemTableFlusher {
         let write_params = WriteParams {
             max_rows_per_file: usize::MAX,
             data_storage_version: Some(self.base_storage_version().await?),
+            // Write the generation through the base's store params + session so it
+            // uses the same store the base was opened with. Adapted for the
+            // generation URI: a path-bound store binding would send this write at
+            // the base table's own path (see [`derived_store_params`]).
+            store_params: self.store_params.as_ref().map(derived_store_params),
+            session: self.session.clone(),
             ..Default::default()
         };
         Dataset::write(reader, &uri, Some(write_params)).await?;
@@ -332,7 +430,7 @@ impl MemTableFlusher {
     async fn write_bloom_filter(
         &self,
         path: &Path,
-        bloom: &lance_index::scalar::bloomfilter::sbbf::Sbbf,
+        bloom: &lance_core::utils::bloomfilter::sbbf::Sbbf,
     ) -> Result<()> {
         let data = bloom.to_bytes();
         self.object_store
@@ -386,7 +484,7 @@ impl MemTableFlusher {
         // Open the dataset once for all index building. Dataset::write already
         // created a v1 manifest with the fragment data.
         let uri = self.path_to_uri(&gen_path);
-        let mut dataset = Dataset::open(&uri).await?;
+        let mut dataset = self.open_generation(&uri).await?;
 
         // Collect all index metadata without committing individually.
         // We write a single manifest containing both data and all indexes.
@@ -409,9 +507,18 @@ impl MemTableFlusher {
                 if let MemIndexConfig::Hnsw(hnsw_config) = config
                     && let Some(mem_index) = registry.get_hnsw(&hnsw_config.name)
                 {
-                    let mut index_meta = self
+                    // `None` → the generation has no indexable vectors (e.g. all
+                    // tombstones); flush the data without an HNSW index for it.
+                    let Some(mut index_meta) = self
                         .create_hnsw_index(&gen_path, hnsw_config, mem_index)
-                        .await?;
+                        .await?
+                    else {
+                        info!(
+                            "Skipped empty HNSW index '{}' on flushed generation {} (no vectors)",
+                            hnsw_config.name, generation
+                        );
+                        continue;
+                    };
 
                     let schema = dataset.schema();
                     let field_idx = schema
@@ -449,6 +556,10 @@ impl MemTableFlusher {
             all_indexes.extend(fts_indexes);
         }
 
+        // Write the standalone primary-key dedup index (sidecar, not a manifest
+        // index — the block-list opens it directly by path).
+        self.create_pk_index(&gen_path, memtable.indexes()).await?;
+
         // Write a single manifest that records the fragments, the
         // within-generation deletion vector, and all indexes, overwriting the
         // data-only v1 manifest created by Dataset::write.
@@ -459,6 +570,10 @@ impl MemTableFlusher {
         self.write_bloom_filter(&bloom_path, memtable.bloom_filter())
             .await?;
 
+        // Warm before commit (zero cold window); no-op without a warmer.
+        let warm_uri = self.path_to_uri(&gen_path);
+        self.warm_generation(&warm_uri).await;
+
         let new_manifest = self
             .update_manifest(
                 epoch,
@@ -543,6 +658,49 @@ impl MemTableFlusher {
         Ok(created_indexes)
     }
 
+    /// Write the standalone primary-key dedup index for this generation.
+    ///
+    /// Unlike user indexes, this is a **sidecar**: it is not registered in the
+    /// manifest. The block-list opens it directly by path
+    /// ([`pk_index_path`]) and probes it with `Equals`. Single-column primary
+    /// keys index the typed value; composite keys index the order-preserving
+    /// `Binary` encoded tuple (see [`super::super::index::encode_pk_tuple`]).
+    /// Row positions line up 1:1 with the forward-written data file, so they are
+    /// the flushed row ids directly. No-op without a primary-key index.
+    async fn create_pk_index(
+        &self,
+        gen_path: &Path,
+        mem_indexes: Option<&super::super::index::IndexStore>,
+    ) -> Result<()> {
+        use datafusion::physical_plan::SendableRecordBatchStream;
+        use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+        use lance_index::scalar::btree::train_btree_index;
+        use lance_index::scalar::lance_format::LanceIndexStore;
+
+        use crate::dataset::mem_wal::util::pk_index_path;
+
+        let Some(registry) = mem_indexes else {
+            return Ok(());
+        };
+        let batches = registry.pk_training_batches(8192)?;
+        if batches.is_empty() {
+            return Ok(());
+        }
+
+        let schema = batches[0].schema();
+        let store = LanceIndexStore::new(
+            self.object_store.clone(),
+            pk_index_path(gen_path),
+            Arc::new(LanceCache::no_cache()),
+        );
+        let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::iter(batches.into_iter().map(Ok)),
+        ));
+        train_btree_index(stream, &store, 8192, None, None).await?;
+        Ok(())
+    }
+
     /// Create FTS (Full-Text Search) indexes from in-memory data (uncommitted).
     ///
     /// Writes the FTS index files and returns index metadata without committing.
@@ -556,7 +714,6 @@ impl MemTableFlusher {
         total_rows: usize,
     ) -> Result<Vec<IndexMetadata>> {
         use lance_index::pbold;
-        use lance_index::scalar::inverted::current_fts_format_version;
         use lance_index::scalar::lance_format::LanceIndexStore;
 
         let fts_configs: Vec<_> = index_configs
@@ -619,6 +776,7 @@ impl MemTableFlusher {
             })?;
 
             let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone();
+            let format_version = fts_cfg.params.resolved_format_version();
 
             let index_meta = IndexMetadata {
                 uuid: index_uuid,
@@ -627,7 +785,7 @@ impl MemTableFlusher {
                 dataset_version: dataset.version().version,
                 fragment_bitmap: Some(fragment_ids),
                 index_details: Some(Arc::new(index_details)),
-                index_version: current_fts_format_version().index_version() as i32,
+                index_version: format_version.index_version() as i32,
                 created_at: None,
                 base_id: None,
                 files: None,
@@ -654,22 +812,50 @@ impl MemTableFlusher {
         use arrow_schema::{DataType, Field, Schema};
         use std::sync::Arc;
 
-        use lance_index::scalar::inverted::TokenSetFormat;
+        use lance_index::scalar::inverted::{
+            FTS_FORMAT_VERSION_KEY, POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1,
+            POSITIONS_LAYOUT_KEY, POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_BLOCK_SIZE_KEY,
+            POSTING_TAIL_CODEC_KEY, TokenSetFormat,
+        };
 
         // Create metadata with params and partitions in schema metadata (this is what InvertedIndex expects)
         let params_json = serde_json::to_string(&config.params)?;
         let partitions_json = serde_json::to_string(&[partition_id])?;
         let token_set_format = TokenSetFormat::default().to_string();
+        let format_version = config.params.resolved_format_version();
+        let mut metadata = [
+            ("params".to_string(), params_json),
+            ("partitions".to_string(), partitions_json),
+            ("token_set_format".to_string(), token_set_format),
+            (
+                POSTING_TAIL_CODEC_KEY.to_string(),
+                format_version.posting_tail_codec().as_str().to_string(),
+            ),
+            (
+                FTS_FORMAT_VERSION_KEY.to_string(),
+                format_version.index_version().to_string(),
+            ),
+            (
+                POSTING_BLOCK_SIZE_KEY.to_string(),
+                config.params.posting_block_size().to_string(),
+            ),
+        ]
+        .into_iter()
+        .collect::<std::collections::HashMap<_, _>>();
+        if config.params.has_positions() && format_version.uses_shared_position_stream() {
+            metadata.insert(
+                POSITIONS_LAYOUT_KEY.to_string(),
+                POSITIONS_LAYOUT_SHARED_STREAM_V2.to_string(),
+            );
+            metadata.insert(
+                POSITIONS_CODEC_KEY.to_string(),
+                POSITIONS_CODEC_PACKED_DELTA_V1.to_string(),
+            );
+        }
 
         let schema = Arc::new(
-            Schema::new(vec![Field::new("_placeholder", DataType::Utf8, true)]).with_metadata(
-                [
-                    ("params".to_string(), params_json),
-                    ("partitions".to_string(), partitions_json),
-                    ("token_set_format".to_string(), token_set_format),
-                ]
-                .into(),
-            ),
+            Schema::new(vec![Field::new("_placeholder", DataType::Utf8, true)])
+                .with_metadata(metadata),
         );
 
         // Create a minimal batch (schema metadata is what matters)
@@ -697,12 +883,21 @@ impl MemTableFlusher {
     /// * `gen_path` - Path to the flushed generation folder
     /// * `config` - HNSW index configuration
     /// * `mem_index` - In-memory HNSW index (snapshotted, not consumed)
+    ///
+    /// # Returns
+    ///
+    /// `Ok(None)` when the memtable holds no indexable vectors — e.g. a
+    /// generation of only tombstones (delete nulls the vector column and the
+    /// HNSW skips nulls), or an all-null vector column. The generation still
+    /// flushes its data; it simply carries no HNSW index, and an index-only
+    /// (`fast_search`) query over it returns empty — no brute-force scan.
+    /// Erroring here would fail the whole flush drain.
     async fn create_hnsw_index(
         &self,
         gen_path: &Path,
         config: &super::super::index::HnswIndexConfig,
         mem_index: &super::super::index::HnswMemIndex,
-    ) -> Result<IndexMetadata> {
+    ) -> Result<Option<IndexMetadata>> {
         use arrow_array::cast::AsArray;
         use arrow_array::types::Float32Type;
         use arrow_array::{FixedSizeListArray, Float32Array, RecordBatch as ArrowRecordBatch};
@@ -739,16 +934,16 @@ impl MemTableFlusher {
         let distance_type = mem_index.distance_type();
         let dim = mem_index.dim();
         if dim == 0 {
-            return Err(Error::invalid_input(
-                "HnswMemIndex has no inserted vectors; nothing to flush",
-            ));
+            // No vector was ever inserted (e.g. an all-tombstone generation):
+            // skip the index, keep the data flush.
+            return Ok(None);
         }
         // Forward-written data: HNSW row ids line up 1:1 with the data file, so
         // no position reversal (pass `None`).
         let Some((hnsw, flat_storage_batch)) = mem_index.to_lance_hnsw(None)? else {
-            return Err(Error::invalid_input(
-                "HnswMemIndex is empty; nothing to flush",
-            ));
+            // Every vector in the generation is null → empty graph; skip the
+            // index rather than failing the flush.
+            return Ok(None);
         };
 
         // Train SQ8 on the full memtable in one pass: learn global min/max
@@ -929,7 +1124,7 @@ impl MemTableFlusher {
             files: None,
         };
 
-        Ok(index_meta)
+        Ok(Some(index_meta))
     }
 
     /// Update the shard manifest with the new flushed generation.
@@ -965,21 +1160,30 @@ impl MemTableFlusher {
     }
 }
 
-/// Message to trigger flush of a frozen memtable to Lance storage.
-pub struct TriggerMemTableFlush {
-    /// The frozen memtable to flush.
-    pub memtable: Arc<MemTable>,
-    /// Optional channel to notify when flush completes.
-    pub done: Option<tokio::sync::oneshot::Sender<Result<FlushResult>>>,
+/// Message driving the background memtable-flush task.
+pub enum TriggerMemTableFlush {
+    /// Flush a frozen memtable to Lance storage.
+    Flush {
+        /// The frozen memtable to flush.
+        memtable: Arc<MemTable>,
+        /// Optional channel to notify when flush completes.
+        done: Option<tokio::sync::oneshot::Sender<Result<FlushResult>>>,
+    },
+    /// Periodic tick: evict frozen memtables whose post-flush grace has elapsed.
+    SweepExpired,
 }
 
 impl std::fmt::Debug for TriggerMemTableFlush {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("TriggerMemTableFlush")
-            .field("memtable_gen", &self.memtable.generation())
-            .field("memtable_rows", &self.memtable.row_count())
-            .field("has_done", &self.done.is_some())
-            .finish()
+        match self {
+            Self::Flush { memtable, done } => f
+                .debug_struct("TriggerMemTableFlush::Flush")
+                .field("memtable_gen", &memtable.generation())
+                .field("memtable_rows", &memtable.row_count())
+                .field("has_done", &done.is_some())
+                .finish(),
+            Self::SweepExpired => f.write_str("TriggerMemTableFlush::SweepExpired"),
+        }
     }
 }
 
@@ -1139,6 +1343,79 @@ mod tests {
         assert_eq!(updated_manifest.flushed_generations.len(), 1);
     }
 
+    /// A `GenerationWarmer` that counts calls and optionally fails.
+    #[derive(Debug)]
+    struct CountingWarmer {
+        calls: Arc<std::sync::atomic::AtomicUsize>,
+        fail: bool,
+    }
+
+    #[async_trait::async_trait]
+    impl GenerationWarmer for CountingWarmer {
+        async fn warm(&self, _path: &str) -> Result<()> {
+            self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+            if self.fail {
+                Err(Error::io("simulated warm failure".to_string()))
+            } else {
+                Ok(())
+            }
+        }
+    }
+
+    /// Warming is a best-effort optimization, never a commit gate: a warmer that
+    /// errors pre-commit must still let the flush commit the generation. The
+    /// warm fires exactly once on the pre-commit path.
+    #[tokio::test]
+    async fn test_flusher_commits_when_warm_fails() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
+            store.clone(),
+            &base_path,
+            shard_id,
+            2,
+        ));
+        let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
+
+        let schema = create_test_schema();
+        let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap();
+        let frag_id = memtable
+            .insert(create_test_batch(&schema, 10))
+            .await
+            .unwrap();
+        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+
+        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let warmer: Arc<dyn GenerationWarmer> = Arc::new(CountingWarmer {
+            calls: calls.clone(),
+            fail: true,
+        });
+
+        let flusher = MemTableFlusher::new(
+            store.clone(),
+            base_path,
+            base_uri,
+            shard_id,
+            manifest_store.clone(),
+        )
+        .with_warmer(Some(warmer));
+        // Flush must succeed despite the warmer erroring.
+        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+
+        assert_eq!(result.generation.generation, 1);
+        assert_eq!(
+            calls.load(std::sync::atomic::Ordering::SeqCst),
+            1,
+            "pre-commit warm fires exactly once"
+        );
+        let updated = manifest_store.read_latest().await.unwrap().unwrap();
+        assert_eq!(
+            updated.flushed_generations.len(),
+            1,
+            "generation still committed after a failed warm"
+        );
+    }
+
     /// Flushing a generation with within-generation duplicate PKs writes a
     /// deletion vector so the flushed dataset exposes newest-per-PK on scan.
     #[tokio::test]
@@ -1227,6 +1504,202 @@ mod tests {
         assert_eq!(rows.get(&3), Some(&"c2".to_string()));
     }
 
+    /// Flushing a memtable with a primary-key index writes a standalone sidecar
+    /// BTree at `{gen}/_pk_index` that the block-list can reopen by path and
+    /// probe by value — including for a within-gen-superseded PK (existence,
+    /// not visibility).
+    #[tokio::test]
+    async fn flushed_pk_index_sidecar_is_probeable() {
+        use lance_core::cache::LanceCache;
+        use lance_index::metrics::NoOpMetricsCollector;
+        use lance_index::registry::IndexPluginRegistry;
+        use lance_index::scalar::lance_format::LanceIndexStore;
+        use lance_index::scalar::{SargableQuery, SearchResult};
+
+        use super::super::super::index::IndexStore;
+        use crate::dataset::mem_wal::util::pk_index_path;
+        use datafusion::common::ScalarValue;
+
+        let (store, base_path, _base_uri, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
+            store.clone(),
+            &base_path,
+            shard_id,
+            2,
+        ));
+        let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
+
+        // Primary-key index on `id`, no user indexes.
+        let schema = create_pk_schema();
+        let mut memtable = MemTable::new(schema.clone(), 1, vec![0]).unwrap();
+        let mut registry = IndexStore::new();
+        registry.enable_pk_index(&[("id".to_string(), 0)]);
+        memtable.set_indexes(registry);
+
+        // id=1 updated in-gen (a -> a2); id=2 unique.
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 1])),
+                Arc::new(StringArray::from(vec!["a", "b", "a2"])),
+            ],
+        )
+        .unwrap();
+        let frag_id = memtable.insert(batch).await.unwrap();
+        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+
+        let flusher = MemTableFlusher::new(
+            store.clone(),
+            base_path.clone(),
+            _base_uri.clone(),
+            shard_id,
+            manifest_store.clone(),
+        );
+        let result = flusher
+            .flush_with_indexes(&memtable, epoch, &[], 1)
+            .await
+            .unwrap();
+
+        // Reopen the sidecar directly by path (the block-list's route).
+        let gen_path = base_path
+            .clone()
+            .join("_mem_wal")
+            .join(shard_id.to_string())
+            .join(result.generation.path.as_str());
+        let index_store = Arc::new(LanceIndexStore::new(
+            store.clone(),
+            pk_index_path(&gen_path),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let registry = IndexPluginRegistry::with_default_plugins();
+        let plugin = registry.get_plugin_by_name("BTree").unwrap();
+        let details =
+            prost_types::Any::from_msg(&lance_index::pbold::BTreeIndexDetails::default()).unwrap();
+        let index = plugin
+            .load_index(index_store, &details, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let contains = |id: i32| {
+            let index = index.clone();
+            async move {
+                let result = index
+                    .search(
+                        &SargableQuery::Equals(ScalarValue::Int32(Some(id))),
+                        &NoOpMetricsCollector,
+                    )
+                    .await
+                    .unwrap();
+                match result {
+                    SearchResult::Exact(s) | SearchResult::AtMost(s) | SearchResult::AtLeast(s) => {
+                        !s.is_empty()
+                    }
+                }
+            }
+        };
+        // Both PKs present (id=1 even though its first version was superseded);
+        // an absent PK is not.
+        assert!(contains(1).await);
+        assert!(contains(2).await);
+        assert!(!contains(99).await);
+    }
+
+    /// Regression: production dispatches a PK-only flush (a primary key, no
+    /// secondary index) to `flush`, not `flush_with_indexes`. `flush` must still
+    /// write the PK dedup sidecar, otherwise cross-generation dedup fails with
+    /// `page_lookup.lance not found`.
+    #[tokio::test]
+    async fn plain_flush_writes_pk_sidecar() {
+        use lance_core::cache::LanceCache;
+        use lance_index::metrics::NoOpMetricsCollector;
+        use lance_index::registry::IndexPluginRegistry;
+        use lance_index::scalar::lance_format::LanceIndexStore;
+        use lance_index::scalar::{SargableQuery, SearchResult};
+
+        use super::super::super::index::IndexStore;
+        use crate::dataset::mem_wal::util::pk_index_path;
+        use datafusion::common::ScalarValue;
+
+        let (store, base_path, _base_uri, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let manifest_store = Arc::new(ShardManifestStore::new(
+            store.clone(),
+            &base_path,
+            shard_id,
+            2,
+        ));
+        let (epoch, _manifest) = manifest_store.claim_epoch(0).await.unwrap();
+
+        // Primary-key index on `id`, no user indexes.
+        let schema = create_pk_schema();
+        let mut memtable = MemTable::new(schema.clone(), 1, vec![0]).unwrap();
+        let mut registry = IndexStore::new();
+        registry.enable_pk_index(&[("id".to_string(), 0)]);
+        memtable.set_indexes(registry);
+
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec!["a", "b"])),
+            ],
+        )
+        .unwrap();
+        let frag_id = memtable.insert(batch).await.unwrap();
+        memtable.mark_wal_flushed(&[frag_id], 1, &[0]);
+
+        let flusher = MemTableFlusher::new(
+            store.clone(),
+            base_path.clone(),
+            _base_uri.clone(),
+            shard_id,
+            manifest_store.clone(),
+        );
+        // The plain-flush path — what the writer dispatches to with no indexes.
+        let result = flusher.flush(&memtable, epoch, 1).await.unwrap();
+
+        let gen_path = base_path
+            .clone()
+            .join("_mem_wal")
+            .join(shard_id.to_string())
+            .join(result.generation.path.as_str());
+        let index_store = Arc::new(LanceIndexStore::new(
+            store.clone(),
+            pk_index_path(&gen_path),
+            Arc::new(LanceCache::no_cache()),
+        ));
+        let registry = IndexPluginRegistry::with_default_plugins();
+        let plugin = registry.get_plugin_by_name("BTree").unwrap();
+        let details =
+            prost_types::Any::from_msg(&lance_index::pbold::BTreeIndexDetails::default()).unwrap();
+        let index = plugin
+            .load_index(index_store, &details, None, &LanceCache::no_cache())
+            .await
+            .unwrap();
+
+        let contains = |id: i32| {
+            let index = index.clone();
+            async move {
+                let result = index
+                    .search(
+                        &SargableQuery::Equals(ScalarValue::Int32(Some(id))),
+                        &NoOpMetricsCollector,
+                    )
+                    .await
+                    .unwrap();
+                match result {
+                    SearchResult::Exact(s) | SearchResult::AtMost(s) | SearchResult::AtLeast(s) => {
+                        !s.is_empty()
+                    }
+                }
+            }
+        };
+        assert!(contains(1).await);
+        assert!(contains(2).await);
+        assert!(!contains(99).await);
+    }
+
     /// Covers `finalize_generation` writing both a deletion vector *and*
     /// indexes into the same manifest — the deletion-only and index-only
     /// paths are exercised by sibling tests.
@@ -1774,7 +2247,7 @@ mod tests {
             "ProjectionExec: expr=[id@2 as id, text@3 as text, _score@1 as _score]
   Take: ...
     CoalesceBatchesExec: ...
-      MatchQuery: column=text, query=hello",
+      MatchQuery: column=text, query=[hello]",
         )
         .await
         .unwrap();
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
index 2c5192e28a1..e1f0d6e689c 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs
@@ -7,21 +7,24 @@ use std::sync::Arc;
 
 use arrow_array::{Array, RecordBatch};
 use arrow_schema::{DataType, Field, SchemaRef};
-use datafusion::common::{ScalarValue, ToDFSchema};
+use datafusion::common::ScalarValue;
 use datafusion::physical_plan::limit::GlobalLimitExec;
 use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream};
 use datafusion::prelude::{Expr, SessionContext};
+use datafusion_physical_expr::PhysicalExprRef;
 use futures::TryStreamExt;
 use lance_core::{Error, ROW_ID, Result};
 use lance_datafusion::expr::safe_coerce_scalar;
 use lance_datafusion::planner::Planner;
+use lance_index::scalar::FullTextSearchQuery;
+use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator};
 use lance_linalg::distance::DistanceType;
 
 use super::exec::{
     BTreeIndexExec, FtsIndexExec, MemTableBruteForceVectorExec, MemTableDedupScanExec,
-    MemTableScanExec, VectorIndexExec,
+    MemTableScanExec, SCORE_COLUMN, VectorIndexExec,
 };
-use crate::dataset::mem_wal::scanner::exec::validate_pk_types;
+use crate::dataset::mem_wal::scanner::{exec::validate_pk_types, parse_filter_expr};
 use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
 
 /// Vector search query parameters.
@@ -58,6 +61,10 @@ pub enum FtsQueryType {
     Match {
         /// The search query string.
         query: String,
+        /// The operator used to combine tokenized query terms.
+        operator: Operator,
+        /// Boost factor applied to the score.
+        boost: f32,
     },
     /// Phrase query with slop.
     Phrase {
@@ -82,8 +89,12 @@ pub enum FtsQueryType {
         /// Maximum edit distance (Levenshtein distance).
         /// None means auto-fuzziness based on token length.
         fuzziness: Option<u32>,
+        /// Number of initial characters that must match exactly.
+        prefix_length: u32,
         /// Maximum number of terms to expand to.
         max_expansions: usize,
+        /// Boost factor applied to the score.
+        boost: f32,
     },
 }
 
@@ -97,6 +108,8 @@ pub struct FtsQuery {
     /// WAND factor for early termination (0.0 to 1.0).
     /// 1.0 = full recall (default), <1.0 = faster but may miss low-scoring results.
     pub wand_factor: f32,
+    /// Query-level result limit.
+    pub limit: Option<usize>,
     /// Whether to also search the mutable tail (rows written since the last
     /// freeze). `true` (default) = read-your-writes; `false` = search only the
     /// immutable frozen partitions (the Lucene model), trading read-recency for
@@ -113,12 +126,23 @@ pub const DEFAULT_WAND_FACTOR: f32 = 1.0;
 impl FtsQuery {
     /// Create a simple term match query.
     pub fn match_query(column: impl Into<String>, query: impl Into<String>) -> Self {
+        Self::match_query_with_operator(column, query, Operator::Or)
+    }
+
+    pub fn match_query_with_operator(
+        column: impl Into<String>,
+        query: impl Into<String>,
+        operator: Operator,
+    ) -> Self {
         Self {
             column: column.into(),
             query_type: FtsQueryType::Match {
                 query: query.into(),
+                operator,
+                boost: 1.0,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -132,6 +156,7 @@ impl FtsQuery {
                 slop,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -151,6 +176,7 @@ impl FtsQuery {
                 must_not,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -167,9 +193,12 @@ impl FtsQuery {
             query_type: FtsQueryType::Fuzzy {
                 query: query.into(),
                 fuzziness: None,
+                prefix_length: 0,
                 max_expansions: DEFAULT_MAX_EXPANSIONS,
+                boost: 1.0,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -185,9 +214,12 @@ impl FtsQuery {
             query_type: FtsQueryType::Fuzzy {
                 query: query.into(),
                 fuzziness: Some(fuzziness),
+                prefix_length: 0,
                 max_expansions: DEFAULT_MAX_EXPANSIONS,
+                boost: 1.0,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -197,6 +229,7 @@ impl FtsQuery {
         column: impl Into<String>,
         query: impl Into<String>,
         fuzziness: Option<u32>,
+        prefix_length: u32,
         max_expansions: usize,
     ) -> Self {
         Self {
@@ -204,9 +237,12 @@ impl FtsQuery {
             query_type: FtsQueryType::Fuzzy {
                 query: query.into(),
                 fuzziness,
+                prefix_length,
                 max_expansions,
+                boost: 1.0,
             },
             wand_factor: DEFAULT_WAND_FACTOR,
+            limit: None,
             include_tail: true,
         }
     }
@@ -221,12 +257,89 @@ impl FtsQuery {
         self
     }
 
+    pub fn with_limit(mut self, limit: Option<usize>) -> Self {
+        self.limit = limit;
+        self
+    }
+
     /// Set whether to search the mutable tail (read-your-writes) or only the
     /// immutable frozen partitions (the Lucene model). Default `true`.
     pub fn with_include_tail(mut self, include_tail: bool) -> Self {
         self.include_tail = include_tail;
         self
     }
+
+    fn with_boost(mut self, boost: f32) -> Self {
+        match &mut self.query_type {
+            FtsQueryType::Match { boost: b, .. } | FtsQueryType::Fuzzy { boost: b, .. } => {
+                *b = boost;
+            }
+            FtsQueryType::Phrase { .. } | FtsQueryType::Boolean { .. } => {}
+        }
+        self
+    }
+}
+
+/// Convert an index-level [`FullTextSearchQuery`] into the MemTable's local
+/// [`FtsQuery`], so the MemTable scanner shares the dataset `Scanner`'s FTS
+/// entry type. Supports match (exact `fuzziness == Some(0)` and fuzzy) and
+/// phrase leaf queries; the column must be bound on the query. Compound queries
+/// (boolean / boost / multi-match) cannot be modeled by the MemTable path and
+/// return a `not_supported` error rather than failing deep in planning.
+fn local_fts_query(query: FullTextSearchQuery) -> Result<FtsQuery> {
+    let wand_factor = query.wand_factor.unwrap_or(DEFAULT_WAND_FACTOR);
+    let limit = query
+        .limit
+        .map(|limit| {
+            if limit < 0 {
+                Err(Error::invalid_input(
+                    "full-text search limit must be non-negative".to_string(),
+                ))
+            } else {
+                Ok(limit as usize)
+            }
+        })
+        .transpose()?;
+    let require_column = |column: Option<String>| {
+        column.ok_or_else(|| {
+            Error::invalid_input(
+                "full-text search requires a column; set it with \
+                 `FullTextSearchQuery::with_column`"
+                    .to_string(),
+            )
+        })
+    };
+    let local = match query.query {
+        IndexFtsQuery::Match(m) => {
+            let column = require_column(m.column)?;
+            match m.fuzziness {
+                // Some(0) is an exact match in the index model.
+                Some(0) => FtsQuery::match_query_with_operator(column, m.terms, m.operator)
+                    .with_boost(m.boost),
+                _ if m.operator != Operator::Or => {
+                    return Err(Error::not_supported(
+                        "MemTable fuzzy full-text search only supports OR match operators"
+                            .to_string(),
+                    ));
+                }
+                fuzziness => FtsQuery::fuzzy_with_options(
+                    column,
+                    m.terms,
+                    fuzziness,
+                    m.prefix_length,
+                    m.max_expansions,
+                )
+                .with_boost(m.boost),
+            }
+        }
+        IndexFtsQuery::Phrase(p) => FtsQuery::phrase(require_column(p.column)?, p.terms, p.slop),
+        other => {
+            return Err(Error::not_supported(format!(
+                "MemTable full-text search supports match and phrase queries, got: {other}"
+            )));
+        }
+    };
+    Ok(local.with_wand_factor(wand_factor).with_limit(limit))
 }
 
 /// Scalar predicate for BTree index queries.
@@ -271,11 +384,15 @@ impl ScalarPredicate {
 ///
 /// # Example
 ///
+/// The builder methods take `&mut self` (mirroring
+/// [`crate::dataset::scanner::Scanner`]), so configure the scanner with
+/// statements rather than a fluent chain:
+///
 /// ```ignore
-/// let scanner = MemTableScanner::new(batch_store, indexes, schema)
-///     .project(&["id", "name"])?
-///     .filter("id > 10")?
-///     .limit(100, None)?;
+/// let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
+/// scanner.project(&["id", "name"])?;
+/// scanner.filter("id > 10")?;
+/// scanner.limit(Some(100), None)?;
 ///
 /// let stream = scanner.try_into_stream().await?;
 /// ```
@@ -300,6 +417,11 @@ pub struct MemTableScanner {
     /// Whether to include _rowaddr column in output.
     /// Same value as _rowid but named for compatibility with LSM scanner.
     with_row_address: bool,
+    /// Primary-key columns, supplied by the LSM planner. When set, a filtered
+    /// vector/FTS search evaluates the predicate against the newest version of
+    /// each PK only, so an in-memtable update whose current version fails the
+    /// predicate is excluded rather than leaking a stale older match.
+    pk_columns: Option<Vec<String>>,
 }
 
 impl MemTableScanner {
@@ -334,18 +456,34 @@ impl MemTableScanner {
             batch_size: None,
             with_row_id: false,
             with_row_address: false,
+            pk_columns: None,
         }
     }
 
-    /// Project only the specified columns.
+    /// Provide the primary-key columns. When set, a filtered vector/FTS search
+    /// evaluates the predicate against the newest version of each PK only,
+    /// preventing a stale older match from leaking past an in-memtable update
+    /// whose current version fails the predicate.
+    pub fn with_pk_columns(&mut self, pk_columns: Vec<String>) -> &mut Self {
+        self.pk_columns = if pk_columns.is_empty() {
+            None
+        } else {
+            Some(pk_columns)
+        };
+        self
+    }
+
+    /// Project only the specified columns. Mirrors
+    /// [`crate::dataset::scanner::Scanner::project`].
     ///
     /// Special columns:
     /// - `_rowid`: Returns the row position (global row offset in MemTable)
-    pub fn project(&mut self, columns: &[&str]) -> &mut Self {
+    pub fn project<T: AsRef<str>>(&mut self, columns: &[T]) -> Result<&mut Self> {
         // Check if _rowid is requested in projection
         let mut filtered_columns = Vec::new();
         for col in columns {
-            if *col == ROW_ID {
+            let col = col.as_ref();
+            if col == ROW_ID {
                 self.with_row_id = true;
             } else {
                 filtered_columns.push(col.to_string());
@@ -355,7 +493,7 @@ impl MemTableScanner {
         if !filtered_columns.is_empty() || self.with_row_id {
             self.projection = Some(filtered_columns);
         }
-        self
+        Ok(self)
     }
 
     /// Include the _rowid column in output.
@@ -366,6 +504,14 @@ impl MemTableScanner {
         self
     }
 
+    /// The `max_visible_batch_position` snapshot this scanner latched at
+    /// construction. A downstream recency filter must key on this same snapshot
+    /// (not a fresh read of the IndexStore watermark, which a concurrent append
+    /// could have advanced) so it stays consistent with the rows the search saw.
+    pub fn max_visible_batch_position(&self) -> usize {
+        self.max_visible_batch_position
+    }
+
     /// Include the _rowaddr column in output.
     ///
     /// Same value as _rowid but named for compatibility with LSM scanner.
@@ -377,15 +523,7 @@ impl MemTableScanner {
 
     /// Set a filter expression using SQL-like syntax.
     pub fn filter(&mut self, filter_expr: &str) -> Result<&mut Self> {
-        let ctx = SessionContext::new();
-        let df_schema = self
-            .schema
-            .clone()
-            .to_dfschema()
-            .map_err(|e| Error::invalid_input(format!("Failed to create DFSchema: {}", e)))?;
-        let expr = ctx.parse_sql_expr(filter_expr, &df_schema).map_err(|e| {
-            Error::invalid_input(format!("Failed to parse filter expression: {}", e))
-        })?;
+        let expr = parse_filter_expr(self.schema.as_ref(), filter_expr)?;
         self.filter = Some(expr);
         Ok(self)
     }
@@ -396,24 +534,50 @@ impl MemTableScanner {
         self
     }
 
-    /// Limit the number of results.
-    pub fn limit(&mut self, limit: usize, offset: Option<usize>) -> &mut Self {
-        self.limit = Some(limit);
-        self.offset = offset;
-        self
+    /// Limit the number of results, with an optional offset. Mirrors
+    /// [`crate::dataset::scanner::Scanner::limit`]: both bounds are `Option<i64>`
+    /// and must be non-negative.
+    pub fn limit(&mut self, limit: Option<i64>, offset: Option<i64>) -> Result<&mut Self> {
+        if let Some(value) = limit
+            && value < 0
+        {
+            return Err(Error::invalid_input(
+                "limit must be non-negative".to_string(),
+            ));
+        }
+        if let Some(value) = offset
+            && value < 0
+        {
+            return Err(Error::invalid_input(
+                "offset must be non-negative".to_string(),
+            ));
+        }
+        self.limit = limit.map(|value| value as usize);
+        self.offset = offset.map(|value| value as usize);
+        Ok(self)
     }
 
-    /// Set up a vector similarity search.
+    /// Set up a vector similarity search. Mirrors
+    /// [`crate::dataset::scanner::Scanner::nearest`] — the query vector is passed
+    /// by reference.
     ///
     /// # Arguments
     ///
     /// * `column` - The name of the vector column to search.
     /// * `query` - The query vector.
     /// * `k` - Number of nearest neighbors to return.
-    pub fn nearest(&mut self, column: &str, query: Arc<dyn Array>, k: usize) -> &mut Self {
+    pub fn nearest(&mut self, column: &str, query: &dyn Array, k: usize) -> Result<&mut Self> {
+        if k == 0 {
+            return Err(Error::invalid_input("k must be positive".to_string()));
+        }
+        if query.is_empty() {
+            return Err(Error::invalid_input(
+                "query vector must have non-zero length".to_string(),
+            ));
+        }
         self.nearest = Some(VectorQuery {
             column: column.to_string(),
-            query_vector: query,
+            query_vector: query.slice(0, query.len()),
             k,
             nprobes: 1,
             maximum_nprobes: None,
@@ -423,7 +587,7 @@ impl MemTableScanner {
             distance_lower_bound: None,
             distance_upper_bound: None,
         });
-        self
+        Ok(self)
     }
 
     /// Set the number of probes for IVF search.
@@ -517,10 +681,15 @@ impl MemTableScanner {
         self
     }
 
-    /// Set up a full-text search with simple term matching.
-    pub fn full_text_search(&mut self, column: &str, query: &str) -> &mut Self {
-        self.full_text_query = Some(FtsQuery::match_query(column, query));
-        self
+    /// Set up a full-text search. Mirrors
+    /// [`crate::dataset::scanner::Scanner::full_text_search`], taking a
+    /// [`FullTextSearchQuery`] whose column is set via
+    /// `FullTextSearchQuery::with_column`. Match (exact/fuzzy) and phrase leaf
+    /// queries are supported; compound queries (boolean/boost/multi-match) are
+    /// not yet supported by the MemTable path and return an error.
+    pub fn full_text_search(&mut self, query: FullTextSearchQuery) -> Result<&mut Self> {
+        self.full_text_query = Some(local_fts_query(query)?);
+        Ok(self)
     }
 
     /// Set up a full-text phrase search.
@@ -607,6 +776,7 @@ impl MemTableScanner {
             column,
             query,
             fuzziness,
+            0,
             max_expansions,
         ));
         self
@@ -756,6 +926,12 @@ impl MemTableScanner {
 
     /// Create the execution plan based on the query configuration.
     pub async fn create_plan(&self) -> Result<Arc<dyn ExecutionPlan>> {
+        if self.nearest.is_some() && self.full_text_query.is_some() {
+            return Err(Error::invalid_input(
+                "MemTableScanner cannot combine vector and full-text search".to_string(),
+            ));
+        }
+
         // Determine which type of plan to create
         if let Some(ref vector_query) = self.nearest {
             return self.plan_vector_search(vector_query).await;
@@ -807,12 +983,12 @@ impl MemTableScanner {
 
         let mut plan: Arc<dyn ExecutionPlan> = Arc::new(scan);
 
-        // Apply limit if present
-        if let Some(limit) = self.limit {
+        // Apply limit / offset if present.
+        if self.limit.is_some() || self.offset.unwrap_or(0) > 0 {
             plan = Arc::new(GlobalLimitExec::new(
                 plan,
                 self.offset.unwrap_or(0),
-                Some(limit),
+                self.limit,
             ));
         }
 
@@ -905,12 +1081,44 @@ impl MemTableScanner {
     /// hasn't reached this writer yet (cold-start, or rows written between an
     /// index commit and the next memtable rotation), KNN must still produce
     /// correct, distance-bearing results so the LSM-level merge stays sound.
+    /// Compile the optional logical `filter` into a physical predicate against
+    /// the memtable schema. Shared by the vector and FTS search arms; mirrors the
+    /// compilation in [`Self::plan_full_scan`] (`optimize_expr` before
+    /// `create_physical_expr` for literal type coercion).
+    fn filter_predicate(&self) -> Result<Option<PhysicalExprRef>> {
+        let Some(ref filter) = self.filter else {
+            return Ok(None);
+        };
+        let planner = Planner::new(self.schema.clone());
+        let optimized = planner.optimize_expr(filter.clone())?;
+        Ok(Some(planner.create_physical_expr(&optimized)?))
+    }
+
     async fn plan_vector_search(&self, query: &VectorQuery) -> Result<Arc<dyn ExecutionPlan>> {
         let max_visible = self.max_visible_batch_position;
         let projection_indices = self.compute_projection_indices()?;
         let base_schema = self.base_output_schema();
+        let filter_predicate = self.filter_predicate()?;
+        if let Some(pk_columns) = &self.pk_columns {
+            validate_pk_types(&self.schema, pk_columns)?;
+        }
 
-        let exec: Arc<dyn ExecutionPlan> = if self.has_vector_index(&query.column) {
+        // With a prefilter we use brute force rather than HNSW because graph
+        // traversal cannot honor an arbitrary predicate. With PK rewrites, we
+        // also need exact newest-before-top-k semantics: a stale near vector
+        // must not consume an HNSW top-k slot and hide the next live row. Pure
+        // append-only PK data can still use HNSW safely. This relies on
+        // `IndexStore` marking PK overrides before advancing the visible batch
+        // watermark, so any snapshot that sees a rewrite also sees the flag.
+        let hnsw_safe_with_pk = self
+            .pk_columns
+            .as_ref()
+            .map(|_| self.indexes.has_pk_index() && !self.indexes.pk_has_overrides())
+            .unwrap_or(true);
+        let exec: Arc<dyn ExecutionPlan> = if filter_predicate.is_none()
+            && hnsw_safe_with_pk
+            && self.has_vector_index(&query.column)
+        {
             Arc::new(VectorIndexExec::new(
                 self.batch_store.clone(),
                 self.indexes.clone(),
@@ -921,14 +1129,18 @@ impl MemTableScanner {
                 self.with_row_id,
             )?)
         } else {
-            Arc::new(MemTableBruteForceVectorExec::new(
-                self.batch_store.clone(),
-                query.clone(),
-                max_visible,
-                projection_indices,
-                base_schema,
-                self.with_row_id,
-            )?)
+            Arc::new(
+                MemTableBruteForceVectorExec::new(
+                    self.batch_store.clone(),
+                    query.clone(),
+                    max_visible,
+                    projection_indices,
+                    base_schema,
+                    self.with_row_id,
+                )?
+                .with_filter(filter_predicate)
+                .with_pk_columns(self.pk_columns.clone()),
+            )
         };
         self.apply_post_index_ops(exec).await
     }
@@ -936,14 +1148,18 @@ impl MemTableScanner {
     /// Plan a full-text search.
     ///
     /// Uses the effective visibility (min of max_visible and max_indexed) to ensure
-    /// queries only see indexed data. Falls back to full scan if no index exists.
+    /// queries only see indexed data.
     async fn plan_fts_search(&self, query: &FtsQuery) -> Result<Arc<dyn ExecutionPlan>> {
         if !self.has_fts_index(&query.column) {
-            return self.plan_full_scan().await;
+            return self.empty_fts_plan();
         }
 
         let max_visible = self.max_visible_batch_position;
         let projection_indices = self.compute_projection_indices()?;
+        let filter_predicate = self.filter_predicate()?;
+        if let Some(pk_columns) = &self.pk_columns {
+            validate_pk_types(&self.schema, pk_columns)?;
+        }
 
         let index_exec = FtsIndexExec::new(
             self.batch_store.clone(),
@@ -953,10 +1169,29 @@ impl MemTableScanner {
             projection_indices,
             self.base_output_schema(),
             self.with_row_id,
-        )?;
+        )?
+        .with_filter(filter_predicate)
+        .with_pk_columns(self.pk_columns.clone());
         self.apply_post_index_ops(Arc::new(index_exec)).await
     }
 
+    fn empty_fts_plan(&self) -> Result<Arc<dyn ExecutionPlan>> {
+        use datafusion::physical_plan::empty::EmptyExec;
+
+        let mut fields: Vec<Field> = self
+            .base_output_schema()
+            .fields()
+            .iter()
+            .map(|f| f.as_ref().clone())
+            .collect();
+        fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true));
+        if self.with_row_id {
+            fields.push(Field::new(ROW_ID, DataType::UInt64, true));
+        }
+        let schema = Arc::new(arrow_schema::Schema::new(fields));
+        Ok(Arc::new(EmptyExec::new(schema)))
+    }
+
     /// Apply limit and other post-processing operations.
     async fn apply_post_index_ops(
         &self,
@@ -964,11 +1199,11 @@ impl MemTableScanner {
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let mut result = plan;
 
-        if let Some(limit) = self.limit {
+        if self.limit.is_some() || self.offset.unwrap_or(0) > 0 {
             result = Arc::new(GlobalLimitExec::new(
                 result,
                 self.offset.unwrap_or(0),
-                Some(limit),
+                self.limit,
             ));
         }
 
@@ -1018,16 +1253,14 @@ impl MemTableScanner {
                                 value: coerced_lit,
                             });
                         }
-                        datafusion::logical_expr::Operator::Lt
-                        | datafusion::logical_expr::Operator::LtEq => {
+                        datafusion::logical_expr::Operator::Lt => {
                             return Some(ScalarPredicate::Range {
                                 column: col.name.clone(),
                                 lower: None,
                                 upper: Some(coerced_lit),
                             });
                         }
-                        datafusion::logical_expr::Operator::Gt
-                        | datafusion::logical_expr::Operator::GtEq => {
+                        datafusion::logical_expr::Operator::GtEq => {
                             return Some(ScalarPredicate::Range {
                                 column: col.name.clone(),
                                 lower: Some(coerced_lit),
@@ -1038,7 +1271,7 @@ impl MemTableScanner {
                     }
                 }
             }
-            Expr::InList(in_list) => {
+            Expr::InList(in_list) if !in_list.negated => {
                 if let Expr::Column(col) = in_list.expr.as_ref() {
                     let values: Vec<ScalarValue> = in_list
                         .list
@@ -1100,7 +1333,7 @@ impl MemTableScanner {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow_array::{Int32Array, StringArray};
+    use arrow_array::{BooleanArray, Int32Array, StringArray};
     use arrow_schema::{DataType, Field, Schema};
 
     fn create_test_schema() -> SchemaRef {
@@ -1205,7 +1438,7 @@ mod tests {
         let indexes = create_index_store_with_batches(&batch_store, &schema, &[(0, 10)]);
 
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
-        scanner.project(&["id"]);
+        scanner.project(&["id"]).unwrap();
 
         let result = scanner.try_into_batch().await.unwrap();
         assert_eq!(result.num_columns(), 1);
@@ -1220,12 +1453,559 @@ mod tests {
         let indexes = create_index_store_with_batches(&batch_store, &schema, &[(0, 100)]);
 
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
-        scanner.limit(10, None);
+        scanner.limit(Some(10), None).unwrap();
 
         let result = scanner.try_into_batch().await.unwrap();
         assert_eq!(result.num_rows(), 10);
     }
 
+    #[tokio::test]
+    async fn test_scanner_offset_without_limit() {
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(100));
+
+        let indexes = create_index_store_with_batches(&batch_store, &schema, &[(0, 10)]);
+
+        let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
+        scanner.limit(Some(3), None).unwrap();
+        scanner.limit(None, Some(2)).unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(ids, vec![2, 3, 4, 5, 6, 7, 8, 9]);
+    }
+
+    #[tokio::test]
+    async fn btree_filter_fallback_preserves_non_representable_predicates() {
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(100));
+        let indexes = create_index_store_with_batches(&batch_store, &schema, &[(0, 10)]);
+
+        async fn ids_for(
+            batch_store: Arc<BatchStore>,
+            indexes: Arc<IndexStore>,
+            schema: SchemaRef,
+            filter: &str,
+        ) -> Vec<i32> {
+            let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
+            scanner.filter(filter).unwrap();
+            scanner
+                .try_into_batch()
+                .await
+                .unwrap()
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap()
+                .values()
+                .to_vec()
+        }
+
+        assert_eq!(
+            ids_for(
+                batch_store.clone(),
+                indexes.clone(),
+                schema.clone(),
+                "id NOT IN (1, 2)"
+            )
+            .await,
+            vec![0, 3, 4, 5, 6, 7, 8, 9]
+        );
+        assert_eq!(
+            ids_for(
+                batch_store.clone(),
+                indexes.clone(),
+                schema.clone(),
+                "id <= 5"
+            )
+            .await,
+            vec![0, 1, 2, 3, 4, 5]
+        );
+        assert_eq!(
+            ids_for(batch_store, indexes, schema, "id > 5").await,
+            vec![6, 7, 8, 9]
+        );
+    }
+
+    /// `full_text_search` now takes a structured `FullTextSearchQuery` (matching
+    /// the dataset `Scanner`); `local_fts_query` maps the supported leaf shapes
+    /// and rejects compound queries and missing columns.
+    #[test]
+    fn local_fts_query_maps_leaf_shapes_and_rejects_the_rest() {
+        use lance_index::scalar::inverted::query::{
+            BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery,
+        };
+
+        // Exact match (default fuzziness Some(0)) -> local Match, preserving the
+        // old `full_text_search(col, terms)` behavior.
+        let q = FullTextSearchQuery::new("hello".to_string())
+            .with_column("text".to_string())
+            .unwrap();
+        let local = local_fts_query(q).unwrap();
+        assert_eq!(local.column, "text");
+        assert!(
+            matches!(local.query_type, FtsQueryType::Match { query, operator, .. }
+                if query == "hello" && operator == Operator::Or)
+        );
+
+        let exact_and = FullTextSearchQuery::new_query(IndexFtsQuery::Match(
+            MatchQuery::new("hello world".to_string())
+                .with_operator(Operator::And)
+                .with_boost(3.0)
+                .with_column(Some("text".to_string())),
+        ));
+        let local = local_fts_query(exact_and).unwrap();
+        assert!(
+            matches!(local.query_type, FtsQueryType::Match { query, operator, boost }
+                if query == "hello world" && operator == Operator::And && boost == 3.0)
+        );
+
+        // Fuzzy match -> local Fuzzy carrying edit distance, prefix length, and boost.
+        let fuzzy = FullTextSearchQuery::new_query(IndexFtsQuery::Match(
+            MatchQuery::new("lance".to_string())
+                .with_fuzziness(Some(2))
+                .with_prefix_length(2)
+                .with_boost(2.5)
+                .with_column(Some("text".to_string())),
+        ));
+        let local = local_fts_query(fuzzy).unwrap();
+        assert!(
+            matches!(local.query_type, FtsQueryType::Fuzzy { fuzziness, prefix_length, boost, .. }
+                if fuzziness == Some(2) && prefix_length == 2 && boost == 2.5)
+        );
+
+        let fuzzy_and = FullTextSearchQuery::new_query(IndexFtsQuery::Match(
+            MatchQuery::new("lance memwal".to_string())
+                .with_operator(Operator::And)
+                .with_fuzziness(Some(1))
+                .with_column(Some("text".to_string())),
+        ));
+        assert!(
+            local_fts_query(fuzzy_and).is_err(),
+            "fuzzy AND cannot be represented by the local memtable query"
+        );
+
+        // Phrase -> local Phrase.
+        let phrase = FullTextSearchQuery::new_query(IndexFtsQuery::Phrase(
+            PhraseQuery::new("quick fox".to_string()).with_column(Some("text".to_string())),
+        ));
+        let local = local_fts_query(phrase).unwrap();
+        assert!(matches!(local.query_type, FtsQueryType::Phrase { .. }));
+
+        // Compound (boolean) -> not supported.
+        let boolean =
+            FullTextSearchQuery::new_query(IndexFtsQuery::Boolean(BooleanQuery::new(vec![(
+                Occur::Must,
+                IndexFtsQuery::Match(
+                    MatchQuery::new("x".to_string()).with_column(Some("text".to_string())),
+                ),
+            )])));
+        assert!(
+            local_fts_query(boolean).is_err(),
+            "boolean must be rejected"
+        );
+
+        // Missing column -> error.
+        let no_col = FullTextSearchQuery::new("hi".to_string());
+        assert!(
+            local_fts_query(no_col).is_err(),
+            "missing column must error"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_honors_query_limit() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "lance",
+                    "lance filler",
+                    "lance filler filler",
+                ])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("lance".to_string())
+                    .with_column("text".to_string())
+                    .unwrap()
+                    .limit(Some(1)),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            result.num_rows(),
+            1,
+            "query-level FTS limit must cap direct MemTableScanner results"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_without_index_returns_empty_score_schema() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec!["needle", "needle"])),
+            ],
+        )
+        .unwrap();
+        batch_store.append(batch).unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(IndexStore::new()), schema);
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("needle".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        assert_eq!(result.num_rows(), 0);
+        assert!(
+            result.schema().field_with_name("_score").is_ok(),
+            "missing FTS indexes should produce an empty FTS-shaped result"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_prefilter_null_predicate_excludes_rows() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("active", DataType::Boolean, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["needle", "needle", "needle"])),
+                Arc::new(BooleanArray::from(vec![None, Some(true), Some(false)])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.filter("active = true").unwrap();
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("needle".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(
+            ids,
+            vec![2],
+            "NULL predicate results must be excluded from FTS prefilter candidates"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_prefilter_disables_wand_pruning() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("active", DataType::Boolean, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec!["alpha beta gamma delta", "alpha"])),
+                Arc::new(BooleanArray::from(vec![Some(false), Some(true)])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.filter("active = true").unwrap();
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("alpha beta gamma delta".to_string())
+                    .with_column("text".to_string())
+                    .unwrap()
+                    .wand_factor(Some(0.99)),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(
+            ids,
+            vec![2],
+            "filtered FTS must not let WAND prune rows before the prefilter is applied"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_append_only_pk_keeps_wand_pruning() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec!["alpha beta gamma delta", "alpha"])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.with_pk_columns(vec!["id".to_string()]);
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("alpha beta gamma delta".to_string())
+                    .with_column("text".to_string())
+                    .unwrap()
+                    .wand_factor(Some(0.99)),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(
+            ids,
+            vec![1],
+            "append-only PK data should keep index WAND pruning enabled"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_with_pk_rewrite_disables_index_limit_pushdown() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "alpha beta gamma delta epsilon",
+                    "other",
+                    "alpha beta gamma delta",
+                    "alpha",
+                ])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.with_pk_columns(vec!["id".to_string()]);
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("alpha beta gamma delta epsilon".to_string())
+                    .with_column("text".to_string())
+                    .unwrap()
+                    .limit(Some(2)),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(
+            ids,
+            vec![2, 3],
+            "FTS-only PK rewrites must disable index limit pushdown so live lower-scoring PKs can backfill"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_with_pk_columns_drops_stale_filtered_hits() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("active", DataType::Boolean, false),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1])),
+                Arc::new(StringArray::from(vec!["needle", "needle"])),
+                Arc::new(BooleanArray::from(vec![true, false])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.with_pk_columns(vec!["id".to_string()]);
+        scanner.filter("active = true").unwrap();
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("needle".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let result = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            result.num_rows(),
+            0,
+            "the older matching version must not leak when the newest PK fails the filter"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_with_pk_columns_falls_back_without_pk_index() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
+                Arc::new(StringArray::from(vec![
+                    "needle stale",
+                    "other",
+                    "other",
+                    "needle fresh",
+                ])),
+            ],
+        )
+        .unwrap();
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let mut scanner = MemTableScanner::new(batch_store, Arc::new(indexes), schema);
+        scanner.with_pk_columns(vec!["id".to_string()]);
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("needle".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let result = scanner
+            .try_into_batch()
+            .await
+            .expect("FTS PK recency should fall back without a PK index");
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        assert_eq!(
+            ids,
+            vec![2],
+            "without a PK index the batch-scan fallback must drop stale id=1 \
+             but keep id=2 whose newest version still matches"
+        );
+    }
+
     #[tokio::test]
     async fn test_scanner_count_rows() {
         let schema = create_test_schema();
@@ -1282,7 +2062,7 @@ mod tests {
 
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
         // Project only "id" and "_rowid"
-        scanner.project(&["id", "_rowid"]);
+        scanner.project(&["id", "_rowid"]).unwrap();
 
         // Verify output schema
         let output_schema = scanner.output_schema();
@@ -1352,7 +2132,7 @@ mod tests {
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
 
         // Project with _rowid should set with_row_id flag
-        scanner.project(&["id", "_rowid"]);
+        scanner.project(&["id", "_rowid"]).unwrap();
 
         // with_row_id should be true now
         assert!(scanner.with_row_id);
@@ -1400,7 +2180,7 @@ mod tests {
         let indexes = create_index_store_with_batches(&batch_store, &schema, &[(0, 10)]);
 
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
-        scanner.project(&["id", "_rowid"]);
+        scanner.project(&["id", "_rowid"]).unwrap();
 
         let plan = scanner.create_plan().await.unwrap();
 
@@ -1579,7 +2359,7 @@ mod tests {
         let mut scanner = MemTableScanner::new(batch_store, indexes, schema.clone());
         let query: Arc<dyn arrow_array::Array> =
             Arc::new(arrow_array::Float32Array::from(vec![0.0_f32, 0.0_f32]));
-        scanner.nearest("vector", query, 5);
+        scanner.nearest("vector", query.as_ref(), 5).unwrap();
 
         let plan = scanner
             .create_plan()
@@ -1592,4 +2372,106 @@ mod tests {
             out_schema
         );
     }
+
+    #[tokio::test]
+    async fn test_nearest_rejects_invalid_query_shape() {
+        let schema: SchemaRef = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(4));
+        let indexes = Arc::new(IndexStore::new());
+
+        let mut scanner =
+            MemTableScanner::new(batch_store.clone(), indexes.clone(), schema.clone());
+        let query: Arc<dyn arrow_array::Array> =
+            Arc::new(arrow_array::Float32Array::from(vec![0.0_f32, 0.0_f32]));
+        let Err(err) = scanner.nearest("vector", query.as_ref(), 0) else {
+            panic!("zero-k vector search should fail");
+        };
+        assert!(
+            err.to_string().contains("k must be positive"),
+            "unexpected zero-k error: {err}"
+        );
+
+        let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
+        let empty_query: Arc<dyn arrow_array::Array> =
+            Arc::new(arrow_array::Float32Array::from(Vec::<f32>::new()));
+        let Err(err) = scanner.nearest("vector", empty_query.as_ref(), 5) else {
+            panic!("empty vector search should fail");
+        };
+        assert!(
+            err.to_string().contains("non-zero length"),
+            "unexpected empty-query error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_create_plan_rejects_vector_and_fts_combination() {
+        let schema: SchemaRef = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("text", DataType::Utf8, true),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(4));
+        let indexes = Arc::new(IndexStore::new());
+
+        let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
+        let query: Arc<dyn arrow_array::Array> =
+            Arc::new(arrow_array::Float32Array::from(vec![0.0_f32, 0.0_f32]));
+        scanner.nearest("vector", query.as_ref(), 5).unwrap();
+        scanner
+            .full_text_search(
+                FullTextSearchQuery::new("needle".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let err = scanner
+            .create_plan()
+            .await
+            .expect_err("vector and FTS search must not be silently combined");
+        assert!(
+            err.to_string().contains("vector and full-text search"),
+            "unexpected combined-search error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_plan_vector_search_validates_pk_types() {
+        let schema: SchemaRef = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Float64, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+        ]));
+        let batch_store = Arc::new(BatchStore::with_capacity(4));
+        let indexes = Arc::new(IndexStore::new());
+
+        let mut scanner = MemTableScanner::new(batch_store, indexes, schema);
+        scanner.with_pk_columns(vec!["id".to_string()]);
+        let query: Arc<dyn arrow_array::Array> =
+            Arc::new(arrow_array::Float32Array::from(vec![0.0_f32, 0.0_f32]));
+        scanner.nearest("vector", query.as_ref(), 5).unwrap();
+
+        let err = scanner
+            .create_plan()
+            .await
+            .expect_err("unsupported vector PK type must be rejected");
+        assert!(
+            err.to_string().contains("unsupported type Float64"),
+            "unexpected error: {err}"
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
index e6d79ba6b48..2a593a0f215 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs
@@ -10,6 +10,12 @@
 //! - `MemTableBruteForceVectorExec` - KNN over the active memtable without an HNSW
 //! - `FtsIndexExec` - Full-text search
 
+use std::collections::{HashMap, HashSet};
+
+use arrow_array::RecordBatch;
+use datafusion::common::ScalarValue;
+use datafusion::error::Result as DataFusionResult;
+
 mod brute_force_vector;
 mod btree;
 mod dedup_scan;
@@ -17,9 +23,54 @@ mod fts;
 mod scan;
 mod vector;
 
+use crate::dataset::mem_wal::scanner::exec::resolve_pk_indices;
+use crate::dataset::mem_wal::write::BatchStore;
+
 pub use brute_force_vector::MemTableBruteForceVectorExec;
 pub use btree::BTreeIndexExec;
 pub use dedup_scan::MemTableDedupScanExec;
-pub use fts::FtsIndexExec;
+pub use fts::{FtsIndexExec, SCORE_COLUMN};
 pub use scan::{MemTableScanExec, ROW_ADDRESS_COLUMN};
 pub use vector::VectorIndexExec;
+
+pub(super) fn newest_pk_positions(
+    batch_store: &BatchStore,
+    pk_columns: &[String],
+    max_visible_batch_position: usize,
+    max_visible_row: u64,
+) -> DataFusionResult<HashSet<u64>> {
+    let mut newest: HashMap<Vec<ScalarValue>, u64> = HashMap::new();
+    let mut current_row: u64 = 0;
+    for (batch_position, stored_batch) in batch_store.iter().enumerate() {
+        let n = stored_batch.num_rows;
+        if n == 0 {
+            continue;
+        }
+        if batch_position > max_visible_batch_position {
+            current_row += n as u64;
+            continue;
+        }
+        let pk_indices = resolve_pk_indices(&stored_batch.data, pk_columns)?;
+        for row in 0..n {
+            let pos = current_row + row as u64;
+            if pos > max_visible_row {
+                break;
+            }
+            let key = pk_key(&stored_batch.data, &pk_indices, row)?;
+            newest.insert(key, pos);
+        }
+        current_row += n as u64;
+    }
+    Ok(newest.into_values().collect())
+}
+
+fn pk_key(
+    batch: &RecordBatch,
+    pk_indices: &[usize],
+    row: usize,
+) -> DataFusionResult<Vec<ScalarValue>> {
+    pk_indices
+        .iter()
+        .map(|&col_idx| ScalarValue::try_from_array(batch.column(col_idx).as_ref(), row))
+        .collect()
+}
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
index 43169f92d2d..e47a715ceaa 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs
@@ -11,11 +11,10 @@
 //! or new rows in the window between commit and next memtable rotation), this
 //! exec keeps KNN correct by computing exact distances row-by-row.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use arrow_array::{Array, Float32Array, RecordBatch, UInt64Array, cast::AsArray};
+use arrow_array::{Array, BooleanArray, Float32Array, RecordBatch, UInt64Array, cast::AsArray};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use datafusion::common::stats::Precision;
 use datafusion::error::Result as DataFusionResult;
@@ -27,12 +26,13 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
     SendableRecordBatchStream, Statistics,
 };
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExprRef};
 use futures::stream::{self, StreamExt};
 use lance_core::{Error, Result};
 use lance_linalg::distance::DistanceType;
 
 use super::super::builder::VectorQuery;
+use super::newest_pk_positions;
 use super::vector::DISTANCE_COLUMN;
 use crate::dataset::mem_wal::write::BatchStore;
 
@@ -53,6 +53,14 @@ pub struct MemTableBruteForceVectorExec {
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
     with_row_id: bool,
+    /// Optional prefilter predicate, compiled against the memtable schema.
+    /// Applied per row before the top-k cut so the KNN only ranks matching
+    /// rows (true prefilter, not a lossy post-filter on the top-k).
+    filter: Option<PhysicalExprRef>,
+    /// Primary-key columns. When set, only the newest version of each PK is
+    /// eligible for top-k. With a filter, this evaluates the predicate against
+    /// the current PK version instead of falling back to a stale older version.
+    pk_columns: Option<Vec<String>>,
 }
 
 impl Debug for MemTableBruteForceVectorExec {
@@ -108,9 +116,49 @@ impl MemTableBruteForceVectorExec {
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
             with_row_id,
+            filter: None,
+            pk_columns: None,
         })
     }
 
+    /// Attach an optional prefilter predicate (compiled against the memtable
+    /// schema). Rows that fail the predicate are excluded before the top-k cut.
+    pub fn with_filter(mut self, filter: Option<PhysicalExprRef>) -> Self {
+        self.filter = filter;
+        self
+    }
+
+    /// Provide the primary-key columns so search keeps only the newest version
+    /// of each PK (see `pk_columns`).
+    pub fn with_pk_columns(mut self, pk_columns: Option<Vec<String>>) -> Self {
+        self.pk_columns = pk_columns.filter(|columns| !columns.is_empty());
+        self
+    }
+
+    /// Evaluate the prefilter predicate against a memtable batch, returning a
+    /// keep-mask (`true` = retain). `Ok(None)` when no filter is configured.
+    fn filter_mask(&self, batch: &RecordBatch) -> Result<Option<BooleanArray>> {
+        let Some(ref predicate) = self.filter else {
+            return Ok(None);
+        };
+        let values = predicate
+            .evaluate(batch)
+            .and_then(|v| v.into_array(batch.num_rows()))
+            .map_err(|e| {
+                Error::invalid_input(format!("vector prefilter evaluation failed: {}", e))
+            })?;
+        let mask = values
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .ok_or_else(|| {
+                Error::invalid_input(
+                    "vector prefilter predicate did not evaluate to boolean".to_string(),
+                )
+            })?
+            .clone();
+        Ok(Some(mask))
+    }
+
     /// Last row position visible under `max_visible_batch_position`, or `None`
     /// if no batches are visible. Identical to `VectorIndexExec`'s helper so
     /// both arms cut at the same MVCC boundary.
@@ -167,6 +215,24 @@ impl MemTableBruteForceVectorExec {
         let distance_type = self.query.distance_type.unwrap_or(DEFAULT_DISTANCE_TYPE);
         let batch_func = distance_type.arrow_batch_func();
 
+        // When PK columns are configured, only the newest version of each PK is
+        // eligible. This keeps top-k slots from being consumed by superseded
+        // rows and makes filtered search evaluate the predicate against the
+        // current version of the PK.
+        let newest_positions = if let Some(pk_columns) = &self.pk_columns {
+            Some(
+                newest_pk_positions(
+                    &self.batch_store,
+                    pk_columns,
+                    self.max_visible_batch_position,
+                    max_visible_row,
+                )
+                .map_err(|e| Error::invalid_input(e.to_string()))?,
+            )
+        } else {
+            None
+        };
+
         // Walk batches in append order. `current_row` is the global row offset
         // of the *next* row about to be visited; rows past `max_visible_row`
         // are dropped before they reach the heap.
@@ -207,11 +273,27 @@ impl MemTableBruteForceVectorExec {
                 ))
             })?;
 
+            // Prefilter: drop rows that fail the predicate before they reach the
+            // top-k heap (a NULL predicate result excludes the row, matching SQL).
+            let filter_mask = self.filter_mask(&stored_batch.data)?;
+
             for row in 0..n {
                 let pos = current_row + row as u64;
                 if pos > max_visible_row {
                     break;
                 }
+                // Skip superseded versions: only the newest version of each PK is
+                // eligible, so a newer non-matching version excludes the PK.
+                if let Some(ref newest) = newest_positions
+                    && !newest.contains(&pos)
+                {
+                    continue;
+                }
+                if let Some(ref mask) = filter_mask
+                    && (!mask.is_valid(row) || !mask.value(row))
+                {
+                    continue;
+                }
                 if distances.is_null(row) {
                     continue;
                 }
@@ -343,10 +425,6 @@ impl ExecutionPlan for MemTableBruteForceVectorExec {
         "MemTableBruteForceVectorExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -383,12 +461,12 @@ impl ExecutionPlan for MemTableBruteForceVectorExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.k),
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -407,10 +485,11 @@ impl ExecutionPlan for MemTableBruteForceVectorExec {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow_array::{FixedSizeListArray, Float32Array, Int32Array};
+    use arrow_array::{BooleanArray, FixedSizeListArray, Float32Array, Int32Array};
     use arrow_schema::{DataType, Field, Schema};
     use datafusion::physical_plan::common::collect;
-    use datafusion::prelude::SessionContext;
+    use datafusion::prelude::{Expr, SessionContext, col, lit};
+    use lance_datafusion::planner::Planner;
 
     fn make_schema() -> SchemaRef {
         Arc::new(Schema::new(vec![
@@ -423,6 +502,18 @@ mod tests {
         ]))
     }
 
+    fn make_schema_with_active() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
+                true,
+            ),
+            Field::new("active", DataType::Boolean, true),
+        ]))
+    }
+
     fn make_batch(schema: SchemaRef, ids: &[i32], vectors: &[[f32; 2]]) -> RecordBatch {
         let id_array = Arc::new(Int32Array::from(ids.to_vec())) as Arc<dyn Array>;
         let values: Vec<f32> = vectors.iter().flat_map(|v| v.iter().copied()).collect();
@@ -434,6 +525,23 @@ mod tests {
         RecordBatch::try_new(schema, vec![id_array, vec_array]).expect("build batch")
     }
 
+    fn make_batch_with_active(
+        schema: SchemaRef,
+        ids: &[i32],
+        vectors: &[[f32; 2]],
+        active: &[Option<bool>],
+    ) -> RecordBatch {
+        let id_array = Arc::new(Int32Array::from(ids.to_vec())) as Arc<dyn Array>;
+        let values: Vec<f32> = vectors.iter().flat_map(|v| v.iter().copied()).collect();
+        let inner = Arc::new(Float32Array::from(values));
+        let field = Arc::new(Field::new("item", DataType::Float32, true));
+        let vec_array =
+            Arc::new(FixedSizeListArray::try_new(field, 2, inner, None).expect("build fsl"))
+                as Arc<dyn Array>;
+        let active_array = Arc::new(BooleanArray::from(active.to_vec())) as Arc<dyn Array>;
+        RecordBatch::try_new(schema, vec![id_array, vec_array, active_array]).expect("build batch")
+    }
+
     fn store_with_batches(batches: Vec<RecordBatch>) -> Arc<BatchStore> {
         let store = Arc::new(BatchStore::with_capacity(batches.len().max(1)));
         for batch in batches {
@@ -458,12 +566,34 @@ mod tests {
         }
     }
 
+    fn physical_filter(schema: SchemaRef, expr: Expr) -> PhysicalExprRef {
+        let planner = Planner::new(schema);
+        let optimized = planner.optimize_expr(expr).expect("optimize filter");
+        planner
+            .create_physical_expr(&optimized)
+            .expect("create physical filter")
+    }
+
     async fn execute_to_batches(exec: Arc<dyn ExecutionPlan>) -> Vec<RecordBatch> {
         let ctx = SessionContext::new();
         let stream = exec.execute(0, ctx.task_ctx()).expect("execute");
         collect(stream).await.expect("collect")
     }
 
+    fn ids_from_batches(batches: &[RecordBatch]) -> Vec<i32> {
+        let mut ids = Vec::new();
+        for batch in batches {
+            let id_arr = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_primitive::<arrow_array::types::Int32Type>();
+            for row in 0..batch.num_rows() {
+                ids.push(id_arr.value(row));
+            }
+        }
+        ids
+    }
+
     #[tokio::test]
     async fn top_k_by_distance() {
         // Five rows; query at (0,0); L2 distances are id² each — expect ids in
@@ -637,4 +767,121 @@ mod tests {
         pairs.sort_by_key(|(id, _)| *id);
         assert_eq!(pairs, vec![(10, 0), (11, 1), (12, 2)]);
     }
+
+    #[tokio::test]
+    async fn prefilter_null_predicate_excludes_rows() {
+        let schema = make_schema_with_active();
+        let batch = make_batch_with_active(
+            schema.clone(),
+            &[1, 2, 3],
+            &[[0.0, 0.0], [3.0, 0.0], [1.0, 0.0]],
+            &[None, Some(true), Some(false)],
+        );
+        let store = store_with_batches(vec![batch]);
+        let query = query_for([0.0, 0.0], 3);
+        let filter = physical_filter(schema.clone(), col("active").eq(lit(true)));
+        let exec = Arc::new(
+            MemTableBruteForceVectorExec::new(store, query, usize::MAX, None, schema, false)
+                .expect("ctor")
+                .with_filter(Some(filter)),
+        );
+
+        let out = execute_to_batches(exec).await;
+        assert_eq!(
+            ids_from_batches(&out),
+            vec![2],
+            "NULL predicate results must be excluded from vector prefilter candidates"
+        );
+    }
+
+    #[tokio::test]
+    async fn prefilter_with_pk_columns_drops_stale_matching_version() {
+        let schema = make_schema_with_active();
+        let batch = make_batch_with_active(
+            schema.clone(),
+            &[5, 5],
+            &[[0.0, 0.0], [10.0, 0.0]],
+            &[Some(true), Some(false)],
+        );
+        let store = store_with_batches(vec![batch]);
+        let query = query_for([0.0, 0.0], 10);
+        let filter = physical_filter(schema.clone(), col("active").eq(lit(true)));
+        let exec = Arc::new(
+            MemTableBruteForceVectorExec::new(store, query, usize::MAX, None, schema, false)
+                .expect("ctor")
+                .with_filter(Some(filter))
+                .with_pk_columns(Some(vec!["id".to_string()])),
+        );
+
+        let out = execute_to_batches(exec).await;
+        assert!(
+            out.iter().all(|batch| batch.num_rows() == 0),
+            "the older matching vector version must not leak when the newest PK fails the filter"
+        );
+    }
+
+    #[tokio::test]
+    async fn pk_columns_keep_newest_version_without_filter() {
+        let schema = make_schema();
+        let batch = make_batch(schema.clone(), &[5, 5], &[[0.0, 0.0], [10.0, 0.0]]);
+        let store = store_with_batches(vec![batch]);
+        let query = query_for([0.0, 0.0], 10);
+        let exec = Arc::new(
+            MemTableBruteForceVectorExec::new(
+                store,
+                query,
+                usize::MAX,
+                None,
+                schema,
+                /* with_row_id = */ true,
+            )
+            .expect("ctor")
+            .with_pk_columns(Some(vec!["id".to_string()])),
+        );
+
+        let out = execute_to_batches(exec).await;
+        let row_ids: Vec<u64> = out
+            .iter()
+            .flat_map(|batch| {
+                let row_ids = batch
+                    .column_by_name(lance_core::ROW_ID)
+                    .unwrap()
+                    .as_primitive::<arrow_array::types::UInt64Type>();
+                (0..batch.num_rows())
+                    .map(|row| row_ids.value(row))
+                    .collect::<Vec<_>>()
+            })
+            .collect();
+        assert_eq!(
+            row_ids,
+            vec![1],
+            "brute-force vector PK recency must keep only the newest duplicate"
+        );
+    }
+
+    #[tokio::test]
+    async fn empty_pk_columns_do_not_collapse_results() {
+        let schema = make_schema();
+        let batch = make_batch(
+            schema.clone(),
+            &[1, 2, 3],
+            &[[0.0, 0.0], [1.0, 0.0], [2.0, 0.0]],
+        );
+        let store = store_with_batches(vec![batch]);
+        let query = query_for([0.0, 0.0], 3);
+        let exec = Arc::new(
+            MemTableBruteForceVectorExec::new(store, query, usize::MAX, None, schema, false)
+                .expect("ctor")
+                .with_pk_columns(Some(vec![])),
+        );
+
+        let out = execute_to_batches(exec).await;
+        let mut ids = ids_from_batches(&out);
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![1, 2, 3],
+            "empty PK columns should behave like no PK columns, not one empty tuple key"
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
index fed61698fab..b9e1e3fa469 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs
@@ -3,7 +3,6 @@
 
 //! BTreeIndexExec - BTree index queries with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -312,10 +311,6 @@ impl ExecutionPlan for BTreeIndexExec {
         "BTreeIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -358,13 +353,13 @@ impl ExecutionPlan for BTreeIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         // We can't know the exact count without querying the index
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
index ba5947e4b12..071581aa065 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs
@@ -16,7 +16,6 @@
 //! forward-aligned mask. A single `filter_record_batch` over the original
 //! batch then emits the survivors with no per-column reverse copy.
 
-use std::any::Any;
 use std::collections::HashSet;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
@@ -155,10 +154,6 @@ impl ExecutionPlan for MemTableDedupScanExec {
         "MemTableDedupScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -280,12 +275,12 @@ impl ExecutionPlan for MemTableDedupScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
index 44b1a097d1a..77c1a27b980 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs
@@ -3,12 +3,12 @@
 
 //! FtsIndexExec - Full-text search with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
-use arrow_array::{Float32Array, RecordBatch, UInt32Array, UInt64Array};
+use arrow_array::{BooleanArray, Float32Array, RecordBatch, UInt32Array, UInt64Array};
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion::common::ScalarValue;
 use datafusion::common::stats::Precision;
 use datafusion::error::Result as DataFusionResult;
 use datafusion::execution::TaskContext;
@@ -19,12 +19,14 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
     SendableRecordBatchStream, Statistics,
 };
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExprRef};
 use futures::stream::{self, StreamExt};
 use lance_core::{Error, Result};
 
 use super::super::builder::{FtsQuery, FtsQueryType};
+use super::newest_pk_positions;
 use crate::dataset::mem_wal::index::{FtsQueryExpr, SearchOptions};
+use crate::dataset::mem_wal::scanner::exec::resolve_pk_indices;
 use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
 
 /// Score column name in output.
@@ -38,6 +40,8 @@ struct BatchRange {
     batch_id: usize,
 }
 
+type MaterializedFtsRows = (Vec<Arc<dyn arrow_array::Array>>, Vec<f32>, Vec<u64>);
+
 /// ExecutionPlan node that queries FTS index with MVCC visibility.
 pub struct FtsIndexExec {
     batch_store: Arc<BatchStore>,
@@ -54,6 +58,13 @@ pub struct FtsIndexExec {
     max_visible_row: Option<u64>,
     /// Whether to include _rowid column (row position) in output.
     with_row_id: bool,
+    /// Optional prefilter predicate, compiled against the memtable schema.
+    /// Applied to the materialized full-schema hits before projection so the
+    /// FTS arm only returns rows matching the predicate.
+    filter: Option<PhysicalExprRef>,
+    /// Primary-key columns. When set, materialized hits are kept only if their
+    /// row position is the newest visible version of that PK.
+    pk_columns: Option<Vec<String>>,
 }
 
 impl Debug for FtsIndexExec {
@@ -162,9 +173,24 @@ impl FtsIndexExec {
             batch_ranges,
             max_visible_row,
             with_row_id,
+            filter: None,
+            pk_columns: None,
         })
     }
 
+    /// Attach an optional prefilter predicate (compiled against the memtable
+    /// schema). Hits that fail the predicate are dropped before projection.
+    pub fn with_filter(mut self, filter: Option<PhysicalExprRef>) -> Self {
+        self.filter = filter;
+        self
+    }
+
+    /// Provide primary-key columns for newest-version filtering.
+    pub fn with_pk_columns(mut self, pk_columns: Option<Vec<String>>) -> Self {
+        self.pk_columns = pk_columns;
+        self
+    }
+
     /// Find batch for a row position using binary search. O(log n).
     #[inline]
     fn find_batch(&self, row_pos: usize) -> Option<&BatchRange> {
@@ -183,7 +209,11 @@ impl FtsIndexExec {
 
         // Convert FtsQueryType to FtsQueryExpr
         let query_expr = match &self.query.query_type {
-            FtsQueryType::Match { query } => FtsQueryExpr::match_query(query),
+            FtsQueryType::Match {
+                query,
+                operator,
+                boost,
+            } => FtsQueryExpr::match_query_with_operator(query, *operator).with_boost(*boost),
             FtsQueryType::Phrase { query, slop } => FtsQueryExpr::phrase_with_slop(query, *slop),
             FtsQueryType::Boolean {
                 must,
@@ -205,15 +235,33 @@ impl FtsIndexExec {
             FtsQueryType::Fuzzy {
                 query,
                 fuzziness,
+                prefix_length,
                 max_expansions,
-            } => FtsQueryExpr::fuzzy_with_options(query, *fuzziness, *max_expansions),
+                boost,
+            } => {
+                FtsQueryExpr::fuzzy_with_options(query, *fuzziness, *prefix_length, *max_expansions)
+                    .with_boost(*boost)
+            }
         };
 
-        // Search the index using the query expression. `include_tail` selects
-        // read-your-writes vs immutable-only; `wand_factor` adds pruning.
-        let options = SearchOptions::new()
-            .with_wand_factor(self.query.wand_factor)
-            .with_include_tail(self.query.include_tail);
+        let all_rows_visible = self.batch_ranges.last().is_none_or(|last| {
+            self.max_visible_row
+                .map(|max_visible| max_visible + 1 >= last.end as u64)
+                .unwrap_or(last.end == 0)
+        });
+        let pk_recency_is_noop = self.pk_columns.is_none()
+            || (self.indexes.has_pk_index() && !self.indexes.pk_has_overrides());
+        let can_prune_in_index = self.filter.is_none() && pk_recency_is_noop && all_rows_visible;
+
+        // Search the index using the query expression. WAND pruning is only
+        // safe when the index search itself sees the final candidate set.
+        let mut options = SearchOptions::new().with_include_tail(self.query.include_tail);
+        if can_prune_in_index {
+            options = options.with_wand_factor(self.query.wand_factor);
+            if let Some(limit) = self.query.limit {
+                options = options.with_limit(limit);
+            }
+        }
         let entries = index.search_with_options(&query_expr, options);
 
         // Convert to (row_position, score) pairs
@@ -298,6 +346,66 @@ impl FtsIndexExec {
             }
         }
 
+        // Prefilter: evaluate the predicate against the full-schema hits and drop
+        // non-matching rows before applying the query limit and projection (a
+        // NULL result excludes the row, matching SQL). When a predicate exists,
+        // query_index deliberately avoids pushing the limit into the index so
+        // this remains an exact prefilter, not a lossy post-filter.
+        let (final_columns, all_scores, all_row_positions) =
+            if let Some(ref predicate) = self.filter {
+                let Some(first) = self.batch_store.get(0) else {
+                    return Ok(vec![]);
+                };
+                let data_batch = RecordBatch::try_new(first.data.schema(), final_columns)?;
+                let mask = predicate
+                    .evaluate(&data_batch)?
+                    .into_array(data_batch.num_rows())?;
+                let mask = mask
+                    .as_any()
+                    .downcast_ref::<BooleanArray>()
+                    .ok_or_else(|| {
+                        datafusion::error::DataFusionError::Internal(
+                            "FTS prefilter predicate did not evaluate to boolean".to_string(),
+                        )
+                    })?;
+                let filtered_columns = data_batch
+                    .columns()
+                    .iter()
+                    .map(|c| arrow_select::filter::filter(c.as_ref(), mask))
+                    .collect::<std::result::Result<Vec<_>, _>>()?;
+                let filtered_scores: Vec<f32> = all_scores
+                    .iter()
+                    .zip(mask.iter())
+                    .filter_map(|(s, keep)| keep.unwrap_or(false).then_some(*s))
+                    .collect();
+                let filtered_positions: Vec<u64> = all_row_positions
+                    .iter()
+                    .zip(mask.iter())
+                    .filter_map(|(p, keep)| keep.unwrap_or(false).then_some(*p))
+                    .collect();
+                (filtered_columns, filtered_scores, filtered_positions)
+            } else {
+                (final_columns, all_scores, all_row_positions)
+            };
+
+        let (mut final_columns, mut all_scores, mut all_row_positions) =
+            self.filter_to_newest_pk(final_columns, all_scores, all_row_positions)?;
+
+        if all_scores.is_empty() {
+            return Ok(vec![]);
+        }
+
+        if let Some(limit) = self.query.limit
+            && all_scores.len() > limit
+        {
+            final_columns = final_columns
+                .into_iter()
+                .map(|column| column.slice(0, limit))
+                .collect();
+            all_scores.truncate(limit);
+            all_row_positions.truncate(limit);
+        }
+
         // Add score column
         final_columns.push(Arc::new(Float32Array::from(all_scores)));
 
@@ -322,6 +430,76 @@ impl FtsIndexExec {
         let batch = RecordBatch::try_new(self.output_schema.clone(), projected_columns)?;
         Ok(vec![batch])
     }
+
+    fn filter_to_newest_pk(
+        &self,
+        final_columns: Vec<Arc<dyn arrow_array::Array>>,
+        all_scores: Vec<f32>,
+        all_row_positions: Vec<u64>,
+    ) -> DataFusionResult<MaterializedFtsRows> {
+        let Some(pk_columns) = &self.pk_columns else {
+            return Ok((final_columns, all_scores, all_row_positions));
+        };
+        if pk_columns.is_empty() || all_scores.is_empty() {
+            return Ok((final_columns, all_scores, all_row_positions));
+        }
+        let Some(max_visible_row) = self.max_visible_row else {
+            return Ok((final_columns, all_scores, all_row_positions));
+        };
+        if self.indexes.has_pk_index() && !self.indexes.pk_has_overrides() {
+            return Ok((final_columns, all_scores, all_row_positions));
+        }
+        let Some(first) = self.batch_store.get(0) else {
+            return Ok((final_columns, all_scores, all_row_positions));
+        };
+        let newest_positions = if self.indexes.has_pk_index() {
+            None
+        } else {
+            Some(newest_pk_positions(
+                &self.batch_store,
+                pk_columns,
+                self.max_visible_batch_position,
+                max_visible_row,
+            )?)
+        };
+
+        let data_batch = RecordBatch::try_new(first.data.schema(), final_columns)?;
+        let pk_indices = resolve_pk_indices(&data_batch, pk_columns)?;
+        let keep = (0..data_batch.num_rows())
+            .map(|row| {
+                Ok(match &newest_positions {
+                    Some(newest) => newest.contains(&all_row_positions[row]),
+                    None => {
+                        let values: Vec<ScalarValue> = pk_indices
+                            .iter()
+                            .map(|&col| ScalarValue::try_from_array(data_batch.column(col), row))
+                            .collect::<DataFusionResult<_>>()?;
+                        self.indexes
+                            .pk_is_newest(&values, all_row_positions[row], max_visible_row)
+                    }
+                })
+            })
+            .collect::<DataFusionResult<Vec<_>>>()?;
+
+        let mask = BooleanArray::from_iter(keep.iter().copied());
+        let filtered_columns = data_batch
+            .columns()
+            .iter()
+            .map(|c| arrow_select::filter::filter(c.as_ref(), &mask))
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+        let filtered_scores = all_scores
+            .into_iter()
+            .zip(keep.iter())
+            .filter_map(|(s, keep)| keep.then_some(s))
+            .collect();
+        let filtered_positions = all_row_positions
+            .into_iter()
+            .zip(keep.iter())
+            .filter_map(|(p, keep)| keep.then_some(p))
+            .collect();
+
+        Ok((filtered_columns, filtered_scores, filtered_positions))
+    }
 }
 
 impl DisplayAs for FtsIndexExec {
@@ -350,10 +528,6 @@ impl ExecutionPlan for FtsIndexExec {
         "FtsIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -399,12 +573,12 @@ impl ExecutionPlan for FtsIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
index c56e960048d..de2f2f05d67 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs
@@ -3,7 +3,6 @@
 
 //! MemTableScanExec - Full table scan with MVCC visibility filtering.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -194,10 +193,6 @@ impl ExecutionPlan for MemTableScanExec {
         "MemTableScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -339,14 +334,14 @@ impl ExecutionPlan for MemTableScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         // Report statistics as Absent to avoid DataFusion analysis bugs
         // with selectivity calculation on in-memory tables.
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Absent,
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
index c3453db68a2..58d4250f0c2 100644
--- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
+++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs
@@ -3,7 +3,6 @@
 
 //! VectorIndexExec - HNSW vector search with MVCC visibility.
 
-use std::any::Any;
 use std::fmt::{Debug, Formatter};
 use std::sync::Arc;
 
@@ -310,10 +309,6 @@ impl ExecutionPlan for VectorIndexExec {
         "VectorIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -355,12 +350,12 @@ impl ExecutionPlan for VectorIndexExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.k),
             total_byte_size: Precision::Absent,
             column_statistics: vec![],
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
diff --git a/rust/lance/src/dataset/mem_wal/scanner.rs b/rust/lance/src/dataset/mem_wal/scanner.rs
index b1766f8525f..f1d84611e04 100644
--- a/rust/lance/src/dataset/mem_wal/scanner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner.rs
@@ -24,13 +24,18 @@
 //! use lance::dataset::mem_wal::scanner::LsmScanner;
 //!
 //! let scanner = LsmScanner::new(base_table, shard_snapshots, vec!["pk".to_string()])
-//!     .project(&["id", "name"])
+//!     .project(&["id", "name"])?
 //!     .filter("id > 10")?
-//!     .limit(100, None);
+//!     .limit(Some(100), None)?;
 //!
 //! let stream = scanner.try_into_stream().await?;
 //! ```
 
+use arrow_schema::Schema as ArrowSchema;
+use datafusion::common::ToDFSchema;
+use datafusion::prelude::{Expr, SessionContext};
+use lance_core::{Error, Result};
+
 mod block_list;
 mod builder;
 mod collector;
@@ -43,13 +48,37 @@ mod point_lookup;
 mod projection;
 mod vector_search;
 
+pub use block_list::write_pk_sidecar;
 pub use builder::LsmScanner;
 pub use collector::{
     ActiveMemTableRef, InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector,
 };
-pub use data_source::{FlushedGeneration, LsmDataSource, LsmGeneration, ShardSnapshot};
-pub use flushed_cache::FlushedMemTableCache;
+pub use data_source::{
+    FlushedGeneration, FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot,
+};
+pub use flushed_cache::{DatasetCache, FlushedMemTableCache, GenerationWarmer};
 pub use fts_search::{LsmFtsSearchPlanner, SCORE_COLUMN};
 pub use point_lookup::LsmPointLookupPlanner;
 pub use projection::DISTANCE_COLUMN;
 pub use vector_search::LsmVectorSearchPlanner;
+
+/// Parse a SQL filter expression against a MemWAL source schema.
+pub fn parse_filter_expr(schema: &ArrowSchema, filter_expr: &str) -> Result<Expr> {
+    let ctx = SessionContext::new();
+    let df_schema = schema
+        .clone()
+        .to_dfschema()
+        .map_err(|e| Error::invalid_input(format!("Failed to create DFSchema: {}", e)))?;
+    ctx.parse_sql_expr(filter_expr, &df_schema)
+        .map_err(|e| Error::invalid_input(format!("Failed to parse filter expression: {}", e)))
+}
+
+pub(crate) fn validate_overfetch_factor(factor: f64) -> Result<f64> {
+    if !factor.is_finite() || factor < 1.0 {
+        return Err(Error::invalid_input(format!(
+            "overfetch_factor must be finite and at least 1.0, got {}",
+            factor
+        )));
+    }
+    Ok(factor)
+}
diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
index 684fde48da1..30a19650005 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs
@@ -3,37 +3,152 @@
 
 //! Per-source block-list construction for LSM vector search.
 //!
-//! A generation's membership is an `Arc<HashSet<u64>>` of PK hashes
-//! ([`compute_pk_hash`]), built once (immutable gens cached). Each source gets a
-//! `Vec<Arc<HashSet<u64>>>` of the newer generations' sets (`NEWER(G)`; base: all
-//! of them) — referenced, never merged. The KNN drops candidates whose PK is in
-//! any (see [`super::exec::PkHashFilterExec`]).
+//! A generation's membership is a [`GenMembership`]: in-memory generations
+//! (active / frozen) are probed by value against their maintained primary-key
+//! index (no per-query set), while flushed generations are probed against their
+//! standalone on-disk PK BTree (the sidecar written at flush, opened by path).
+//! Probing is batched — [`GenMembership::contains_keys`] tests a whole batch of
+//! keys per generation in one pass. Each source gets a `Vec<GenMembership>` of
+//! the newer generations (`NEWER(G)`; base: all of them); the KNN drops a
+//! candidate whose PK any of them contains (see
+//! [`super::exec::PkBlockFilterExec`]).
 //!
-//! Cross-generation only: within-gen dups share a hash and fall to the global
-//! dedup's `(generation, freshness)` tiebreaker.
+//! Cross-generation only: within-gen dups collapse via the global dedup's
+//! `(generation, freshness)` tiebreaker.
+
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock};
+
+use datafusion::common::ScalarValue;
+use lance_core::{Error, Result};
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::registry::IndexPluginRegistry;
+use lance_index::scalar::btree::BTreeIndex;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_index::scalar::{
+    IndexStore as ScalarIndexStore, SargableQuery, ScalarIndex, SearchResult,
+};
+use uuid::Uuid;
 
-use std::collections::{HashMap, HashSet};
-use std::sync::Arc;
+use super::data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration};
+use super::flushed_cache::{DatasetCache, open_flushed_dataset};
+use crate::dataset::mem_wal::index::encode_pk_tuple;
+use crate::dataset::mem_wal::util::PK_INDEX_DIR;
+use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
+
+/// Default-plugin registry, used only to load the standalone PK BTree by its
+/// `BTreeIndexDetails` type. Built once.
+static PK_BTREE_REGISTRY: LazyLock<Arc<IndexPluginRegistry>> =
+    LazyLock::new(IndexPluginRegistry::with_default_plugins);
+
+/// One newer generation's PK membership, used to decide whether it shadows an
+/// older source's row.
+#[derive(Clone, Debug)]
+pub enum GenMembership {
+    /// Probe the in-memory memtable's primary-key index, bounded to its visible
+    /// prefix (so a not-yet-visible write can't shadow an older visible copy).
+    InMemory {
+        index_store: Arc<IndexStore>,
+        /// Inclusive visible row watermark; `None` when no rows are visible.
+        max_visible_row: Option<u64>,
+    },
+    /// Probe the flushed generation's standalone on-disk PK BTree.
+    OnDisk(Arc<dyn ScalarIndex>),
+}
 
-use arrow_array::RecordBatch;
-use futures::TryStreamExt;
-use lance_core::Result;
+impl GenMembership {
+    /// Whether this generation visibly contains the primary `key` — the typed
+    /// value for a single-column PK, the encoded `Binary` tuple for a composite
+    /// one (built by [`on_disk_pk_key`]). The same key probes the in-memory
+    /// BTree and the flushed on-disk BTree, which now share a key space.
+    pub async fn contains(&self, key: &ScalarValue) -> Result<bool> {
+        match self {
+            Self::InMemory {
+                index_store,
+                max_visible_row,
+            } => Ok(max_visible_row.is_some_and(|max| index_store.pk_contains_key(key, max))),
+            Self::OnDisk(index) => {
+                let result = index
+                    .search(&SargableQuery::Equals(key.clone()), &NoOpMetricsCollector)
+                    .await
+                    .map_err(|e| Error::io(e.to_string()))?;
+                Ok(!search_is_empty(&result))
+            }
+        }
+    }
 
-use uuid::Uuid;
+    /// Batched [`Self::contains`]: for each key in `keys`, whether this
+    /// generation visibly contains it, returned as a mask aligned to `keys`.
+    ///
+    /// One probe replaces N. The on-disk arm issues a single
+    /// [`BTreeIndex::contains_keys`] (no per-key `SearchResult` allocation); the
+    /// in-memory arm maps the sync, allocation-free PK lookup over the slice.
+    /// Keys are in the index's key space (see [`on_disk_pk_key`]).
+    pub async fn contains_keys(&self, keys: &[ScalarValue]) -> Result<Vec<bool>> {
+        match self {
+            Self::InMemory {
+                index_store,
+                max_visible_row,
+            } => Ok(keys
+                .iter()
+                .map(|key| max_visible_row.is_some_and(|max| index_store.pk_contains_key(key, max)))
+                .collect()),
+            Self::OnDisk(index) => {
+                // The flushed PK sidecar is always a BTree (built via
+                // `PK_BTREE_REGISTRY`); downcast to reach the batched probe.
+                let btree = index.as_any().downcast_ref::<BTreeIndex>().ok_or_else(|| {
+                    Error::io("flushed PK dedup index is not a BTree".to_string())
+                })?;
+                btree
+                    .contains_keys(keys, &NoOpMetricsCollector)
+                    .await
+                    .map_err(|e| Error::io(e.to_string()))
+            }
+        }
+    }
 
-use super::data_source::{LsmDataSource, LsmGeneration};
-use super::exec::{compute_pk_hash, resolve_pk_indices};
-use super::flushed_cache::{FlushedMemTableCache, open_flushed_dataset};
-use crate::dataset::Dataset;
-use crate::dataset::mem_wal::write::BatchStore;
-use crate::session::Session;
+    /// Whether this generation has no (visible) membership — used to skip adding
+    /// an empty blocked set. A flushed generation always has rows (flush rejects
+    /// an empty memtable), so it is never empty.
+    fn is_empty(&self) -> bool {
+        match self {
+            Self::InMemory {
+                index_store,
+                max_visible_row,
+            } => max_visible_row.is_none() || index_store.pk_is_empty(),
+            Self::OnDisk(_) => false,
+        }
+    }
+}
+
+/// Whether a scalar search returned no rows (existence test for the block-list).
+fn search_is_empty(result: &SearchResult) -> bool {
+    match result {
+        SearchResult::Exact(set) | SearchResult::AtMost(set) | SearchResult::AtLeast(set) => {
+            set.is_empty()
+        }
+    }
+}
 
-/// Per-source blocked PK-hash sets, keyed by `(shard_id, generation)`. Each
-/// value is the membership sets of the generations newer than that source.
-pub type SourceBlockLists = HashMap<(Option<Uuid>, LsmGeneration), Vec<Arc<HashSet<u64>>>>;
+/// The probe key for the on-disk PK BTree: a single-column PK indexes its typed
+/// value directly; a composite PK indexes the order-preserving encoded tuple as
+/// `Binary` (matching what flush wrote — see [`encode_pk_tuple`]).
+pub fn on_disk_pk_key(values: &[ScalarValue]) -> Result<ScalarValue> {
+    match values {
+        [single] => Ok(single.clone()),
+        _ => Ok(ScalarValue::Binary(Some(encode_pk_tuple(values)?))),
+    }
+}
 
-/// A shard's generations paired with their PK-hash membership, before sorting.
-type ShardGenSets = HashMap<Uuid, Vec<(LsmGeneration, Arc<HashSet<u64>>)>>;
+/// Per-source blocked memberships, keyed by `(shard_id, generation)`. Each value
+/// is the memberships of the generations newer than that source.
+pub type SourceBlockLists = HashMap<(Option<Uuid>, LsmGeneration), Vec<GenMembership>>;
+
+/// A shard's generations paired with their membership, before sorting.
+type ShardGenSets = HashMap<Uuid, Vec<(LsmGeneration, GenMembership)>>;
 
 /// Per-source `NEWER(G)`, keyed by `(shard_id, generation)`. Generations are
 /// per-shard, so a source is superseded only by strictly-newer generations of
@@ -42,59 +157,65 @@ type ShardGenSets = HashMap<Uuid, Vec<(LsmGeneration, Arc<HashSet<u64>>)>>;
 /// Only superseded sources get an entry; the newest of each shard never does.
 pub async fn compute_source_block_lists(
     sources: &[LsmDataSource],
-    pk_columns: &[String],
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<FlushedMemTableCache>>,
+    store_params: Option<&ObjectStoreParams>,
+    flushed_cache: Option<&Arc<dyn DatasetCache>>,
 ) -> Result<SourceBlockLists> {
-    // Hash each non-base source's membership, grouped by shard (generations are
+    // Membership per non-base source, grouped by shard (generations are
     // per-shard, so supersession is within-shard only).
     let mut by_shard: ShardGenSets = HashMap::new();
     let mut has_base = false;
+    // Flushed PK-BTree opens are cold S3 reads; overlap them with
+    // `try_join_all`. Order is irrelevant — gens are sorted per-shard below.
+    let mut flushed_loads = Vec::new();
     for source in sources {
         match source {
             LsmDataSource::BaseTable { .. } => has_base = true,
             LsmDataSource::ActiveMemTable {
                 batch_store,
+                index_store,
                 shard_id,
                 generation,
                 ..
             } => {
-                let hashes = Arc::new(pk_hashes_from_batch_store(batch_store, pk_columns)?);
+                let membership = in_memory_membership(batch_store, index_store);
                 by_shard
                     .entry(*shard_id)
                     .or_default()
-                    .push((*generation, hashes));
+                    .push((*generation, membership));
             }
             LsmDataSource::FlushedMemTable {
                 path,
                 shard_id,
                 generation,
                 ..
-            } => {
-                // Cached by immutable path so repeated searches skip the PK scan.
-                let hashes = flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?;
-                by_shard
-                    .entry(*shard_id)
-                    .or_default()
-                    .push((*generation, hashes));
-            }
+            } => flushed_loads.push(async move {
+                let index = open_pk_index(path, session, store_params, flushed_cache).await?;
+                Ok::<_, Error>((*shard_id, *generation, GenMembership::OnDisk(index)))
+            }),
         }
     }
+    for (shard_id, generation, membership) in futures::future::try_join_all(flushed_loads).await? {
+        by_shard
+            .entry(shard_id)
+            .or_default()
+            .push((generation, membership));
+    }
 
     let mut blocked: SourceBlockLists = HashMap::new();
     // Base (shardless, oldest) is superseded by every non-base generation.
-    let mut base_blocked: Vec<Arc<HashSet<u64>>> = Vec::new();
+    let mut base_blocked: Vec<GenMembership> = Vec::new();
     for (shard, mut gens) in by_shard {
         // Newest-first: a gen's blocked list is its own shard's newer gens.
         gens.sort_by_key(|(generation, _)| std::cmp::Reverse(*generation));
-        let mut newer: Vec<Arc<HashSet<u64>>> = Vec::new();
-        for (generation, hashes) in gens {
+        let mut newer: Vec<GenMembership> = Vec::new();
+        for (generation, membership) in gens {
             if !newer.is_empty() {
                 blocked.insert((Some(shard), generation), newer.clone());
             }
-            if !hashes.is_empty() {
-                base_blocked.push(hashes.clone());
-                newer.push(hashes);
+            if !membership.is_empty() {
+                base_blocked.push(membership.clone());
+                newer.push(membership);
             }
         }
     }
@@ -104,260 +225,358 @@ pub async fn compute_source_block_lists(
     Ok(blocked)
 }
 
-/// The fresh-tier block-list: one membership set per generation that shadows the
-/// base table — active + frozen memtables (hashed now) and flushed generations
-/// (from the cache). Same `Vec<Arc<HashSet<u64>>>` shape the vector-search filter
-/// consumes; a base/external reader can drop any row whose PK is in one of them.
-/// The base source, if present, is skipped (it is what gets shadowed).
+/// The fresh-tier block-list: one [`GenMembership`] per generation that shadows
+/// the base table — active + frozen memtables (probed against their index) and
+/// flushed generations (probed against their on-disk PK BTree). A base/external
+/// reader can test any PK against these (via [`GenMembership::contains`]) to
+/// decide whether the fresh tier shadows it. The base source, if present, is
+/// skipped (it is what gets shadowed).
+///
+/// When `watermarks` carries a watermark for a source's shard, membership is
+/// bounded to it (see [`FreshTierWatermark`]): higher generations are excluded,
+/// the active generation is bounded to its first `active_batch_count` batches,
+/// and lower generations (frozen and flushed) are immutable and included whole.
+/// A shard absent from `watermarks` (or `watermarks == None`) uses the live tier.
 pub async fn fresh_tier_block_list(
     sources: &[LsmDataSource],
-    pk_columns: &[String],
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<FlushedMemTableCache>>,
-) -> Result<Vec<Arc<HashSet<u64>>>> {
-    let mut sets = Vec::new();
+    store_params: Option<&ObjectStoreParams>,
+    flushed_cache: Option<&Arc<dyn DatasetCache>>,
+    watermarks: Option<&HashMap<Uuid, FreshTierWatermark>>,
+) -> Result<Vec<GenMembership>> {
+    // Membership per source, in source order (`None` = skipped). Flushed
+    // PK-BTree opens are cold S3 reads, so collect them tagged with their slot
+    // and overlap with `try_join_all` rather than opening one at a time.
+    let mut slots: Vec<Option<GenMembership>> = Vec::with_capacity(sources.len());
+    let mut flushed_loads = Vec::new();
     for source in sources {
-        let set = match source {
-            LsmDataSource::BaseTable { .. } => continue,
-            LsmDataSource::ActiveMemTable { batch_store, .. } => {
-                Arc::new(pk_hashes_from_batch_store(batch_store, pk_columns)?)
+        match source {
+            LsmDataSource::BaseTable { .. } => slots.push(None),
+            LsmDataSource::ActiveMemTable {
+                batch_store,
+                index_store,
+                shard_id,
+                generation,
+                ..
+            } => {
+                let membership = match watermarks.and_then(|m| m.get(shard_id)) {
+                    None => Some(in_memory_membership(batch_store, index_store)),
+                    Some(watermark) => {
+                        let g = generation.as_u64();
+                        if g > watermark.active_generation {
+                            // Rolled in after the snapshot; the arm never saw it.
+                            None
+                        } else if g == watermark.active_generation {
+                            // Bound the active generation to the batches the arm saw.
+                            Some(bounded_in_memory_membership(
+                                batch_store,
+                                index_store,
+                                watermark.active_batch_count,
+                            ))
+                        } else {
+                            // Lower (frozen) generations are immutable — include all.
+                            Some(in_memory_membership(batch_store, index_store))
+                        }
+                    }
+                };
+                slots.push(membership);
             }
-            LsmDataSource::FlushedMemTable { path, .. } => {
-                flushed_pk_hashes(path, pk_columns, session, flushed_cache).await?
+            LsmDataSource::FlushedMemTable {
+                path,
+                shard_id,
+                generation,
+                ..
+            } => {
+                // A generation at or above the active one was flushed after the
+                // snapshot; exclude it. Lower generations are immutable. The
+                // `==` case is the active generation flushed between the two
+                // reads: excluding the flushed copy loses nothing, since its
+                // rows are already captured by the in-memory arm above (bounded
+                // to `active_batch_count`).
+                let flushed_after_snapshot = watermarks
+                    .and_then(|m| m.get(shard_id))
+                    .is_some_and(|watermark| generation.as_u64() >= watermark.active_generation);
+                if flushed_after_snapshot {
+                    slots.push(None);
+                } else {
+                    let slot = slots.len();
+                    slots.push(None);
+                    flushed_loads.push(async move {
+                        let index =
+                            open_pk_index(path, session, store_params, flushed_cache).await?;
+                        Ok::<_, Error>((slot, GenMembership::OnDisk(index)))
+                    });
+                }
             }
-        };
-        if !set.is_empty() {
-            sets.push(set);
         }
     }
-    Ok(sets)
+    for (slot, membership) in futures::future::try_join_all(flushed_loads).await? {
+        slots[slot] = Some(membership);
+    }
+    Ok(slots
+        .into_iter()
+        .flatten()
+        .filter(|membership| !membership.is_empty())
+        .collect())
 }
 
-/// Hash the PK membership of an in-memory memtable (active or frozen) from its
-/// committed `BatchStore` rows.
-pub fn pk_hashes_from_batch_store(
-    store: &BatchStore,
-    pk_columns: &[String],
-) -> Result<HashSet<u64>> {
-    let mut batches: Vec<RecordBatch> = Vec::with_capacity(store.len());
-    for i in 0..store.len() {
-        if let Some(stored) = store.get(i) {
-            batches.push(stored.data.clone());
-        }
+/// Cross-source membership of an in-memory (active / frozen) memtable: a
+/// snapshot-bounded probe of its maintained primary-key index. A memtable
+/// without a primary-key index can't be probed, so it blocks nothing — the
+/// production vector-search path always enables the index.
+fn in_memory_membership(
+    batch_store: &Arc<BatchStore>,
+    index_store: &Arc<IndexStore>,
+) -> GenMembership {
+    let max_visible_row = batch_store.max_visible_row(index_store.max_visible_batch_position());
+    GenMembership::InMemory {
+        index_store: index_store.clone(),
+        max_visible_row,
     }
-    pk_hashes_from_batches(&batches, pk_columns)
 }
 
-/// Hash every row's primary key across `batches` into a membership set.
-fn pk_hashes_from_batches(batches: &[RecordBatch], pk_columns: &[String]) -> Result<HashSet<u64>> {
-    let mut pk_hashes = HashSet::new();
-    for batch in batches {
-        if batch.num_rows() == 0 {
-            continue;
-        }
-        let pk_indices = resolve_pk_indices(batch, pk_columns)
-            .map_err(|e| lance_core::Error::invalid_input(e.to_string()))?;
-        for row_idx in 0..batch.num_rows() {
-            pk_hashes.insert(compute_pk_hash(batch, &pk_indices, row_idx));
-        }
+/// As-of variant of [`in_memory_membership`] for the active generation under a
+/// watermark: bounds visibility to the first `batch_count` batches — those a
+/// prior scan observed before the memtable grew. A later append lands at a
+/// higher row position and is excluded by the probe, so it can't shadow a base
+/// row whose replacement the scan never delivered. `batch_count == 0` leaves the
+/// membership empty.
+fn bounded_in_memory_membership(
+    batch_store: &Arc<BatchStore>,
+    index_store: &Arc<IndexStore>,
+    batch_count: u64,
+) -> GenMembership {
+    let max_visible_row = batch_count
+        .checked_sub(1)
+        .and_then(|last_batch| batch_store.max_visible_row(last_batch as usize));
+    GenMembership::InMemory {
+        index_store: index_store.clone(),
+        max_visible_row,
     }
-    Ok(pk_hashes)
 }
 
-/// Build (or fetch the cached) PK-hash membership for one flushed generation.
-/// Cached by immutable path (single-flight); the build scans the flushed
-/// dataset's PK columns.
-async fn flushed_pk_hashes(
+/// Open the standalone PK BTree at `{flushed gen}/_pk_index` for one flushed
+/// generation. Reuses the flushed dataset's (session-configured) object store
+/// and **its index cache**, then loads the sidecar directly by path through the
+/// BTree plugin — it is not a manifest index. The opened index and its pages
+/// are cached in the session's index cache (keyed by the immutable flushed
+/// path), so repeated probes reuse them with no separate cache path and no
+/// upfront scan; concurrent first-opens may each load before the cache fills.
+/// A stable cache UUID for a non-manifest index identified only by its path.
+///
+/// `DSIndexCache::for_index` keys by `&Uuid`, but the flushed PK sidecar has no
+/// manifest UUID — its identity is its immutable path. Derive a deterministic
+/// UUID from the path so the cache namespace is per-path and stable across
+/// probes (the `uuid` crate lacks the `v5` "name-based" feature here, so hash to
+/// a `u128` instead).
+fn path_cache_uuid(path: &str) -> Uuid {
+    use std::hash::{Hash, Hasher};
+    let mut lo = std::collections::hash_map::DefaultHasher::new();
+    path.hash(&mut lo);
+    let mut hi = std::collections::hash_map::DefaultHasher::new();
+    // Seed the high half differently so it never equals the low half.
+    "lance/flushed-pk-index".hash(&mut hi);
+    path.hash(&mut hi);
+    Uuid::from_u128(((hi.finish() as u128) << 64) | lo.finish() as u128)
+}
+
+async fn open_pk_index(
     path: &str,
-    pk_columns: &[String],
     session: Option<&Arc<Session>>,
-    flushed_cache: Option<&Arc<FlushedMemTableCache>>,
-) -> Result<Arc<HashSet<u64>>> {
-    match flushed_cache {
-        Some(cache) => {
-            let build_cache = cache.clone();
-            let build_path = path.to_string();
-            let build_session = session.cloned();
-            let build_pk = pk_columns.to_vec();
-            cache
-                .get_or_build_pk_hashes(
-                    path,
-                    // `Box::pin` keeps this build future off the caller's future
-                    // (avoids `clippy::large_futures`).
-                    Box::pin(async move {
-                        let dataset = open_flushed_dataset(
-                            &build_path,
-                            build_session.as_ref(),
-                            Some(&build_cache),
-                        )
-                        .await?;
-                        scan_pk_hashes(&dataset, &build_pk).await
-                    }),
-                )
-                .await
-        }
-        None => {
-            let dataset = open_flushed_dataset(path, session, None).await?;
-            Ok(Arc::new(scan_pk_hashes(&dataset, pk_columns).await?))
-        }
+    store_params: Option<&ObjectStoreParams>,
+    flushed_cache: Option<&Arc<dyn DatasetCache>>,
+) -> Result<Arc<dyn ScalarIndex>> {
+    let dataset = open_flushed_dataset(path, session, store_params, flushed_cache, None).await?;
+    // Namespace the session index cache by the (immutable) flushed path so this
+    // sidecar's pages live alongside every other index instead of a bespoke
+    // cache. `fri_uuid` is None — flushed generations carry no fragment-reuse.
+    let index_cache = dataset.index_cache.for_index(&path_cache_uuid(path), None);
+    let index_dir = dataset.base.clone().join(PK_INDEX_DIR);
+    let store: Arc<dyn ScalarIndexStore> = Arc::new(LanceIndexStore::new(
+        dataset.object_store.clone(),
+        index_dir,
+        Arc::new(index_cache.clone()),
+    ));
+
+    let plugin = PK_BTREE_REGISTRY.get_plugin_by_name("BTree")?;
+    // Cache the opened index in the session cache (mirrors `open_scalar_index`).
+    if let Some(index) = plugin
+        .get_from_cache(store.clone(), None, &index_cache)
+        .await?
+    {
+        return Ok(index);
     }
+    let details = prost_types::Any::from_msg(&lance_index::pbold::BTreeIndexDetails::default())
+        .map_err(|e| Error::io(e.to_string()))?;
+    let index = plugin
+        .load_index(store, &details, None, &index_cache)
+        .await?;
+    plugin.put_in_cache(&index_cache, index.clone()).await?;
+    Ok(index)
 }
 
-/// Scan a dataset's PK columns and fold them into a membership set, one batch
-/// resident at a time (no full PK-column buffer).
-async fn scan_pk_hashes(dataset: &Dataset, pk_columns: &[String]) -> Result<HashSet<u64>> {
-    let pk_refs: Vec<&str> = pk_columns.iter().map(String::as_str).collect();
-    let mut scanner = dataset.scan();
-    scanner.project(&pk_refs)?;
-    let mut stream = scanner.try_into_stream().await?;
-    let mut hashes = HashSet::new();
-    while let Some(batch) = stream.try_next().await? {
-        if batch.num_rows() == 0 {
-            continue;
-        }
-        let pk_indices = resolve_pk_indices(&batch, pk_columns)
-            .map_err(|e| lance_core::Error::invalid_input(e.to_string()))?;
-        for row in 0..batch.num_rows() {
-            hashes.insert(compute_pk_hash(&batch, &pk_indices, row));
-        }
+/// Write a flushed generation's standalone PK sidecar at `{uri}/_pk_index` from
+/// `batches`, mirroring what flush does in production. `pk_columns` are the
+/// primary-key column names (field ids are synthesized by position — `insert`
+/// resolves columns by name). A no-op when no batch carries the PK columns.
+///
+/// Used by Rust scanner tests and by the Python test-support binding to stage
+/// faithful flushed generations (a flushed dataset alone, with no sidecar, is
+/// not a state production ever produces).
+pub async fn write_pk_sidecar(
+    uri: &str,
+    batches: &[arrow_array::RecordBatch],
+    pk_columns: &[&str],
+) -> Result<()> {
+    use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+    use lance_core::cache::LanceCache;
+    use lance_index::scalar::btree::train_btree_index;
+    use lance_io::object_store::ObjectStore;
+
+    use crate::dataset::mem_wal::util::pk_index_path;
+
+    let pk: Vec<(String, i32)> = pk_columns
+        .iter()
+        .enumerate()
+        .map(|(i, c)| (c.to_string(), i as i32))
+        .collect();
+    let mut index = IndexStore::new();
+    index.enable_pk_index(&pk);
+    let mut offset = 0u64;
+    for batch in batches {
+        index.insert(batch, offset)?;
+        offset += batch.num_rows() as u64;
     }
-    Ok(hashes)
+
+    let training = index.pk_training_batches(8192)?;
+    if training.is_empty() {
+        return Ok(());
+    }
+    let schema = training[0].schema();
+    let (object_store, base_path) = ObjectStore::from_uri(uri).await?;
+    let store = LanceIndexStore::new(
+        object_store,
+        pk_index_path(&base_path),
+        Arc::new(LanceCache::no_cache()),
+    );
+    let stream = Box::pin(RecordBatchStreamAdapter::new(
+        schema,
+        futures::stream::iter(training.into_iter().map(Ok)),
+    ));
+    // `train_btree_index` now returns the written index files; the sidecar
+    // writer only needs success/failure.
+    train_btree_index(stream, &store, 8192, None, None).await?;
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow_array::Int32Array;
+    use crate::dataset::mem_wal::scanner::data_source::{LsmDataSource, LsmGeneration};
+    use crate::dataset::mem_wal::write::IndexStore;
+    use arrow_array::{Int32Array, RecordBatch};
     use arrow_schema::{DataType, Field, Schema};
     use std::sync::Arc;
+    use uuid::Uuid;
 
     fn id_batch(ids: &[i32]) -> RecordBatch {
         let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
         RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(ids.to_vec()))]).unwrap()
     }
 
-    /// Hash a single Int32 `id` PK the way the planner does, so a test can probe
-    /// a returned blocked set by value.
-    fn hash_id(id: i32) -> u64 {
-        let batch = id_batch(&[id]);
-        let pk_indices = resolve_pk_indices(&batch, &["id".to_string()]).unwrap();
-        compute_pk_hash(&batch, &pk_indices, 0)
-    }
-
-    /// Whether `id`'s PK hash is blocked by any of a source's newer-gen sets.
-    fn blocks(sets: &[Arc<HashSet<u64>>], id: i32) -> bool {
-        sets.iter().any(|s| s.contains(&hash_id(id)))
-    }
-
-    #[test]
-    fn pk_hashes_collapse_within_gen_duplicates() {
-        // Two rows share pk=1 (a within-gen duplicate); pk=2 is unique.
-        let hashes = pk_hashes_from_batches(&[id_batch(&[1, 2, 1])], &["id".to_string()]).unwrap();
-        assert_eq!(hashes.len(), 2); // distinct pks: 1, 2
+    /// An active/frozen memtable source whose PK index holds one row per id in
+    /// `ids` (positions 0..n), all committed and visible.
+    fn active_source(shard: Uuid, generation: u64, ids: &[i32]) -> LsmDataSource {
+        let store = BatchStore::with_capacity(16);
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        for &id in ids {
+            let b = id_batch(&[id]);
+            let (bp, off, _) = store.append(b.clone()).unwrap();
+            index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
+        }
+        LsmDataSource::ActiveMemTable {
+            batch_store: Arc::new(store),
+            index_store: Arc::new(index),
+            schema: id_batch(&[1]).schema(),
+            shard_id: shard,
+            generation: LsmGeneration::memtable(generation),
+        }
     }
 
-    #[test]
-    fn empty_batches_yield_empty_membership() {
-        let hashes = pk_hashes_from_batches(&[id_batch(&[])], &["id".to_string()]).unwrap();
-        assert!(hashes.is_empty());
+    /// Whether `id`'s PK is blocked by any of a source's newer-gen memberships.
+    async fn blocks(memberships: &[GenMembership], id: i32) -> bool {
+        let key = on_disk_pk_key(&[ScalarValue::Int32(Some(id))]).unwrap();
+        for m in memberships {
+            if m.contains(&key).await.unwrap() {
+                return true;
+            }
+        }
+        false
     }
 
     #[test]
-    fn batch_store_membership_collapses_within_gen_dups() {
-        let store = BatchStore::with_capacity(8);
-        // Two single-row batches, both pk=1 (a within-gen update).
-        store.append(id_batch(&[1])).unwrap();
-        store.append(id_batch(&[1])).unwrap();
-        // A two-row batch: pk=2, pk=3.
-        store.append(id_batch(&[2, 3])).unwrap();
-
-        let hashes = pk_hashes_from_batch_store(&store, &["id".to_string()]).unwrap();
-        assert_eq!(hashes.len(), 3); // distinct pks: 1, 2, 3
+    fn on_disk_key_is_typed_for_single_and_binary_for_composite() {
+        // Single-column → the typed value; composite → encoded Binary.
+        let single = [ScalarValue::Int32(Some(7))];
+        assert_eq!(
+            on_disk_pk_key(&single).unwrap(),
+            ScalarValue::Int32(Some(7))
+        );
+        let composite = [ScalarValue::Int32(Some(1)), ScalarValue::from("a")];
+        assert!(matches!(
+            on_disk_pk_key(&composite).unwrap(),
+            ScalarValue::Binary(Some(_))
+        ));
     }
 
     #[tokio::test]
-    async fn fresh_tier_block_list_one_set_per_in_memory_gen() {
-        use crate::dataset::mem_wal::scanner::data_source::{LsmDataSource, LsmGeneration};
-        use crate::dataset::mem_wal::write::IndexStore;
-        use uuid::Uuid;
-
+    async fn fresh_tier_block_list_one_membership_per_in_memory_gen() {
         let shard = Uuid::new_v4();
-        let mk = |ids: &[i32], generation: u64| {
-            let store = BatchStore::with_capacity(8);
-            store.append(id_batch(ids)).unwrap();
-            LsmDataSource::ActiveMemTable {
-                batch_store: Arc::new(store),
-                index_store: Arc::new(IndexStore::new()),
-                schema: id_batch(&[1]).schema(),
-                shard_id: shard,
-                generation: LsmGeneration::memtable(generation),
-            }
-        };
         // Active gen 2: pk=1,2. Frozen gen 1: pk=3.
-        let sources = vec![mk(&[1, 2], 2), mk(&[3], 1)];
+        let sources = vec![
+            active_source(shard, 2, &[1, 2]),
+            active_source(shard, 1, &[3]),
+        ];
 
-        let sets = fresh_tier_block_list(&sources, &["id".to_string()], None, None)
+        let memberships = fresh_tier_block_list(&sources, None, None, None, None)
             .await
             .unwrap();
 
-        // One set per generation; together they cover pk=1,2,3 (not 4).
-        assert_eq!(sets.len(), 2);
+        // One membership per generation; together they cover pk=1,2,3 (not 4).
+        assert_eq!(memberships.len(), 2);
         for id in [1, 2, 3] {
-            assert!(blocks(&sets, id));
+            assert!(blocks(&memberships, id).await);
         }
-        assert!(!blocks(&sets, 4));
+        assert!(!blocks(&memberships, 4).await);
     }
 
     #[tokio::test]
     async fn block_lists_suppress_stale_across_in_memory_gens() {
-        use crate::dataset::mem_wal::scanner::data_source::{LsmDataSource, LsmGeneration};
-        use crate::dataset::mem_wal::write::IndexStore;
-        use uuid::Uuid;
-
         let shard = Uuid::new_v4();
-        let mk = |batches: &[&[i32]], generation: u64| {
-            let store = BatchStore::with_capacity(8);
-            for ids in batches {
-                store.append(id_batch(ids)).unwrap();
-            }
-            LsmDataSource::ActiveMemTable {
-                batch_store: Arc::new(store),
-                index_store: Arc::new(IndexStore::new()),
-                schema: id_batch(&[1]).schema(),
-                shard_id: shard,
-                generation: LsmGeneration::memtable(generation),
-            }
-        };
-
-        // Frozen gen 1: stale pk=1.
-        // Active gen 2: pk=1 re-written, pk=2 new.
-        let sources = vec![mk(&[&[1]], 1), mk(&[&[1], &[2]], 2)];
+        // Frozen gen 1: stale pk=1. Active gen 2: pk=1 re-written, pk=2 new.
+        let sources = vec![
+            active_source(shard, 1, &[1]),
+            active_source(shard, 2, &[1, 2]),
+        ];
 
-        let blocked = Box::pin(compute_source_block_lists(
-            &sources,
-            &["id".to_string()],
-            None,
-            None,
-        ))
-        .await
-        .unwrap();
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
+            .await
+            .unwrap();
 
         let g1 = LsmGeneration::memtable(1);
         let g2 = LsmGeneration::memtable(2);
         // The newer active write supersedes the frozen copy: gen 1 is blocked on
         // pk=1, so its KNN drops pk=1.
-        assert!(blocks(&blocked[&(Some(shard), g1)], 1));
+        assert!(blocks(&blocked[&(Some(shard), g1)], 1).await);
         // The active (newest) generation is superseded by nothing — no entry.
         assert!(!blocked.contains_key(&(Some(shard), g2)));
     }
 
     #[tokio::test]
     async fn block_lists_suppress_stale_base_row() {
-        use crate::dataset::mem_wal::scanner::data_source::{LsmDataSource, LsmGeneration};
-        use crate::dataset::mem_wal::write::IndexStore;
         use crate::dataset::{Dataset, WriteParams};
         use arrow_array::RecordBatchIterator;
-        use uuid::Uuid;
 
         // Base (gen 0): pk=1 (stale), pk=3 (live).
         let base_batch = id_batch(&[1, 3]);
@@ -372,89 +591,239 @@ mod tests {
         );
 
         // Active gen 1: pk=1 re-written, pk=2 new.
-        let store = BatchStore::with_capacity(8);
-        store.append(id_batch(&[1])).unwrap();
-        store.append(id_batch(&[2])).unwrap();
-
         let sources = vec![
             LsmDataSource::BaseTable { dataset: base },
-            LsmDataSource::ActiveMemTable {
-                batch_store: Arc::new(store),
-                index_store: Arc::new(IndexStore::new()),
-                schema,
-                shard_id: Uuid::new_v4(),
-                generation: LsmGeneration::memtable(1),
-            },
+            active_source(Uuid::new_v4(), 1, &[1, 2]),
         ];
 
-        let blocked = Box::pin(compute_source_block_lists(
-            &sources,
-            &["id".to_string()],
-            None,
-            None,
-        ))
-        .await
-        .unwrap();
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
+            .await
+            .unwrap();
 
         // Base is blocked by every newer gen: pk=1 (re-written in gen 1) is
-        // blocked, pk=3 (base-only) is not. End-to-end drop: vector_search specs.
+        // blocked, pk=3 (base-only) is not.
         let base_blocked = blocked
             .get(&(None, LsmGeneration::BASE_TABLE))
             .expect("base has a blocked set");
-        assert!(blocks(base_blocked, 1));
-        assert!(!blocks(base_blocked, 3));
+        assert!(blocks(base_blocked, 1).await);
+        assert!(!blocks(base_blocked, 3).await);
     }
 
     #[tokio::test]
     async fn block_lists_are_keyed_per_shard() {
         // Regression: generations are per-shard, so a source must only be blocked
-        // by newer generations of its OWN shard. A generation-only key would
-        // cross-block same-generation sources from different shards.
-        use crate::dataset::mem_wal::scanner::data_source::{LsmDataSource, LsmGeneration};
-        use crate::dataset::mem_wal::write::IndexStore;
-        use uuid::Uuid;
-
-        let mk = |shard: Uuid, ids: &[i32], generation: u64| {
-            let store = BatchStore::with_capacity(8);
-            store.append(id_batch(ids)).unwrap();
-            LsmDataSource::ActiveMemTable {
-                batch_store: Arc::new(store),
-                index_store: Arc::new(IndexStore::new()),
-                schema: id_batch(&[1]).schema(),
-                shard_id: shard,
-                generation: LsmGeneration::memtable(generation),
-            }
-        };
-
-        // Two shards, each: frozen gen 1 (stale) + active gen 2 (re-write).
-        // Shard A keys pk=1; shard B keys pk=2 (disjoint partitions).
+        // by newer generations of its OWN shard.
         let a = Uuid::new_v4();
         let b = Uuid::new_v4();
+        // Two shards, each: frozen gen 1 (stale) + active gen 2 (re-write).
+        // Shard A keys pk=1; shard B keys pk=2 (disjoint partitions).
         let sources = vec![
-            mk(a, &[1], 1),
-            mk(a, &[1], 2),
-            mk(b, &[2], 1),
-            mk(b, &[2], 2),
+            active_source(a, 1, &[1]),
+            active_source(a, 2, &[1]),
+            active_source(b, 1, &[2]),
+            active_source(b, 2, &[2]),
         ];
 
-        let blocked = Box::pin(compute_source_block_lists(
-            &sources,
-            &["id".to_string()],
-            None,
-            None,
-        ))
-        .await
-        .unwrap();
+        let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None))
+            .await
+            .unwrap();
 
         let g1 = LsmGeneration::memtable(1);
         let g2 = LsmGeneration::memtable(2);
         // Each shard's gen 1 is blocked by its OWN gen 2 only.
-        assert!(blocks(&blocked[&(Some(a), g1)], 1));
-        assert!(!blocks(&blocked[&(Some(a), g1)], 2));
-        assert!(blocks(&blocked[&(Some(b), g1)], 2));
-        assert!(!blocks(&blocked[&(Some(b), g1)], 1));
+        assert!(blocks(&blocked[&(Some(a), g1)], 1).await);
+        assert!(!blocks(&blocked[&(Some(a), g1)], 2).await);
+        assert!(blocks(&blocked[&(Some(b), g1)], 2).await);
+        assert!(!blocks(&blocked[&(Some(b), g1)], 1).await);
         // The newest generation of each shard is superseded by nothing.
         assert!(!blocked.contains_key(&(Some(a), g2)));
         assert!(!blocked.contains_key(&(Some(b), g2)));
     }
+
+    #[tokio::test]
+    async fn index_membership_is_snapshot_bounded() {
+        // The index-sourced membership only counts a PK whose version is visible
+        // at the source's watermark, so a newer generation's not-yet-visible
+        // write can't shadow an older generation's visible copy.
+        let shard = Uuid::new_v4();
+        let schema = id_batch(&[1]).schema();
+
+        // Older frozen gen 1: pk=1.
+        let g1 = active_source(shard, 1, &[1]);
+
+        // Newer active gen 2: pk=99 visible at position 0, then pk=1 written at
+        // position 1 but with the watermark left at batch 0 (so pk=1 is in the
+        // index yet not visible) — the concurrent-write race.
+        let g2_store = BatchStore::with_capacity(8);
+        let mut g2_index = IndexStore::new();
+        g2_index.enable_pk_index(&[("id".to_string(), 0)]);
+        let b0 = id_batch(&[99]);
+        let (bp0, off0, _) = g2_store.append(b0.clone()).unwrap();
+        g2_index
+            .insert_with_batch_position(&b0, off0, Some(bp0)) // advances watermark to 0
+            .unwrap();
+        let b1 = id_batch(&[1]);
+        let (_, off1, _) = g2_store.append(b1.clone()).unwrap();
+        g2_index
+            .insert_with_batch_position(&b1, off1, None) // index updated, watermark unchanged
+            .unwrap();
+        let g2 = LsmDataSource::ActiveMemTable {
+            batch_store: Arc::new(g2_store),
+            index_store: Arc::new(g2_index),
+            schema,
+            shard_id: shard,
+            generation: LsmGeneration::memtable(2),
+        };
+
+        let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None, None))
+            .await
+            .unwrap();
+
+        let g1_block = &blocked[&(Some(shard), LsmGeneration::memtable(1))];
+        // pk=99 is visible in gen 2 → it blocks gen 1's pk=99.
+        assert!(blocks(g1_block, 99).await);
+        // pk=1's only gen-2 copy is not yet visible → it must NOT shadow gen 1.
+        assert!(
+            !blocks(g1_block, 1).await,
+            "a not-yet-visible newer write must not shadow an older visible copy"
+        );
+    }
+
+    /// A fresh-tier watermark bounds the active generation to the first
+    /// `active_batch_count` batches — those the arm observed before the memtable
+    /// grew. A later append is invisible, so a base row is never dropped without
+    /// the arm having delivered its replacement.
+    #[tokio::test]
+    async fn fresh_tier_watermark_bounds_active_memtable_by_batch_count() {
+        use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark;
+        use std::collections::HashMap;
+
+        let shard = Uuid::new_v4();
+        // Three single-row batches: pk=1 at batch 0, pk=2 at batch 1, pk=3 at
+        // batch 2 (appended after the arm).
+        let sources = vec![active_source(shard, 1, &[1, 2, 3])];
+
+        // Watermark at 2 batches of gen 1: pk=1,2 are members; pk=3 (batch 2) is not.
+        let watermarks: HashMap<Uuid, FreshTierWatermark> = [(
+            shard,
+            FreshTierWatermark {
+                active_generation: 1,
+                active_batch_count: 2,
+            },
+        )]
+        .into_iter()
+        .collect();
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks))
+            .await
+            .unwrap();
+        assert!(blocks(&sets, 1).await);
+        assert!(blocks(&sets, 2).await);
+        assert!(!blocks(&sets, 3).await);
+
+        // No watermark → live tier: all three are members.
+        let sets = fresh_tier_block_list(&sources, None, None, None, None)
+            .await
+            .unwrap();
+        for id in [1, 2, 3] {
+            assert!(blocks(&sets, id).await);
+        }
+    }
+
+    /// A generation above the active one rolled in after the snapshot and is
+    /// excluded whole; a lower one is immutable (frozen) and included whole
+    /// regardless of the active batch count.
+    #[tokio::test]
+    async fn fresh_tier_watermark_excludes_newer_gen_includes_lower_gen() {
+        use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark;
+        use std::collections::HashMap;
+
+        let shard = Uuid::new_v4();
+        // gen 3 newer (after snapshot), gen 2 == active (bounded to 1 batch),
+        // gen 1 lower/immutable (whole). Each id is its own batch.
+        let sources = vec![
+            active_source(shard, 3, &[100]),
+            active_source(shard, 2, &[20, 21]),
+            active_source(shard, 1, &[1, 2]),
+        ];
+
+        let watermarks: HashMap<Uuid, FreshTierWatermark> = [(
+            shard,
+            FreshTierWatermark {
+                active_generation: 2,
+                active_batch_count: 1,
+            },
+        )]
+        .into_iter()
+        .collect();
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks))
+            .await
+            .unwrap();
+        assert!(blocks(&sets, 1).await); // gen 1, whole
+        assert!(blocks(&sets, 2).await); // gen 1, whole
+        assert!(blocks(&sets, 20).await); // gen 2, batch 0
+        assert!(!blocks(&sets, 21).await); // gen 2, batch 1 — past the watermark
+        assert!(!blocks(&sets, 100).await); // gen 3 — after the snapshot
+    }
+
+    /// A flushed generation at or above the active generation was produced by a
+    /// flush after the snapshot and is excluded; one strictly below it is
+    /// immutable and included.
+    #[tokio::test]
+    async fn fresh_tier_watermark_excludes_flushed_at_or_above_active() {
+        use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark;
+        use crate::dataset::{Dataset, WriteParams};
+        use arrow_array::RecordBatchIterator;
+        use std::collections::HashMap;
+
+        // A flushed generation 2 holding pk=5, staged as a flushed dataset with
+        // its standalone PK sidecar (what the on-disk membership probes).
+        let flushed_batch = id_batch(&[5]);
+        let schema = flushed_batch.schema();
+        let tmp = tempfile::tempdir().unwrap();
+        let path = format!("{}/gen2", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(flushed_batch.clone())], schema.clone());
+        Dataset::write(reader, &path, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        write_pk_sidecar(&path, &[flushed_batch], &["id"])
+            .await
+            .unwrap();
+
+        let shard = Uuid::new_v4();
+        let sources = vec![LsmDataSource::FlushedMemTable {
+            path,
+            shard_id: shard,
+            generation: LsmGeneration::memtable(2),
+        }];
+
+        // active_generation 2 (gen 2 flushed at/after the snapshot): excluded.
+        let at: HashMap<Uuid, FreshTierWatermark> = [(
+            shard,
+            FreshTierWatermark {
+                active_generation: 2,
+                active_batch_count: u64::MAX,
+            },
+        )]
+        .into_iter()
+        .collect();
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&at))
+            .await
+            .unwrap();
+        assert!(!blocks(&sets, 5).await);
+
+        // active_generation 3 (gen 2 strictly below, immutable): included.
+        let above: HashMap<Uuid, FreshTierWatermark> = [(
+            shard,
+            FreshTierWatermark {
+                active_generation: 3,
+                active_batch_count: u64::MAX,
+            },
+        )]
+        .into_iter()
+        .collect();
+        let sets = fresh_tier_block_list(&sources, None, None, None, Some(&above))
+            .await
+            .unwrap();
+        assert!(blocks(&sets, 5).await);
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
index ade4164d485..b433ad911d4 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs
@@ -7,25 +7,51 @@ use std::collections::HashMap;
 use std::collections::hash_map::Entry;
 use std::sync::Arc;
 
-use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+use arrow_array::cast::AsArray;
+use arrow_array::types::Float32Type;
+use arrow_array::{Array, FixedSizeListArray, RecordBatch};
+use arrow_schema::{DataType, SchemaRef};
 use datafusion::common::ScalarValue;
-use datafusion::common::ToDFSchema;
 use datafusion::logical_expr::Operator;
 use datafusion::physical_plan::limit::GlobalLimitExec;
 use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream};
 use datafusion::prelude::{Expr, SessionContext};
 use futures::TryStreamExt;
 use lance_core::{Error, Result, is_system_column};
+use lance_index::scalar::FullTextSearchQuery;
+use lance_linalg::distance::DistanceType;
 use uuid::Uuid;
 
 use super::collector::{InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector};
-use super::data_source::ShardSnapshot;
-use super::flushed_cache::FlushedMemTableCache;
+use super::data_source::{FreshTierWatermark, ShardSnapshot};
+use super::flushed_cache::{DatasetCache, GenerationWarmer};
 use super::planner::LsmScanPlanner;
 use super::point_lookup::LsmPointLookupPlanner;
+use super::projection::validate_projection_names;
 use crate::dataset::Dataset;
+use crate::dataset::mem_wal::util::derived_store_params;
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
+
+/// Vector (KNN) search state, set by [`LsmScanner::nearest`] and friends. Mirrors
+/// the subset of `lance::dataset::scanner::Query` the LSM vector planner honors.
+#[derive(Debug, Clone)]
+struct LsmVectorQuery {
+    /// Vector column to search.
+    column: String,
+    /// Flat query vector (one vector; validated against the column dim in `create_plan`).
+    key: Arc<dyn Array>,
+    /// Number of nearest neighbors to fetch per source before the global merge.
+    k: usize,
+    /// Number of IVF partitions to probe on the base arm.
+    nprobes: usize,
+    /// Re-rank base candidates with exact distances when set (refine factor is
+    /// treated as a boolean; the LSM merge needs exact base distances).
+    refine: bool,
+    /// Distance metric; `None` defaults to L2 (matching the unindexed memtable arm).
+    metric_type: Option<DistanceType>,
+}
 
 /// If `filter` is a point-lookup shape on `pk_col` — `pk = lit` (either
 /// operand order) or `pk IN (lit, …)` — return the literal key values. Any
@@ -72,6 +98,60 @@ enum BaseSource {
     PathOnly(String),
 }
 
+/// The fixed-size-list dimension of `column` in `schema`. Errors if the column
+/// is missing or is not a fixed-size-list vector column.
+fn vector_dim(schema: &arrow_schema::Schema, column: &str) -> Result<i32> {
+    let field = schema
+        .field_with_name(column)
+        .map_err(|_| Error::invalid_input(format!("vector column '{}' not found", column)))?;
+    match field.data_type() {
+        DataType::FixedSizeList(_, dim) => Ok(*dim),
+        other => Err(Error::invalid_input(format!(
+            "column '{}' is not a fixed-size-list vector (got {:?})",
+            column, other
+        ))),
+    }
+}
+
+/// Wrap a flat `Float32` query vector into a single-row `FixedSizeListArray` of
+/// `dim`, as the vector planner expects. A pre-built single-row FSL passes
+/// through unchanged. The query length must match the column dimension.
+fn key_to_fsl(key: &dyn Array, dim: i32) -> Result<FixedSizeListArray> {
+    if let Some(fsl) = key.as_any().downcast_ref::<FixedSizeListArray>() {
+        // The LSM cross-source merge assumes a single query vector; a multi-row
+        // FSL would otherwise flow through and silently produce wrong results.
+        if fsl.len() != 1 {
+            return Err(Error::invalid_input(format!(
+                "LSM vector search supports a single query vector, got {} rows",
+                fsl.len()
+            )));
+        }
+        if fsl.value_length() != dim {
+            return Err(Error::invalid_input(format!(
+                "query vector dimension {} does not match column dimension {}",
+                fsl.value_length(),
+                dim
+            )));
+        }
+        return Ok(fsl.clone());
+    }
+    let values = key
+        .as_primitive_opt::<Float32Type>()
+        .ok_or_else(|| Error::invalid_input("query vector must be Float32".to_string()))?;
+    if values.len() != dim as usize {
+        return Err(Error::invalid_input(format!(
+            "query vector dimension {} does not match column dimension {}",
+            values.len(),
+            dim
+        )));
+    }
+    let mut builder =
+        FixedSizeListBuilder::with_capacity(Float32Builder::with_capacity(dim as usize), dim, 1);
+    builder.values().append_slice(values.values());
+    builder.append(true);
+    Ok(builder.finish())
+}
+
 /// Scanner for LSM tree data spanning base table, flushed MemTables, and active MemTable.
 ///
 /// This scanner provides a unified interface for querying data across multiple
@@ -87,12 +167,18 @@ enum BaseSource {
 ///
 /// ```ignore
 /// let scanner = LsmScanner::new(base_table, shard_snapshots, vec!["pk".to_string()])
-///     .project(&["id", "name"])
+///     .project(&["id", "name"])?
 ///     .filter("id > 10")?
-///     .limit(100, None);
+///     .limit(Some(100), None)?;
 ///
 /// let results = scanner.try_into_batch().await?;
 /// ```
+///
+/// The query-building methods mirror [`crate::dataset::scanner::Scanner`]:
+/// [`Self::nearest`] (+ [`Self::nprobes`] / [`Self::refine`] /
+/// [`Self::distance_metric`]) for vector search and [`Self::full_text_search`]
+/// for FTS are state setters, and [`Self::create_plan`] dispatches to the right
+/// planner — so an LSM read reads like a normal scan.
 pub struct LsmScanner {
     // Data sources
     base: BaseSource,
@@ -110,6 +196,10 @@ pub struct LsmScanner {
     filter: Option<Expr>,
     limit: Option<usize>,
     offset: Option<usize>,
+    /// Vector (KNN) search; when set, `create_plan` routes to the vector planner.
+    nearest: Option<LsmVectorQuery>,
+    /// Full-text search; when set, `create_plan` routes to the FTS planner.
+    full_text_query: Option<FullTextSearchQuery>,
 
     // Internal columns
     with_row_address: bool,
@@ -118,13 +208,20 @@ pub struct LsmScanner {
     // Primary key columns (required for deduplication)
     pk_columns: Vec<String>,
 
-    /// Session threaded into flushed-generation opens so the first open of
-    /// each generation populates the shared index / file-metadata caches.
-    /// Defaults to the base table's session when one is present.
+    /// Session for opening flushed generations (shares the base's caches).
+    /// Defaults to the base table's session.
     session: Option<Arc<Session>>,
+    /// Store params for opening flushed generations, reusing the base dataset's
+    /// store. Defaults to the base table's params.
+    store_params: Option<ObjectStoreParams>,
     /// Cache of opened flushed-generation datasets. When set, repeated
     /// queries against the same generation skip the manifest read entirely.
-    flushed_cache: Option<Arc<FlushedMemTableCache>>,
+    flushed_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of a flushed generation.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Over-fetch multiple for block-listed sources in search plans
+    /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]).
+    overfetch_factor: Option<f64>,
 }
 
 impl LsmScanner {
@@ -146,6 +243,10 @@ impl LsmScanner {
         // the shared index / metadata caches without extra wiring. An
         // explicit `with_session` still overrides this.
         let session = Some(base_table.session());
+        // The scanner only ever opens flushed generations with these — the base
+        // table is already open and handed in — so they must not carry a
+        // path-bound store binding.
+        let store_params = base_table.store_params().map(derived_store_params);
         Self {
             base: BaseSource::Table(base_table),
             schema: Arc::new(arrow_schema),
@@ -155,11 +256,16 @@ impl LsmScanner {
             filter: None,
             limit: None,
             offset: None,
+            nearest: None,
+            full_text_query: None,
             with_row_address: false,
             with_memtable_gen: false,
             pk_columns,
             session,
+            store_params,
             flushed_cache: None,
+            warmer: None,
+            overfetch_factor: None,
         }
     }
 
@@ -193,11 +299,16 @@ impl LsmScanner {
             filter: None,
             limit: None,
             offset: None,
+            nearest: None,
+            full_text_query: None,
             with_row_address: false,
             with_memtable_gen: false,
             pk_columns,
             session: None,
+            store_params: None,
             flushed_cache: None,
+            warmer: None,
+            overfetch_factor: None,
         }
     }
 
@@ -230,52 +341,67 @@ impl LsmScanner {
         self
     }
 
-    /// Thread an existing session into flushed-generation opens.
-    ///
-    /// The first open of each flushed generation then populates the shared
-    /// index / file-metadata caches, so later queries skip re-decoding them.
-    /// When a base table is configured this defaults to its session; call
-    /// this to override (e.g. on a fresh-tier-only scanner that owns its own
-    /// long-lived session).
+    /// Set the session used to open flushed generations. Defaults to the base
+    /// table's; set explicitly on a fresh-tier-only scanner (no base table).
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
+    /// Set the store params used to open flushed generations. Defaults to the
+    /// base table's; set explicitly on a fresh-tier-only scanner (no base table).
+    ///
+    /// Pass the params the *base* was opened with. As in [`Self::new`], they are
+    /// adapted for generation URIs: a path-bound `object_store` binding would
+    /// redirect every generation open at the base table itself, so it is dropped
+    /// while storage options, wrapper, and credentials carry over.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(derived_store_params(&store_params));
+        self
+    }
+
     /// Inject a cache of opened flushed-generation datasets.
     ///
     /// With a cache, repeated queries against the same generation become a
     /// pure `Arc::clone` with no manifest read or object-store I/O. The cache
-    /// is owned and sized by the caller (see [`FlushedMemTableCache`]); not
-    /// set by default, so behavior is unchanged unless opted in.
-    pub fn with_flushed_cache(mut self, cache: Arc<FlushedMemTableCache>) -> Self {
+    /// is owned and sized by the caller (any [`DatasetCache`] impl, e.g.
+    /// [`FlushedMemTableCache`](super::FlushedMemTableCache)); not set by
+    /// default, so behavior is unchanged unless opted in.
+    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
         self.flushed_cache = Some(cache);
         self
     }
 
+    /// Inject the warmer fired on first open of a flushed generation. Not set by
+    /// default, so behavior is unchanged unless opted in.
+    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+        self.warmer = Some(warmer);
+        self
+    }
+
+    /// Set the over-fetch multiple block-listed sources use in search plans
+    /// so they still yield `k` live rows after cross-generation dedup.
+    /// Threaded into the LSM search planners; values below `1.0` are rejected
+    /// when the plan is created.
+    pub fn with_overfetch_factor(mut self, factor: f64) -> Self {
+        self.overfetch_factor = Some(factor);
+        self
+    }
+
     /// Project specific columns.
     ///
     /// If not called, all columns from the base schema are included.
     /// Primary key columns are always included for deduplication.
-    pub fn project(mut self, columns: &[&str]) -> Self {
-        self.projection = Some(columns.iter().map(|s| s.to_string()).collect());
-        self
+    pub fn project<T: AsRef<str>>(mut self, columns: &[T]) -> Result<Self> {
+        self.projection = Some(columns.iter().map(|s| s.as_ref().to_string()).collect());
+        Ok(self)
     }
 
     /// Set filter expression using SQL-like syntax.
     ///
     /// The filter is pushed down to each data source when possible.
     pub fn filter(mut self, filter_expr: &str) -> Result<Self> {
-        let ctx = SessionContext::new();
-        let df_schema = self
-            .schema
-            .as_ref()
-            .clone()
-            .to_dfschema()
-            .map_err(|e| Error::invalid_input(format!("Failed to create DFSchema: {}", e)))?;
-        let expr = ctx.parse_sql_expr(filter_expr, &df_schema).map_err(|e| {
-            Error::invalid_input(format!("Failed to parse filter expression: {}", e))
-        })?;
+        let expr = super::parse_filter_expr(self.schema.as_ref(), filter_expr)?;
         self.filter = Some(expr);
         Ok(self)
     }
@@ -286,10 +412,80 @@ impl LsmScanner {
         self
     }
 
-    /// Limit the number of results.
-    pub fn limit(mut self, limit: usize, offset: Option<usize>) -> Self {
-        self.limit = Some(limit);
-        self.offset = offset;
+    /// Limit the number of results, with an optional offset. Matches
+    /// [`crate::dataset::scanner::Scanner::limit`]: both bounds are `Option<i64>`
+    /// and must be non-negative.
+    pub fn limit(mut self, limit: Option<i64>, offset: Option<i64>) -> Result<Self> {
+        if let Some(value) = limit
+            && value < 0
+        {
+            return Err(Error::invalid_input(
+                "limit must be non-negative".to_string(),
+            ));
+        }
+        if let Some(value) = offset
+            && value < 0
+        {
+            return Err(Error::invalid_input(
+                "offset must be non-negative".to_string(),
+            ));
+        }
+        self.limit = limit.map(|value| value as usize);
+        self.offset = offset.map(|value| value as usize);
+        Ok(self)
+    }
+
+    /// Find the `k` nearest neighbors of `key` in `column`. Routes `create_plan`
+    /// through the LSM vector planner (base ∪ flushed ∪ in-memory). Mirrors
+    /// [`crate::dataset::scanner::Scanner::nearest`]; the LSM path supports a
+    /// single Float32 query vector. When combined with an offset, the LSM path
+    /// fetches `k + offset` per source before applying the final page. Tune with
+    /// [`Self::nprobes`], [`Self::refine`], and [`Self::distance_metric`].
+    pub fn nearest(mut self, column: &str, key: &dyn Array, k: usize) -> Result<Self> {
+        if k == 0 {
+            return Err(Error::invalid_input("k must be positive".to_string()));
+        }
+        if key.is_empty() {
+            return Err(Error::invalid_input(
+                "query vector must have non-zero length".to_string(),
+            ));
+        }
+        self.nearest = Some(LsmVectorQuery {
+            column: column.to_string(),
+            key: key.slice(0, key.len()),
+            k,
+            nprobes: 1,
+            refine: false,
+            metric_type: None,
+        });
+        Ok(self)
+    }
+
+    /// Number of IVF partitions to probe on the base arm (default 1). No-op
+    /// unless [`Self::nearest`] was called.
+    pub fn nprobes(mut self, nprobes: usize) -> Self {
+        if let Some(q) = self.nearest.as_mut() {
+            q.nprobes = nprobes;
+        }
+        self
+    }
+
+    /// Re-rank base-table candidates with exact distances so they are directly
+    /// comparable to the exact memtable distances in the cross-source merge.
+    /// The factor is treated as a boolean today. No-op unless `nearest` was set.
+    pub fn refine(mut self, refine_factor: u32) -> Self {
+        if let Some(q) = self.nearest.as_mut() {
+            q.refine = refine_factor > 0;
+        }
+        self
+    }
+
+    /// Distance metric for the vector search. Defaults to L2. No-op unless
+    /// [`Self::nearest`] was called.
+    pub fn distance_metric(mut self, metric: DistanceType) -> Self {
+        if let Some(q) = self.nearest.as_mut() {
+            q.metric_type = Some(metric);
+        }
         self
     }
 
@@ -329,8 +525,169 @@ impl LsmScanner {
 
     /// Create the execution plan.
     pub async fn create_plan(&self) -> Result<Arc<dyn ExecutionPlan>> {
+        // Dispatch by builder state, mirroring `Scanner::create_plan` and
+        // `MemTableScanner::create_plan`: vector search, then full-text search,
+        // then the (point-lookup or union) plain scan.
+        // The LSM path has no hybrid mode, so reject combined search rather than
+        // silently dropping the full-text query under the vector dispatch.
+        if self.nearest.is_some() && self.full_text_query.is_some() {
+            return Err(Error::invalid_input(
+                "LSM scanner does not support combined vector and full-text search".to_string(),
+            ));
+        }
+        if self.nearest.is_some() {
+            return self.plan_vector().await;
+        }
+        if self.full_text_query.is_some() {
+            return self.plan_fts().await;
+        }
+        self.plan_scan().await
+    }
+
+    /// Apply the builder's `limit`/`offset` on top of a search plan. The plain
+    /// scan applies them inside the planner; the vector/FTS arms wrap here.
+    fn apply_limit_offset(&self, plan: Arc<dyn ExecutionPlan>) -> Arc<dyn ExecutionPlan> {
+        let skip = self.offset.unwrap_or(0);
+        if skip == 0 && self.limit.is_none() {
+            return plan;
+        }
+        Arc::new(GlobalLimitExec::new(plan, skip, self.limit))
+    }
+
+    /// Vector (KNN) search across base ∪ flushed ∪ in-memory, via the LSM vector
+    /// planner. Honors the builder filter as a prefilter.
+    async fn plan_vector(&self) -> Result<Arc<dyn ExecutionPlan>> {
+        let nearest = self
+            .nearest
+            .as_ref()
+            .expect("plan_vector requires a nearest query");
+        let base_schema = self.schema();
+        let dim = vector_dim(base_schema.as_ref(), &nearest.column)?;
+        let query_fsl = key_to_fsl(nearest.key.as_ref(), dim)?;
+        let distance_type = nearest.metric_type.unwrap_or(DistanceType::L2);
+
+        let collector = self.build_collector();
+        let mut planner = super::LsmVectorSearchPlanner::new(
+            collector,
+            self.pk_columns.clone(),
+            base_schema,
+            nearest.column.clone(),
+            distance_type,
+        )
+        .with_filter(self.filter.clone());
+        if let BaseSource::Table(dataset) = &self.base {
+            planner = planner.with_dataset(dataset.clone());
+        }
+        if let Some(session) = &self.session {
+            planner = planner.with_session(session.clone());
+        }
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
+        if let Some(cache) = &self.flushed_cache {
+            planner = planner.with_flushed_cache(cache.clone());
+        }
+        if let Some(warmer) = &self.warmer {
+            planner = planner.with_warmer(warmer.clone());
+        }
+        // Over-fetch by `offset` so the per-source top-k still yields `k` live
+        // rows after `apply_limit_offset` skips the first `offset` (mirrors the
+        // FTS arm, which folds `offset` into its `k`).
+        let per_source_k = nearest.k.saturating_add(self.offset.unwrap_or(0));
+        let overfetch_factor = self.overfetch_factor.unwrap_or(1.0);
+        let plan = planner
+            .plan_search(
+                &query_fsl,
+                per_source_k,
+                nearest.nprobes,
+                self.projection.as_deref(),
+                nearest.refine,
+                overfetch_factor,
+            )
+            .await?;
+        Ok(self.apply_limit_offset(plan))
+    }
+
+    /// Full-text search across base ∪ flushed ∪ in-memory, via the LSM FTS
+    /// planner. Query/scanner limits bound per-source fetches when present;
+    /// otherwise the search remains unbounded and any offset is applied above.
+    async fn plan_fts(&self) -> Result<Arc<dyn ExecutionPlan>> {
+        let query = self
+            .full_text_query
+            .as_ref()
+            .expect("plan_fts requires a full-text query");
+        let columns: Vec<String> = query.columns().into_iter().collect();
+        if columns.len() > 1 {
+            return Err(Error::invalid_input(
+                "LSM full-text search supports a single column".to_string(),
+            ));
+        }
+        let column = columns.into_iter().next().ok_or_else(|| {
+            Error::invalid_input(
+                "full_text_search requires a column; set it with `FullTextSearchQuery::with_column`"
+                    .to_string(),
+            )
+        })?;
+        let base_schema = self.schema();
+        base_schema.field_with_name(&column).map_err(|_| {
+            Error::invalid_input(format!("Column '{}' not found in schema", column))
+        })?;
+        let query_limit = query
+            .limit
+            .map(|limit| {
+                if limit < 0 {
+                    Err(Error::invalid_input(
+                        "full-text search limit must be non-negative".to_string(),
+                    ))
+                } else {
+                    Ok(limit as usize)
+                }
+            })
+            .transpose()?;
+        // Match the dataset Scanner: a query-level FTS limit is the hard
+        // search cap, and scanner offset pages within that capped result set.
+        let source_limit = match query_limit {
+            Some(limit) => Some(limit),
+            None => self
+                .limit
+                .map(|limit| limit.saturating_add(self.offset.unwrap_or(0))),
+        };
+
+        let collector = self.build_collector();
+        let mut planner =
+            super::LsmFtsSearchPlanner::new(collector, self.pk_columns.clone(), base_schema)
+                .with_filter(self.filter.clone());
+        if let Some(session) = &self.session {
+            planner = planner.with_session(session.clone());
+        }
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
+        if let Some(cache) = &self.flushed_cache {
+            planner = planner.with_flushed_cache(cache.clone());
+        }
+        if let Some(warmer) = &self.warmer {
+            planner = planner.with_warmer(warmer.clone());
+        }
+        if let Some(factor) = self.overfetch_factor {
+            planner = planner.with_overfetch_factor(factor);
+        }
+        let plan = planner
+            .plan_search(
+                &column,
+                query.clone(),
+                source_limit,
+                self.projection.as_deref(),
+            )
+            .await?;
+        Ok(self.apply_limit_offset(plan))
+    }
+
+    /// Plain (filter / projection / limit) scan over base ∪ flushed ∪ in-memory.
+    async fn plan_scan(&self) -> Result<Arc<dyn ExecutionPlan>> {
         let collector = self.build_collector();
         let base_schema = self.schema();
+        validate_projection_names(self.projection.as_deref(), &base_schema, &[])?;
 
         // Fast point-lookup routing: a `pk = lit` / `pk IN (..)` filter on the
         // single pk column — with no offset and no scan-only system columns —
@@ -351,9 +708,15 @@ impl LsmScanner {
             if let Some(session) = &self.session {
                 planner = planner.with_session(session.clone());
             }
+            if let Some(store_params) = &self.store_params {
+                planner = planner.with_store_params(store_params.clone());
+            }
             if let Some(cache) = &self.flushed_cache {
                 planner = planner.with_flushed_cache(cache.clone());
             }
+            if let Some(warmer) = &self.warmer {
+                planner = planner.with_warmer(warmer.clone());
+            }
             let plan = planner
                 .plan_point_lookup(&keys, self.projection.as_deref())
                 .await?;
@@ -367,9 +730,18 @@ impl LsmScanner {
         if let Some(session) = &self.session {
             planner = planner.with_session(session.clone());
         }
+        if let Some(store_params) = &self.store_params {
+            planner = planner.with_store_params(store_params.clone());
+        }
         if let Some(cache) = &self.flushed_cache {
             planner = planner.with_flushed_cache(cache.clone());
         }
+        if let Some(warmer) = &self.warmer {
+            planner = planner.with_warmer(warmer.clone());
+        }
+        if let Some(factor) = self.overfetch_factor {
+            planner = planner.with_overfetch_factor(factor);
+        }
 
         planner
             .plan_scan(
@@ -383,31 +755,17 @@ impl LsmScanner {
             .await
     }
 
-    /// Build a local-scoring FTS plan spanning base + flushed + active sources.
-    ///
-    /// Routes through [`super::LsmFtsSearchPlanner`]. Output schema is
-    /// `projection ∪ pk_columns + _score`; per-source local BM25 `_score`
-    /// is merged DESC and capped at `k`. `column` must be FTS-indexed on
-    /// the queried sources.
-    pub async fn full_text_search(
-        &self,
-        column: &str,
-        query: lance_index::scalar::FullTextSearchQuery,
-        k: usize,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        let collector = self.build_collector();
-        let base_schema = self.schema();
-        let mut planner =
-            super::LsmFtsSearchPlanner::new(collector, self.pk_columns.clone(), base_schema);
-        if let Some(session) = &self.session {
-            planner = planner.with_session(session.clone());
-        }
-        if let Some(cache) = &self.flushed_cache {
-            planner = planner.with_flushed_cache(cache.clone());
-        }
-        planner
-            .plan_search(column, query, k, self.projection.as_deref())
-            .await
+    /// Find rows matching a full-text query. Routes `create_plan` through the
+    /// LSM FTS planner (base ∪ flushed ∪ in-memory), local-scored by BM25 and
+    /// merged by `_score` DESC. Mirrors
+    /// [`crate::dataset::scanner::Scanner::full_text_search`]: the searched
+    /// column(s) come from the query (set via `FullTextSearchQuery::with_column`);
+    /// the query limit, when present, controls the FTS source/merge top-k;
+    /// scanner limit/offset still apply to the final merged result. The LSM
+    /// path supports a single FTS-indexed column.
+    pub fn full_text_search(mut self, query: FullTextSearchQuery) -> Result<Self> {
+        self.full_text_query = Some(query);
+        Ok(self)
     }
 
     /// Execute the scan and return a stream of record batches.
@@ -422,14 +780,14 @@ impl LsmScanner {
     /// Execute the scan and collect all results into a single RecordBatch.
     pub async fn try_into_batch(&self) -> Result<RecordBatch> {
         let stream = self.try_into_stream().await?;
+        let output_schema = stream.schema();
         let batches: Vec<RecordBatch> = stream
             .try_collect()
             .await
             .map_err(|e| Error::io(format!("Failed to collect batches: {}", e)))?;
 
         if batches.is_empty() {
-            let schema = self.schema();
-            return Ok(RecordBatch::new_empty(schema));
+            return Ok(RecordBatch::new_empty(output_schema));
         }
 
         let schema = batches[0].schema();
@@ -454,24 +812,66 @@ impl LsmScanner {
     /// the primary-key columns; the returned `Vec<bool>` is aligned with its
     /// rows. Hashing matches the scanner's internal dedup, so the caller never
     /// hashes PKs itself. Flushed membership comes from the injected
-    /// [`FlushedMemTableCache`] when one is set.
+    /// [`DatasetCache`] when one is set.
     pub async fn contains_pks(&self, pks: &RecordBatch) -> Result<Vec<bool>> {
+        self.contains_pks_at(pks, None).await
+    }
+
+    /// As-of variant of [`Self::contains_pks`]. Membership is evaluated against
+    /// a per-shard watermark on the fresh tier, supplied via `watermarks` (see
+    /// [`FreshTierWatermark`]), matching the tier a prior scan observed and
+    /// avoiding the two-snapshot skew that would drop a base row with no
+    /// delivered replacement. `None` evaluates against the live tier.
+    pub async fn contains_pks_at(
+        &self,
+        pks: &RecordBatch,
+        watermarks: Option<&HashMap<Uuid, FreshTierWatermark>>,
+    ) -> Result<Vec<bool>> {
         let sources = self.build_collector().collect()?;
-        let sets = super::block_list::fresh_tier_block_list(
+        let memberships = super::block_list::fresh_tier_block_list(
             &sources,
-            &self.pk_columns,
             self.session.as_ref(),
+            self.store_params.as_ref(),
             self.flushed_cache.as_ref(),
+            watermarks,
         )
         .await?;
         let pk_indices = super::exec::resolve_pk_indices(pks, &self.pk_columns)
             .map_err(|e| Error::invalid_input(e.to_string()))?;
-        Ok((0..pks.num_rows())
+        // One key per row, in the index key space (typed value, or encoded
+        // `Binary` tuple for a composite PK).
+        let keys: Vec<ScalarValue> = (0..pks.num_rows())
             .map(|row| {
-                let hash = super::exec::compute_pk_hash(pks, &pk_indices, row);
-                sets.iter().any(|set| set.contains(&hash))
+                let values: Vec<ScalarValue> = pk_indices
+                    .iter()
+                    .map(|&col| ScalarValue::try_from_array(pks.column(col), row))
+                    .collect::<std::result::Result<_, _>>()
+                    .map_err(|e| Error::invalid_input(e.to_string()))?;
+                super::block_list::on_disk_pk_key(&values)
             })
-            .collect())
+            .collect::<Result<_>>()?;
+
+        // A row is contained if any generation contains its key. Probe each
+        // generation once (batched), narrowing to still-unfound rows.
+        let mut contained = vec![false; keys.len()];
+        let mut live: Vec<usize> = (0..keys.len()).collect();
+        for membership in &memberships {
+            if live.is_empty() {
+                break;
+            }
+            let live_keys: Vec<ScalarValue> = live.iter().map(|&i| keys[i].clone()).collect();
+            let mask = membership.contains_keys(&live_keys).await?;
+            let mut next_live = Vec::with_capacity(live.len());
+            for (pos, &row) in live.iter().enumerate() {
+                if mask[pos] {
+                    contained[row] = true;
+                } else {
+                    next_live.push(row);
+                }
+            }
+            live = next_live;
+        }
+        Ok(contained)
     }
 
     /// Build the data source collector.
@@ -572,35 +972,160 @@ mod tests {
         assert_eq!(memtable_ref.generation, 10);
     }
 
-    #[tokio::test]
-    async fn contains_pks_reports_fresh_tier_membership() {
-        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
-        use arrow_array::Int32Array;
+    /// Single-column `id: Int32` schema used by the PK-membership tests.
+    fn pk_schema() -> SchemaRef {
         use arrow_schema::{DataType, Field, Schema};
+        Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]))
+    }
 
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        let id_batch = |ids: &[i32]| {
-            RecordBatch::try_new(
-                schema.clone(),
-                vec![Arc::new(Int32Array::from(ids.to_vec()))],
-            )
-            .unwrap()
+    /// A `RecordBatch` of `id` values against [`pk_schema`].
+    fn id_pk_batch(ids: &[i32]) -> RecordBatch {
+        use arrow_array::Int32Array;
+        RecordBatch::try_new(pk_schema(), vec![Arc::new(Int32Array::from(ids.to_vec()))]).unwrap()
+    }
+
+    /// An active/frozen memtable holding `ids` at `generation`, with a single
+    /// batch and a maintained primary-key index on `id`.
+    fn mk_pk_memtable(ids: &[i32], generation: u64) -> InMemoryMemTableRef {
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        let store = BatchStore::with_capacity(8);
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        let b = id_pk_batch(ids);
+        let (bp, off, _) = store.append(b.clone()).unwrap();
+        index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
+        InMemoryMemTableRef {
+            batch_store: Arc::new(store),
+            index_store: Arc::new(index),
+            schema: pk_schema(),
+            generation,
+        }
+    }
+
+    #[tokio::test]
+    async fn invalid_overfetch_factor_is_rejected() {
+        let shard = Uuid::new_v4();
+        let scanner = LsmScanner::without_base_table(
+            pk_schema(),
+            "memory://t",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_in_memory_memtables(
+            shard,
+            InMemoryMemTables {
+                active: mk_pk_memtable(&[1, 2], 2),
+                frozen: vec![],
+            },
+        )
+        .with_overfetch_factor(0.5);
+
+        let Err(err) = scanner.try_into_stream().await else {
+            panic!("invalid overfetch factor should fail planning");
         };
-        let mk = |ids: &[i32], generation: u64| {
-            let store = BatchStore::with_capacity(8);
-            store.append(id_batch(ids)).unwrap();
-            InMemoryMemTableRef {
-                batch_store: Arc::new(store),
-                index_store: Arc::new(IndexStore::new()),
-                schema: schema.clone(),
-                generation,
-            }
+        assert!(
+            err.to_string().contains("overfetch_factor"),
+            "unexpected error for invalid overfetch factor: {err}"
+        );
+
+        let empty_scanner = LsmScanner::without_base_table(
+            pk_schema(),
+            "memory://empty",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_overfetch_factor(0.5);
+        let Err(err) = empty_scanner.try_into_stream().await else {
+            panic!("invalid overfetch factor should fail even when there are no sources");
         };
+        assert!(
+            err.to_string().contains("overfetch_factor"),
+            "unexpected error for invalid empty-source overfetch factor: {err}"
+        );
+    }
 
+    #[tokio::test]
+    async fn unknown_scan_projection_column_is_rejected() {
+        let scanner = LsmScanner::without_base_table(
+            pk_schema(),
+            "memory://t",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .project(&["missing"])
+        .unwrap();
+
+        let Err(err) = scanner.try_into_stream().await else {
+            panic!("unknown projection column should fail planning");
+        };
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing-column projection error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn point_lookup_fast_route_rejects_missing_projection_column() {
+        let shard = Uuid::new_v4();
+        let scanner = LsmScanner::without_base_table(
+            pk_schema(),
+            "memory://t",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_in_memory_memtables(
+            shard,
+            InMemoryMemTables {
+                active: mk_pk_memtable(&[1, 2], 2),
+                frozen: vec![],
+            },
+        )
+        .project(&["missing"])
+        .unwrap()
+        .filter("id = 1")
+        .unwrap();
+
+        let Err(err) = scanner.try_into_stream().await else {
+            panic!("unknown projection column should fail on point-lookup fast route");
+        };
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing-column point lookup error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_missing_column_is_rejected() {
+        use arrow_schema::{DataType, Field};
+
+        let scanner = LsmScanner::without_base_table(
+            pk_schema_with(Field::new("text", DataType::Utf8, true)),
+            "memory://t",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .full_text_search(
+            FullTextSearchQuery::new("lance".to_string())
+                .with_column("missing".to_string())
+                .unwrap(),
+        )
+        .unwrap();
+
+        let Err(err) = scanner.try_into_stream().await else {
+            panic!("unknown FTS column should fail planning");
+        };
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing FTS column error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn contains_pks_reports_fresh_tier_membership() {
         // Fresh-tier only: active gen 2 (pk=1,2) + frozen gen 1 (pk=3).
         let shard = Uuid::new_v4();
         let scanner = LsmScanner::without_base_table(
-            schema.clone(),
+            pk_schema(),
             "memory://t",
             vec![],
             vec!["id".to_string()],
@@ -608,16 +1133,708 @@ mod tests {
         .with_in_memory_memtables(
             shard,
             InMemoryMemTables {
-                active: mk(&[1, 2], 2),
-                frozen: vec![mk(&[3], 1)],
+                active: mk_pk_memtable(&[1, 2], 2),
+                frozen: vec![mk_pk_memtable(&[3], 1)],
             },
         );
 
         // pk=1 (active), pk=4 (absent), pk=3 (frozen).
-        let result = scanner.contains_pks(&id_batch(&[1, 4, 3])).await.unwrap();
+        let result = scanner
+            .contains_pks(&id_pk_batch(&[1, 4, 3]))
+            .await
+            .unwrap();
         assert_eq!(result, vec![true, false, true]);
     }
 
+    /// `contains_pks_at` probes each generation once over the still-unfound
+    /// rows, so a multi-PK batch spanning several generations resolves to the
+    /// right per-row mask — and a watermark bounds which generations count.
+    #[tokio::test]
+    async fn contains_pks_at_batched_probe_respects_watermark() {
+        use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark;
+
+        // active gen 2 (pk=1,2) + frozen gen 1 (pk=3,4).
+        let shard = Uuid::new_v4();
+        let scanner = LsmScanner::without_base_table(
+            pk_schema(),
+            "memory://t",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_in_memory_memtables(
+            shard,
+            InMemoryMemTables {
+                active: mk_pk_memtable(&[1, 2], 2),
+                frozen: vec![mk_pk_memtable(&[3, 4], 1)],
+            },
+        );
+
+        // Duplicate and out-of-order keys exercise the live-row narrowing: each
+        // generation only re-probes the rows earlier generations didn't claim.
+        let probe = id_pk_batch(&[4, 1, 9, 3, 2, 1]);
+
+        // watermark=None → live tier: every PK present in either generation.
+        let live = scanner.contains_pks_at(&probe, None).await.unwrap();
+        assert_eq!(live, vec![true, true, false, true, true, true]);
+
+        // watermark at gen 1 → active gen 2 rolled in after the snapshot and is
+        // excluded; only the frozen gen 1 keys (3,4) remain members.
+        let watermarks: HashMap<Uuid, FreshTierWatermark> = [(
+            shard,
+            FreshTierWatermark {
+                active_generation: 1,
+                active_batch_count: u64::MAX,
+            },
+        )]
+        .into_iter()
+        .collect();
+        let bounded = scanner
+            .contains_pks_at(&probe, Some(&watermarks))
+            .await
+            .unwrap();
+        assert_eq!(bounded, vec![true, false, false, true, false, false]);
+    }
+
+    fn pk_schema_with(extra: arrow_schema::Field) -> SchemaRef {
+        use arrow_schema::{DataType, Field, Schema};
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            extra,
+        ]))
+    }
+
+    /// `LsmScanner::nearest(..).create_plan()` must route through the vector
+    /// planner (exercising the Scanner-aligned facade, `key_to_fsl`, and
+    /// `vector_dim`) and surface the in-memory match.
+    #[tokio::test]
+    async fn nearest_dispatches_through_facade() {
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use crate::dataset::{Dataset, WriteParams};
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_array::{Float32Array, Int32Array, RecordBatchIterator};
+        use arrow_schema::{DataType, Field};
+
+        let schema = pk_schema_with(Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ));
+        let make_batch = |ids: &[i32]| -> RecordBatch {
+            let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+            for id in ids {
+                let base = *id as f32 * 0.1;
+                for d in 0..4 {
+                    vb.values().append_value(base + d as f32 * 0.1);
+                }
+                vb.append(true);
+            }
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(ids.to_vec())),
+                    Arc::new(vb.finish()),
+                ],
+            )
+            .unwrap()
+        };
+
+        // Far, unindexed base rows contribute nothing under fast_search.
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(make_batch(&[100, 200]))], schema.clone());
+        let base = Arc::new(
+            Dataset::write(reader, &uri, Some(WriteParams::default()))
+                .await
+                .unwrap(),
+        );
+
+        let store = Arc::new(BatchStore::with_capacity(16));
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        index.add_hnsw(
+            "vec_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            DistanceType::L2,
+            64,
+            8,
+        );
+        let batch = make_batch(&[1, 2, 3]);
+        store.append(batch.clone()).unwrap();
+        index
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store: store,
+                        index_store: Arc::new(index),
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            )
+            // A flat query vector (closest to id=1); the facade wraps it into an FSL.
+            .nearest(
+                "vector",
+                &Float32Array::from(vec![0.1f32, 0.2, 0.3, 0.4]),
+                3,
+            )
+            .unwrap();
+
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let ids: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(
+            ids.first().copied(),
+            Some(1),
+            "nearest neighbor via the facade should be id=1; got {ids:?}"
+        );
+    }
+
+    /// Vector pagination: `limit(Some(2), Some(1))` over-fetches `k + offset`
+    /// per source, then skips `offset` and caps at `limit`. With ids ordered by
+    /// distance (id=1 nearest), the page must be the 2nd–3rd nearest (id=2, id=3),
+    /// not the 1st–2nd. Covers `plan_vector`'s `per_source_k` + `apply_limit_offset`.
+    #[tokio::test]
+    async fn nearest_pagination_skips_offset_and_caps_limit() {
+        use crate::dataset::{Dataset, WriteParams};
+        use crate::index::DatasetIndexExt;
+        use crate::index::vector::VectorIndexParams;
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_array::{Float32Array, Int32Array, RecordBatchIterator};
+        use arrow_schema::{DataType, Field};
+        use lance_index::IndexType;
+
+        let schema = pk_schema_with(Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ));
+        let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        let ids: Vec<i32> = (1..=6).collect();
+        for id in &ids {
+            let base = *id as f32 * 0.1;
+            for d in 0..4 {
+                vb.values().append_value(base + d as f32 * 0.1);
+            }
+            vb.append(true);
+        }
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids.clone())),
+                Arc::new(vb.finish()),
+            ],
+        )
+        .unwrap();
+
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        // `ivf_flat(1)` is exhaustive within its single partition, so ordering is exact.
+        let mut base = Dataset::write(reader, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        let ivf_flat = VectorIndexParams::ivf_flat(1, DistanceType::L2);
+        base.create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+        let base = Arc::new(base);
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .nearest(
+                "vector",
+                &Float32Array::from(vec![0.1f32, 0.2, 0.3, 0.4]),
+                2,
+            )
+            .unwrap()
+            .limit(Some(2), Some(1))
+            .unwrap();
+
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let mut out: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        out.sort();
+        assert_eq!(
+            out,
+            vec![2, 3],
+            "offset=1, limit=2 over k=2 must return the 2nd-3rd nearest (id=2, id=3); got {out:?}"
+        );
+    }
+
+    /// FTS pagination: `limit(Some(1), Some(1))` fetches `limit + offset` per
+    /// source then skips `offset`. With BM25 ranking id=1 > id=2 > id=3 (by doc
+    /// length), the page must be the 2nd-ranked hit (id=2). Covers `plan_fts`'s
+    /// `k = limit + offset` + `apply_limit_offset`.
+    #[tokio::test]
+    async fn full_text_search_pagination_skips_offset_and_caps_limit() {
+        use crate::dataset::{Dataset, WriteParams};
+        use crate::index::DatasetIndexExt;
+        use arrow_array::{Int32Array, RecordBatchIterator, StringArray};
+        use arrow_schema::{DataType, Field};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = pk_schema_with(Field::new("text", DataType::Utf8, true));
+        // Shorter docs score higher under BM25 length normalization, so the
+        // ranking is deterministically id=1 > id=2 > id=3.
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec![
+                    "lance",
+                    "lance filler",
+                    "lance filler filler",
+                ])),
+            ],
+        )
+        .unwrap();
+
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut base = Dataset::write(reader, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        base.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+        let base = Arc::new(Dataset::open(&uri).await.unwrap());
+
+        let query_limited = LsmScanner::new(base.clone(), vec![], vec!["id".to_string()])
+            .full_text_search(
+                FullTextSearchQuery::new("lance".to_string())
+                    .with_column("text".to_string())
+                    .unwrap()
+                    .limit(Some(1)),
+            )
+            .unwrap();
+        let batches: Vec<RecordBatch> = query_limited
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let out: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(
+            out,
+            vec![1],
+            "query-level FTS limit=1 must cap the unpaginated scanner result; got {out:?}"
+        );
+
+        let query_limited_with_offset =
+            LsmScanner::new(base.clone(), vec![], vec!["id".to_string()])
+                .full_text_search(
+                    FullTextSearchQuery::new("lance".to_string())
+                        .with_column("text".to_string())
+                        .unwrap()
+                        .limit(Some(2)),
+                )
+                .unwrap()
+                .limit(None, Some(1))
+                .unwrap();
+        let batches: Vec<RecordBatch> = query_limited_with_offset
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let out: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(
+            out,
+            vec![2],
+            "query-level FTS limit=2 plus offset=1 must page within the top 2; got {out:?}"
+        );
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .full_text_search(
+                FullTextSearchQuery::new("lance".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap()
+            .limit(Some(1), Some(1))
+            .unwrap();
+
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let out: Vec<i32> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(
+            out,
+            vec![2],
+            "offset=1, limit=1 must return the 2nd-ranked hit (id=2); got {out:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn full_text_search_without_limit_returns_all_matches() {
+        use crate::dataset::{Dataset, WriteParams};
+        use crate::index::DatasetIndexExt;
+        use arrow_array::{Int32Array, RecordBatchIterator, StringArray};
+        use arrow_schema::{DataType, Field};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = pk_schema_with(Field::new("text", DataType::Utf8, true));
+        let ids: Vec<i32> = (0..12).collect();
+        let texts: Vec<&str> = (0..12).map(|_| "lance").collect();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(texts)),
+            ],
+        )
+        .unwrap();
+
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut base = Dataset::write(reader, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        base.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+        let base = Arc::new(Dataset::open(&uri).await.unwrap());
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .full_text_search(
+                FullTextSearchQuery::new("lance".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            total, 12,
+            "unbounded LSM FTS must not apply the old default top-10 cap"
+        );
+    }
+
+    /// Setting both `nearest` and `full_text_search` must error rather than
+    /// silently running vector search and dropping the full-text query (the LSM
+    /// path has no hybrid mode).
+    #[tokio::test]
+    async fn combined_vector_and_fts_is_rejected() {
+        use crate::dataset::{Dataset, WriteParams};
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_array::{Float32Array, Int32Array, RecordBatchIterator};
+        use arrow_schema::{DataType, Field};
+
+        let schema = pk_schema_with(Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ));
+        let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for d in 0..4 {
+            vb.values().append_value(d as f32);
+        }
+        vb.append(true);
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1])), Arc::new(vb.finish())],
+        )
+        .unwrap();
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let base = Arc::new(
+            Dataset::write(reader, &uri, Some(WriteParams::default()))
+                .await
+                .unwrap(),
+        );
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .nearest(
+                "vector",
+                &Float32Array::from(vec![0.0f32, 1.0, 2.0, 3.0]),
+                1,
+            )
+            .unwrap()
+            .full_text_search(
+                FullTextSearchQuery::new("lance".to_string())
+                    .with_column("vector".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+        let err = scanner.create_plan().await.unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("combined vector and full-text search"),
+            "expected combined-search rejection, got: {err}"
+        );
+    }
+
+    /// A multi-row query vector must be rejected, not silently flow through the
+    /// single-vector LSM merge as wrong results.
+    #[tokio::test]
+    async fn multi_row_query_vector_is_rejected() {
+        use crate::dataset::{Dataset, WriteParams};
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_array::{Int32Array, RecordBatchIterator};
+        use arrow_schema::{DataType, Field};
+
+        let schema = pk_schema_with(Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            false,
+        ));
+        let mut col = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for d in 0..4 {
+            col.values().append_value(d as f32);
+        }
+        col.append(true);
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1])), Arc::new(col.finish())],
+        )
+        .unwrap();
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let base = Arc::new(
+            Dataset::write(reader, &uri, Some(WriteParams::default()))
+                .await
+                .unwrap(),
+        );
+
+        // A two-row FSL query vector.
+        let mut q = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for _ in 0..2 {
+            for d in 0..4 {
+                q.values().append_value(d as f32);
+            }
+            q.append(true);
+        }
+        let query = q.finish();
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .nearest("vector", &query, 1)
+            .unwrap();
+        let err = scanner.create_plan().await.unwrap_err();
+        assert!(
+            err.to_string().contains("single query vector"),
+            "expected single-query-vector rejection, got: {err}"
+        );
+    }
+
+    /// `LsmScanner::full_text_search(query).create_plan()` must route through the
+    /// FTS planner and surface a memtable-only match.
+    #[tokio::test]
+    async fn full_text_search_dispatches_through_facade() {
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use crate::dataset::{Dataset, WriteParams};
+        use crate::index::DatasetIndexExt;
+        use arrow_array::{Int32Array, RecordBatchIterator, StringArray};
+        use arrow_schema::{DataType, Field};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = pk_schema_with(Field::new("text", DataType::Utf8, true));
+        let make_batch = |ids: &[i32], texts: &[&str]| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(ids.to_vec())),
+                    Arc::new(StringArray::from(texts.to_vec())),
+                ],
+            )
+            .unwrap()
+        };
+
+        let tmp = tempfile::tempdir().unwrap();
+        let uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let reader =
+            RecordBatchIterator::new(vec![Ok(make_batch(&[1], &["alpha"]))], schema.clone());
+        let mut base = Dataset::write(reader, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        base.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+        let base = Arc::new(Dataset::open(&uri).await.unwrap());
+
+        let store = Arc::new(BatchStore::with_capacity(16));
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        index.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let batch = make_batch(&[99], &["zebra"]);
+        store.append(batch.clone()).unwrap();
+        index
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store: store,
+                        index_store: Arc::new(index),
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            )
+            .full_text_search(
+                FullTextSearchQuery::new("zebra".to_string())
+                    .with_column("text".to_string())
+                    .unwrap(),
+            )
+            .unwrap();
+
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            rows, 1,
+            "facade FTS should surface the memtable 'zebra' row"
+        );
+    }
+
+    /// Empty search results must still preserve the execution plan schema.
+    #[tokio::test]
+    async fn try_into_batch_empty_fts_keeps_score_schema() {
+        use arrow_schema::{DataType, Field};
+
+        let schema = pk_schema_with(Field::new("text", DataType::Utf8, true));
+        let scanner = LsmScanner::without_base_table(
+            schema,
+            "memory://empty",
+            vec![],
+            vec!["id".to_string()],
+        )
+        .full_text_search(
+            FullTextSearchQuery::new("missing".to_string())
+                .with_column("text".to_string())
+                .unwrap(),
+        )
+        .unwrap();
+
+        let batch = scanner.try_into_batch().await.unwrap();
+        assert_eq!(batch.num_rows(), 0);
+        assert!(
+            batch.schema().field_with_name("_score").is_ok(),
+            "empty FTS batch must keep the planned _score column"
+        );
+    }
+
     /// One active memtable with a maintained BTree on `id`, all rows visible.
     fn mk_indexed_memtable(schema: &SchemaRef, ids: &[i32], names: &[&str]) -> InMemoryMemTableRef {
         use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
@@ -686,6 +1903,29 @@ mod tests {
                 rows.iter().map(|b| b.num_rows()).sum::<usize>()
             }
         };
+        let collect_ids = |plan: Arc<dyn ExecutionPlan>| {
+            let ctx = ctx.clone();
+            async move {
+                let rows: Vec<RecordBatch> = plan
+                    .execute(0, ctx.task_ctx())
+                    .unwrap()
+                    .try_collect()
+                    .await
+                    .unwrap();
+                let mut ids = Vec::new();
+                for batch in rows {
+                    let id_array = batch
+                        .column_by_name("id")
+                        .unwrap()
+                        .as_any()
+                        .downcast_ref::<arrow_array::Int32Array>()
+                        .unwrap();
+                    ids.extend(id_array.values().iter().copied());
+                }
+                ids.sort_unstable();
+                ids
+            }
+        };
 
         // `id = 2` routes to the direct point-lookup node (OneShotStream), not the
         // union/dedup scan, and returns the one matching row.
@@ -725,5 +1965,21 @@ mod tests {
             "range filter must not route to the point-lookup node"
         );
         assert_eq!(count(plan).await, 3); // 3,4,5
+
+        // Offset changes the semantics, so even a point-lookup-shaped filter
+        // must bypass the direct lookup route and use the general scan path.
+        let plan = scanner()
+            .filter_expr(col("id").in_list(vec![lit(1i32), lit(3i32), lit(5i32)], false))
+            .limit(None, Some(1))
+            .unwrap()
+            .create_plan()
+            .await
+            .unwrap();
+        let disp = format!("{}", displayable(plan.as_ref()).indent(true));
+        assert!(
+            !disp.contains("OneShotStream"),
+            "offset point-lookup filters must use the scan path: {disp}"
+        );
+        assert_eq!(collect_ids(plan).await, vec![3, 5]);
     }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
index 2db4b4f277d..6645f159b12 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs
@@ -229,6 +229,19 @@ impl LsmDataSourceCollector {
             .collect()
     }
 
+    /// True when `generation` for `shard_id` is still pinned in memory as a
+    /// frozen memtable. During the post-flush grace window a generation is both
+    /// committed to the manifest (a flushed source) and held in memory (an
+    /// in-memory source); it must be served only from memory — which preserves
+    /// the per-batch boundaries the flushed dataset has lost, so as-of reads
+    /// stay snapshot-bounded — and its on-disk copy skipped to avoid scanning
+    /// the generation twice. See `ShardWriterConfig::frozen_memtable_grace`.
+    fn flushed_gen_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool {
+        self.in_memory_memtables
+            .get(shard_id)
+            .is_some_and(|mems| mems.frozen.iter().any(|f| f.generation == generation))
+    }
+
     /// Collect all data sources.
     ///
     /// Returns sources in a consistent order:
@@ -246,6 +259,9 @@ impl LsmDataSourceCollector {
 
         for snapshot in &self.shard_snapshots {
             for flushed in &snapshot.flushed_generations {
+                if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) {
+                    continue;
+                }
                 let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
                 sources.push(LsmDataSource::FlushedMemTable {
                     path,
@@ -284,6 +300,9 @@ impl LsmDataSourceCollector {
             }
 
             for flushed in &snapshot.flushed_generations {
+                if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) {
+                    continue;
+                }
                 let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path);
                 sources.push(LsmDataSource::FlushedMemTable {
                     path,
@@ -443,4 +462,53 @@ mod tests {
             3
         );
     }
+
+    /// During the post-flush grace window a generation is both committed to the
+    /// manifest (a flushed source) and still pinned in memory (a frozen
+    /// source). The collector must emit it once, from memory — so as-of reads
+    /// keep batch-resolved membership — and skip the on-disk copy. Flushed
+    /// generations NOT pinned in memory are still emitted from disk.
+    #[test]
+    fn test_collect_suppresses_flushed_gen_pinned_in_memory() {
+        let shard = Uuid::new_v4();
+        // Manifest lists gens 1 and 2 as flushed; gen 2 is still pinned in
+        // memory (just flushed, within grace), gen 1 has been swept.
+        let snapshot = ShardSnapshot {
+            shard_id: shard,
+            spec_id: 0,
+            current_generation: 3,
+            flushed_generations: vec![
+                FlushedGeneration {
+                    generation: 1,
+                    path: "gen_1".to_string(),
+                },
+                FlushedGeneration {
+                    generation: 2,
+                    path: "gen_2".to_string(),
+                },
+            ],
+        };
+        let mems = InMemoryMemTables {
+            active: memtable_ref(3),
+            frozen: vec![memtable_ref(2)],
+        };
+        let collector = LsmDataSourceCollector::without_base_table("/tmp/x", vec![snapshot])
+            .with_in_memory_memtables(shard, mems);
+
+        let sources = collector.collect().unwrap();
+        // gen 1: on-disk (not pinned). gen 2: in-memory only (pinned, disk
+        // copy suppressed). gen 3: active. No duplicate gen 2.
+        let flushed: Vec<u64> = sources
+            .iter()
+            .filter(|s| !s.is_active_memtable())
+            .map(|s| s.generation().as_u64())
+            .collect();
+        let in_memory: Vec<u64> = sources
+            .iter()
+            .filter(|s| s.is_active_memtable())
+            .map(|s| s.generation().as_u64())
+            .collect();
+        assert_eq!(flushed, vec![1], "only the unpinned flushed gen from disk");
+        assert_eq!(in_memory, vec![2, 3], "pinned gen 2 served from memory");
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
index 1a6207f27e3..0d5f3fdc925 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs
@@ -11,6 +11,29 @@ use uuid::Uuid;
 use crate::dataset::Dataset;
 use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
 
+/// A watermark marking how far into one shard's fresh tier a prior scan
+/// observed, so membership can be evaluated as of that point (see
+/// [`super::builder::LsmScanner::contains_pks_at`]).
+///
+/// Only the active memtable grows between two reads (appended batches, and a new
+/// generation when it rolls); everything at a lower generation — frozen and
+/// flushed — is immutable and was fully observed. The watermark includes lower
+/// generations whole, the active generation up to `active_batch_count` batches,
+/// and excludes higher generations (which appeared after it). It uses only the
+/// batch count and generation — both always available, unlike per-batch WAL
+/// positions, which the write path does not track. The bound only excludes rows
+/// the scan did not observe, so a stale watermark under-counts (a tolerable
+/// stale read) rather than dropping a row with no replacement.
+#[derive(Debug, Clone, Copy)]
+pub struct FreshTierWatermark {
+    /// Active generation the scan observed. Higher generations are excluded;
+    /// lower ones are immutable and included whole.
+    pub active_generation: u64,
+    /// Active-memtable batch count at snapshot time. Within the active
+    /// generation, only batches at index `< active_batch_count` were observed.
+    pub active_batch_count: u64,
+}
+
 /// Generation number in LSM tree.
 ///
 /// The base table has generation 0. MemTables have positive integers
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/scanner/exec.rs
index 88fd617dc0a..1498c5f60ea 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec.rs
@@ -9,15 +9,13 @@
 //! - [`MemtableGenTagExec`]: Wraps a scan to add `_memtable_gen` column
 //! - [`BloomFilterGuardExec`]: Guards child execution with bloom filter check
 //! - [`CoalesceFirstExec`]: Returns first non-empty result with short-circuit
-//! - [`WithinSourceDedupExec`]: Deduplicates rows with the same PK from a single source
-//! - [`PkHashFilterExec`]: Drops rows whose PK hash was superseded by a newer generation (the cross-generation block-list)
+//! - [`PkBlockFilterExec`]: Drops rows whose PK was superseded by a newer generation (the cross-generation block-list)
 
 mod bloom_guard;
 mod coalesce_first;
 mod generation_tag;
 mod pk;
-mod pk_hash_filter;
-mod within_source_dedup;
+mod pk_block_filter;
 
 pub use bloom_guard::{BloomFilterGuardExec, compute_pk_hash_from_scalars};
 pub use coalesce_first::CoalesceFirstExec;
@@ -26,5 +24,4 @@ pub use pk::{
     ROW_ADDRESS_COLUMN, compute_pk_hash, is_supported_pk_type, resolve_pk_indices,
     validate_pk_types,
 };
-pub use pk_hash_filter::PkHashFilterExec;
-pub use within_source_dedup::{DedupDirection, WithinSourceDedupExec};
+pub use pk_block_filter::PkBlockFilterExec;
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
index 6039eed1629..e3a710bd367 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs
@@ -5,7 +5,6 @@
 //!
 //! Used in point lookup queries to skip generations that definitely don't contain the key.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -21,7 +20,7 @@ use datafusion::physical_plan::{
     SendableRecordBatchStream,
 };
 use futures::Stream;
-use lance_index::scalar::bloomfilter::sbbf::Sbbf;
+use lance_core::utils::bloomfilter::sbbf::Sbbf;
 
 /// Guards a child execution node with a bloom filter check.
 ///
@@ -134,10 +133,6 @@ impl ExecutionPlan for BloomFilterGuardExec {
         "BloomFilterGuardExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
index 9e158c86b4a..c212b47c013 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs
@@ -5,7 +5,6 @@
 //!
 //! Used in point lookup queries to stop searching after finding the first match.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -111,10 +110,6 @@ impl ExecutionPlan for CoalesceFirstExec {
         "CoalesceFirstExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
index ba9d565316f..9c1d0060a78 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs
@@ -3,7 +3,6 @@
 
 //! MemTable generation tagging execution node.
 
-use std::any::Any;
 use std::fmt;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -100,10 +99,6 @@ impl ExecutionPlan for MemtableGenTagExec {
         "MemtableGenTagExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.schema.clone()
     }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk.rs
index 523dd30bf82..af1e7696181 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk.rs
@@ -4,7 +4,7 @@
 //! Shared primary-key helpers for the LSM scanner execution nodes.
 //!
 //! Centralizes PK column resolution and per-row hashing so that every
-//! consumer (e.g. [`super::WithinSourceDedupExec`], [`super::PkHashFilterExec`])
+//! consumer (e.g. [`super::PkBlockFilterExec`])
 //! resolves and hashes a primary key the same way. The row hash is kept
 //! consistent with the variants supported by [`super::compute_pk_hash_from_scalars`]
 //! so a single PK produces the same hash regardless of which exec consumes it.
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
new file mode 100644
index 00000000000..4903f0de3a8
--- /dev/null
+++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs
@@ -0,0 +1,374 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Drop superseded rows from a per-source result by primary-key membership.
+//!
+//! Drops a row when any newer generation's membership ([`GenMembership`])
+//! contains its primary key — in-memory generations probe their PK index by
+//! value, flushed generations probe their on-disk PK BTree. Each generation is
+//! probed once per batch (see the perf note below). Used both as the KNN
+//! post-filter (vector search, with over-fetch) and the cross-generation scan
+//! filter (`k = 0`).
+//!
+//! Cross-generation only: within-gen duplicates collapse via the global dedup's
+//! `(generation, freshness)` tiebreaker.
+//!
+//! Post-filters an over-fetched KNN (the planner's `overfetch_factor`); warns
+//! when a source had >= k candidates but < k survived (over-fetch too small).
+//!
+//! Perf note: each generation is probed once per batch via
+//! [`GenMembership::contains_keys`] — a batched existence check over the
+//! batch's keys — not once per row. The on-disk arm issues a single
+//! `BTreeIndex::contains_keys` (one page pass, no per-key `SearchResult`
+//! allocation); the in-memory arm maps a sync PK lookup over the keys. Probes
+//! are not disk-bound in steady state: the opened index and its (small,
+//! memtable-sized) pages are held by the injected `FlushedMemTableCache` /
+//! `LanceCache`, so after the first touch every probe is memory-resident.
+//! Already-blocked rows are dropped from the key set before probing older
+//! generations, preserving the per-row short-circuit.
+
+use std::fmt;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use arrow::compute::filter_record_batch;
+use arrow_array::{BooleanArray, RecordBatch};
+use arrow_schema::SchemaRef;
+use datafusion::common::ScalarValue;
+use datafusion::error::{DataFusionError, Result as DFResult};
+use datafusion::execution::TaskContext;
+use datafusion::physical_expr::EquivalenceProperties;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+    SendableRecordBatchStream,
+};
+use futures::future::BoxFuture;
+use futures::{FutureExt, Stream, StreamExt};
+use tracing::warn;
+
+use super::super::block_list::{GenMembership, on_disk_pk_key};
+use super::pk::resolve_pk_indices;
+
+/// Filters out rows whose PK is contained in any newer generation's membership.
+#[derive(Debug)]
+pub struct PkBlockFilterExec {
+    input: Arc<dyn ExecutionPlan>,
+    pk_columns: Vec<String>,
+    /// Newer generations' membership; a row is blocked if any contains its PK.
+    blocked: Vec<GenMembership>,
+    /// Target neighbor count, used only to warn on a per-source under-fetch.
+    k: usize,
+    properties: Arc<PlanProperties>,
+}
+
+impl PkBlockFilterExec {
+    pub fn new(
+        input: Arc<dyn ExecutionPlan>,
+        pk_columns: Vec<String>,
+        blocked: Vec<GenMembership>,
+        k: usize,
+    ) -> Self {
+        // A filter preserves the input schema and partitioning.
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(input.schema()),
+            input.output_partitioning().clone(),
+            input.pipeline_behavior(),
+            input.boundedness(),
+        ));
+        Self {
+            input,
+            pk_columns,
+            blocked,
+            k,
+            properties,
+        }
+    }
+}
+
+impl DisplayAs for PkBlockFilterExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+        match t {
+            DisplayFormatType::Default
+            | DisplayFormatType::Verbose
+            | DisplayFormatType::TreeRender => {
+                write!(
+                    f,
+                    "PkBlockFilterExec: pk_cols=[{}], gens={}",
+                    self.pk_columns.join(", "),
+                    self.blocked.len(),
+                )
+            }
+        }
+    }
+}
+
+impl ExecutionPlan for PkBlockFilterExec {
+    fn name(&self) -> &str {
+        "PkBlockFilterExec"
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> DFResult<Arc<dyn ExecutionPlan>> {
+        if children.len() != 1 {
+            return Err(DataFusionError::Internal(
+                "PkBlockFilterExec requires exactly one child".to_string(),
+            ));
+        }
+        Ok(Arc::new(Self::new(
+            children[0].clone(),
+            self.pk_columns.clone(),
+            self.blocked.clone(),
+            self.k,
+        )))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> DFResult<SendableRecordBatchStream> {
+        let input_stream = self.input.execute(partition, context)?;
+        Ok(Box::pin(PkBlockFilterStream {
+            input: input_stream,
+            config: Arc::new(FilterConfig {
+                pk_columns: self.pk_columns.clone(),
+                blocked: self.blocked.clone(),
+            }),
+            k: self.k,
+            schema: self.schema(),
+            pending: None,
+            input_seen: 0,
+            kept: 0,
+            warned: false,
+        }))
+    }
+}
+
+/// Immutable per-stream filter config. Shared into each batch's `'static` async
+/// future by a single `Arc` clone, rather than deep-cloning the PK columns and
+/// memberships per batch.
+struct FilterConfig {
+    pk_columns: Vec<String>,
+    blocked: Vec<GenMembership>,
+}
+
+struct PkBlockFilterStream {
+    input: SendableRecordBatchStream,
+    config: Arc<FilterConfig>,
+    k: usize,
+    schema: SchemaRef,
+    /// The in-flight filter for the batch currently being processed (the probe
+    /// is async, so a batch is filtered off-poll and resumed here).
+    pending: Option<BoxFuture<'static, DFResult<RecordBatch>>>,
+    input_seen: usize,
+    kept: usize,
+    warned: bool,
+}
+
+/// Keep only the rows no newer-gen membership contains. Async because flushed
+/// generations are probed against their on-disk PK BTree.
+async fn filter_batch(batch: RecordBatch, config: Arc<FilterConfig>) -> DFResult<RecordBatch> {
+    let FilterConfig {
+        pk_columns,
+        blocked,
+    } = config.as_ref();
+    if blocked.is_empty() || batch.num_rows() == 0 {
+        return Ok(batch);
+    }
+    let pk_indices = resolve_pk_indices(&batch, pk_columns)?;
+    let to_df = |e: lance_core::Error| DataFusionError::Execution(e.to_string());
+
+    // One key per row, in the index key space.
+    let keys: Vec<ScalarValue> = (0..batch.num_rows())
+        .map(|row| {
+            let values: Vec<ScalarValue> = pk_indices
+                .iter()
+                .map(|&col| ScalarValue::try_from_array(batch.column(col), row))
+                .collect::<DFResult<_>>()?;
+            on_disk_pk_key(&values).map_err(to_df)
+        })
+        .collect::<DFResult<_>>()?;
+
+    // A row is dropped if any newer generation contains its key. Probe each
+    // generation once (batched) rather than once per row, narrowing to the
+    // still-live rows so an already-blocked row isn't re-probed against older
+    // generations.
+    let mut blocked_row = vec![false; keys.len()];
+    let mut live: Vec<usize> = (0..keys.len()).collect();
+    for membership in blocked {
+        if live.is_empty() {
+            break;
+        }
+        let live_keys: Vec<ScalarValue> = live.iter().map(|&i| keys[i].clone()).collect();
+        let mask = membership.contains_keys(&live_keys).await.map_err(to_df)?;
+        let mut next_live = Vec::with_capacity(live.len());
+        for (pos, &row) in live.iter().enumerate() {
+            if mask[pos] {
+                blocked_row[row] = true;
+            } else {
+                next_live.push(row);
+            }
+        }
+        live = next_live;
+    }
+
+    let keep = BooleanArray::from_iter(blocked_row.into_iter().map(|b| Some(!b)));
+    filter_record_batch(&batch, &keep).map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+}
+
+impl Stream for PkBlockFilterStream {
+    type Item = DFResult<RecordBatch>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+        loop {
+            // Drive an in-flight filter to completion before pulling more input.
+            if let Some(fut) = this.pending.as_mut() {
+                return match fut.as_mut().poll(cx) {
+                    Poll::Ready(Ok(out)) => {
+                        this.pending = None;
+                        this.kept += out.num_rows();
+                        Poll::Ready(Some(Ok(out)))
+                    }
+                    Poll::Ready(Err(e)) => {
+                        this.pending = None;
+                        Poll::Ready(Some(Err(e)))
+                    }
+                    Poll::Pending => Poll::Pending,
+                };
+            }
+
+            match this.input.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok(batch))) => {
+                    this.input_seen += batch.num_rows();
+                    this.pending = Some(filter_batch(batch, this.config.clone()).boxed());
+                    // Loop to poll the just-created future.
+                }
+                Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))),
+                Poll::Ready(None) => {
+                    // >= k candidates in, < k out: over-fetch missed superseded
+                    // rows. Each is a PK shadowed by a newer generation — an
+                    // update (replaced elsewhere) or a delete (a tombstone that
+                    // emits nothing, so it is pure, uncompensated subtraction).
+                    // A burst of deletes between compactions is the likeliest
+                    // cause of repeated warnings.
+                    if !this.warned && this.input_seen >= this.k && this.kept < this.k {
+                        warn!(
+                            k = this.k,
+                            fetched = this.input_seen,
+                            kept = this.kept,
+                            "LSM vector search: < k live rows survived the PK post-filter \
+                             (superseded by newer-generation updates or deletes); \
+                             raise the over-fetch factor or use a true KNN prefilter."
+                        );
+                        this.warned = true;
+                    }
+                    return Poll::Ready(None);
+                }
+                Poll::Pending => return Poll::Pending,
+            }
+        }
+    }
+}
+
+impl datafusion::physical_plan::RecordBatchStream for PkBlockFilterStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+    use arrow_array::Int32Array;
+    use arrow_schema::{DataType, Field, Schema};
+    use datafusion::prelude::SessionContext;
+    use datafusion_physical_plan::test::TestMemoryExec;
+    use futures::TryStreamExt;
+
+    fn int_batch(ids: &[i32]) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(ids.to_vec()))]).unwrap()
+    }
+
+    /// An in-memory membership whose PK index holds `ids` (positions 0..n).
+    fn membership(ids: &[i32]) -> GenMembership {
+        let store = BatchStore::with_capacity(16);
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        for &id in ids {
+            let b = int_batch(&[id]);
+            let (bp, off, _) = store.append(b.clone()).unwrap();
+            index.insert_with_batch_position(&b, off, Some(bp)).unwrap();
+        }
+        let max_visible_row = store.max_visible_row(index.max_visible_batch_position());
+        GenMembership::InMemory {
+            index_store: Arc::new(index),
+            max_visible_row,
+        }
+    }
+
+    async fn run(exec: PkBlockFilterExec) -> Vec<i32> {
+        let ctx = SessionContext::new();
+        let out: Vec<RecordBatch> = exec
+            .execute(0, ctx.task_ctx())
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        out.iter()
+            .flat_map(|b| {
+                b.column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect()
+    }
+
+    #[tokio::test]
+    async fn drops_rows_blocked_by_a_newer_generation() {
+        let b = int_batch(&[10, 20, 30]);
+        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
+        let exec =
+            PkBlockFilterExec::new(input, vec!["id".to_string()], vec![membership(&[20])], 1);
+        assert_eq!(run(exec).await, vec![10, 30]);
+    }
+
+    #[tokio::test]
+    async fn blocks_a_pk_present_in_any_generation() {
+        // Two newer-gen memberships: a row is dropped if either contains its PK.
+        let b = int_batch(&[10, 20, 30]);
+        let blocked = vec![membership(&[10]), membership(&[30])];
+        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
+        let exec = PkBlockFilterExec::new(input, vec!["id".to_string()], blocked, 1);
+        assert_eq!(run(exec).await, vec![20]);
+    }
+
+    #[tokio::test]
+    async fn empty_blocked_keeps_all_rows() {
+        let b = int_batch(&[1, 2, 3]);
+        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
+        let exec = PkBlockFilterExec::new(input, vec!["id".to_string()], Vec::new(), 1);
+        assert_eq!(run(exec).await, vec![1, 2, 3]);
+    }
+}
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_hash_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_hash_filter.rs
deleted file mode 100644
index ee473047d01..00000000000
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_hash_filter.rs
+++ /dev/null
@@ -1,350 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-//! Drop superseded rows from a per-source KNN result by primary-key hash.
-//!
-//! Drops a row when its PK hash ([`super::compute_pk_hash`]) is in any `blocked`
-//! set — the newer generations' membership (`Arc<HashSet>`, shared, never merged;
-//! base table: all generations). Only the KNN output is hashed.
-//!
-//! Cross-generation only: within-gen duplicates share a hash, so the global
-//! dedup's `(generation, freshness)` tiebreaker collapses those instead.
-//!
-//! Post-filters an over-fetched KNN (the planner's `overfetch_factor`); warns
-//! when a source had >= k candidates but < k survived (over-fetch too small).
-
-use std::any::Any;
-use std::collections::HashSet;
-use std::fmt;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::task::{Context, Poll};
-
-use arrow::compute::filter_record_batch;
-use arrow_array::{BooleanArray, RecordBatch};
-use arrow_schema::SchemaRef;
-use datafusion::error::{DataFusionError, Result as DFResult};
-use datafusion::execution::TaskContext;
-use datafusion::physical_expr::EquivalenceProperties;
-use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
-    SendableRecordBatchStream,
-};
-use futures::{Stream, StreamExt};
-use tracing::warn;
-
-use super::pk::{compute_pk_hash, resolve_pk_indices};
-
-/// Filters out rows whose PK hash is in any set of `blocked`.
-#[derive(Debug)]
-pub struct PkHashFilterExec {
-    input: Arc<dyn ExecutionPlan>,
-    pk_columns: Vec<String>,
-    /// Newer generations' membership; a row is blocked if any set holds its hash.
-    blocked: Vec<Arc<HashSet<u64>>>,
-    /// Target neighbor count, used only to warn on a per-source under-fetch.
-    k: usize,
-    properties: Arc<PlanProperties>,
-}
-
-impl PkHashFilterExec {
-    pub fn new(
-        input: Arc<dyn ExecutionPlan>,
-        pk_columns: Vec<String>,
-        blocked: Vec<Arc<HashSet<u64>>>,
-        k: usize,
-    ) -> Self {
-        // A filter preserves the input schema and partitioning.
-        let properties = Arc::new(PlanProperties::new(
-            EquivalenceProperties::new(input.schema()),
-            input.output_partitioning().clone(),
-            input.pipeline_behavior(),
-            input.boundedness(),
-        ));
-        Self {
-            input,
-            pk_columns,
-            blocked,
-            k,
-            properties,
-        }
-    }
-}
-
-impl DisplayAs for PkHashFilterExec {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
-        match t {
-            DisplayFormatType::Default
-            | DisplayFormatType::Verbose
-            | DisplayFormatType::TreeRender => {
-                let total: usize = self.blocked.iter().map(|s| s.len()).sum();
-                write!(
-                    f,
-                    "PkHashFilterExec: pk_cols=[{}], gens={}, blocked={}",
-                    self.pk_columns.join(", "),
-                    self.blocked.len(),
-                    total,
-                )
-            }
-        }
-    }
-}
-
-impl ExecutionPlan for PkHashFilterExec {
-    fn name(&self) -> &str {
-        "PkHashFilterExec"
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.input.schema()
-    }
-
-    fn properties(&self) -> &Arc<PlanProperties> {
-        &self.properties
-    }
-
-    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        vec![&self.input]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> DFResult<Arc<dyn ExecutionPlan>> {
-        if children.len() != 1 {
-            return Err(DataFusionError::Internal(
-                "PkHashFilterExec requires exactly one child".to_string(),
-            ));
-        }
-        Ok(Arc::new(Self::new(
-            children[0].clone(),
-            self.pk_columns.clone(),
-            self.blocked.clone(),
-            self.k,
-        )))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        context: Arc<TaskContext>,
-    ) -> DFResult<SendableRecordBatchStream> {
-        let input_stream = self.input.execute(partition, context)?;
-        Ok(Box::pin(PkHashFilterStream {
-            input: input_stream,
-            pk_columns: self.pk_columns.clone(),
-            blocked: self.blocked.clone(),
-            k: self.k,
-            schema: self.schema(),
-            input_seen: 0,
-            kept: 0,
-            warned: false,
-        }))
-    }
-}
-
-struct PkHashFilterStream {
-    input: SendableRecordBatchStream,
-    pk_columns: Vec<String>,
-    blocked: Vec<Arc<HashSet<u64>>>,
-    k: usize,
-    schema: SchemaRef,
-    input_seen: usize,
-    kept: usize,
-    warned: bool,
-}
-
-impl PkHashFilterStream {
-    fn filter_batch(&self, batch: RecordBatch) -> DFResult<RecordBatch> {
-        if self.blocked.is_empty() || batch.num_rows() == 0 {
-            return Ok(batch);
-        }
-        let pk_indices = resolve_pk_indices(&batch, &self.pk_columns)?;
-        let keep: BooleanArray = (0..batch.num_rows())
-            .map(|row| {
-                let hash = compute_pk_hash(&batch, &pk_indices, row);
-                !self.blocked.iter().any(|set| set.contains(&hash))
-            })
-            .collect();
-        filter_record_batch(&batch, &keep)
-            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
-    }
-}
-
-impl Stream for PkHashFilterStream {
-    type Item = DFResult<RecordBatch>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        match self.input.poll_next_unpin(cx) {
-            Poll::Ready(Some(Ok(batch))) => {
-                self.input_seen += batch.num_rows();
-                match self.filter_batch(batch) {
-                    Ok(out) => {
-                        self.kept += out.num_rows();
-                        Poll::Ready(Some(Ok(out)))
-                    }
-                    Err(e) => Poll::Ready(Some(Err(e))),
-                }
-            }
-            Poll::Ready(None) => {
-                // >= k candidates in, < k out: the over-fetch missed superseded rows.
-                if !self.warned && self.input_seen >= self.k && self.kept < self.k {
-                    warn!(
-                        k = self.k,
-                        fetched = self.input_seen,
-                        kept = self.kept,
-                        "LSM vector search: < k live rows survived the PK-hash post-filter; \
-                         raise the over-fetch factor or use a true KNN prefilter."
-                    );
-                    self.warned = true;
-                }
-                Poll::Ready(None)
-            }
-            other => other,
-        }
-    }
-}
-
-impl datafusion::physical_plan::RecordBatchStream for PkHashFilterStream {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow_array::{Int32Array, StringArray};
-    use arrow_schema::{DataType, Field, Schema};
-    use datafusion::prelude::SessionContext;
-    use datafusion_physical_plan::test::TestMemoryExec;
-    use futures::TryStreamExt;
-
-    /// Hash a single-column Int32 PK value the way the exec does, so a test can
-    /// build blocked sets from values rather than hand-computed hashes.
-    fn hash_int_pk(id: i32) -> u64 {
-        let batch = int_batch(&[id]);
-        let pk_indices = resolve_pk_indices(&batch, &["id".to_string()]).unwrap();
-        compute_pk_hash(&batch, &pk_indices, 0)
-    }
-
-    fn int_batch(ids: &[i32]) -> RecordBatch {
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
-        RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(ids.to_vec()))]).unwrap()
-    }
-
-    fn blocked(ids: &[i32]) -> Vec<Arc<HashSet<u64>>> {
-        vec![Arc::new(ids.iter().map(|&id| hash_int_pk(id)).collect())]
-    }
-
-    async fn run(exec: PkHashFilterExec) -> Vec<i32> {
-        let ctx = SessionContext::new();
-        let out: Vec<RecordBatch> = exec
-            .execute(0, ctx.task_ctx())
-            .unwrap()
-            .try_collect()
-            .await
-            .unwrap();
-        out.iter()
-            .flat_map(|b| {
-                b.column_by_name("id")
-                    .unwrap()
-                    .as_any()
-                    .downcast_ref::<Int32Array>()
-                    .unwrap()
-                    .values()
-                    .to_vec()
-            })
-            .collect()
-    }
-
-    #[tokio::test]
-    async fn drops_rows_with_blocked_pk_hash() {
-        let b = int_batch(&[10, 20, 30]);
-        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
-        let exec = PkHashFilterExec::new(input, vec!["id".to_string()], blocked(&[20]), 1);
-        assert_eq!(run(exec).await, vec![10, 30]);
-    }
-
-    #[tokio::test]
-    async fn blocks_a_pk_present_in_any_generation_set() {
-        // Two newer-gen sets: a row is dropped if either contains its PK.
-        let b = int_batch(&[10, 20, 30]);
-        let sets = vec![
-            Arc::new(HashSet::from([hash_int_pk(10)])),
-            Arc::new(HashSet::from([hash_int_pk(30)])),
-        ];
-        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
-        let exec = PkHashFilterExec::new(input, vec!["id".to_string()], sets, 1);
-        assert_eq!(run(exec).await, vec![20]);
-    }
-
-    #[tokio::test]
-    async fn empty_blocked_keeps_all_rows() {
-        let b = int_batch(&[1, 2, 3]);
-        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
-        let exec = PkHashFilterExec::new(input, vec!["id".to_string()], Vec::new(), 1);
-        assert_eq!(run(exec).await, vec![1, 2, 3]);
-    }
-
-    #[tokio::test]
-    async fn null_pk_is_hashed_consistently_and_blockable() {
-        // A null PK hashes deterministically (compute_pk_hash hashes is_null),
-        // so a superseded null-key row can be dropped like any other.
-        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, true)]));
-        let with_null = |ids: Vec<Option<i32>>| {
-            RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(ids))]).unwrap()
-        };
-        let pk = vec!["id".to_string()];
-        let null_row = with_null(vec![None]);
-        let pk_indices = resolve_pk_indices(&null_row, &pk).unwrap();
-        let sets = vec![Arc::new(HashSet::from([compute_pk_hash(
-            &null_row,
-            &pk_indices,
-            0,
-        )]))];
-
-        // Rows: 10, NULL, 30 — only the NULL-key row is dropped.
-        let b = with_null(vec![Some(10), None, Some(30)]);
-        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
-        let exec = PkHashFilterExec::new(input, pk, sets, 1);
-        assert_eq!(run(exec).await, vec![10, 30]);
-    }
-
-    #[tokio::test]
-    async fn composite_pk_hash_matches_block_set() {
-        // Composite PK (id, name): block the (2, "b") tuple only.
-        let schema = Arc::new(Schema::new(vec![
-            Field::new("id", DataType::Int32, false),
-            Field::new("name", DataType::Utf8, false),
-        ]));
-        let mk = |ids: &[i32], names: &[&str]| {
-            RecordBatch::try_new(
-                schema.clone(),
-                vec![
-                    Arc::new(Int32Array::from(ids.to_vec())),
-                    Arc::new(StringArray::from(names.to_vec())),
-                ],
-            )
-            .unwrap()
-        };
-        let pk = vec!["id".to_string(), "name".to_string()];
-        let one_row = mk(&[2], &["b"]);
-        let pk_indices = resolve_pk_indices(&one_row, &pk).unwrap();
-        let sets = vec![Arc::new(HashSet::from([compute_pk_hash(
-            &one_row,
-            &pk_indices,
-            0,
-        )]))];
-
-        // (1,"a") and (2,"a") survive; only the exact (2,"b") tuple is dropped.
-        let b = mk(&[1, 2, 2], &["a", "a", "b"]);
-        let input = TestMemoryExec::try_new_exec(&[vec![b.clone()]], b.schema(), None).unwrap();
-        let exec = PkHashFilterExec::new(input, pk, sets, 1);
-        assert_eq!(run(exec).await, vec![1, 2]);
-    }
-}
diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/within_source_dedup.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/within_source_dedup.rs
deleted file mode 100644
index be5dae6a668..00000000000
--- a/rust/lance/src/dataset/mem_wal/scanner/exec/within_source_dedup.rs
+++ /dev/null
@@ -1,432 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The Lance Authors
-
-//! WithinSourceDedupExec - Deduplicates rows with the same primary key from a
-//! single LSM source, keeping the newest insert.
-//!
-//! In MemWAL/LSM mode the same primary key can be written multiple times into
-//! the same memtable. The active memtable stores rows in insert order (larger
-//! `_rowaddr` = newer), while flushed memtables are reverse-written so that
-//! within a flushed file the smallest `_rowid` is the newest insert (see
-//! `memtable/flush.rs:152` and `hnsw/storage.rs:307`). Point lookup uses this
-//! node to collapse such duplicates *within a single source* so that the
-//! downstream `CoalesceFirstExec` / `LIMIT` sees at most one row per primary
-//! key per source.
-
-use std::any::Any;
-use std::collections::HashMap;
-use std::fmt;
-use std::pin::Pin;
-use std::sync::Arc;
-use std::task::{Context, Poll};
-
-use arrow_array::{Array, RecordBatch, UInt64Array};
-use arrow_schema::SchemaRef;
-use datafusion::error::Result as DFResult;
-use datafusion::execution::TaskContext;
-use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
-use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
-    SendableRecordBatchStream,
-};
-use futures::{Stream, StreamExt, ready};
-
-use super::pk::{compute_pk_hash, resolve_pk_indices};
-
-/// Among rows that share a primary key, which row-address extreme identifies
-/// the newest insert to keep. The kept row is always the freshest; only the
-/// row address (`_rowaddr`/`_rowid`) used to find it differs by source.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum DedupDirection {
-    /// Keep the row with the largest row-address value (active memtable: larger
-    /// `_rowaddr` = inserted later).
-    KeepMaxRowAddr,
-    /// Keep the row with the smallest row-address value (flushed memtable under
-    /// reverse-write: smaller `_rowid` = inserted later).
-    KeepMinRowAddr,
-}
-
-/// Deduplicates rows from a single source by primary key, keeping the row
-/// whose `row_addr_column` value wins per [`DedupDirection`].
-///
-/// # Required columns
-///
-/// The input must expose:
-/// - All `pk_columns`
-/// - `row_addr_column` of `UInt64` type
-///
-/// The output schema is unchanged from the input. Callers that need to hide
-/// the row-address column from downstream consumers should compose this node
-/// with `project_to_canonical` or `null_columns`.
-///
-/// # Performance
-///
-/// Memory: `O(unique primary keys in input)`. For point lookup the input is
-/// already filtered to a single primary key so the map holds at most one
-/// entry.
-#[derive(Debug)]
-pub struct WithinSourceDedupExec {
-    input: Arc<dyn ExecutionPlan>,
-    pk_columns: Vec<String>,
-    row_addr_column: String,
-    direction: DedupDirection,
-    schema: SchemaRef,
-    properties: Arc<PlanProperties>,
-}
-
-impl WithinSourceDedupExec {
-    pub fn new(
-        input: Arc<dyn ExecutionPlan>,
-        pk_columns: Vec<String>,
-        row_addr_column: impl Into<String>,
-        direction: DedupDirection,
-    ) -> Self {
-        let schema = input.schema();
-        let properties = Arc::new(PlanProperties::new(
-            EquivalenceProperties::new(schema.clone()),
-            Partitioning::UnknownPartitioning(1),
-            input.pipeline_behavior(),
-            input.boundedness(),
-        ));
-        Self {
-            input,
-            pk_columns,
-            row_addr_column: row_addr_column.into(),
-            direction,
-            schema,
-            properties,
-        }
-    }
-
-    pub fn pk_columns(&self) -> &[String] {
-        &self.pk_columns
-    }
-
-    pub fn row_addr_column(&self) -> &str {
-        &self.row_addr_column
-    }
-
-    pub fn direction(&self) -> DedupDirection {
-        self.direction
-    }
-}
-
-impl DisplayAs for WithinSourceDedupExec {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
-        match t {
-            DisplayFormatType::Default
-            | DisplayFormatType::Verbose
-            | DisplayFormatType::TreeRender => {
-                write!(
-                    f,
-                    "WithinSourceDedupExec: pk=[{}], row_addr={}, direction={:?}",
-                    self.pk_columns.join(", "),
-                    self.row_addr_column,
-                    self.direction,
-                )
-            }
-        }
-    }
-}
-
-impl ExecutionPlan for WithinSourceDedupExec {
-    fn name(&self) -> &str {
-        "WithinSourceDedupExec"
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-
-    fn properties(&self) -> &Arc<PlanProperties> {
-        &self.properties
-    }
-
-    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        vec![&self.input]
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> DFResult<Arc<dyn ExecutionPlan>> {
-        if children.len() != 1 {
-            return Err(datafusion::error::DataFusionError::Internal(
-                "WithinSourceDedupExec requires exactly one child".to_string(),
-            ));
-        }
-        Ok(Arc::new(Self::new(
-            children[0].clone(),
-            self.pk_columns.clone(),
-            self.row_addr_column.clone(),
-            self.direction,
-        )))
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        context: Arc<TaskContext>,
-    ) -> DFResult<SendableRecordBatchStream> {
-        let input_stream = self.input.execute(partition, context)?;
-        Ok(Box::pin(WithinSourceDedupStream {
-            input: input_stream,
-            pk_columns: self.pk_columns.clone(),
-            row_addr_column: self.row_addr_column.clone(),
-            direction: self.direction,
-            schema: self.schema.clone(),
-            winners: HashMap::new(),
-            emitted: false,
-        }))
-    }
-}
-
-/// One winning row, materialized as a single-row `RecordBatch` so we don't
-/// have to keep the source batch alive after we've picked the winner.
-struct Winner {
-    batch: RecordBatch,
-    row_addr: u64,
-}
-
-struct WithinSourceDedupStream {
-    input: SendableRecordBatchStream,
-    pk_columns: Vec<String>,
-    row_addr_column: String,
-    direction: DedupDirection,
-    schema: SchemaRef,
-    winners: HashMap<u64, Winner>,
-    emitted: bool,
-}
-
-impl WithinSourceDedupStream {
-    fn consume_batch(&mut self, batch: RecordBatch) -> DFResult<()> {
-        if batch.num_rows() == 0 {
-            return Ok(());
-        }
-        let pk_indices = resolve_pk_indices(&batch, &self.pk_columns)?;
-        let row_addr_array = batch
-            .column_by_name(&self.row_addr_column)
-            .ok_or_else(|| {
-                datafusion::error::DataFusionError::Internal(format!(
-                    "Row-address column '{}' not found in batch",
-                    self.row_addr_column
-                ))
-            })?
-            .as_any()
-            .downcast_ref::<UInt64Array>()
-            .ok_or_else(|| {
-                datafusion::error::DataFusionError::Internal(format!(
-                    "Row-address column '{}' is not UInt64",
-                    self.row_addr_column
-                ))
-            })?;
-
-        for row_idx in 0..batch.num_rows() {
-            if row_addr_array.is_null(row_idx) {
-                // A NULL row address can't be ordered against a real one. Skip
-                // rather than guess — callers should always project a real
-                // row-address column for dedup-eligible sources.
-                continue;
-            }
-            let row_addr = row_addr_array.value(row_idx);
-            let pk_hash = compute_pk_hash(&batch, &pk_indices, row_idx);
-
-            let take_row = match self.winners.get(&pk_hash) {
-                None => true,
-                Some(existing) => match self.direction {
-                    DedupDirection::KeepMaxRowAddr => row_addr > existing.row_addr,
-                    DedupDirection::KeepMinRowAddr => row_addr < existing.row_addr,
-                },
-            };
-
-            if take_row {
-                let single = batch.slice(row_idx, 1);
-                self.winners.insert(
-                    pk_hash,
-                    Winner {
-                        batch: single,
-                        row_addr,
-                    },
-                );
-            }
-        }
-        Ok(())
-    }
-
-    fn finalize(&mut self) -> DFResult<RecordBatch> {
-        if self.winners.is_empty() {
-            return Ok(RecordBatch::new_empty(self.schema.clone()));
-        }
-        let batches: Vec<RecordBatch> = self.winners.drain().map(|(_, w)| w.batch).collect();
-        let batch_refs: Vec<&RecordBatch> = batches.iter().collect();
-        arrow_select::concat::concat_batches(&self.schema, batch_refs)
-            .map_err(|e| datafusion::error::DataFusionError::ArrowError(Box::new(e), None))
-    }
-}
-
-impl Stream for WithinSourceDedupStream {
-    type Item = DFResult<RecordBatch>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        loop {
-            if self.emitted {
-                return Poll::Ready(None);
-            }
-            match ready!(self.input.poll_next_unpin(cx)) {
-                Some(Ok(batch)) => {
-                    if let Err(e) = self.consume_batch(batch) {
-                        self.emitted = true;
-                        return Poll::Ready(Some(Err(e)));
-                    }
-                }
-                Some(Err(e)) => {
-                    self.emitted = true;
-                    return Poll::Ready(Some(Err(e)));
-                }
-                None => {
-                    self.emitted = true;
-                    return Poll::Ready(Some(self.finalize()));
-                }
-            }
-        }
-    }
-}
-
-impl datafusion::physical_plan::RecordBatchStream for WithinSourceDedupStream {
-    fn schema(&self) -> SchemaRef {
-        self.schema.clone()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use arrow_array::{Float32Array, Int32Array, StringArray};
-    use arrow_schema::{DataType, Field, Schema};
-    use datafusion::prelude::SessionContext;
-    use datafusion_physical_plan::test::TestMemoryExec;
-    use futures::TryStreamExt;
-
-    fn create_test_schema() -> SchemaRef {
-        Arc::new(Schema::new(vec![
-            Field::new("id", DataType::Int32, false),
-            Field::new("name", DataType::Utf8, true),
-            Field::new("_distance", DataType::Float32, true),
-            Field::new("_row_addr", DataType::UInt64, true),
-        ]))
-    }
-
-    fn batch(ids: &[i32], names: &[&str], distances: &[f32], row_addr: &[u64]) -> RecordBatch {
-        let schema = create_test_schema();
-        RecordBatch::try_new(
-            schema,
-            vec![
-                Arc::new(Int32Array::from(ids.to_vec())),
-                Arc::new(StringArray::from(names.to_vec())),
-                Arc::new(Float32Array::from(distances.to_vec())),
-                Arc::new(UInt64Array::from(row_addr.to_vec())),
-            ],
-        )
-        .unwrap()
-    }
-
-    async fn run(batches: Vec<RecordBatch>, direction: DedupDirection) -> Vec<RecordBatch> {
-        let schema = create_test_schema();
-        let input = TestMemoryExec::try_new_exec(&[batches], schema, None).unwrap();
-        let exec =
-            WithinSourceDedupExec::new(input, vec!["id".to_string()], "_row_addr", direction);
-        let ctx = SessionContext::new();
-        let stream = exec.execute(0, ctx.task_ctx()).unwrap();
-        stream.try_collect().await.unwrap()
-    }
-
-    fn extract(batches: &[RecordBatch]) -> Vec<(i32, String, u64)> {
-        let mut out = Vec::new();
-        for b in batches {
-            let ids = b.column(0).as_any().downcast_ref::<Int32Array>().unwrap();
-            let names = b.column(1).as_any().downcast_ref::<StringArray>().unwrap();
-            let addr = b.column(3).as_any().downcast_ref::<UInt64Array>().unwrap();
-            for i in 0..b.num_rows() {
-                out.push((ids.value(i), names.value(i).to_string(), addr.value(i)));
-            }
-        }
-        out.sort_by_key(|(id, _, _)| *id);
-        out
-    }
-
-    #[tokio::test]
-    async fn keep_max_picks_largest_row_addr() {
-        // Active-memtable case: same pk inserted twice; newer = larger _rowaddr.
-        let b1 = batch(
-            &[1, 1, 2],
-            &["old", "new", "two"],
-            &[0.1, 0.2, 0.3],
-            &[10, 99, 5],
-        );
-        let out = run(vec![b1], DedupDirection::KeepMaxRowAddr).await;
-        let rows = extract(&out);
-        assert_eq!(rows.len(), 2);
-        assert_eq!(rows[0], (1, "new".to_string(), 99));
-        assert_eq!(rows[1], (2, "two".to_string(), 5));
-    }
-
-    #[tokio::test]
-    async fn keep_min_picks_smallest_row_addr() {
-        // Flushed-memtable case under reverse-write: newer = smaller _rowid.
-        let b1 = batch(
-            &[1, 1, 2],
-            &["old", "new", "two"],
-            &[0.1, 0.2, 0.3],
-            &[99, 10, 5],
-        );
-        let out = run(vec![b1], DedupDirection::KeepMinRowAddr).await;
-        let rows = extract(&out);
-        assert_eq!(rows.len(), 2);
-        assert_eq!(rows[0], (1, "new".to_string(), 10));
-        assert_eq!(rows[1], (2, "two".to_string(), 5));
-    }
-
-    #[tokio::test]
-    async fn dedup_across_batches() {
-        let b1 = batch(&[1, 2], &["a", "b"], &[0.1, 0.2], &[1, 1]);
-        let b2 = batch(&[1, 3], &["a_new", "c"], &[0.5, 0.4], &[7, 1]);
-        let out = run(vec![b1, b2], DedupDirection::KeepMaxRowAddr).await;
-        let rows = extract(&out);
-        assert_eq!(rows.len(), 3);
-        assert_eq!(rows[0], (1, "a_new".to_string(), 7));
-        assert_eq!(rows[1], (2, "b".to_string(), 1));
-        assert_eq!(rows[2], (3, "c".to_string(), 1));
-    }
-
-    #[tokio::test]
-    async fn empty_input() {
-        let out = run(vec![], DedupDirection::KeepMaxRowAddr).await;
-        let total: usize = out.iter().map(|b| b.num_rows()).sum();
-        assert_eq!(total, 0);
-    }
-
-    #[tokio::test]
-    async fn null_row_addr_skipped() {
-        // Rows with NULL row address can't be ordered — they're dropped so they
-        // don't accidentally become winners against real values.
-        let schema = create_test_schema();
-        let b = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from(vec![1, 1])),
-                Arc::new(StringArray::from(vec!["nulladdr", "real"])),
-                Arc::new(Float32Array::from(vec![0.1, 0.2])),
-                Arc::new(UInt64Array::from(vec![None, Some(5)])),
-            ],
-        )
-        .unwrap();
-        let out = run(vec![b], DedupDirection::KeepMaxRowAddr).await;
-        let rows = extract(&out);
-        assert_eq!(rows.len(), 1);
-        assert_eq!(rows[0], (1, "real".to_string(), 5));
-    }
-}
diff --git a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs b/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs
index 39abf7e8c71..7a011078571 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs
@@ -22,7 +22,9 @@
 use std::collections::HashSet;
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use lance_core::{Error, Result};
+use lance_io::object_store::ObjectStoreParams;
 
 use crate::dataset::{Dataset, DatasetBuilder};
 use crate::session::Session;
@@ -38,15 +40,22 @@ use crate::session::Session;
 /// The key is the resolved absolute flushed path
 /// (`{base}/_mem_wal/{shard}/{folder}`), which is globally unique, so a single
 /// cache can safely span multiple tables.
+///
+/// `store_params` is deliberately *not* part of the key: the first caller to
+/// open a path binds the store that every later hit reuses. Credential rotation
+/// still works — a vended-credential store holds the live
+/// `StorageOptionsAccessor` and re-resolves per request, so a cached handle
+/// never carries expired credentials. What this does assume is that a given
+/// path is only ever served under one store configuration. Serving one table
+/// through a single cache under two different `ObjectStoreParams` would hand
+/// every caller the store the first one opened with.
 pub struct FlushedMemTableCache {
     // `moka`'s async cache gives a bounded size plus single-flight
     // `try_get_with`, so concurrent first-queries on a just-flushed
-    // generation open the dataset exactly once.
+    // generation open the dataset exactly once. The opened dataset carries the
+    // session index cache, which also backs each generation's standalone PK
+    // dedup index (see `block_list::open_pk_index`) — no separate cache path.
     inner: moka::future::Cache<String, Arc<Dataset>>,
-    // Per-generation set of PK hashes for the vector-search block-list, keyed by
-    // the same immutable flushed path. Built lazily on the first query that needs
-    // it (single-flight) so repeated searches skip re-scanning the PK column.
-    pk_hashes: moka::future::Cache<String, Arc<HashSet<u64>>>,
 }
 
 impl FlushedMemTableCache {
@@ -63,24 +72,18 @@ impl FlushedMemTableCache {
                 // into at build time.
                 .support_invalidation_closures()
                 .build(),
-            pk_hashes: moka::future::Cache::builder()
-                .max_capacity(max_entries)
-                .support_invalidation_closures()
-                .build(),
         }
     }
 
     /// Get the dataset for `path`, opening it (exactly once) on a miss.
-    ///
-    /// `session` is threaded into the open so the first open populates the
-    /// shared index / file-metadata caches; subsequent hits are a pure
-    /// `Arc::clone` with zero object-store I/O. Concurrent callers for the
-    /// same path share a single open via `moka`'s single-flight
-    /// `try_get_with`.
+    /// Concurrent callers share a single open via `moka`'s single-flight
+    /// `try_get_with`; hits are a pure `Arc::clone`. `session` / `store_params`
+    /// configure the open.
     pub async fn get_or_open(
         &self,
         path: &str,
         session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
     ) -> Result<Arc<Dataset>> {
         self.inner
             .try_get_with(path.to_string(), async move {
@@ -88,6 +91,9 @@ impl FlushedMemTableCache {
                 if let Some(session) = session {
                     builder = builder.with_session(session);
                 }
+                if let Some(store_params) = store_params {
+                    builder = builder.with_store_params(store_params);
+                }
                 builder.load().await.map(Arc::new)
             })
             .await
@@ -96,21 +102,6 @@ impl FlushedMemTableCache {
             .map_err(|e: Arc<Error>| Error::cloned(e.to_string()))
     }
 
-    /// Get the cached set of PK hashes for `path`, building it (exactly once) on
-    /// a miss via `build`. The flushed path is immutable, so a cached set is
-    /// never stale; concurrent first-queries share one build via `moka`'s
-    /// single-flight `try_get_with`.
-    pub async fn get_or_build_pk_hashes(
-        &self,
-        path: &str,
-        build: impl std::future::Future<Output = Result<HashSet<u64>>>,
-    ) -> Result<Arc<HashSet<u64>>> {
-        self.pk_hashes
-            .try_get_with(path.to_string(), async move { build.await.map(Arc::new) })
-            .await
-            .map_err(|e: Arc<Error>| Error::cloned(e.to_string()))
-    }
-
     /// Drop cached entries whose path is not in `live_paths`.
     ///
     /// Called by the consumer after compaction retires generations. Purely a
@@ -125,10 +116,6 @@ impl FlushedMemTableCache {
         let _ = self
             .inner
             .invalidate_entries_if(move |path, _| !live.contains(path));
-        let live = live_paths.clone();
-        let _ = self
-            .pk_hashes
-            .invalidate_entries_if(move |path, _| !live.contains(path));
     }
 }
 
@@ -140,29 +127,110 @@ impl std::fmt::Debug for FlushedMemTableCache {
     }
 }
 
+/// Caching of opened flushed-generation datasets, keyed by immutable path. The
+/// opened dataset carries the session index cache, which also backs each
+/// generation's secondary indexes and its PK dedup sidecar (see
+/// `block_list::open_pk_index`) — so a single `get_or_open` is the
+/// whole caching surface. Implemented by [`FlushedMemTableCache`]; a
+/// [`GenerationWarmer`] composes one to warm through it, and a consumer may
+/// supply its own implementation.
+#[async_trait]
+pub trait DatasetCache: Send + Sync + std::fmt::Debug {
+    async fn get_or_open(
+        &self,
+        path: &str,
+        session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Arc<Dataset>>;
+
+    /// Drop cached entries whose path is not in `live_paths`. Async so an
+    /// implementation can evict retired generations' index objects (e.g.
+    /// `Session::invalidate_index_prefix`) without a later breaking signature
+    /// change; [`FlushedMemTableCache`]'s own eviction is synchronous.
+    async fn retain_paths(&self, live_paths: &HashSet<String>);
+}
+
+#[async_trait]
+impl DatasetCache for FlushedMemTableCache {
+    async fn get_or_open(
+        &self,
+        path: &str,
+        session: Option<Arc<Session>>,
+        store_params: Option<ObjectStoreParams>,
+    ) -> Result<Arc<Dataset>> {
+        Self::get_or_open(self, path, session, store_params).await
+    }
+
+    async fn retain_paths(&self, live_paths: &HashSet<String>) {
+        Self::retain_paths(self, live_paths)
+    }
+}
+
+/// Proactively warms a flushed generation into the shared caches: open the
+/// dataset and pre-load its secondary indexes and PK dedup sidecar so the first
+/// query sees no cold reads. This is the **seam** the flush and read paths fire
+/// — lance defines it; the consumer (e.g. the WAL pod) implements it. `None` =>
+/// no warming, generations warm lazily on first read.
+///
+/// Everything a warmer touches is keyed by the immutable generation `path`
+/// (opened dataset, its secondary indexes, its PK dedup sidecar), so `path` is
+/// the only input it needs.
+///
+/// `warm` is fired fire-and-forget from every read path that opens a generation
+/// (all four LSM planners) as well as pre-commit on flush, so the same path may
+/// be warmed concurrently and repeatedly. Implementations **must be idempotent
+/// and cheap when the path is already warm** (e.g. dedup in-flight and
+/// completed paths) — a redundant call must not re-do work or fail.
+#[async_trait]
+pub trait GenerationWarmer: Send + Sync + std::fmt::Debug {
+    async fn warm(&self, path: &str) -> Result<()>;
+}
+
 /// Open a flushed-generation dataset, shared by all three LSM open sites
 /// (scan, point lookup, vector search).
 ///
-/// - `cache` present: route through [`FlushedMemTableCache`] (single-flight,
-///   shared `Arc`, manifest read amortized across queries).
+/// - `cache` present: route through a [`DatasetCache`] (e.g.
+///   [`FlushedMemTableCache`]: single-flight, shared `Arc`, manifest read
+///   amortized across queries).
 /// - `cache` absent: cold open via [`DatasetBuilder`]. Passing `session`
 ///   still reuses the shared index / metadata caches; `None`/`None`
 ///   reproduces the original per-query cold-open behavior exactly.
+/// - `warmer` present: fire a fire-and-forget warm-on-open backstop behind the
+///   returned handle (the warmer dedups already-warm paths). `None` => no warming.
 pub async fn open_flushed_dataset(
     path: &str,
     session: Option<&Arc<Session>>,
-    cache: Option<&Arc<FlushedMemTableCache>>,
+    store_params: Option<&ObjectStoreParams>,
+    cache: Option<&Arc<dyn DatasetCache>>,
+    warmer: Option<&Arc<dyn GenerationWarmer>>,
 ) -> Result<Arc<Dataset>> {
-    match cache {
-        Some(cache) => cache.get_or_open(path, session.cloned()).await,
+    let dataset = match cache {
+        Some(cache) => {
+            cache
+                .get_or_open(path, session.cloned(), store_params.cloned())
+                .await?
+        }
         None => {
             let mut builder = DatasetBuilder::from_uri(path);
             if let Some(session) = session {
                 builder = builder.with_session(session.clone());
             }
-            Ok(Arc::new(builder.load().await?))
+            if let Some(store_params) = store_params {
+                builder = builder.with_store_params(store_params.clone());
+            }
+            Arc::new(builder.load().await?)
         }
+    };
+    if let Some(warmer) = warmer {
+        let warmer = Arc::clone(warmer);
+        let path = path.to_string();
+        tokio::spawn(async move {
+            if let Err(e) = warmer.warm(&path).await {
+                tracing::debug!(generation = %path, error = %e, "warm-on-open failed");
+            }
+        });
     }
+    Ok(dataset)
 }
 
 #[cfg(test)]
@@ -200,8 +268,8 @@ mod tests {
         write_dataset(&uri, &[1, 2, 3]).await;
 
         let cache = FlushedMemTableCache::new(8);
-        let first = cache.get_or_open(&uri, None).await.unwrap();
-        let second = cache.get_or_open(&uri, None).await.unwrap();
+        let first = cache.get_or_open(&uri, None, None).await.unwrap();
+        let second = cache.get_or_open(&uri, None, None).await.unwrap();
 
         assert!(
             Arc::ptr_eq(&first, &second),
@@ -232,7 +300,7 @@ mod tests {
             let calls = calls.clone();
             handles.push(tokio::spawn(async move {
                 calls.fetch_add(1, Ordering::SeqCst);
-                cache.get_or_open(&uri, None).await.unwrap()
+                cache.get_or_open(&uri, None, None).await.unwrap()
             }));
         }
 
@@ -250,34 +318,6 @@ mod tests {
         assert_eq!(cache.inner.entry_count(), 1, "exactly one entry cached");
     }
 
-    #[tokio::test]
-    async fn pk_hashes_cached_reuses_first_build() {
-        // The PK-hash set is keyed by the immutable flushed path: a hit returns
-        // the first-built set and never runs the second build closure.
-        let cache = FlushedMemTableCache::new(8);
-        let path = "memory://shard/gen_1";
-        let first = cache
-            .get_or_build_pk_hashes(path, async { Ok(HashSet::from([1u64, 2])) })
-            .await
-            .unwrap();
-        let second = cache
-            .get_or_build_pk_hashes(path, async {
-                // Different contents; must be ignored because the path is cached.
-                Ok(HashSet::from([9u64]))
-            })
-            .await
-            .unwrap();
-        assert!(
-            Arc::ptr_eq(&first, &second),
-            "a PK-hash cache hit must reuse the first-built set"
-        );
-        assert_eq!(
-            second.len(),
-            2,
-            "cached set keeps the first build's contents"
-        );
-    }
-
     #[tokio::test]
     async fn test_retain_paths_drops_unreferenced() {
         let temp_dir = tempfile::tempdir().unwrap();
@@ -288,8 +328,8 @@ mod tests {
         write_dataset(&drop_uri, &[2]).await;
 
         let cache = FlushedMemTableCache::new(8);
-        cache.get_or_open(&keep_uri, None).await.unwrap();
-        cache.get_or_open(&drop_uri, None).await.unwrap();
+        cache.get_or_open(&keep_uri, None, None).await.unwrap();
+        cache.get_or_open(&drop_uri, None, None).await.unwrap();
         cache.inner.run_pending_tasks().await;
         assert_eq!(cache.inner.entry_count(), 2);
 
@@ -310,8 +350,12 @@ mod tests {
         let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap());
         write_dataset(&uri, &[7, 8, 9]).await;
 
-        let a = open_flushed_dataset(&uri, None, None).await.unwrap();
-        let b = open_flushed_dataset(&uri, None, None).await.unwrap();
+        let a = open_flushed_dataset(&uri, None, None, None, None)
+            .await
+            .unwrap();
+        let b = open_flushed_dataset(&uri, None, None, None, None)
+            .await
+            .unwrap();
         assert!(
             !Arc::ptr_eq(&a, &b),
             "no-cache path must cold-open each call"
@@ -319,13 +363,57 @@ mod tests {
         assert_eq!(a.count_rows(None).await.unwrap(), 3);
 
         // With a cache, the second call is a shared clone.
-        let cache = Arc::new(FlushedMemTableCache::new(8));
-        let c = open_flushed_dataset(&uri, None, Some(&cache))
+        let cache: Arc<dyn DatasetCache> = Arc::new(FlushedMemTableCache::new(8));
+        let c = open_flushed_dataset(&uri, None, None, Some(&cache), None)
             .await
             .unwrap();
-        let d = open_flushed_dataset(&uri, None, Some(&cache))
+        let d = open_flushed_dataset(&uri, None, None, Some(&cache), None)
             .await
             .unwrap();
         assert!(Arc::ptr_eq(&c, &d), "cached path must reuse the Arc");
     }
+
+    /// A warmer that records calls and signals each one.
+    #[derive(Debug)]
+    struct NotifyingWarmer {
+        calls: Arc<AtomicUsize>,
+        notify: Arc<tokio::sync::Notify>,
+    }
+
+    #[async_trait]
+    impl GenerationWarmer for NotifyingWarmer {
+        async fn warm(&self, _path: &str) -> Result<()> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            self.notify.notify_one();
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_open_flushed_dataset_fires_warm_on_open() {
+        // The warm-on-open backstop fires the warmer (fire-and-forget) when a
+        // generation is opened, so generations the flusher never warmed still
+        // get warmed lazily on first read.
+        let temp_dir = tempfile::tempdir().unwrap();
+        let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap());
+        write_dataset(&uri, &[1, 2, 3]).await;
+
+        let calls = Arc::new(AtomicUsize::new(0));
+        let notify = Arc::new(tokio::sync::Notify::new());
+        let warmer: Arc<dyn GenerationWarmer> = Arc::new(NotifyingWarmer {
+            calls: calls.clone(),
+            notify: notify.clone(),
+        });
+
+        let ds = open_flushed_dataset(&uri, None, None, None, Some(&warmer))
+            .await
+            .unwrap();
+        assert_eq!(ds.count_rows(None).await.unwrap(), 3);
+
+        // The warm is spawned fire-and-forget; wait (bounded) for it to run.
+        tokio::time::timeout(std::time::Duration::from_secs(5), notify.notified())
+            .await
+            .expect("warm-on-open must fire");
+        assert_eq!(calls.load(Ordering::SeqCst), 1, "warmer fired once on open");
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
index aa086a80e66..60000666f52 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs
@@ -44,23 +44,67 @@ use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
 use datafusion::physical_plan::union::UnionExec;
+use datafusion::prelude::Expr;
 use lance_core::{Error, Result, is_system_column};
 use lance_index::scalar::FullTextSearchQuery;
-use lance_index::scalar::inverted::query::FtsQuery as IndexFtsQuery;
+use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator};
 use tracing::instrument;
 
+use super::block_list::compute_source_block_lists;
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
-use super::flushed_cache::{FlushedMemTableCache, open_flushed_dataset};
-use super::projection::project_to_canonical;
+use super::exec::PkBlockFilterExec;
+use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
+use super::projection::{project_to_canonical, validate_projection_names};
 use crate::dataset::mem_wal::memtable::scanner::MemTableScanner;
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// `_score` column name in FTS results — kept aligned with
 /// `lance_index::scalar::inverted::SCORE_COL` so this module doesn't
 /// require an import for one string constant.
 pub const SCORE_COLUMN: &str = "_score";
 
+/// Default over-fetch multiple for blocked sources. `1.0` keeps cross-generation
+/// dedup on with no over-fetch; callers (e.g. the sophon WAL handler) raise it
+/// so a blocked source still yields `k` live rows after the block-list filter.
+const DEFAULT_OVERFETCH_FACTOR: f64 = 1.0;
+
+fn validate_lsm_fts_query(query: &FullTextSearchQuery) -> Result<()> {
+    match &query.query {
+        IndexFtsQuery::Match(m) => {
+            if m.fuzziness != Some(0) && m.operator != Operator::Or {
+                return Err(Error::not_supported(
+                    "LSM fuzzy full-text search only supports OR match operators".to_string(),
+                ));
+            }
+            Ok(())
+        }
+        IndexFtsQuery::Phrase(_) => Ok(()),
+        _ => Err(Error::not_supported(
+            "LSM full-text search only supports match and phrase leaf queries".to_string(),
+        )),
+    }
+}
+
+fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool {
+    match source {
+        LsmDataSource::ActiveMemTable {
+            batch_store,
+            index_store,
+            ..
+        } => {
+            index_store
+                .get_fts_by_column(column)
+                .is_some_and(|index| !index.is_empty())
+                && batch_store
+                    .max_visible_row(index_store.max_visible_batch_position())
+                    .is_some()
+        }
+        _ => false,
+    }
+}
+
 /// Plans local-scoring FTS queries over LSM data.
 pub struct LsmFtsSearchPlanner {
     collector: LsmDataSourceCollector,
@@ -68,8 +112,18 @@ pub struct LsmFtsSearchPlanner {
     base_schema: SchemaRef,
     /// Session threaded into flushed-generation opens (shared caches).
     session: Option<Arc<Session>>,
+    /// Store params for opening flushed generations, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
     /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<FlushedMemTableCache>>,
+    flushed_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of a flushed generation.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Over-fetch multiple for blocked sources.
+    overfetch_factor: f64,
+    /// Optional prefilter predicate applied to every source arm so FTS hits
+    /// failing the predicate are dropped. Base/flushed arms use the dataset
+    /// scanner's native filter; memtable arms filter the materialized hits.
+    filter: Option<Expr>,
 }
 
 impl LsmFtsSearchPlanner {
@@ -84,24 +138,55 @@ impl LsmFtsSearchPlanner {
             pk_columns,
             base_schema,
             session: None,
+            store_params: None,
             flushed_cache: None,
+            warmer: None,
+            overfetch_factor: DEFAULT_OVERFETCH_FACTOR,
+            filter: None,
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Attach an optional prefilter predicate. Every source arm restricts its
+    /// FTS hits to rows matching the predicate, matching a normal filtered
+    /// full-text scan over base ∪ flushed ∪ in-memory data.
+    pub fn with_filter(mut self, filter: Option<Expr>) -> Self {
+        self.filter = filter;
+        self
+    }
+
+    /// Set the over-fetch multiple for blocked sources so they still yield `k`
+    /// live rows after cross-generation block-list filtering. Values below
+    /// `1.0` are rejected by [`Self::plan_search`].
+    pub fn with_overfetch_factor(mut self, factor: f64) -> Self {
+        self.overfetch_factor = factor;
+        self
+    }
+
+    /// Set the session used to open flushed generations.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
+    /// Set the store params used to open flushed generations.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
     /// Inject a cache of opened flushed-generation datasets, making repeated
     /// searches against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<FlushedMemTableCache>) -> Self {
+    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
         self.flushed_cache = Some(cache);
         self
     }
 
+    /// Inject the warmer fired on first open of a flushed generation.
+    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+        self.warmer = Some(warmer);
+        self
+    }
+
     /// Build the FTS execution plan (local scoring).
     ///
     /// # Arguments
@@ -110,39 +195,114 @@ impl LsmFtsSearchPlanner {
     /// * `query` — the FTS query (match / phrase / boolean / fuzzy for
     ///   base/flushed Lance sources; the active memtable currently
     ///   supports `MatchQuery`).
-    /// * `k` — global top-k to return.
+    /// * `limit` — optional global top-k to return.
     /// * `projection` — user columns to project. PK columns are
     ///   auto-included; `_score` is always appended.
     ///
     /// Each source is scored independently (local BM25), normalized to a
-    /// canonical schema, unioned, and merged by `_score` DESC with a
-    /// top-k cap pushed into each partition.
+    /// canonical schema, unioned, and merged by `_score` DESC. When a finite
+    /// limit is supplied, top-k caps are pushed into each partition.
     #[instrument(
         name = "lsm_fts_search",
         level = "info",
         skip_all,
-        fields(column = %column, k)
+        fields(column = %column, limit)
     )]
     pub async fn plan_search(
         &self,
         column: &str,
         query: FullTextSearchQuery,
-        k: usize,
+        limit: Option<usize>,
         projection: Option<&[String]>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let sources = self.collector.collect()?;
+        if sources
+            .iter()
+            .any(|source| active_source_can_execute_fts(source, column))
+        {
+            validate_lsm_fts_query(&query)?;
+        }
+        validate_projection_names(projection, &self.base_schema, &[SCORE_COLUMN])?;
         let target_schema = self.canonical_fts_schema(projection);
+        let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?;
 
         if sources.is_empty() {
             return self.empty_plan(&target_schema);
         }
 
+        // Per-source PK block sets for cross-generation dedup (NEWER(G) per
+        // shard; base = union of all gens). Query-type-agnostic — same call the
+        // vector planner makes. `Box::pin` keeps the future off
+        // `clippy::large_futures`.
+        let block_lists = Box::pin(compute_source_block_lists(
+            &sources,
+            self.session.as_ref(),
+            self.store_params.as_ref(),
+            self.flushed_cache.as_ref(),
+        ))
+        .await?;
+
+        // Stage the per-source over-fetch decisions, then build every source
+        // plan concurrently — the builds are independent and a sequential loop
+        // was the dominant serial planning cost at multiple generations.
+        let arm_inputs: Vec<_> = sources
+            .iter()
+            .map(|source| {
+                let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. });
+                let blocked = block_lists.get(&(source.shard_id(), source.generation()));
+                let active_needs_uncapped_recency = match source {
+                    LsmDataSource::ActiveMemTable { index_store, .. }
+                        if !self.pk_columns.is_empty() =>
+                    {
+                        !index_store.has_pk_index() || index_store.pk_has_overrides()
+                    }
+                    _ => false,
+                };
+                // Active PK arms only need to stay uncapped when recency
+                // filtering may drop hits. Append-only PK memtables can safely
+                // pass the limit through and let FtsIndexExec use WAND/top-k.
+                // Blocked non-active sources use heuristic over-fetch because
+                // their newer generation membership may also drop candidates.
+                let fetch_limit = if active_needs_uncapped_recency {
+                    None
+                } else if blocked.is_some() && !self.pk_columns.is_empty() {
+                    limit.map(|limit| ((limit as f64) * overfetch).ceil() as usize)
+                } else {
+                    limit
+                };
+                (source, is_active, blocked, fetch_limit)
+            })
+            .collect();
+        let built =
+            futures::future::try_join_all(arm_inputs.iter().map(|(source, _, _, fetch_limit)| {
+                Box::pin(self.build_source_plan(source, column, &query, *fetch_limit, projection))
+            }))
+            .await?;
+
         let mut per_source_plans: Vec<Arc<dyn ExecutionPlan>> = Vec::with_capacity(sources.len());
-        for source in &sources {
-            let plan = self
-                .build_source_plan(source, column, &query, k, projection)
-                .await?;
-            let normalized = project_to_canonical(plan, &target_schema)?;
+        for ((_, _, blocked, _), plan) in arm_inputs.iter().zip(built) {
+            let blocked = *blocked;
+            // Dedup, mirroring LsmVectorSearchPlanner:
+            //  * each memtable: `FtsIndexExec` drops superseded PK versions
+            //    within that memtable before the query limit whenever PK
+            //    columns are present.
+            //  * any source with a block-list: drop rows superseded by a newer
+            //    generation, including frozen in-memory memtables.
+            let deduped = if let Some(set) = blocked
+                && !self.pk_columns.is_empty()
+            {
+                Arc::new(PkBlockFilterExec::new(
+                    plan,
+                    self.pk_columns.clone(),
+                    set.clone(),
+                    limit.unwrap_or(usize::MAX),
+                )) as Arc<dyn ExecutionPlan>
+            } else {
+                plan
+            };
+
+            // Normalize to the canonical FTS schema before merging sources.
+            let normalized = project_to_canonical(deduped, &target_schema)?;
             per_source_plans.push(normalized);
         }
 
@@ -151,8 +311,11 @@ impl LsmFtsSearchPlanner {
             per_source_plans.into_iter().next().unwrap()
         } else {
             #[allow(deprecated)]
-            let union: Arc<dyn ExecutionPlan> = Arc::new(UnionExec::new(per_source_plans));
-            union
+            // The downstream `SortPreservingMergeExec` already spawns one driver
+            // task per input partition (one per union arm) via `spawn_buffered`,
+            // so each arm's per-arm CPU (posting decode, BM25) runs on its own
+            // task without an extra repartition.
+            Arc::new(UnionExec::new(per_source_plans))
         };
 
         let score_idx = merged.schema().index_of(SCORE_COLUMN).map_err(|_| {
@@ -179,10 +342,10 @@ impl LsmFtsSearchPlanner {
         let per_partition_sorted: Arc<dyn ExecutionPlan> = Arc::new(
             SortExec::new(lex_ordering.clone(), merged)
                 .with_preserve_partitioning(true)
-                .with_fetch(Some(k)),
+                .with_fetch(limit),
         );
         let merged_sorted: Arc<dyn ExecutionPlan> = Arc::new(
-            SortPreservingMergeExec::new(lex_ordering, per_partition_sorted).with_fetch(Some(k)),
+            SortPreservingMergeExec::new(lex_ordering, per_partition_sorted).with_fetch(limit),
         );
 
         Ok(merged_sorted)
@@ -193,7 +356,7 @@ impl LsmFtsSearchPlanner {
         source: &LsmDataSource,
         column: &str,
         query: &FullTextSearchQuery,
-        k: usize,
+        limit: Option<usize>,
         projection: Option<&[String]>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         match source {
@@ -201,24 +364,46 @@ impl LsmFtsSearchPlanner {
                 let mut scanner = dataset.scan();
                 let cols = self.fts_scanner_projection(projection);
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
-                let bound_query = query
-                    .clone()
-                    .with_column(column.to_string())?
-                    .limit(Some(k as i64));
+                if let Some(ref filter) = self.filter {
+                    // `prefilter(true)` is required: without it the scanner
+                    // post-filters the unfiltered BM25 top-k, dropping matching
+                    // rows that scored below non-matching ones.
+                    scanner.filter_expr(filter.clone());
+                    scanner.prefilter(true);
+                }
+                let mut bound_query = query.clone().with_column(column.to_string())?;
+                if let Some(limit) = limit {
+                    bound_query = bound_query.limit(Some(limit as i64));
+                } else {
+                    bound_query = bound_query.limit(None);
+                }
                 scanner.full_text_search(bound_query)?;
                 scanner.create_plan().await
             }
             LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset =
-                    open_flushed_dataset(path, self.session.as_ref(), self.flushed_cache.as_ref())
-                        .await?;
+                let dataset = open_flushed_dataset(
+                    path,
+                    self.session.as_ref(),
+                    self.store_params.as_ref(),
+                    self.flushed_cache.as_ref(),
+                    self.warmer.as_ref(),
+                )
+                .await?;
                 let mut scanner = dataset.scan();
                 let cols = self.fts_scanner_projection(projection);
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
-                let bound_query = query
-                    .clone()
-                    .with_column(column.to_string())?
-                    .limit(Some(k as i64));
+                if let Some(ref filter) = self.filter {
+                    // See the base arm: `prefilter(true)` makes this a true
+                    // prefilter rather than a lossy post-filter on the BM25 top-k.
+                    scanner.filter_expr(filter.clone());
+                    scanner.prefilter(true);
+                }
+                let mut bound_query = query.clone().with_column(column.to_string())?;
+                if let Some(limit) = limit {
+                    bound_query = bound_query.limit(Some(limit as i64));
+                } else {
+                    bound_query = bound_query.limit(None);
+                }
                 scanner.full_text_search(bound_query)?;
                 scanner.create_plan().await
             }
@@ -228,28 +413,35 @@ impl LsmFtsSearchPlanner {
                 schema,
                 ..
             } => {
+                if !active_source_can_execute_fts(source, column) {
+                    return self.empty_plan(&self.canonical_fts_schema(projection));
+                }
+                validate_lsm_fts_query(query)?;
                 let mut scanner =
                     MemTableScanner::new(batch_store.clone(), index_store.clone(), schema.clone());
                 let cols = self.fts_scanner_projection(projection);
-                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>());
-                // `MemTableScanner::full_text_search` takes a raw match
-                // string; richer query shapes (phrase/boolean/fuzzy) can
-                // be plumbed through once the MemTable scanner accepts a
-                // structured query.
-                let match_str = match &query.query {
-                    IndexFtsQuery::Match(m) => m.terms.clone(),
-                    other => {
-                        return Err(Error::not_supported(format!(
-                            "Active memtable FTS via LsmFtsSearchPlanner currently only \
-                             supports MatchQuery, got: {other:?}"
-                        )));
-                    }
-                };
-                let _ = scanner.full_text_search(column, &match_str);
-                // Active arm doesn't take a top-K hint via the builder
-                // today; the per-partition Sort+fetch above bounds the
-                // emitted rows.
-                let _ = k;
+                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
+                if let Some(ref filter) = self.filter {
+                    // Honored inside `plan_fts_search`: the materialized hits are
+                    // masked by the predicate before projection.
+                    scanner.filter_expr(filter.clone());
+                }
+                // The append-only inverted index keeps an updated row's old
+                // postings live, so the memtable FTS exec needs PK columns to
+                // drop stale hits before it applies the query limit.
+                if !self.pk_columns.is_empty() {
+                    scanner.with_pk_columns(self.pk_columns.clone());
+                }
+                // `MemTableScanner::full_text_search` now takes a structured
+                // `FullTextSearchQuery` (match/phrase); it rejects compound
+                // shapes the MemTable path can't model.
+                let mut bound_query = query.clone().with_column(column.to_string())?;
+                if let Some(limit) = limit {
+                    bound_query = bound_query.limit(Some(limit as i64));
+                } else {
+                    bound_query = bound_query.limit(None);
+                }
+                scanner.full_text_search(bound_query)?;
                 scanner.create_plan().await
             }
         }
@@ -327,7 +519,7 @@ mod tests {
     use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
     use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
     use crate::dataset::{Dataset, WriteParams};
-    use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray};
+    use arrow_array::{BooleanArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray};
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use futures::TryStreamExt;
     use std::collections::HashMap;
@@ -345,6 +537,20 @@ mod tests {
         ]))
     }
 
+    fn fts_tombstone_schema() -> Arc<ArrowSchema> {
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_meta);
+        Arc::new(ArrowSchema::new(vec![
+            id_field,
+            Field::new("text", DataType::Utf8, true),
+            Field::new(crate::dataset::mem_wal::TOMBSTONE, DataType::Boolean, false),
+        ]))
+    }
+
     fn make_batch(schema: &ArrowSchema, ids: &[i32], texts: &[&str]) -> RecordBatch {
         RecordBatch::try_new(
             Arc::new(schema.clone()),
@@ -356,12 +562,61 @@ mod tests {
         .unwrap()
     }
 
+    fn make_tombstone_batch(
+        schema: &ArrowSchema,
+        rows: &[(i32, Option<&str>, bool)],
+    ) -> RecordBatch {
+        let ids: Vec<i32> = rows.iter().map(|(id, _, _)| *id).collect();
+        let texts: Vec<Option<&str>> = rows.iter().map(|(_, text, _)| *text).collect();
+        let tombstones: Vec<bool> = rows.iter().map(|(_, _, tombstone)| *tombstone).collect();
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(texts)),
+                Arc::new(BooleanArray::from(tombstones)),
+            ],
+        )
+        .unwrap()
+    }
+
     async fn write_dataset(uri: &str, batches: Vec<RecordBatch>) -> Dataset {
         let schema = batches[0].schema();
-        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
-        Dataset::write(reader, uri, Some(WriteParams::default()))
+        let has_id = schema.column_with_name("id").is_some();
+        let reader = RecordBatchIterator::new(batches.clone().into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, uri, Some(WriteParams::default()))
             .await
-            .unwrap()
+            .unwrap();
+        if has_id {
+            crate::dataset::mem_wal::scanner::block_list::write_pk_sidecar(uri, &batches, &["id"])
+                .await
+                .unwrap();
+        }
+        dataset
+    }
+
+    #[tokio::test]
+    async fn rejects_missing_projection_column() {
+        let schema = fts_schema();
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]);
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+
+        let projection = vec!["missing".to_string()];
+        let err = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(1),
+                Some(&projection),
+            )
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing-column projection error: {err}"
+        );
     }
 
     #[tokio::test]
@@ -404,6 +659,7 @@ mod tests {
         // Active memtable with its own FTS index, containing a matching row.
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
         indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
         let active_batch = make_batch(
             &schema,
@@ -434,7 +690,7 @@ mod tests {
             .plan_search(
                 "text",
                 FullTextSearchQuery::new("lance".to_string()),
-                10,
+                Some(10),
                 None,
             )
             .await
@@ -478,77 +734,1204 @@ mod tests {
         assert!(ids.contains(&3), "missing active hit id=3; got ids={ids:?}");
     }
 
+    /// A prefilter on a full-text search must drop hits failing the predicate
+    /// from both the base arm (native scanner filter) and the active memtable
+    /// arm (materialized-hit mask), even though they match the query text.
     #[tokio::test]
-    async fn local_mode_active_memtable_only_returns_score_sorted_hits() {
+    async fn prefilter_drops_nonmatching_hits_across_base_and_active() {
+        use crate::index::DatasetIndexExt;
+        use datafusion::prelude::{col, lit};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
         let schema = fts_schema();
+        let tmp = tempfile::tempdir().unwrap();
+
+        // Base rows 1 and 2 both contain "lance".
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let mut base_ds = write_dataset(
+            &base_uri,
+            vec![make_batch(&schema, &[1, 2], &["lance one", "lance two"])],
+        )
+        .await;
+        base_ds
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap());
+
+        // Active memtable rows 0 and 3 both contain "lance".
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut indexes = IndexStore::new();
-        // text column has field_id 1 in fts_schema()
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
         indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
-        let batch = make_batch(
-            &schema,
-            &[1, 2, 3, 4],
-            &[
-                "lance is a columnar data format",
-                "memwal handles streaming writes",
-                "lance memwal lance lance",
-                "completely unrelated",
-            ],
-        );
-        batch_store.append(batch.clone()).unwrap();
+        let active_batch = make_batch(&schema, &[0, 3], &["lance zero", "lance three"]);
+        batch_store.append(active_batch.clone()).unwrap();
         indexes
-            .insert_with_batch_position(&batch, 0, Some(0))
+            .insert_with_batch_position(&active_batch, 0, Some(0))
             .unwrap();
         let indexes = Arc::new(indexes);
 
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store: indexes,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            // `id >= 2` keeps base id=2 and active id=3; drops base id=1 and active id=0.
+            .with_filter(Some(col("id").gt_eq(lit(2i32))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered base+active plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        ids.sort();
+        assert_eq!(
+            ids,
+            vec![2, 3],
+            "prefilter must keep only id>=2 across base (id=2) and active (id=3), \
+            dropping base id=1 and active id=0; got {ids:?}"
+        );
+    }
+
+    /// The flushed arm must apply the filter as a true FTS prefilter, and that
+    /// prefiltered candidate set must compose with cross-generation block-list
+    /// filtering plus over-fetch. Gen 1's best predicate-matching hit (id=3) is
+    /// superseded by gen 2; with over-fetch, gen 1 should still contribute id=4.
+    #[tokio::test]
+    async fn prefilter_on_flushed_composes_with_block_list() {
+        use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
+        use crate::index::DatasetIndexExt;
+        use datafusion::prelude::{col, lit};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = fts_schema();
         let tmp = tempfile::tempdir().unwrap();
         let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
-        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+        let shard_id = uuid::Uuid::new_v4();
+
+        // Gen 1: id=1 matches strongly but fails the predicate. id=3 matches
+        // strongly but is stale (blocked by gen 2). id=4 is the next live
+        // predicate match that only survives if the flushed arm prefilters and
+        // over-fetches before the block-list drops id=3.
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
+        let mut gen1 = write_dataset(
+            &gen1_uri,
+            vec![make_batch(
+                &schema,
+                &[1, 3, 4],
+                &["lance lance lance lance", "lance lance lance", "lance"],
+            )],
+        )
+        .await;
+        gen1.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        // Gen 2: newer id=3 shadows gen 1's match but does not match the query.
+        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
+        let mut gen2 =
+            write_dataset(&gen2_uri, vec![make_batch(&schema, &[3], &["other text"])]).await;
+        gen2.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(3)
+            .with_flushed_generation(1, "gen_1".to_string())
+            .with_flushed_generation(2, "gen_2".to_string());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]);
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            .with_filter(Some(col("id").gt_eq(lit(3i32))))
+            .with_overfetch_factor(2.0);
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(1),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered flushed plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids,
+            vec![4],
+            "flushed FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn active_tombstone_masks_base_fts_hit() {
+        use crate::index::DatasetIndexExt;
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let base_schema = fts_schema();
+        let mem_schema = fts_tombstone_schema();
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+
+        let mut base = write_dataset(
+            &base_uri,
+            vec![make_batch(&base_schema, &[1, 2], &["lance lance", "lance"])],
+        )
+        .await;
+        base.create_index(
+            &["text"],
+            IndexType::Inverted,
+            Some("text_fts".to_string()),
+            &InvertedIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        let active_tombstone = make_tombstone_batch(&mem_schema, &[(1, None, true)]);
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        index_store.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let (_, row_offset, batch_position) = batch_store.append(active_tombstone.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&active_tombstone, row_offset, Some(batch_position))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(Arc::new(base), vec![])
             .with_in_memory_memtables(
                 uuid::Uuid::new_v4(),
                 InMemoryMemTables {
                     active: InMemoryMemTableRef {
                         batch_store,
-                        index_store: indexes,
-                        schema: schema.clone(),
+                        index_store,
+                        schema: mem_schema,
                         generation: 1,
                     },
                     frozen: vec![],
                 },
             );
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], base_schema)
+            .with_overfetch_factor(2.0);
 
-        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
         let plan = planner
             .plan_search(
                 "text",
                 FullTextSearchQuery::new("lance".to_string()),
-                10,
+                Some(1),
                 None,
             )
             .await
-            .expect("local mode planner should produce a plan");
-
-        // Plan executes and emits _score-sorted rows.
-        let ctx = datafusion::prelude::SessionContext::new();
-        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+            .unwrap();
+        let stream = plan
+            .execute(0, datafusion::prelude::SessionContext::new().task_ctx())
+            .unwrap();
         let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
-        assert!(
-            total >= 2,
-            "expected at least the 2 'lance' rows, got {total}"
-        );
-
-        // Schema must include _score and the PK id.
-        let out = batches[0].schema();
-        assert!(out.field_with_name(SCORE_COLUMN).is_ok());
-        assert!(out.field_with_name("id").is_ok());
 
-        // _score must be non-ascending across the result.
-        let mut prev_score: Option<f32> = None;
-        for batch in &batches {
-            let score = batch
-                .column_by_name(SCORE_COLUMN)
-                .unwrap()
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids,
+            vec![2],
+            "active tombstone for id=1 must block the older base FTS hit; got {ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn active_update_and_tombstone_mask_frozen_fts_hits() {
+        let base_schema = fts_schema();
+        let mem_schema = fts_tombstone_schema();
+
+        let frozen_batch = make_tombstone_batch(
+            &mem_schema,
+            &[
+                (1, Some("lance stale update"), false),
+                (2, Some("lance live"), false),
+                (3, Some("lance deleted"), false),
+            ],
+        );
+        let frozen_batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut frozen_index_store = IndexStore::new();
+        frozen_index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        frozen_index_store.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let (_, frozen_row_offset, frozen_batch_position) =
+            frozen_batch_store.append(frozen_batch.clone()).unwrap();
+        frozen_index_store
+            .insert_with_batch_position(
+                &frozen_batch,
+                frozen_row_offset,
+                Some(frozen_batch_position),
+            )
+            .unwrap();
+
+        let active_batch = make_tombstone_batch(
+            &mem_schema,
+            &[(1, Some("fresh other text"), false), (3, None, true)],
+        );
+        let active_batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut active_index_store = IndexStore::new();
+        active_index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        active_index_store.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let (_, active_row_offset, active_batch_position) =
+            active_batch_store.append(active_batch.clone()).unwrap();
+        active_index_store
+            .insert_with_batch_position(
+                &active_batch,
+                active_row_offset,
+                Some(active_batch_position),
+            )
+            .unwrap();
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let shard_id = uuid::Uuid::new_v4();
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                shard_id,
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store: active_batch_store,
+                        index_store: Arc::new(active_index_store),
+                        schema: mem_schema.clone(),
+                        generation: 2,
+                    },
+                    frozen: vec![InMemoryMemTableRef {
+                        batch_store: frozen_batch_store,
+                        index_store: Arc::new(frozen_index_store),
+                        schema: mem_schema,
+                        generation: 1,
+                    }],
+                },
+            );
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], base_schema);
+
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .unwrap();
+        let stream = plan
+            .execute(0, datafusion::prelude::SessionContext::new().task_ctx())
+            .unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![2],
+            "active update id=1 and tombstone id=3 must block stale frozen FTS hits; got {ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn active_filtered_search_without_pk_applies_small_limit_after_filter() {
+        use datafusion::prelude::{col, lit};
+
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let active_batch = make_batch(
+            &schema,
+            &[1, 2, 3],
+            &["lance", "lance filler", "lance filler filler"],
+        );
+        batch_store.append(active_batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&active_batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec![], schema)
+            .with_filter(Some(col("id").gt_eq(lit(1i32))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(2),
+                None,
+            )
+            .await
+            .expect("planner should produce an active-only filtered plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![1, 2],
+            "no-PK filtered active search must apply the limit after filtering \
+             and keep the top-scoring matching hits; got ids={ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn fuzzy_and_query_is_rejected_when_active_memtable_is_present() {
+        use lance_index::scalar::inverted::query::{
+            FtsQuery as IndexFtsQuery, MatchQuery, Operator,
+        };
+
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let active_batch = make_batch(&schema, &[1], &["lance memwal"]);
+        let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&active_batch, row_offset, Some(batch_position))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+        let planner = LsmFtsSearchPlanner::new(collector, vec![], schema);
+        let query = FullTextSearchQuery::new_query(IndexFtsQuery::Match(
+            MatchQuery::new("lance memwal".to_string())
+                .with_operator(Operator::And)
+                .with_fuzziness(Some(1)),
+        ));
+
+        let err = planner
+            .plan_search("text", query, Some(10), None)
+            .await
+            .expect_err("fuzzy AND should be rejected consistently");
+        assert!(
+            err.to_string().contains("fuzzy full-text search"),
+            "unexpected error for fuzzy AND query: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn base_only_boolean_query_uses_dataset_scanner_support() {
+        use crate::index::DatasetIndexExt;
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::query::{
+            BooleanQuery, FtsQuery as IndexFtsQuery, MatchQuery, Occur,
+        };
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = fts_schema();
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let mut base_ds = write_dataset(
+            &base_uri,
+            vec![make_batch(
+                &schema,
+                &[1, 2],
+                &["lance rocks", "unrelated text"],
+            )],
+        )
+        .await;
+        base_ds
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap());
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]);
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+
+        let query = FullTextSearchQuery::new_query(IndexFtsQuery::Boolean(BooleanQuery::new(
+            vec![(Occur::Must, MatchQuery::new("lance".to_string()).into())],
+        )));
+        let plan = planner
+            .plan_search("text", query, Some(10), Some(&["id".to_string()]))
+            .await
+            .expect("base-only boolean query should be delegated to dataset scanner");
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let ids: Vec<i32> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(ids, vec![1]);
+    }
+
+    #[tokio::test]
+    async fn boolean_query_ignores_active_memtable_without_relevant_fts_index() {
+        use crate::index::DatasetIndexExt;
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::query::{
+            BooleanQuery, FtsQuery as IndexFtsQuery, MatchQuery, Occur,
+        };
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = fts_schema();
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let mut base_ds = write_dataset(
+            &base_uri,
+            vec![make_batch(
+                &schema,
+                &[1, 2],
+                &["lance rocks", "unrelated text"],
+            )],
+        )
+        .await;
+        base_ds
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap());
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        let active_batch = make_batch(&schema, &[99], &["active text has no fts index"]);
+        let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&active_batch, row_offset, Some(batch_position))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store: indexes,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+
+        let query = FullTextSearchQuery::new_query(IndexFtsQuery::Boolean(BooleanQuery::new(
+            vec![(Occur::Must, MatchQuery::new("lance".to_string()).into())],
+        )));
+        let plan = planner
+            .plan_search("text", query, Some(10), Some(&["id".to_string()]))
+            .await
+            .expect("irrelevant active memtable must not reject base-supported boolean query");
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let ids: Vec<i32> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert_eq!(ids, vec![1]);
+    }
+
+    /// The base arm must apply the filter as a true *prefilter*, not a
+    /// post-filter on the BM25 top-k. With `k = 1` and the higher-scoring base
+    /// doc failing the predicate, a post-filter would return zero rows; a
+    /// prefilter restricts BM25 to matching rows and returns the lower-scoring
+    /// match. Regression for a missing `scanner.prefilter(true)` on the base arm.
+    #[tokio::test]
+    async fn prefilter_on_base_is_not_a_lossy_postfilter() {
+        use crate::index::DatasetIndexExt;
+        use datafusion::prelude::{col, lit};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let schema = fts_schema();
+        let tmp = tempfile::tempdir().unwrap();
+
+        // id=1 is a short doc ("lance") so it scores higher under BM25 length
+        // normalization; id=2 buries "lance" among filler so it scores lower.
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let mut base_ds = write_dataset(
+            &base_uri,
+            vec![make_batch(
+                &schema,
+                &[1, 2],
+                &["lance", "lance filler filler filler filler filler"],
+            )],
+        )
+        .await;
+        base_ds
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap());
+
+        // Base-only collector (no in-memory memtables): isolates the base arm.
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]);
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            // Keeps only id=2, which is the *lower*-scoring match. A post-filter
+            // on the top-1 (id=1) would drop everything.
+            .with_filter(Some(col("id").gt_eq(lit(2i32))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(1),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered base plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids,
+            vec![2],
+            "prefilter must return the lower-scoring match id=2, not post-filter \
+             the top-1 (id=1) down to nothing; got {ids:?}"
+        );
+    }
+
+    /// The active memtable FTS arm must also apply the predicate before its
+    /// top-k cap. With `k = 1` and the higher-scoring active doc failing the
+    /// predicate, pushing the limit into the index would return zero rows.
+    #[tokio::test]
+    async fn prefilter_on_active_is_not_a_lossy_postfilter() {
+        use datafusion::prelude::{col, lit};
+
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec![
+                    "lance",
+                    "lance filler filler filler filler filler",
+                ])),
+                Arc::new(StringArray::from(vec!["archived", "active"])),
+            ],
+        )
+        .unwrap();
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            .with_filter(Some(col("status").eq(lit("active"))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(1),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered active plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids,
+            vec![2],
+            "active FTS prefilter must return the lower-scoring matching row, \
+             not post-filter the top-1 down to nothing; got {ids:?}"
+        );
+    }
+
+    /// The active FTS arm must also avoid capping before newest-PK filtering.
+    /// A stale high-scoring hit can be removed by `FtsIndexExec`; a lower
+    /// scoring live hit must still be available for the final global top-k.
+    #[tokio::test]
+    async fn active_limit_applies_after_newest_pk_recency_filter() {
+        use datafusion::prelude::{col, lit};
+
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 1])),
+                Arc::new(StringArray::from(vec![
+                    "lance",
+                    "lance filler filler filler filler filler",
+                    "other text",
+                ])),
+                Arc::new(StringArray::from(vec!["active", "active", "active"])),
+            ],
+        )
+        .unwrap();
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            .with_filter(Some(col("status").eq(lit("active"))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(1),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered active plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids,
+            vec![2],
+            "active FTS limit must apply after newest-PK filtering; got {ids:?}"
+        );
+    }
+
+    /// An in-memtable update whose *newest* version fails the prefilter must
+    /// exclude the PK, not leak the stale older hit that still passes. Both
+    /// versions of pk=5 match the query text "lance", but only the older one is
+    /// "active"; the current version is "archived" and must be dropped.
+    /// Regression for filter-before-dedup on the active FTS arm.
+    #[tokio::test]
+    async fn prefilter_excludes_pk_whose_newest_version_fails() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use arrow_schema::{DataType, Field};
+        use datafusion::prelude::{col, lit};
+
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        let make_row = |statuses: &[&str]| -> RecordBatch {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![5; statuses.len()])),
+                    Arc::new(StringArray::from(vec!["lance text"; statuses.len()])),
+                    Arc::new(StringArray::from(statuses.to_vec())),
+                ],
+            )
+            .unwrap()
+        };
+
+        // Base unindexed → contributes nothing; isolate the active arm.
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let base_batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![999])),
+                Arc::new(StringArray::from(vec!["unrelated"])),
+                Arc::new(StringArray::from(vec!["active"])),
+            ],
+        )
+        .unwrap();
+        let base_ds = Arc::new(write_dataset(&base_uri, vec![base_batch]).await);
+
+        // Active memtable: pk=5 appended twice (active then archived), both "lance".
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let active_batch = make_row(&["active", "archived"]);
+        batch_store.append(active_batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&active_batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store: indexes,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            .with_filter(Some(col("status").eq(lit("active"))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered active plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            total, 0,
+            "pk=5's current version is 'archived' and must be excluded; the stale \
+             'active' older hit must not leak (filter evaluated on newest version)"
+        );
+    }
+
+    /// Cross-arm stale hits must be blocked even if the newer active row fails
+    /// the prefilter. The base copy of pk=5 matches both text and status, but
+    /// the newer active copy is archived; pk=5 must not leak from the base arm.
+    #[tokio::test]
+    async fn prefilter_blocks_base_hit_when_active_newest_fails() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use crate::index::DatasetIndexExt;
+        use datafusion::prelude::{col, lit};
+        use lance_index::IndexType;
+        use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
+
+        let mut id_meta = HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new("text", DataType::Utf8, true),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        let make_rows = |rows: &[(i32, &str, &str)]| -> RecordBatch {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(
+                        rows.iter().map(|(id, _, _)| *id).collect::<Vec<_>>(),
+                    )),
+                    Arc::new(StringArray::from(
+                        rows.iter().map(|(_, text, _)| *text).collect::<Vec<_>>(),
+                    )),
+                    Arc::new(StringArray::from(
+                        rows.iter()
+                            .map(|(_, _, status)| *status)
+                            .collect::<Vec<_>>(),
+                    )),
+                ],
+            )
+            .unwrap()
+        };
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let mut base_ds = write_dataset(
+            &base_uri,
+            vec![make_rows(&[
+                (5, "lance base stale", "active"),
+                (6, "lance base live", "active"),
+            ])],
+        )
+        .await;
+        base_ds
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap());
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let active_batch = make_rows(&[(5, "lance active newest", "archived")]);
+        batch_store.append(active_batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&active_batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store: indexes,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema)
+            .with_filter(Some(col("status").eq(lit("active"))));
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce a filtered base+active plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let mut ids = Vec::new();
+        for batch in &batches {
+            let id_array = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for row in 0..batch.num_rows() {
+                ids.push(id_array.value(row));
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![6],
+            "base pk=5 passes the filter but is superseded by active archived pk=5; got {ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn local_mode_active_memtable_only_returns_score_sorted_hits() {
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        // text column has field_id 1 in fts_schema()
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let batch = make_batch(
+            &schema,
+            &[1, 2, 3, 4],
+            &[
+                "lance is a columnar data format",
+                "memwal handles streaming writes",
+                "lance memwal lance lance",
+                "completely unrelated",
+            ],
+        );
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("local mode planner should produce a plan");
+
+        // Plan executes and emits _score-sorted rows.
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert!(
+            total >= 2,
+            "expected at least the 2 'lance' rows, got {total}"
+        );
+
+        // Schema must include _score and the PK id.
+        let out = batches[0].schema();
+        assert!(out.field_with_name(SCORE_COLUMN).is_ok());
+        assert!(out.field_with_name("id").is_ok());
+
+        // _score must be non-ascending across the result.
+        let mut prev_score: Option<f32> = None;
+        for batch in &batches {
+            let score = batch
+                .column_by_name(SCORE_COLUMN)
+                .unwrap()
                 .as_any()
                 .downcast_ref::<arrow_array::Float32Array>()
                 .unwrap();
@@ -561,4 +1944,345 @@ mod tests {
             }
         }
     }
+
+    #[tokio::test]
+    async fn active_match_query_preserves_and_operator() {
+        use lance_index::scalar::inverted::query::{
+            FtsQuery as IndexFtsQuery, MatchQuery, Operator,
+        };
+
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let batch = make_batch(
+            &schema,
+            &[1, 2, 3],
+            &["lance only", "memwal only", "lance memwal"],
+        );
+        batch_store.append(batch.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+        let query = FullTextSearchQuery::new_query(IndexFtsQuery::Match(
+            MatchQuery::new("lance memwal".to_string())
+                .with_operator(Operator::And)
+                .with_column(Some("text".to_string())),
+        ));
+        let plan = planner
+            .plan_search("text", query, Some(10), None)
+            .await
+            .expect("planner should produce an active-only plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![3],
+            "AND query must only return rows containing both terms; got ids={ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn local_mode_active_dedups_updated_pk_keeping_newest() {
+        // The active memtable is an append log and the FTS index is
+        // append-only, so a PK updated before flush is searchable as two
+        // row-positions. WithinSourceDedupExec(KeepMaxRowAddr) must collapse
+        // them to the newest insert. Without it the same PK would surface
+        // twice (criterion 2 violation).
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+
+        // First append (positions 0,1): id=1 is the stale version of the PK.
+        let batch_old = make_batch(&schema, &[1, 2], &["lance stale version", "other doc"]);
+        batch_store.append(batch_old.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch_old, 0, Some(0))
+            .unwrap();
+
+        // Second append (position 2): id=1 updated — same PK, later row.
+        let batch_new = make_batch(&schema, &[1], &["lance fresh version"]);
+        batch_store.append(batch_new.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&batch_new, 2, Some(1))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("lance".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce an active-only plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut rows: Vec<(i32, String)> = Vec::new();
+        for b in &batches {
+            let ids = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            let texts = b
+                .column_by_name("text")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                rows.push((ids.value(i), texts.value(i).to_string()));
+            }
+        }
+
+        // id=1 must appear exactly once, and it must be the *newest* version.
+        let id1: Vec<&(i32, String)> = rows.iter().filter(|(id, _)| *id == 1).collect();
+        assert_eq!(
+            id1.len(),
+            1,
+            "updated PK id=1 must be deduped to one row; got {rows:?}"
+        );
+        assert_eq!(
+            id1[0].1, "lance fresh version",
+            "dedup must keep the newest (max row-position) version"
+        );
+    }
+
+    #[tokio::test]
+    async fn active_stale_update_predicate_crossing_leaks() {
+        // A PK update that crosses out of the match set: pk=1 inserted as
+        // "alpha lance", then updated to "beta lance". The append-only inverted
+        // index keeps the old "alpha" posting live, so an "alpha" search still
+        // matches the STALE pk=1 row — and the fresh "beta lance" row isn't even
+        // a candidate, so a result-set dedup has nothing to suppress it against.
+        // `FtsIndexExec` drops it predicate-independently: pk=1's newest visible
+        // row is "beta lance", so the "alpha" hit is not the newest.
+        let schema = fts_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut indexes = IndexStore::new();
+        indexes.enable_pk_index(&[("id".to_string(), 0)]);
+        indexes.add_fts("text_fts".to_string(), 1, "text".to_string());
+
+        // Insert pk=1 ("alpha lance") and an unrelated live pk=2 ("alpha foo").
+        let b1 = make_batch(&schema, &[1, 2], &["alpha lance", "alpha foo"]);
+        let (bp1, off1, _) = batch_store.append(b1.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&b1, off1, Some(bp1))
+            .unwrap();
+
+        // Update pk=1 → "beta lance" (no longer matches "alpha").
+        let b2 = make_batch(&schema, &[1], &["beta lance"]);
+        let (bp2, off2, _) = batch_store.append(b2.clone()).unwrap();
+        indexes
+            .insert_with_batch_position(&b2, off2, Some(bp2))
+            .unwrap();
+        let indexes = Arc::new(indexes);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store: indexes,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("alpha".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce a plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+
+        assert!(
+            !ids.contains(&1),
+            "stale pk=1 (now 'beta lance') leaked on an 'alpha' search; got ids={ids:?}"
+        );
+        assert!(
+            ids.contains(&2),
+            "live pk=2 ('alpha foo') must still match 'alpha'; got ids={ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn cross_gen_stale_update_blocked_by_newer_memtable() {
+        // The cross-generation analog of `active_stale_update_predicate_crossing_leaks`:
+        // pk=1's stale "alpha" version lives in a FROZEN memtable and its fresh
+        // "beta" version in the ACTIVE one. The frozen arm's recency filter is
+        // per-generation and can't see the newer gen, so only the cross-gen
+        // block-list can drop the stale "alpha" hit. The cluster constantly
+        // freezes memtables, so an insert and its later update/delete split
+        // across in-memory generations — this is the residual fuzz phantom.
+        let schema = fts_schema();
+
+        // Frozen gen=1: pk=1 "alpha lance" (matches), pk=2 "alpha foo" (live).
+        let frozen_store = Arc::new(BatchStore::with_capacity(16));
+        let mut frozen_idx = IndexStore::new();
+        frozen_idx.enable_pk_index(&[("id".to_string(), 0)]);
+        frozen_idx.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let fb = make_batch(&schema, &[1, 2], &["alpha lance", "alpha foo"]);
+        let (bp, off, _) = frozen_store.append(fb.clone()).unwrap();
+        frozen_idx
+            .insert_with_batch_position(&fb, off, Some(bp))
+            .unwrap();
+
+        // Active gen=2: pk=1 updated to "beta lance" (no longer matches "alpha").
+        let active_store = Arc::new(BatchStore::with_capacity(16));
+        let mut active_idx = IndexStore::new();
+        active_idx.enable_pk_index(&[("id".to_string(), 0)]);
+        active_idx.add_fts("text_fts".to_string(), 1, "text".to_string());
+        let ab = make_batch(&schema, &[1], &["beta lance"]);
+        let (bp, off, _) = active_store.append(ab.clone()).unwrap();
+        active_idx
+            .insert_with_batch_position(&ab, off, Some(bp))
+            .unwrap();
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store: active_store,
+                        index_store: Arc::new(active_idx),
+                        schema: schema.clone(),
+                        generation: 2,
+                    },
+                    frozen: vec![InMemoryMemTableRef {
+                        batch_store: frozen_store,
+                        index_store: Arc::new(frozen_idx),
+                        schema: schema.clone(),
+                        generation: 1,
+                    }],
+                },
+            );
+
+        let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema);
+        let plan = planner
+            .plan_search(
+                "text",
+                FullTextSearchQuery::new("alpha".to_string()),
+                Some(10),
+                None,
+            )
+            .await
+            .expect("planner should produce a plan");
+
+        let ctx = datafusion::prelude::SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+
+        assert!(
+            !ids.contains(&1),
+            "stale frozen pk=1 ('alpha lance', now 'beta lance' in the active gen) \
+             leaked on an 'alpha' search; got ids={ids:?}"
+        );
+        assert!(
+            ids.contains(&2),
+            "live pk=2 ('alpha foo', only in the frozen gen) must still match; got ids={ids:?}"
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
index f3f15e2e680..b915f635784 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs
@@ -9,18 +9,33 @@ use arrow_schema::{DataType, Field, Schema, SchemaRef};
 use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
 use datafusion::physical_plan::union::UnionExec;
 use datafusion::physical_plan::{ExecutionPlan, limit::GlobalLimitExec};
-use datafusion::prelude::Expr;
+use datafusion::prelude::{Expr, col};
 use lance_core::Result;
 use tracing::instrument;
 
+use crate::dataset::mem_wal::TOMBSTONE;
+
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
-use super::exec::{MEMTABLE_GEN_COLUMN, MemtableGenTagExec, PkHashFilterExec, ROW_ADDRESS_COLUMN};
-use super::flushed_cache::{FlushedMemTableCache, open_flushed_dataset};
+use super::exec::{MEMTABLE_GEN_COLUMN, MemtableGenTagExec, PkBlockFilterExec, ROW_ADDRESS_COLUMN};
+use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
     build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical,
+    validate_projection_names,
 };
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
+
+/// Combine the user filter (if any) with `NOT _tombstone` so tombstone rows are
+/// dropped from a WAL-arm scan. Used only for sources whose schema carries the
+/// column (active / flushed generations written since deletes existed).
+fn fold_not_tombstone(filter: Option<&Expr>) -> Expr {
+    let live = !col(TOMBSTONE);
+    match filter {
+        Some(expr) => expr.clone().and(live),
+        None => live,
+    }
+}
 
 /// Plans scan queries over LSM data.
 pub struct LsmScanPlanner {
@@ -32,8 +47,25 @@ pub struct LsmScanPlanner {
     base_schema: SchemaRef,
     /// Session threaded into flushed-generation opens (shared caches).
     session: Option<Arc<Session>>,
+    /// Store params for opening flushed generations, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
     /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<FlushedMemTableCache>>,
+    flushed_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of a flushed generation.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Over-fetch multiple for the per-source limit pushdown: block-listed
+    /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops
+    /// still leave enough live rows. Clamped to `>= 1.0`.
+    ///
+    /// This headroom must also absorb deletes: a tombstone shadows the older
+    /// real row without emitting a replacement (shadow-without-replace), so it
+    /// is pure subtraction from a block-listed source. If delete density inside
+    /// a source's fetch window exceeds `factor - 1`, that source can deliver
+    /// `< k` live rows (a recall shortfall, not wrong content — see the
+    /// under-fetch `warn!` in `PkBlockFilterExec`). Steady-state this is
+    /// self-limiting because L0→base compaction drains tombstones from the
+    /// fresh tier; the exposure is a delete-heavy burst between compactions.
+    overfetch_factor: f64,
 }
 
 impl LsmScanPlanner {
@@ -48,24 +80,46 @@ impl LsmScanPlanner {
             pk_columns,
             base_schema,
             session: None,
+            store_params: None,
             flushed_cache: None,
+            warmer: None,
+            overfetch_factor: 1.0,
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open flushed generations.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
+    /// Set the store params used to open flushed generations.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
     /// Inject a cache of opened flushed-generation datasets, making repeated
     /// queries against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<FlushedMemTableCache>) -> Self {
+    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
         self.flushed_cache = Some(cache);
         self
     }
 
+    /// Inject the warmer fired on first open of a flushed generation.
+    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+        self.warmer = Some(warmer);
+        self
+    }
+
+    /// Set the over-fetch multiple for the per-source limit pushdown
+    /// (see the field docs). Values below `1.0` are rejected by
+    /// [`Self::plan_scan`].
+    pub fn with_overfetch_factor(mut self, factor: f64) -> Self {
+        self.overfetch_factor = factor;
+        self
+    }
+
     /// Create scan plan with deduplication.
     ///
     /// # Arguments
@@ -82,7 +136,7 @@ impl LsmScanPlanner {
     /// Each source is independently newest-per-PK (active via the fused
     /// [`MemTableDedupScanExec`](super::super::memtable::scanner), flushed via
     /// its within-generation deletion vector) and a cross-generation block-list
-    /// ([`PkHashFilterExec`]) drops any PK superseded by a newer generation.
+    /// ([`PkBlockFilterExec`]) drops any PK superseded by a newer generation.
     /// Each PK therefore survives in exactly one source, so a plain
     /// `UnionExec` carries at most one row per PK — no cross-source dedup,
     /// sort, or merge needed. `_memtable_gen` / `_rowaddr` are output-only and
@@ -105,9 +159,11 @@ impl LsmScanPlanner {
             .map(|p| p.iter().any(|c| c == ROW_ADDRESS_COLUMN))
             .unwrap_or(false);
         let keep_row_address = keep_row_address || user_wants_rowaddr;
+        validate_projection_names(projection, &self.base_schema, &[])?;
 
         // 1. Collect all data sources
         let sources = self.collector.collect()?;
+        let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?;
 
         if sources.is_empty() {
             // Return empty plan
@@ -119,8 +175,8 @@ impl LsmScanPlanner {
         // `Box::pin` keeps the future off `clippy::large_futures`.
         let block_lists = Box::pin(super::block_list::compute_source_block_lists(
             &sources,
-            &self.pk_columns,
             self.session.as_ref(),
+            self.store_params.as_ref(),
             self.flushed_cache.as_ref(),
         ))
         .await?;
@@ -130,21 +186,57 @@ impl LsmScanPlanner {
         // cross-gen block-list, not from output ordering.
         let sources: Vec<_> = sources.into_iter().rev().collect();
 
+        // Per-source limit pushdown: an unordered LIMIT needs only
+        // `offset + limit` live rows from EACH source to fill the global
+        // limit after dedup (any-N semantics), so cap every on-disk source
+        // instead of scanning whole generations and trimming above the
+        // union. Block-listed sources over-fetch by `overfetch_factor` so
+        // cross-gen dedup drops still leave `n_needed` live rows; the
+        // PkBlockFilter warns when that was not enough. The active memtable
+        // is in-memory and within-gen append duplicates are resolved by its
+        // own dedup, so it is never capped here.
+        let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0)));
+
         let mut source_plans = Vec::new();
         for source in sources {
             let is_base = matches!(source, LsmDataSource::BaseTable { .. });
-            let scan = self.build_source_scan(&source, projection, filter).await?;
+            let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. });
+            let blocked = block_lists
+                .get(&(source.shard_id(), source.generation()))
+                .cloned();
+            let fetch = match (n_needed, is_active) {
+                (Some(n), false) => Some(if blocked.is_some() && !self.pk_columns.is_empty() {
+                    ((n as f64) * overfetch).ceil() as usize
+                } else {
+                    n
+                }),
+                _ => None,
+            };
+            let scan = self
+                .build_source_scan(&source, projection, filter, fetch)
+                .await?;
 
             // Drop cross-generation stale rows (PKs superseded by a newer gen).
-            // `k = 0`: there is no top-k, so the under-fetch warning never fires.
-            let scan = match block_lists.get(&(source.shard_id(), source.generation())) {
-                Some(set) => Arc::new(PkHashFilterExec::new(
+            // With a limit, `k = n_needed` arms the under-fetch warning; with
+            // no limit `k = 0` keeps it silent.
+            let scan = match blocked {
+                Some(set) if !self.pk_columns.is_empty() => Arc::new(PkBlockFilterExec::new(
                     scan,
                     self.pk_columns.clone(),
-                    set.clone(),
-                    0,
-                )) as Arc<dyn ExecutionPlan>,
-                None => scan,
+                    set,
+                    n_needed.unwrap_or(0),
+                ))
+                    as Arc<dyn ExecutionPlan>,
+                _ => scan,
+            };
+
+            // Post-block-list cap: each source contributes at most `n_needed`
+            // live rows toward the global limit.
+            let scan: Arc<dyn ExecutionPlan> = match n_needed {
+                Some(n) if !is_active => Arc::new(
+                    datafusion::physical_plan::limit::LocalLimitExec::new(scan, n),
+                ),
+                _ => scan,
             };
 
             // When `_rowaddr` is surfaced, NULL it for non-base arms: only base
@@ -183,9 +275,9 @@ impl LsmScanPlanner {
             &self.canonical_scan_schema(projection, with_memtable_gen, keep_row_address),
         )?;
 
-        // 6. Add limit if specified
-        if let Some(limit) = limit {
-            plan = Arc::new(GlobalLimitExec::new(plan, offset.unwrap_or(0), Some(limit)));
+        // 6. Add limit / offset if specified
+        if limit.is_some() || offset.unwrap_or(0) > 0 {
+            plan = Arc::new(GlobalLimitExec::new(plan, offset.unwrap_or(0), limit));
         }
 
         Ok(plan)
@@ -229,6 +321,7 @@ impl LsmScanPlanner {
         source: &LsmDataSource,
         projection: Option<&[String]>,
         filter: Option<&Expr>,
+        fetch: Option<usize>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         match source {
             LsmDataSource::BaseTable { dataset } => {
@@ -247,13 +340,23 @@ impl LsmScanPlanner {
                 if let Some(expr) = filter {
                     scanner.filter_expr(expr.clone());
                 }
+                // Per-source limit pushdown (post-filter rows): bounds the
+                // physical scan instead of trimming above the union.
+                if let Some(fetch) = fetch {
+                    scanner.limit(Some(fetch as i64), None)?;
+                }
 
                 scanner.create_plan().await
             }
             LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset =
-                    open_flushed_dataset(path, self.session.as_ref(), self.flushed_cache.as_ref())
-                        .await?;
+                let dataset = open_flushed_dataset(
+                    path,
+                    self.session.as_ref(),
+                    self.store_params.as_ref(),
+                    self.flushed_cache.as_ref(),
+                    self.warmer.as_ref(),
+                )
+                .await?;
                 let mut scanner = dataset.scan();
 
                 let cols =
@@ -261,9 +364,27 @@ impl LsmScanPlanner {
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
                 scanner.with_row_address();
 
-                if let Some(expr) = filter {
+                // Drop tombstones: fold `NOT _tombstone` into the predicate so
+                // it runs before the pushdown limit (counting only live rows).
+                // The older real row a tombstone supersedes is dropped by the
+                // cross-gen block-list, not by this filter. Gen written before
+                // deletes existed lack the column → no fold, nothing to drop.
+                let folded;
+                let effective: Option<&Expr> = if dataset.schema().field(TOMBSTONE).is_some() {
+                    folded = fold_not_tombstone(filter);
+                    Some(&folded)
+                } else {
+                    filter
+                };
+                if let Some(expr) = effective {
                     scanner.filter_expr(expr.clone());
                 }
+                // Per-source limit pushdown: flushed generations are
+                // within-gen live (dedup-on-flush deletion vectors), so any
+                // `fetch` post-filter rows are valid contributions.
+                if let Some(fetch) = fetch {
+                    scanner.limit(Some(fetch as i64), None)?;
+                }
 
                 scanner.create_plan().await
             }
@@ -280,13 +401,24 @@ impl LsmScanPlanner {
 
                 let cols =
                     build_scanner_projection(projection, &self.base_schema, &self.pk_columns);
-                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
                 scanner.with_row_address();
 
                 // The dedup scan applies the filter post-dedup; pushing it
                 // into the raw scan would resurrect older versions of PKs
-                // whose newest version fails the predicate.
-                if let Some(expr) = filter {
+                // whose newest version fails the predicate. Folding
+                // `NOT _tombstone` here is correct: a tombstone wins the
+                // position-based dedup (suppressing the older real row) and is
+                // then dropped by this predicate. A memtable without the column
+                // (legacy / test) gets no fold.
+                let folded;
+                let effective: Option<&Expr> = if schema.column_with_name(TOMBSTONE).is_some() {
+                    folded = fold_not_tombstone(filter);
+                    Some(&folded)
+                } else {
+                    filter
+                };
+                if let Some(expr) = effective {
                     scanner.filter_expr(expr.clone());
                 }
 
@@ -413,13 +545,36 @@ mod integration_tests {
         .unwrap()
     }
 
-    /// Create a dataset at the given URI with the provided batches.
+    /// Create a dataset at the given URI with the provided batches. Also writes
+    /// the standalone PK sidecar (on `id`) so a flushed-generation source can be
+    /// probed by the block-list; harmless for a base table (never probed).
     async fn create_dataset(uri: &str, batches: Vec<RecordBatch>) -> Dataset {
         let schema = batches[0].schema();
-        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
-        Dataset::write(reader, uri, Some(WriteParams::default()))
+        let has_id = schema.column_with_name("id").is_some();
+        let reader = RecordBatchIterator::new(batches.clone().into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, uri, Some(WriteParams::default()))
             .await
-            .unwrap()
+            .unwrap();
+        if has_id {
+            super::super::block_list::write_pk_sidecar(uri, &batches, &["id"])
+                .await
+                .unwrap();
+        }
+        dataset
+    }
+
+    /// Build an in-memory memtable's `(batch_store, index_store)` with the PK
+    /// index enabled and populated (mirrors production — the block-list needs
+    /// the PK index to dedup in-memory generations).
+    fn pk_indexed(batches: &[RecordBatch]) -> (Arc<BatchStore>, Arc<IndexStore>) {
+        let batch_store = Arc::new(BatchStore::with_capacity(100));
+        let mut index = IndexStore::new();
+        index.enable_pk_index(&[("id".to_string(), 0)]);
+        for b in batches {
+            let (bp, off, _) = batch_store.append(b.clone()).unwrap();
+            index.insert_with_batch_position(b, off, Some(bp)).unwrap();
+        }
+        (batch_store, Arc::new(index))
     }
 
     /// Setup a multi-level LSM structure with:
@@ -470,10 +625,8 @@ mod integration_tests {
             .with_flushed_generation(2, "gen_2".to_string());
 
         // Create active memtable
-        let batch_store = Arc::new(BatchStore::with_capacity(100));
-        let index_store = Arc::new(IndexStore::new());
-        let active_batch = create_test_batch(&schema, &[5, 6, 7], "active");
-        let _ = batch_store.append(active_batch);
+        let (batch_store, index_store) =
+            pk_indexed(&[create_test_batch(&schema, &[5, 6, 7], "active")]);
 
         let active_memtable = InMemoryMemTables {
             active: InMemoryMemTableRef {
@@ -515,18 +668,18 @@ mod integration_tests {
         // Verify the plan (gen DESC order: active -> gen2 -> gen1 -> base):
         // - plain UnionExec at top
         // - active arm: MemTableDedupScanExec (newest gen, not block-listed)
-        // - older arms: PkHashFilterExec (cross-gen block-list) -> LanceRead
+        // - older arms: PkBlockFilterExec (cross-gen block-list) -> LanceRead
         assert_plan_node_equals(
             plan,
             "ProjectionExec:...
   CoalescePartitionsExec
     UnionExec
     MemTableDedupScanExec: projection=[id, name, _rowaddr], with_row_id=false, with_row_address=true
-    PkHashFilterExec: pk_cols=[id]...
+    PkBlockFilterExec: pk_cols=[id]...
       LanceRead:...gen_2...
-    PkHashFilterExec: pk_cols=[id]...
+    PkBlockFilterExec: pk_cols=[id]...
       LanceRead:...gen_1...
-    PkHashFilterExec: pk_cols=[id]...
+    PkBlockFilterExec: pk_cols=[id]...
       LanceRead:...base/data...refine_filter=--",
         )
         .await
@@ -549,9 +702,9 @@ mod integration_tests {
 
         // Verify the plan with `_memtable_gen` tags (gen DESC order):
         // - plain UnionExec at top
-        // - each arm: MemtableGenTagExec -> (PkHashFilterExec ->) data source
+        // - each arm: MemtableGenTagExec -> (PkBlockFilterExec ->) data source
         //   - gen3 (active): MemtableGenTagExec -> MemTableDedupScanExec
-        //   - gen2/gen1/base: MemtableGenTagExec -> PkHashFilterExec -> LanceRead
+        //   - gen2/gen1/base: MemtableGenTagExec -> PkBlockFilterExec -> LanceRead
         assert_plan_node_equals(
             plan,
             "ProjectionExec:...
@@ -560,13 +713,13 @@ mod integration_tests {
     MemtableGenTagExec: gen=gen3
       MemTableDedupScanExec: projection=[id, name, _rowaddr], with_row_id=false, with_row_address=true
     MemtableGenTagExec: gen=gen2
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...gen_2...
     MemtableGenTagExec: gen=gen1
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...gen_1...
     MemtableGenTagExec: gen=base
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...base/data...refine_filter=--",
         )
         .await
@@ -647,14 +800,14 @@ mod integration_tests {
         }
 
         // base/gen1/gen2 all hold PKs superseded by a newer generation, so each
-        // is wrapped in a `PkHashFilterExec`; the newest (active) arm is not.
+        // is wrapped in a `PkBlockFilterExec`; the newest (active) arm is not.
         let plan = scanner.create_plan().await.unwrap();
         let plan_str = format!(
             "{}",
             datafusion::physical_plan::displayable(plan.as_ref()).indent(true)
         );
         assert!(
-            plan_str.contains("PkHashFilterExec"),
+            plan_str.contains("PkBlockFilterExec"),
             "filtered-read plan must apply the cross-gen block-list, got:\n{}",
             plan_str
         );
@@ -730,21 +883,21 @@ mod integration_tests {
             .with_flushed_generation(2, "gen_2".to_string());
 
         // Frozen gen3 (sealed, NOT in the manifest) and active gen4.
-        let frozen_store = Arc::new(BatchStore::with_capacity(100));
-        let _ = frozen_store.append(create_test_batch(&schema, &[6, 7], "frozen"));
+        let (frozen_store, frozen_index) =
+            pk_indexed(&[create_test_batch(&schema, &[6, 7], "frozen")]);
         let frozen = InMemoryMemTableRef {
             batch_store: frozen_store,
-            index_store: Arc::new(IndexStore::new()),
+            index_store: frozen_index,
             schema: schema.clone(),
             generation: 3,
         };
 
-        let active_store = Arc::new(BatchStore::with_capacity(100));
-        let _ = active_store.append(create_test_batch(&schema, &[7, 8], "active"));
+        let (active_store, active_index) =
+            pk_indexed(&[create_test_batch(&schema, &[7, 8], "active")]);
         let in_memory = InMemoryMemTables {
             active: InMemoryMemTableRef {
                 batch_store: active_store,
-                index_store: Arc::new(IndexStore::new()),
+                index_store: active_index,
                 schema: schema.clone(),
                 generation: 4,
             },
@@ -801,8 +954,9 @@ mod integration_tests {
             setup_multi_level_lsm().await;
 
         // Create scanner with projection (only id column)
-        let mut scanner =
-            LsmScanner::new(base_dataset, shard_snapshots, pk_columns).project(&["id"]);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
+            .project(&["id"])
+            .unwrap();
         if let Some((shard_id, memtable)) = active_memtable {
             scanner = scanner.with_in_memory_memtables(shard_id, memtable);
         }
@@ -832,7 +986,9 @@ mod integration_tests {
             setup_multi_level_lsm().await;
 
         // Create scanner with limit
-        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns).limit(3, None);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
+            .limit(Some(3), None)
+            .unwrap();
         if let Some((shard_id, memtable)) = active_memtable {
             scanner = scanner.with_in_memory_memtables(shard_id, memtable);
         }
@@ -851,6 +1007,35 @@ mod integration_tests {
         assert_eq!(total_rows, 3, "Should have 3 rows due to limit");
     }
 
+    #[tokio::test]
+    async fn test_lsm_scan_with_offset_without_limit() {
+        let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
+            setup_multi_level_lsm().await;
+
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
+            .limit(Some(3), None)
+            .unwrap()
+            .limit(None, Some(2))
+            .unwrap();
+        if let Some((shard_id, memtable)) = active_memtable {
+            scanner = scanner.with_in_memory_memtables(shard_id, memtable);
+        }
+
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            total_rows, 5,
+            "offset-only scan should skip 2 rows from the 7-row deduped result"
+        );
+    }
+
     #[tokio::test]
     async fn test_lsm_scan_base_only() {
         let (base_dataset, _, _, pk_columns, _temp_path) = setup_multi_level_lsm().await;
@@ -969,12 +1154,12 @@ mod integration_tests {
     ProjectionExec: expr=[id@0 as id, name@1 as name, NULL as _rowaddr]
       MemTableDedupScanExec: projection=[id, name, _rowaddr], with_row_id=false, with_row_address=true
     ProjectionExec: expr=[id@0 as id, name@1 as name, NULL as _rowaddr]
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...gen_2...
     ProjectionExec: expr=[id@0 as id, name@1 as name, NULL as _rowaddr]
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...gen_1...
-    PkHashFilterExec: pk_cols=[id]...
+    PkBlockFilterExec: pk_cols=[id]...
       LanceRead:...base/data...refine_filter=--",
         )
         .await
@@ -1037,14 +1222,14 @@ mod integration_tests {
         MemTableDedupScanExec: projection=[id, name, _rowaddr], with_row_id=false, with_row_address=true
     MemtableGenTagExec: gen=gen2
       ProjectionExec: expr=[id@0 as id, name@1 as name, NULL as _rowaddr]
-        PkHashFilterExec: pk_cols=[id]...
+        PkBlockFilterExec: pk_cols=[id]...
           LanceRead:...gen_2...
     MemtableGenTagExec: gen=gen1
       ProjectionExec: expr=[id@0 as id, name@1 as name, NULL as _rowaddr]
-        PkHashFilterExec: pk_cols=[id]...
+        PkBlockFilterExec: pk_cols=[id]...
           LanceRead:...gen_1...
     MemtableGenTagExec: gen=base
-      PkHashFilterExec: pk_cols=[id]...
+      PkBlockFilterExec: pk_cols=[id]...
         LanceRead:...base/data...refine_filter=--",
         )
         .await
@@ -1113,6 +1298,8 @@ mod integration_tests {
         let mut index_store = IndexStore::new();
         // Add BTree index on id column (field_id=0)
         index_store.add_btree("id_idx".to_string(), 0, "id".to_string());
+        // Reuse it as the PK index so the block-list can dedup this generation.
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
 
         let active_batch = create_test_batch(&schema, &[5, 6, 7], "active");
         let _ = batch_store.append(active_batch.clone());
@@ -1177,7 +1364,7 @@ mod integration_tests {
         // 1. Verify overall structure
         assert!(plan_str.contains("UnionExec"), "Should have UnionExec");
         assert!(
-            plan_str.contains("PkHashFilterExec"),
+            plan_str.contains("PkBlockFilterExec"),
             "older generations should be block-list filtered"
         );
         assert!(
@@ -1365,7 +1552,6 @@ mod integration_tests {
 
         // Active memtable: id=10 inserted ("keep") then updated to NULL within
         // the same generation; id=20 ("active_20") is a control that matches.
-        let batch_store = Arc::new(BatchStore::with_capacity(16));
         let active_batch = RecordBatch::try_new(
             schema.clone(),
             vec![
@@ -1378,12 +1564,12 @@ mod integration_tests {
             ],
         )
         .unwrap();
-        batch_store.append(active_batch).unwrap();
+        let (batch_store, index_store) = pk_indexed(&[active_batch]);
 
         let in_memory = InMemoryMemTables {
             active: InMemoryMemTableRef {
                 batch_store,
-                index_store: Arc::new(IndexStore::new()),
+                index_store,
                 schema: schema.clone(),
                 generation: 1,
             },
@@ -1603,13 +1789,9 @@ mod integration_tests {
         let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) =
             setup_multi_level_lsm().await;
 
-        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns).project(&[
-            "id",
-            "_rowoffset",
-            "name",
-            "_rowaddr",
-            "_rowid",
-        ]);
+        let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
+            .project(&["id", "_rowoffset", "name", "_rowaddr", "_rowid"])
+            .unwrap();
         if let Some((shard_id, memtable)) = active_memtable {
             scanner = scanner.with_in_memory_memtables(shard_id, memtable);
         }
@@ -1699,7 +1881,8 @@ mod integration_tests {
             setup_multi_level_lsm().await;
 
         let mut scanner = LsmScanner::new(base_dataset, shard_snapshots, pk_columns)
-            .project(&["id", "_rowid", "name"]);
+            .project(&["id", "_rowid", "name"])
+            .unwrap();
         if let Some((shard_id, memtable)) = active_memtable {
             scanner = scanner.with_in_memory_memtables(shard_id, memtable);
         }
@@ -1762,7 +1945,8 @@ mod integration_tests {
 
         let scanner =
             LsmScanner::without_base_table(schema, base_uri, vec![], vec!["id".to_string()])
-                .project(&["id", "_rowaddr", "name", "_rowid"]);
+                .project(&["id", "_rowaddr", "name", "_rowid"])
+                .unwrap();
         let plan = scanner.create_plan().await.unwrap();
 
         let names: Vec<String> = plan
@@ -1806,4 +1990,207 @@ mod integration_tests {
             "rejection message should mention the offending column, got: {msg}",
         );
     }
+
+    // ----- tombstone (delete) filtered-read tests -----
+
+    /// Memtable / generation schema: base (`id`, `name`) + `_tombstone`.
+    fn ts_pk_schema() -> Arc<ArrowSchema> {
+        let mut id_metadata = HashMap::new();
+        id_metadata.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let id = Field::new("id", DataType::Int32, false).with_metadata(id_metadata);
+        Arc::new(ArrowSchema::new(vec![
+            id,
+            Field::new("name", DataType::Utf8, true),
+            Field::new(crate::dataset::mem_wal::TOMBSTONE, DataType::Boolean, false),
+        ]))
+    }
+
+    /// Build a `_tombstone`-bearing batch from `(id, name, tombstone)` rows.
+    fn ts_batch(schema: &Arc<ArrowSchema>, rows: &[(i32, Option<&str>, bool)]) -> RecordBatch {
+        let ids: Vec<i32> = rows.iter().map(|(i, _, _)| *i).collect();
+        let names: Vec<Option<String>> = rows
+            .iter()
+            .map(|(_, n, _)| n.map(|s| s.to_string()))
+            .collect();
+        let ts: Vec<bool> = rows.iter().map(|(_, _, t)| *t).collect();
+        RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+                Arc::new(arrow_array::BooleanArray::from(ts)),
+            ],
+        )
+        .unwrap()
+    }
+
+    fn collect_sorted_ids(batches: &[RecordBatch]) -> Vec<i32> {
+        let mut ids: Vec<i32> = Vec::new();
+        for b in batches {
+            let arr = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            ids.extend((0..arr.len()).map(|i| arr.value(i)));
+        }
+        ids.sort_unstable();
+        ids
+    }
+
+    #[tokio::test]
+    async fn test_lsm_scan_active_tombstone_masks_base() {
+        // Active gen tombstones id=2 (present in base) and adds real id=4. The
+        // result drops id=2 (block-list masks the base row; the active arm's
+        // folded `NOT _tombstone` drops the tombstone itself) and never
+        // surfaces the tombstone row.
+        let base_schema = create_pk_schema();
+        let mem_schema = ts_pk_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+        let base = Arc::new(
+            create_dataset(
+                &base_uri,
+                vec![create_test_batch(&base_schema, &[1, 2, 3], "base")],
+            )
+            .await,
+        );
+
+        let active_batch = ts_batch(
+            &mem_schema,
+            &[(4, Some("active_4"), false), (2, None, true)],
+        );
+        let (bs, ix) = pk_indexed(&[active_batch]);
+        let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store: bs,
+                        index_store: ix,
+                        schema: mem_schema,
+                        generation: 2,
+                    },
+                    frozen: vec![],
+                },
+            );
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        assert_eq!(
+            collect_sorted_ids(&batches),
+            vec![1, 3, 4],
+            "id=2 deleted; tombstone row not surfaced"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_lsm_scan_flushed_tombstone_masks_base() {
+        // A tombstone living in a flushed generation masks the older base row by
+        // PK presence (block-list) and is itself dropped by the folded predicate.
+        let base_schema = create_pk_schema();
+        let mem_schema = ts_pk_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_path = temp.path().to_str().unwrap();
+        let base_uri = format!("{}/base", base_path);
+        let base = Arc::new(
+            create_dataset(
+                &base_uri,
+                vec![create_test_batch(&base_schema, &[1, 2, 3], "base")],
+            )
+            .await,
+        );
+
+        // Flushed gen 1 holds only a tombstone for id=2 (written with the
+        // `_tombstone` schema, so the flushed arm folds `NOT _tombstone`).
+        let shard_id = Uuid::new_v4();
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
+        create_dataset(&gen1_uri, vec![ts_batch(&mem_schema, &[(2, None, true)])]).await;
+        let shard_snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(2)
+            .with_flushed_generation(1, "gen_1".to_string());
+
+        let scanner = LsmScanner::new(base, vec![shard_snapshot], vec!["id".to_string()]);
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        assert_eq!(
+            collect_sorted_ids(&batches),
+            vec![1, 3],
+            "id=2 deleted via flushed-generation tombstone"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_lsm_scan_tombstone_does_not_consume_limit() {
+        // A single (newest) flushed generation holds both tombstones and live
+        // rows. With LIMIT 2 the folded `NOT _tombstone` runs *before* the
+        // per-source pushdown limit, so the limit counts only live rows — we get
+        // 2 live rows, not 0 (which is what a post-limit tombstone filter, or a
+        // limit that counted the tombstones, would yield).
+        let base_schema = create_pk_schema();
+        let mem_schema = ts_pk_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+
+        // gen_1 file order: two tombstones first, then two live rows.
+        let shard_id = Uuid::new_v4();
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
+        create_dataset(
+            &gen1_uri,
+            vec![ts_batch(
+                &mem_schema,
+                &[
+                    (1, None, true),
+                    (2, None, true),
+                    (3, Some("live_3"), false),
+                    (4, Some("live_4"), false),
+                ],
+            )],
+        )
+        .await;
+        let shard_snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(2)
+            .with_flushed_generation(1, "gen_1".to_string());
+
+        let scanner = LsmScanner::without_base_table(
+            base_schema,
+            base_uri,
+            vec![shard_snapshot],
+            vec!["id".to_string()],
+        )
+        .limit(Some(2), None)
+        .unwrap();
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let ids = collect_sorted_ids(&batches);
+        assert_eq!(
+            ids.len(),
+            2,
+            "limit honored with live rows only, got {:?}",
+            ids
+        );
+        assert!(
+            !ids.contains(&1) && !ids.contains(&2),
+            "deleted ids must not appear, got {:?}",
+            ids
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
index a6063c2930c..d80e0e26795 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs
@@ -8,35 +8,39 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use arrow_array::{Array, RecordBatch};
-use arrow_schema::SchemaRef;
+use arrow_array::{Array, BooleanArray, RecordBatch};
+use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions};
 use datafusion::common::ScalarValue;
 use datafusion::execution::TaskContext;
+use datafusion::physical_expr::expressions::{Column, Literal, NotExpr};
+use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr};
 use datafusion::physical_plan::ExecutionPlan;
+use datafusion::physical_plan::filter::FilterExec;
 use datafusion::physical_plan::limit::GlobalLimitExec;
+use datafusion::physical_plan::projection::ProjectionExec;
+use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::prelude::{Expr, SessionContext};
 use futures::TryStreamExt;
+use lance_core::utils::bloomfilter::sbbf::Sbbf;
 use lance_core::{Result, is_system_column};
 use lance_datafusion::exec::OneShotExec;
-use lance_index::scalar::bloomfilter::sbbf::Sbbf;
 use tracing::instrument;
 
+use crate::dataset::mem_wal::TOMBSTONE;
 use crate::dataset::mem_wal::index::IndexStore;
 use crate::dataset::mem_wal::memtable::batch_store::BatchStore;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
-use super::exec::{
-    BloomFilterGuardExec, CoalesceFirstExec, DedupDirection, WithinSourceDedupExec,
-    compute_pk_hash_from_scalars,
-};
-use super::flushed_cache::{FlushedMemTableCache, open_flushed_dataset};
+use super::exec::{BloomFilterGuardExec, CoalesceFirstExec, compute_pk_hash_from_scalars};
+use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
-    build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical,
-    wants_row_address, wants_row_id,
+    DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns,
+    project_to_canonical, validate_projection_names, wants_row_address, wants_row_id,
 };
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Plans point lookup queries over LSM data.
 ///
@@ -86,8 +90,12 @@ pub struct LsmPointLookupPlanner {
     bloom_filters: std::collections::HashMap<u64, Arc<Sbbf>>,
     /// Session threaded into flushed-generation opens (shared caches).
     session: Option<Arc<Session>>,
+    /// Store params for opening flushed generations, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
     /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<FlushedMemTableCache>>,
+    flushed_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of a flushed generation.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
     /// Precomputed canonical output schema for the no-projection case, so the
     /// hot `lookup(.., None)` path clones an `Arc` instead of rebuilding the
     /// schema on every call.
@@ -119,29 +127,42 @@ impl LsmPointLookupPlanner {
             base_schema,
             bloom_filters: std::collections::HashMap::new(),
             session: None,
+            store_params: None,
             flushed_cache: None,
+            warmer: None,
             none_target,
             task_ctx: SessionContext::new().task_ctx(),
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Set the session used to open flushed generations.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
+    /// Set the store params used to open flushed generations.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
     /// Inject a cache of opened flushed-generation datasets, making repeated
     /// lookups against the same generation a pure `Arc::clone`. Populate it up
     /// front during scan setup via
     /// [`DatasetMemWalExt::prewarm_mem_wal`](crate::dataset::mem_wal::DatasetMemWalExt::prewarm_mem_wal)
     /// so the first gen-key lookup does not pay the dataset open.
-    pub fn with_flushed_cache(mut self, cache: Arc<FlushedMemTableCache>) -> Self {
+    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
         self.flushed_cache = Some(cache);
         self
     }
 
+    /// Inject the warmer fired on first open of a flushed generation.
+    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+        self.warmer = Some(warmer);
+        self
+    }
+
     /// Add a bloom filter for a generation.
     ///
     /// Bloom filters are optional but improve performance by skipping
@@ -177,6 +198,34 @@ impl LsmPointLookupPlanner {
         pk_values: &[ScalarValue],
         projection: Option<&[String]>,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        match self.plan_lookup_coalesced(pk_values, projection).await? {
+            // Tombstones are dropped AFTER the coalesce, never per-source: the
+            // tombstone wins the newest-first coalesce (its source is the newest
+            // non-empty arm), so filtering it here yields "not found". Filtering
+            // per-source would empty the newest arm and let `CoalesceFirstExec`
+            // fall through to an older arm — resurrecting the deleted row. Then
+            // the carried `_tombstone` column is projected away.
+            Some(coalesced) => {
+                let canonical =
+                    canonical_output_schema(projection, &self.base_schema, &self.pk_columns, false);
+                filter_tombstones_after_coalesce(coalesced, &canonical)
+            }
+            None => self.empty_plan(projection),
+        }
+    }
+
+    /// Build the coalesced point-lookup plan: each source scanned newest-first,
+    /// unioned under `CoalesceFirstExec`, output in the *carry* schema (canonical
+    /// output + the `_tombstone` marker). `None` when there are no sources at
+    /// all. [`Self::plan_lookup`] drops the tombstone on top of this;
+    /// [`Self::lookup_keep_tombstone`] keeps it so partial-update merge can tell
+    /// a fresh-deleted key from an absent one.
+    async fn plan_lookup_coalesced(
+        &self,
+        pk_values: &[ScalarValue],
+        projection: Option<&[String]>,
+    ) -> Result<Option<Arc<dyn ExecutionPlan>>> {
+        validate_projection_names(projection, &self.base_schema, &[])?;
         if pk_values.len() != self.pk_columns.len() {
             return Err(lance_core::Error::invalid_input(format!(
                 "Expected {} primary key values, got {}",
@@ -190,7 +239,7 @@ impl LsmPointLookupPlanner {
         let sources = self.collector.collect()?;
 
         if sources.is_empty() {
-            return self.empty_plan(projection);
+            return Ok(None);
         }
 
         // Sort by generation DESC (newest first)
@@ -224,13 +273,66 @@ impl LsmPointLookupPlanner {
             source_plans.push(guarded_plan);
         }
 
-        let plan: Arc<dyn ExecutionPlan> = if source_plans.len() == 1 {
-            source_plans.remove(0)
-        } else {
-            Arc::new(CoalesceFirstExec::new(source_plans))
+        // Always coalesce, even for a single source: besides picking the newest
+        // non-empty arm, `CoalesceFirstExec` normalizes child statistics to a
+        // schema-sized "unknown", which the downstream tombstone `FilterExec`
+        // needs (the in-memory mem_wal execs report empty column statistics, and
+        // datafusion's projection statistics would index out of bounds without
+        // this normalization).
+        Ok(Some(Arc::new(CoalesceFirstExec::new(source_plans))))
+    }
+
+    /// Like [`Self::lookup`] but does NOT drop a tombstone: the returned 1-row
+    /// batch carries the `_tombstone` marker (`true` ⇒ the key's newest fresh
+    /// version is a delete); `None` still means the key is absent from every
+    /// source. Partial-update merge uses this to treat a fresh-deleted PK as
+    /// absent, so it never resurrects stale, not-yet-compacted base columns.
+    /// Always plans (no in-memory fast path) — used off the hot read path, on
+    /// small partial-update batches.
+    pub async fn lookup_keep_tombstone(
+        &self,
+        pk_values: &[ScalarValue],
+        projection: Option<&[String]>,
+    ) -> Result<Option<RecordBatch>> {
+        let Some(plan) = self.plan_lookup_coalesced(pk_values, projection).await? else {
+            return Ok(None);
         };
+        let batches: Vec<RecordBatch> = plan
+            .execute(0, self.task_ctx.clone())?
+            .try_collect()
+            .await?;
+        for batch in batches {
+            if batch.num_rows() > 0 {
+                return Ok(Some(batch.slice(0, 1)));
+            }
+        }
+        Ok(None)
+    }
 
-        Ok(plan)
+    /// Tombstone-preserving [`Self::lookup_many`] for single- or multi-column
+    /// keys: resolves each key with [`Self::lookup_keep_tombstone`] and
+    /// concatenates the hits in the carry schema (canonical output +
+    /// `_tombstone`); keys absent from every source are omitted. Per-key (no
+    /// batched fast path) — the partial-update batches that use it are small.
+    pub async fn lookup_many_keep_tombstone(
+        &self,
+        keys: &[Vec<ScalarValue>],
+        projection: Option<&[String]>,
+    ) -> Result<RecordBatch> {
+        let canonical =
+            canonical_output_schema(projection, &self.base_schema, &self.pk_columns, false);
+        let target = carry_schema(&canonical);
+        let mut out: Vec<RecordBatch> = Vec::with_capacity(keys.len());
+        for key in keys {
+            if let Some(b) = self.lookup_keep_tombstone(key, projection).await? {
+                out.push(b);
+            }
+        }
+        match out.len() {
+            0 => Ok(RecordBatch::new_empty(target)),
+            1 => Ok(out.pop().unwrap()),
+            _ => Ok(arrow_select::concat::concat_batches(&target, &out)?),
+        }
     }
 
     /// Resolve a single-row point lookup, returning the newest matching row (a
@@ -299,6 +401,7 @@ impl LsmPointLookupPlanner {
                             target,
                         )? {
                             Probe::Hit(batch) => Ok(Some(FastOutcome::Hit(batch))),
+                            Probe::Deleted => Ok(Some(FastOutcome::Deleted)),
                             Probe::Miss => Ok(None),
                             Probe::NoIndex => Ok(Some(FastOutcome::NeedsFallback)),
                         }
@@ -306,6 +409,9 @@ impl LsmPointLookupPlanner {
                 )?;
                 match outcome {
                     Some(FastOutcome::Hit(batch)) => return Ok(Some(batch)),
+                    // Newest version is a tombstone → deleted; do not consult
+                    // older (on-disk) sources.
+                    Some(FastOutcome::Deleted) => return Ok(None),
                     Some(FastOutcome::NeedsFallback) => { /* fall through to plan */ }
                     None => {
                         // Every in-memory memtable missed. If there is no
@@ -403,7 +509,12 @@ impl LsmPointLookupPlanner {
             for (ri, m) in refs.iter().enumerate() {
                 match probe_position(&m.batch_store, &m.index_store, pk_col, key)? {
                     ProbePos::Found { batch_idx, row } => {
-                        hits.entry((ri, batch_idx)).or_default().push(row as u32);
+                        // Newest version is a tombstone → the key is deleted:
+                        // resolve it as a miss (emit nothing) and do not fall
+                        // through to an older source.
+                        if !is_tombstone_at(&m.batch_store, batch_idx, row)? {
+                            hits.entry((ri, batch_idx)).or_default().push(row as u32);
+                        }
                         resolved = true;
                         break;
                     }
@@ -546,10 +657,19 @@ impl LsmPointLookupPlanner {
                 scanner.create_plan().await?
             }
             LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset =
-                    open_flushed_dataset(path, self.session.as_ref(), self.flushed_cache.as_ref())
-                        .await?;
+                let dataset = open_flushed_dataset(
+                    path,
+                    self.session.as_ref(),
+                    self.store_params.as_ref(),
+                    self.flushed_cache.as_ref(),
+                    self.warmer.as_ref(),
+                )
+                .await?;
                 let mut scanner = dataset.scan();
+                // Carry `_tombstone` through so the post-coalesce filter can drop
+                // a deleted key (gen written before deletes existed lack it →
+                // `project_to_carry` synthesizes `false`).
+                let cols = cols_with_tombstone(&cols, dataset.schema().field(TOMBSTONE).is_some());
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
                 scanner.filter_expr(filter.clone());
                 scanner.create_plan().await?
@@ -564,7 +684,10 @@ impl LsmPointLookupPlanner {
 
                 let mut scanner =
                     MemTableScanner::new(batch_store.clone(), index_store.clone(), schema.clone());
-                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>());
+                // Carry `_tombstone` through so the post-coalesce filter can drop
+                // a deleted key; it survives the sort below.
+                let cols = cols_with_tombstone(&cols, schema.column_with_name(TOMBSTONE).is_some());
+                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
                 scanner.filter_expr(filter.clone());
                 // Expose `_rowid` (the BatchStore row offset, monotonic with
                 // insert order) so we can pick the most recently inserted
@@ -573,22 +696,35 @@ impl LsmPointLookupPlanner {
                 // multiple rows sharing the target primary key.
                 scanner.with_row_id();
                 let raw = scanner.create_plan().await?;
-                // Within the active memtable, larger `_rowid` = newer
-                // insert. After dedup there is exactly one row per PK.
-                let deduped: Arc<dyn ExecutionPlan> = Arc::new(WithinSourceDedupExec::new(
-                    raw,
-                    self.pk_columns.clone(),
-                    lance_core::ROW_ID,
-                    DedupDirection::KeepMaxRowAddr,
-                ));
+                // The filter already restricts to the exact PK value, so the
+                // scan yields that key's insert history. Within the active
+                // memtable larger `_rowid` = newer insert, so sorting `_rowid`
+                // DESC and keeping the first row picks the newest version — one
+                // row per (value-exact) PK.
+                let rowid_idx = raw.schema().index_of(lance_core::ROW_ID)?;
+                let ordering = LexOrdering::new(vec![PhysicalSortExpr {
+                    expr: Arc::new(Column::new(lance_core::ROW_ID, rowid_idx)),
+                    options: SortOptions {
+                        descending: true,
+                        nulls_first: false,
+                    },
+                }])
+                .ok_or_else(|| {
+                    lance_core::Error::internal("point-lookup: failed to build _rowid ordering")
+                })?;
+                let newest: Arc<dyn ExecutionPlan> =
+                    Arc::new(SortExec::new(ordering, raw).with_fetch(Some(1)));
                 // Per-source `_rowid` would collide with the base table's;
                 // NULL it before canonicalization (the value is internal to
                 // this arm). project_to_canonical drops it entirely when
                 // the user didn't request `_rowid` in the projection.
-                null_columns(deduped, &[lance_core::ROW_ID])?
+                null_columns(newest, &[lance_core::ROW_ID])?
             }
         };
-        project_to_canonical(scan, &target)
+        // Output carries `_tombstone` (canonical + the marker) so it survives
+        // the union/coalesce to the post-coalesce filter; base / legacy sources
+        // that lack the column get a synthesized `false`.
+        project_to_carry(scan, &target)
     }
 
     /// Create an empty execution plan with the canonical output schema.
@@ -601,10 +737,113 @@ impl LsmPointLookupPlanner {
     }
 }
 
+/// Append `_tombstone` to a scanner projection when the source carries it, so
+/// the column survives to the post-coalesce tombstone filter. Sources without
+/// it (base table, generations written before deletes existed) are left alone
+/// and have `false` synthesized by [`project_to_carry`].
+fn cols_with_tombstone(cols: &[String], present: bool) -> Vec<String> {
+    if !present {
+        return cols.to_vec();
+    }
+    let mut out = cols.to_vec();
+    if !out.iter().any(|c| c == TOMBSTONE) {
+        out.push(TOMBSTONE.to_string());
+    }
+    out
+}
+
+/// Carry schema = canonical output + a trailing non-nullable `_tombstone`
+/// Boolean. Non-nullable so the base arm's synthesized `Literal(false)` matches
+/// the WAL arms' real column under `CoalesceFirstExec`'s exact-schema check.
+fn carry_schema(canonical: &SchemaRef) -> SchemaRef {
+    let mut fields: Vec<Arc<Field>> = canonical.fields().iter().cloned().collect();
+    fields.push(Arc::new(Field::new(TOMBSTONE, DataType::Boolean, false)));
+    Arc::new(Schema::new(fields))
+}
+
+/// Project a source scan to the carry schema: existing columns are forwarded, a
+/// missing `_tombstone` becomes `false` (base table / legacy generations carry
+/// no tombstones), and missing system / `_distance` columns are NULL-filled
+/// (mirroring [`project_to_canonical`]).
+fn project_to_carry(
+    plan: Arc<dyn ExecutionPlan>,
+    canonical: &SchemaRef,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    let input = plan.schema();
+    let carry = carry_schema(canonical);
+    let mut project_exprs: Vec<(Arc<dyn PhysicalExpr>, String)> =
+        Vec::with_capacity(carry.fields().len());
+    for field in carry.fields() {
+        let name = field.name();
+        let expr: Arc<dyn PhysicalExpr> = match input.column_with_name(name) {
+            Some((idx, _)) => Arc::new(Column::new(name, idx)),
+            None if name == TOMBSTONE => Arc::new(Literal::new(ScalarValue::Boolean(Some(false)))),
+            None if is_system_column(name) => Arc::new(Literal::new(ScalarValue::UInt64(None))),
+            None if name == DISTANCE_COLUMN => Arc::new(Literal::new(ScalarValue::Float32(None))),
+            None => {
+                return Err(lance_core::Error::internal(format!(
+                    "Column '{}' missing from point-lookup carry source schema (have: {:?})",
+                    name,
+                    input
+                        .fields()
+                        .iter()
+                        .map(|f| f.name().clone())
+                        .collect::<Vec<_>>()
+                )));
+            }
+        };
+        project_exprs.push((expr, name.clone()));
+    }
+    Ok(Arc::new(
+        ProjectionExec::try_new(project_exprs, plan).map_err(|e| {
+            lance_core::Error::internal(format!("Failed to build carry ProjectionExec: {}", e))
+        })?,
+    ))
+}
+
+/// Drop tombstone rows after `CoalesceFirstExec` has already picked the newest
+/// source, then project the carried `_tombstone` column away (back to the
+/// canonical schema).
+fn filter_tombstones_after_coalesce(
+    plan: Arc<dyn ExecutionPlan>,
+    canonical: &SchemaRef,
+) -> Result<Arc<dyn ExecutionPlan>> {
+    let idx = plan.schema().index_of(TOMBSTONE).map_err(|e| {
+        lance_core::Error::internal(format!("point-lookup carry plan missing _tombstone: {}", e))
+    })?;
+    let predicate: Arc<dyn PhysicalExpr> =
+        Arc::new(NotExpr::new(Arc::new(Column::new(TOMBSTONE, idx))));
+    let filtered: Arc<dyn ExecutionPlan> =
+        Arc::new(FilterExec::try_new(predicate, plan).map_err(|e| {
+            lance_core::Error::internal(format!("Failed to build tombstone FilterExec: {}", e))
+        })?);
+    project_to_canonical(filtered, canonical)
+}
+
+/// Whether the row at `(batch_idx, row)` of `batch_store` is a tombstone.
+/// Memtables without the `_tombstone` column (legacy / direct-construction
+/// tests) are treated as carrying no tombstones.
+fn is_tombstone_at(batch_store: &BatchStore, batch_idx: usize, row: usize) -> Result<bool> {
+    let stored = batch_store.get(batch_idx).ok_or_else(|| {
+        lance_core::Error::internal("point-lookup: tombstone-check batch missing")
+    })?;
+    let Some(col) = stored.data.column_by_name(TOMBSTONE) else {
+        return Ok(false);
+    };
+    let arr = col.as_any().downcast_ref::<BooleanArray>().ok_or_else(|| {
+        lance_core::Error::internal("point-lookup: _tombstone column is not Boolean")
+    })?;
+    Ok(arr.is_valid(row) && arr.value(row))
+}
+
 /// Result of probing the in-memory memtables newest-first in `lookup()`.
 enum FastOutcome {
     /// A visible row was found; here it is, projected.
     Hit(RecordBatch),
+    /// The newest visible version of the key is a tombstone — the key is
+    /// deleted. The search stops here (no fall-through to older sources) and
+    /// resolves to "not found".
+    Deleted,
     /// A memtable could not be probed directly (no BTree on the key) — the
     /// caller must fall back to the plan path.
     NeedsFallback,
@@ -614,6 +853,9 @@ enum FastOutcome {
 enum Probe {
     /// The key was found; here is the newest visible row, projected.
     Hit(RecordBatch),
+    /// The newest visible version of the key is a tombstone — deleted. Stop the
+    /// search and return "not found".
+    Deleted,
     /// The key is not present in this memtable (but may be in an older source).
     Miss,
     /// This memtable has no BTree on the key column, so it cannot be probed
@@ -642,10 +884,6 @@ fn probe_position(
     pk_column: &str,
     pk_value: &ScalarValue,
 ) -> Result<ProbePos> {
-    let Some(btree) = index_store.get_btree_by_column(pk_column) else {
-        return Ok(ProbePos::NoIndex);
-    };
-
     // Visible batches are the committed prefix [0, last_visible_idx]; each
     // `StoredBatch` carries its cumulative `row_offset`, so visibility and the
     // position→batch mapping are O(1)/O(log) with no per-probe allocation.
@@ -661,22 +899,37 @@ fn probe_position(
     if visible_end == 0 {
         return Ok(ProbePos::Miss);
     }
+    let max_visible_row = visible_end - 1;
 
-    // Newest visible position of the key — a single seek-and-stop on the
-    // ordered skiplist (largest key ≤ (value, max_visible_row)). No range
-    // collect, no allocation.
-    let Some(pos) = btree.get_newest_visible(pk_value, visible_end - 1) else {
+    // A single-column primary key always has a value-keyed BTree (reused or
+    // auto-created — see `IndexStore::enable_pk_index`): collision-free, so one
+    // seek yields the answer with no re-check. Absent only when the table has no
+    // PK index, where the caller falls back to the plan path.
+    let Some(btree) = index_store.get_btree_by_column(pk_column) else {
+        return Ok(ProbePos::NoIndex);
+    };
+    let Some(pos) = btree.get_newest_visible(pk_value, max_visible_row) else {
         return Ok(ProbePos::Miss);
     };
+    let (batch_idx, row) = resolve_position(batch_store, last_visible_idx, pos)?;
+    Ok(ProbePos::Found { batch_idx, row })
+}
 
-    // Binary-search the owning batch by `row_offset` (appended in order).
+/// Map a global row `position` to its `(batch_idx, row_in_batch)` by binary
+/// searching the visible batch prefix on cumulative `row_offset` (batches are
+/// appended in order).
+fn resolve_position(
+    batch_store: &BatchStore,
+    last_visible_idx: usize,
+    position: u64,
+) -> Result<(usize, usize)> {
     let (mut lo, mut hi) = (0usize, last_visible_idx);
     while lo < hi {
         let mid = lo + (hi - lo).div_ceil(2);
         let off = batch_store.get(mid).map(|b| b.row_offset).ok_or_else(|| {
             lance_core::Error::internal("point-lookup: batch index out of range during search")
         })?;
-        if off <= pos {
+        if off <= position {
             lo = mid;
         } else {
             hi = mid - 1;
@@ -685,10 +938,7 @@ fn probe_position(
     let stored = batch_store
         .get(lo)
         .ok_or_else(|| lance_core::Error::internal("point-lookup: resolved batch missing"))?;
-    Ok(ProbePos::Found {
-        batch_idx: lo,
-        row: (pos - stored.row_offset) as usize,
-    })
+    Ok((lo, (position - stored.row_offset) as usize))
 }
 
 /// Gather `rows` from `batch_store`'s batch `batch_idx` into the `target`
@@ -745,12 +995,20 @@ fn probe_memtable(
     match probe_position(batch_store, index_store, pk_column, pk_value)? {
         ProbePos::NoIndex => Ok(Probe::NoIndex),
         ProbePos::Miss => Ok(Probe::Miss),
-        ProbePos::Found { batch_idx, row } => Ok(Probe::Hit(gather_rows(
-            batch_store,
-            batch_idx,
-            &[row as u32],
-            target,
-        )?)),
+        ProbePos::Found { batch_idx, row } => {
+            // The newest visible version is a tombstone → the key is deleted.
+            // Stop here rather than materializing or falling through to an
+            // older source.
+            if is_tombstone_at(batch_store, batch_idx, row)? {
+                return Ok(Probe::Deleted);
+            }
+            Ok(Probe::Hit(gather_rows(
+                batch_store,
+                batch_idx,
+                &[row as u32],
+                target,
+            )?))
+        }
     }
 }
 
@@ -1092,13 +1350,34 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_point_lookup_rejects_missing_projection_column() {
+        let schema = create_pk_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]);
+        let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema);
+
+        let projection = vec!["missing".to_string()];
+        let pk_values = vec![ScalarValue::Int32(Some(2))];
+        let err = planner
+            .plan_lookup(&pk_values, Some(&projection))
+            .await
+            .expect_err("unknown projection column should fail planning");
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing-column projection error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_point_lookup_active_memtable_returns_newest_duplicate() {
         // Regression: same primary key inserted twice into one active
         // memtable must return the *newest* row. The bug was that
         // `FilterExec → LIMIT 1` over an insert-ordered scan returned the
-        // first (oldest) match. `WithinSourceDedupExec` collapses by PK,
-        // keeping the row with the largest `_rowid` (insert order).
+        // first (oldest) match. The plan-path active arm now sorts `_rowid`
+        // DESC and keeps the first row (largest `_rowid` = newest insert).
         use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
         use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
         use futures::TryStreamExt;
@@ -1118,17 +1397,17 @@ mod tests {
         let b_old = create_test_batch(&schema, &[1], "old");
         let b_new = create_test_batch(&schema, &[1], "new");
         let b_other = create_test_batch(&schema, &[2], "two");
-        let (_, _, bp_old) = batch_store.append(b_old.clone()).unwrap();
+        let (bp_old, off_old, _) = batch_store.append(b_old.clone()).unwrap();
         index_store
-            .insert_with_batch_position(&b_old, 0, Some(bp_old))
+            .insert_with_batch_position(&b_old, off_old, Some(bp_old))
             .unwrap();
-        let (_, _, bp_new) = batch_store.append(b_new.clone()).unwrap();
+        let (bp_new, off_new, _) = batch_store.append(b_new.clone()).unwrap();
         index_store
-            .insert_with_batch_position(&b_new, 1, Some(bp_new))
+            .insert_with_batch_position(&b_new, off_new, Some(bp_new))
             .unwrap();
-        let (_, _, bp_other) = batch_store.append(b_other.clone()).unwrap();
+        let (bp_other, off_other, _) = batch_store.append(b_other.clone()).unwrap();
         index_store
-            .insert_with_batch_position(&b_other, 2, Some(bp_other))
+            .insert_with_batch_position(&b_other, off_other, Some(bp_other))
             .unwrap();
         let index_store = Arc::new(index_store);
 
@@ -1168,6 +1447,88 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_point_lookup_probes_auto_created_pk_btree() {
+        // No user `add_btree` on the PK column — only `enable_pk_index`, which
+        // auto-creates a BTree on the primary key (the production default). The
+        // fast probe must resolve the newest visible version through that
+        // collision-free BTree rather than falling back to the plan path.
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+
+        let schema = create_pk_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        // No `add_btree` — `enable_pk_index` auto-creates the PK BTree.
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+
+        // pk=1 written twice (the newer second), plus an unrelated pk=2.
+        let b_old = create_test_batch(&schema, &[1], "old");
+        let b_new = create_test_batch(&schema, &[1], "new");
+        let b_other = create_test_batch(&schema, &[2], "two");
+        let (bp_old, off_old, _) = batch_store.append(b_old.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&b_old, off_old, Some(bp_old))
+            .unwrap();
+        let (bp_new, off_new, _) = batch_store.append(b_new.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&b_new, off_new, Some(bp_new))
+            .unwrap();
+        let (bp_other, off_other, _) = batch_store.append(b_other.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&b_other, off_other, Some(bp_other))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let shard_id = Uuid::new_v4();
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                shard_id,
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+        let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema);
+
+        // `lookup` takes the fast probe path (single-column PK, no system cols).
+        let hit = planner
+            .lookup(&[ScalarValue::Int32(Some(1))], None)
+            .await
+            .unwrap()
+            .expect("pk=1 must be found via the PK-position index probe");
+        assert_eq!(hit.num_rows(), 1);
+        let name = hit
+            .column_by_name("name")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        assert_eq!(
+            name.value(0),
+            "new_1",
+            "probe must return the newest version"
+        );
+
+        // An absent key resolves to None (no on-disk sources to consult).
+        assert!(
+            planner
+                .lookup(&[ScalarValue::Int32(Some(999))], None)
+                .await
+                .unwrap()
+                .is_none(),
+            "absent key must miss"
+        );
+    }
+
     #[tokio::test]
     async fn test_point_lookup_flushed_memtable_returns_newest_duplicate() {
         // Regression / invariant pin: when a flushed memtable contains two
@@ -1661,4 +2022,265 @@ mod tests {
             "absent key must miss"
         );
     }
+
+    // ----- tombstone (delete) point-lookup tests -----
+
+    /// Memtable schema = base (`id`, `name`) + the `_tombstone` marker.
+    fn pk_ts_schema() -> Arc<ArrowSchema> {
+        let mut id_metadata = HashMap::new();
+        id_metadata.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let id = Field::new("id", DataType::Int32, false).with_metadata(id_metadata);
+        Arc::new(ArrowSchema::new(vec![
+            id,
+            Field::new("name", DataType::Utf8, true),
+            Field::new(TOMBSTONE, DataType::Boolean, false),
+        ]))
+    }
+
+    fn ts_real(schema: &Arc<ArrowSchema>, ids: &[i32], prefix: &str) -> RecordBatch {
+        let names: Vec<String> = ids.iter().map(|i| format!("{}_{}", prefix, i)).collect();
+        RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids.to_vec())),
+                Arc::new(StringArray::from(names)),
+                Arc::new(arrow_array::BooleanArray::from(vec![false; ids.len()])),
+            ],
+        )
+        .unwrap()
+    }
+
+    fn ts_tomb(schema: &Arc<ArrowSchema>, ids: &[i32]) -> RecordBatch {
+        let names: Vec<Option<String>> = ids.iter().map(|_| None).collect();
+        RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids.to_vec())),
+                Arc::new(StringArray::from(names)),
+                Arc::new(arrow_array::BooleanArray::from(vec![true; ids.len()])),
+            ],
+        )
+        .unwrap()
+    }
+
+    /// Active-only planner whose memtable carries `_tombstone`; the planner's
+    /// base schema stays tombstone-free (as the base table is in production).
+    fn active_ts_planner(batches: &[RecordBatch]) -> LsmPointLookupPlanner {
+        use crate::dataset::mem_wal::scanner::collector::InMemoryMemTables;
+        let base_schema = create_pk_schema();
+        let mem_schema = pk_ts_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+        let active = active_memtable_ref(&mem_schema, batches, 1);
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                Uuid::new_v4(),
+                InMemoryMemTables {
+                    active,
+                    frozen: vec![],
+                },
+            );
+        LsmPointLookupPlanner::new(collector, vec!["id".to_string()], base_schema)
+    }
+
+    /// Read the `_tombstone` marker from row 0 of a keep-tombstone result.
+    fn tombstone_at(b: &RecordBatch) -> bool {
+        let idx = b.schema().index_of(TOMBSTONE).expect("_tombstone column");
+        b.column(idx)
+            .as_any()
+            .downcast_ref::<arrow_array::BooleanArray>()
+            .expect("_tombstone is Boolean")
+            .value(0)
+    }
+
+    #[tokio::test]
+    async fn test_lookup_keep_tombstone_returns_deleted_row_with_marker() {
+        // The tombstone-preserving variant keeps a deleted key that the filtered
+        // `lookup` would drop: id=1 deleted, id=2 live, id=99 absent.
+        let schema = pk_ts_schema();
+        let planner = active_ts_planner(&[ts_real(&schema, &[1, 2], "v"), ts_tomb(&schema, &[1])]);
+
+        // Deleted key: present with `_tombstone = true` (vs `None` from `lookup`).
+        let deleted = planner
+            .lookup_keep_tombstone(&[ScalarValue::Int32(Some(1))], None)
+            .await
+            .unwrap()
+            .expect("deleted key kept by lookup_keep_tombstone");
+        assert_eq!(id_at(&deleted), 1);
+        assert!(
+            tombstone_at(&deleted),
+            "deleted key carries _tombstone = true"
+        );
+
+        // Live key: present with `_tombstone = false`.
+        let live = planner
+            .lookup_keep_tombstone(&[ScalarValue::Int32(Some(2))], None)
+            .await
+            .unwrap()
+            .expect("live key found");
+        assert_eq!(id_at(&live), 2);
+        assert!(!tombstone_at(&live), "live key carries _tombstone = false");
+
+        // Absent key: still `None` — no fresh entry at all to distinguish.
+        assert!(
+            planner
+                .lookup_keep_tombstone(&[ScalarValue::Int32(Some(99))], None)
+                .await
+                .unwrap()
+                .is_none(),
+            "absent key has no fresh entry"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_lookup_many_keep_tombstone_includes_tombstoned_keys() {
+        // Batched variant: the tombstoned key is INCLUDED (carrying its marker),
+        // unlike `lookup_many` which omits it. id=2 deleted; 1 and 3 live.
+        let schema = pk_ts_schema();
+        let planner =
+            active_ts_planner(&[ts_real(&schema, &[1, 2, 3], "v"), ts_tomb(&schema, &[2])]);
+        let keys = vec![
+            vec![ScalarValue::Int32(Some(1))],
+            vec![ScalarValue::Int32(Some(2))],
+            vec![ScalarValue::Int32(Some(3))],
+        ];
+        let batch = planner
+            .lookup_many_keep_tombstone(&keys, None)
+            .await
+            .unwrap();
+        assert_eq!(
+            batch.num_rows(),
+            3,
+            "all three keys kept, including the tombstone"
+        );
+        // Exactly one row (id=2) is marked deleted.
+        let deleted: Vec<i32> = (0..batch.num_rows())
+            .map(|r| batch.slice(r, 1))
+            .filter(tombstone_at)
+            .map(|b| id_at(&b))
+            .collect();
+        assert_eq!(deleted, vec![2], "only id=2 is marked deleted");
+    }
+
+    #[tokio::test]
+    async fn test_lookup_tombstone_within_active_not_found() {
+        // Fast path: id=1 written then tombstoned (newer); id=2 is a control.
+        let schema = pk_ts_schema();
+        let planner = active_ts_planner(&[ts_real(&schema, &[1, 2], "v"), ts_tomb(&schema, &[1])]);
+
+        assert!(
+            planner
+                .lookup(&[ScalarValue::Int32(Some(1))], None)
+                .await
+                .unwrap()
+                .is_none(),
+            "deleted key must resolve to not-found, not the older real row"
+        );
+        let row = planner
+            .lookup(&[ScalarValue::Int32(Some(2))], None)
+            .await
+            .unwrap()
+            .expect("untouched key still found");
+        assert_eq!(id_at(&row), 2);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_tombstone_then_reinsert() {
+        // delete then re-insert the same key → the re-insert is newest.
+        let schema = pk_ts_schema();
+        let planner = active_ts_planner(&[
+            ts_real(&schema, &[1], "old"),
+            ts_tomb(&schema, &[1]),
+            ts_real(&schema, &[1], "new"),
+        ]);
+        let row = planner
+            .lookup(&[ScalarValue::Int32(Some(1))], None)
+            .await
+            .unwrap()
+            .expect("re-inserted key must be found");
+        assert_eq!(name_at(&row), "new_1");
+    }
+
+    #[tokio::test]
+    async fn test_lookup_tombstone_of_absent_key_is_noop() {
+        // Deleting a key that never existed is a no-op miss.
+        let schema = pk_ts_schema();
+        let planner = active_ts_planner(&[ts_real(&schema, &[1], "v"), ts_tomb(&schema, &[99])]);
+        assert!(
+            planner
+                .lookup(&[ScalarValue::Int32(Some(99))], None)
+                .await
+                .unwrap()
+                .is_none()
+        );
+        let row = planner
+            .lookup(&[ScalarValue::Int32(Some(1))], None)
+            .await
+            .unwrap()
+            .expect("unrelated key unaffected");
+        assert_eq!(id_at(&row), 1);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_tombstone_plan_path_resurrection_guard() {
+        // Force the plan path (project a system column) so the after-coalesce
+        // filter — not the fast-path short-circuit — must drop the tombstone.
+        // Real row lives in base; the newer active arm holds only its tombstone.
+        use crate::dataset::mem_wal::scanner::collector::InMemoryMemTables;
+        let base_schema = create_pk_schema();
+        let mem_schema = pk_ts_schema();
+        let temp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp.path().to_str().unwrap());
+        let base = Arc::new(
+            create_dataset(
+                &base_uri,
+                vec![create_test_batch(&base_schema, &[1, 2], "base")],
+            )
+            .await,
+        );
+        let active = active_memtable_ref(&mem_schema, &[ts_tomb(&mem_schema, &[1])], 2);
+        let collector = LsmDataSourceCollector::new(base, vec![]).with_in_memory_memtables(
+            Uuid::new_v4(),
+            InMemoryMemTables {
+                active,
+                frozen: vec![],
+            },
+        );
+        let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], base_schema);
+
+        let proj = vec!["id".to_string(), "_rowid".to_string()];
+        assert!(
+            planner
+                .lookup(&[ScalarValue::Int32(Some(1))], Some(&proj))
+                .await
+                .unwrap()
+                .is_none(),
+            "plan path must not fall through to the base row for a deleted key"
+        );
+        let row = planner
+            .lookup(&[ScalarValue::Int32(Some(2))], Some(&proj))
+            .await
+            .unwrap()
+            .expect("untouched base row found");
+        assert_eq!(id_at(&row), 2);
+    }
+
+    #[tokio::test]
+    async fn test_lookup_many_skips_tombstoned_keys() {
+        // Batched lookup: a tombstoned key is omitted, others resolved.
+        let schema = pk_ts_schema();
+        let planner =
+            active_ts_planner(&[ts_real(&schema, &[1, 2, 3], "v"), ts_tomb(&schema, &[2])]);
+        let keys = [
+            ScalarValue::Int32(Some(1)),
+            ScalarValue::Int32(Some(2)),
+            ScalarValue::Int32(Some(3)),
+        ];
+        let batch = planner.lookup_many(&keys, None).await.unwrap();
+        assert_eq!(batch.num_rows(), 2, "the tombstoned key is omitted");
+        assert_eq!(sorted_ids(&batch), vec![1, 3]);
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/scanner/projection.rs b/rust/lance/src/dataset/mem_wal/scanner/projection.rs
index 20d1b1a403d..0ec482aebf8 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/projection.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/projection.rs
@@ -73,12 +73,37 @@ pub fn build_scanner_projection(
     cols
 }
 
+/// Validate user-facing projection names before constructing a canonical schema.
+pub fn validate_projection_names(
+    user_projection: Option<&[String]>,
+    base_schema: &SchemaRef,
+    extra_columns: &[&str],
+) -> Result<()> {
+    let Some(user_projection) = user_projection else {
+        return Ok(());
+    };
+    for name in user_projection {
+        if is_system_column(name)
+            || extra_columns.contains(&name.as_str())
+            || base_schema.field_with_name(name).is_ok()
+        {
+            continue;
+        }
+        return Err(lance_core::Error::invalid_input(format!(
+            "Column '{}' not found in schema",
+            name
+        )));
+    }
+    Ok(())
+}
+
 /// Canonical output schema honoring user column order.
 ///
 /// System cols → nullable `UInt64` at user position (filled by
 /// `project_to_canonical`). `_distance` (when `include_distance`) →
 /// nullable `Float32` at user position, appended if absent. PKs appended.
-/// Unknown names are silently dropped.
+/// Callers should run [`validate_projection_names`] before using this with a
+/// user-supplied projection.
 pub fn canonical_output_schema(
     user_projection: Option<&[String]>,
     base_schema: &SchemaRef,
diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
index b6b1f952b25..2705383f0b5 100644
--- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
+++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs
@@ -18,8 +18,9 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
 use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
 use datafusion::physical_plan::union::UnionExec;
-use lance_core::Result;
+use datafusion::prelude::Expr;
 use lance_core::datatypes::OnMissing;
+use lance_core::{Error, Result};
 use tracing::instrument;
 
 use crate::dataset::Dataset;
@@ -27,23 +28,24 @@ use crate::io::exec::TakeExec;
 
 use super::collector::LsmDataSourceCollector;
 use super::data_source::LsmDataSource;
-use super::exec::{DedupDirection, WithinSourceDedupExec};
-use super::flushed_cache::{FlushedMemTableCache, open_flushed_dataset};
+use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset};
 use super::projection::{
     DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns,
-    project_to_canonical, wants_row_id,
+    project_to_canonical, validate_projection_names, wants_row_id,
 };
 use crate::session::Session;
+use lance_io::object_store::ObjectStoreParams;
 
 /// Plans vector search queries over LSM data.
 ///
 /// Each source is independently newest-per-PK before the union — the active
-/// memtable via an over-fetched KNN + within-source dedup, flushed generations
-/// via their within-generation deletion vector — and the cross-generation
-/// block-list ([`super::exec::PkHashFilterExec`]) drops any PK superseded by a
-/// newer generation. So each PK reaches the union from exactly one source and a
-/// distance-ordered merge yields the global top-k; no cross-source dedup is
-/// needed.
+/// memtable via exact brute-force KNN when PK rewrites or a filter require it
+/// (append-only active data can still use HNSW),
+/// flushed generations via their within-generation deletion vector — and the
+/// cross-generation block-list ([`super::exec::PkBlockFilterExec`]) drops any
+/// PK superseded by a newer generation. So each PK reaches the union from
+/// exactly one source and a distance-ordered merge yields the global top-k; no
+/// cross-source dedup is needed.
 ///
 /// # Query Plan Structure
 ///
@@ -54,15 +56,14 @@ use crate::session::Session;
 ///       UnionExec
 ///         ProjectionExec (canonical output schema)
 ///           SortExec(_distance, fetch=k)
-///             WithinSourceDedupExec: KeepMaxRowAddr           (active)
-///               KNNExec: active memtable, fetch=ceil(k*overfetch)
+///               MemTableBruteForceVectorExec or VectorIndexExec: active memtable KNN
 ///         ProjectionExec (canonical output schema)
 ///           ProjectionExec (null_columns _rowid)
-///             PkHashFilterExec: block-list                   (flushed)
+///             PkBlockFilterExec: block-list                   (flushed)
 ///               KNNExec: flushed gen N, fetch=ceil(k*overfetch) (fast_search)
 ///         … one per flushed gen …
 ///         ProjectionExec (canonical output schema)
-///           PkHashFilterExec: block-list                     (base)
+///           PkBlockFilterExec: block-list                     (base)
 ///             KNNExec: base table, k (fast_search)[.refine()?]
 /// ```
 ///
@@ -91,8 +92,17 @@ pub struct LsmVectorSearchPlanner {
     dataset: Option<Arc<Dataset>>,
     /// Session threaded into flushed-generation opens (shared caches).
     session: Option<Arc<Session>>,
+    /// Store params for opening flushed generations, reusing the base dataset's store.
+    store_params: Option<ObjectStoreParams>,
     /// Cache of opened flushed-generation datasets.
-    flushed_cache: Option<Arc<FlushedMemTableCache>>,
+    flushed_cache: Option<Arc<dyn DatasetCache>>,
+    /// Optional warmer fired on first open of a flushed generation.
+    warmer: Option<Arc<dyn GenerationWarmer>>,
+    /// Optional prefilter predicate applied to every source arm before its KNN
+    /// search, so rows failing the predicate never enter the top-k. Base and
+    /// flushed arms use the dataset scanner's native prefilter; memtable arms
+    /// route to a filtered brute-force scan.
+    filter: Option<Expr>,
 }
 
 impl LsmVectorSearchPlanner {
@@ -120,24 +130,46 @@ impl LsmVectorSearchPlanner {
             distance_type,
             dataset: None,
             session: None,
+            store_params: None,
             flushed_cache: None,
+            warmer: None,
+            filter: None,
         }
     }
 
-    /// Thread a session into flushed-generation opens so the first open
-    /// populates the shared index / file-metadata caches.
+    /// Attach an optional prefilter predicate. Every source arm restricts its
+    /// KNN to rows matching the predicate (true prefilter), so results match a
+    /// normal filtered vector scan over base ∪ flushed ∪ in-memory data.
+    pub fn with_filter(mut self, filter: Option<Expr>) -> Self {
+        self.filter = filter;
+        self
+    }
+
+    /// Set the session used to open flushed generations.
     pub fn with_session(mut self, session: Arc<Session>) -> Self {
         self.session = Some(session);
         self
     }
 
+    /// Set the store params used to open flushed generations.
+    pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self {
+        self.store_params = Some(store_params);
+        self
+    }
+
     /// Inject a cache of opened flushed-generation datasets, making repeated
     /// searches against the same generation a pure `Arc::clone`.
-    pub fn with_flushed_cache(mut self, cache: Arc<FlushedMemTableCache>) -> Self {
+    pub fn with_flushed_cache(mut self, cache: Arc<dyn DatasetCache>) -> Self {
         self.flushed_cache = Some(cache);
         self
     }
 
+    /// Inject the warmer fired on first open of a flushed generation.
+    pub fn with_warmer(mut self, warmer: Arc<dyn GenerationWarmer>) -> Self {
+        self.warmer = Some(warmer);
+        self
+    }
+
     /// Set the base dataset for post-rerank take.
     ///
     /// After global PK dedup and sort, a `TakeExec` against this dataset
@@ -161,17 +193,13 @@ impl LsmVectorSearchPlanner {
     /// * `refine_base_table` - When true, the base-table arm re-ranks its
     ///   candidates with exact distances (refine factor 1). Useful when the base
     ///   table uses an approximate index (IVF-PQ) so cross-source distance
-    ///   comparison is exact. Memtable arms use exact HNSW search and never need
-    ///   refine. Auto-enabled whenever stale filtering is on (see below).
-    /// * `overfetch_factor` - A single knob that controls **both** whether stale
-    ///   rows are filtered and how aggressively sources over-fetch to backfill
-    ///   the rows that filtering drops:
+    ///   comparison is exact. Memtable arms already use exact distances and
+    ///   never need refine. Auto-enabled whenever stale filtering is on (see
+    ///   below).
+    /// * `overfetch_factor` - Controls how aggressively sources over-fetch to
+    ///   backfill the rows dropped by stale-row filtering. Values below `1.0`
+    ///   are rejected; stale filtering is always enabled.
     ///
-    ///   - `factor < 1.0` (e.g. `0.0`): **stale filtering off.** The per-source
-    ///     block-list / [`super::exec::PkHashFilterExec`] is not built or applied,
-    ///     so rows superseded by a newer generation can surface. The global PK
-    ///     dedup still runs, so it still suppresses stale copies in the cases
-    ///     where both the stale and the fresh row reach it.
     ///   - `factor == 1.0`: **stale filtering on, no over-fetch.** Each source
     ///     that has superseded rows fetches exactly `k` candidates, drops the
     ///     stale ones, and may therefore return fewer than `k` live rows.
@@ -179,15 +207,10 @@ impl LsmVectorSearchPlanner {
     ///     fetches `ceil(k * factor)` candidates so that dropping the stale ones
     ///     still leaves `k` live rows for the merge.
     ///
-    ///   There is intentionally no separate on/off flag: over-fetch is only ever
-    ///   meaningful while filtering, so the factor encodes both. A true KNN
-    ///   prefilter would remove the need for over-fetch entirely.
-    ///
     /// # Returns
     ///
     /// An execution plan that returns the top-K nearest neighbors across all
-    /// LSM levels, with stale results filtered out (unless `overfetch_factor`
-    /// disables filtering).
+    /// LSM levels, with stale results filtered out.
     #[instrument(name = "lsm_vector_search", level = "info", skip_all, fields(k, nprobes, vector_column = %self.vector_column, distance_type = ?self.distance_type))]
     pub async fn plan_search(
         &self,
@@ -198,24 +221,30 @@ impl LsmVectorSearchPlanner {
         refine_base_table: bool,
         overfetch_factor: f64,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        if k == 0 {
+            return Err(Error::invalid_input("k must be positive".to_string()));
+        }
+        if nprobes == 0 {
+            return Err(Error::invalid_input("nprobes must be positive".to_string()));
+        }
+
         let sources = self.collector.collect()?;
+        validate_projection_names(projection, &self.base_schema, &[DISTANCE_COLUMN])?;
+        // The block-list is the sole cross-generation dedup mechanism, so it
+        // runs unconditionally; `overfetch_factor` only tunes the over-fetch
+        // multiple for blocked sources.
+        let overfetch_factor = super::validate_overfetch_factor(overfetch_factor)?;
 
         if sources.is_empty() {
             return self.empty_plan(projection);
         }
 
-        // The block-list is the sole cross-generation dedup mechanism, so it
-        // runs unconditionally; `overfetch_factor` only tunes the over-fetch
-        // multiple and is clamped to >= 1.0 so blocked sources still yield k
-        // live candidates after the post-filter.
-        let overfetch_factor = overfetch_factor.max(1.0);
-
-        // Per-source PK-hash block sets (`NEWER(G)`; base = union of all gens).
+        // Per-source PK block sets (`NEWER(G)`; base = union of all gens).
         // `Box::pin` keeps the future off `clippy::large_futures`.
         let block_lists = Box::pin(super::block_list::compute_source_block_lists(
             &sources,
-            &self.pk_columns,
             self.session.as_ref(),
+            self.store_params.as_ref(),
             self.flushed_cache.as_ref(),
         ))
         .await?;
@@ -233,56 +262,61 @@ impl LsmVectorSearchPlanner {
         // `block_lists` is non-empty exactly when a newer generation exists.
         let refine_base = refine_base_table || !block_lists.is_empty();
 
+        // Stage per-source over-fetch decisions, then build every KNN plan
+        // concurrently — the builds are independent and a sequential loop was
+        // the dominant serial planning cost at multiple generations.
+        let arm_inputs: Vec<_> = sources
+            .iter()
+            .map(|source| {
+                let generation = source.generation();
+                let is_base = matches!(source, LsmDataSource::BaseTable { .. });
+                let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. });
+                // Over-fetch when the post-source block-list can drop
+                // candidates. Active memtable PK recency is handled inside the
+                // exact brute-force exec before top-k, so it does not need
+                // source over-fetch.
+                let blocked = block_lists.get(&(source.shard_id(), generation));
+                let fetch_k = if blocked.is_some() && !self.pk_columns.is_empty() {
+                    ((k as f64) * overfetch_factor).ceil() as usize
+                } else {
+                    k
+                };
+                (source, is_base, is_active, blocked, fetch_k)
+            })
+            .collect();
+        let built = futures::future::try_join_all(arm_inputs.iter().map(
+            |(source, is_base, _, _, fetch_k)| {
+                Box::pin(self.build_knn_plan(
+                    source,
+                    query_vector,
+                    *fetch_k,
+                    nprobes,
+                    projection,
+                    *is_base && refine_base,
+                ))
+            },
+        ))
+        .await?;
+
         let mut knn_plans = Vec::new();
-        for source in &sources {
-            let generation = source.generation();
-            let is_base = matches!(source, LsmDataSource::BaseTable { .. });
-            let is_active = matches!(source, LsmDataSource::ActiveMemTable { .. });
-            // Over-fetch when the post-source filter can drop candidates: a
-            // blocked source loses superseded rows; the active source's
-            // within-source dedup collapses duplicate-PK HNSW nodes. Block
-            // lookup is per shard — generations are per-shard.
-            let blocked = block_lists.get(&(source.shard_id(), generation));
-            let fetch_k = if blocked.is_some() || is_active {
-                ((k as f64) * overfetch_factor).ceil() as usize
-            } else {
-                k
-            };
-            let knn = Box::pin(self.build_knn_plan(
-                source,
-                query_vector,
-                fetch_k,
-                nprobes,
-                projection,
-                is_base && refine_base,
-            ))
-            .await?;
+        for ((_, is_base, _, blocked, _), knn) in arm_inputs.iter().zip(built) {
+            let is_base = *is_base;
+            let blocked = *blocked;
             // Make each source independently newest-per-PK before the union:
-            //  * active: the append-only HNSW returns one node per inserted
-            //    version, so collapse duplicate PKs to the newest insert
-            //    (KeepMaxRowAddr on `_rowid`) and re-sort by distance. This
-            //    stays probabilistic — a fresh version evicted from the
-            //    over-fetched top-k still leaks.
+            //  * active: append-only memtables can use HNSW directly; once a
+            //    PK rewrite is observed, `MemTableBruteForceVectorExec` drops
+            //    superseded versions before the top-k cut.
             //  * flushed/base: drop cross-gen superseded rows via the
             //    block-list (within-gen is handled by the flushed DV).
-            let knn = if is_active {
-                let deduped: Arc<dyn ExecutionPlan> = Arc::new(WithinSourceDedupExec::new(
+            let knn = match blocked {
+                Some(_) if self.pk_columns.is_empty() => knn,
+                Some(set) => Arc::new(super::exec::PkBlockFilterExec::new(
                     knn,
                     self.pk_columns.clone(),
-                    lance_core::ROW_ID,
-                    DedupDirection::KeepMaxRowAddr,
-                ));
-                sort_by_distance(deduped, k)?
-            } else {
-                match blocked {
-                    Some(set) => Arc::new(super::exec::PkHashFilterExec::new(
-                        knn,
-                        self.pk_columns.clone(),
-                        set.clone(),
-                        k,
-                    )) as Arc<dyn ExecutionPlan>,
-                    None => knn,
-                }
+                    set.clone(),
+                    k,
+                )) as Arc<dyn ExecutionPlan>,
+                None => knn,
             };
             // Lance's `fast_search()` and the active scan both produce a
             // per-source `_rowid` that would collide with base row ids in the
@@ -301,6 +335,10 @@ impl LsmVectorSearchPlanner {
         // No cross-source dedup needed (see struct doc): SortExec(per partition)
         // + SortPreservingMerge does the p-way distance-ordered top-k merge.
         #[allow(deprecated)]
+        // The downstream `SortPreservingMergeExec` already spawns one driver
+        // task per input partition (one per union arm) via `spawn_buffered`, so
+        // each arm's per-arm CPU (HNSW search, distance refine) runs on its own
+        // task without an extra repartition.
         let merged: Arc<dyn ExecutionPlan> = Arc::new(UnionExec::new(knn_plans));
 
         let distance_idx = merged.schema().index_of(DISTANCE_COLUMN).map_err(|_| {
@@ -364,11 +402,12 @@ impl LsmVectorSearchPlanner {
             merged_sorted
         };
 
-        // Under-fetch is warned per-source inside `PkHashFilterExec`.
+        // Under-fetch is warned per-source inside `PkBlockFilterExec`.
         Ok(result)
     }
 
     /// Build KNN plan for a single data source.
+    ///
     async fn build_knn_plan(
         &self,
         source: &LsmDataSource,
@@ -384,6 +423,15 @@ impl LsmVectorSearchPlanner {
                 let cols =
                     build_scanner_projection(projection, &self.base_schema, &self.pk_columns);
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
+                if let Some(ref filter) = self.filter {
+                    // Native scanner prefilter: the ANN runs over rows matching
+                    // the predicate, so the top-k holds only matching rows.
+                    // `prefilter(true)` is required — without it the scanner
+                    // post-filters the unfiltered top-k, dropping matching rows
+                    // that ranked below non-matching ones.
+                    scanner.filter_expr(filter.clone());
+                    scanner.prefilter(true);
+                }
                 // Only the base produces a meaningful `_rowid`. `_rowaddr`
                 // can't be combined with `fast_search()` — the IVF index
                 // doesn't preserve it and `TakeExec` refuses to insert it
@@ -405,13 +453,24 @@ impl LsmVectorSearchPlanner {
                 scanner.create_plan().await
             }
             LsmDataSource::FlushedMemTable { path, .. } => {
-                let dataset =
-                    open_flushed_dataset(path, self.session.as_ref(), self.flushed_cache.as_ref())
-                        .await?;
+                let dataset = open_flushed_dataset(
+                    path,
+                    self.session.as_ref(),
+                    self.store_params.as_ref(),
+                    self.flushed_cache.as_ref(),
+                    self.warmer.as_ref(),
+                )
+                .await?;
                 let mut scanner = dataset.scan();
                 let cols =
                     build_scanner_projection(projection, &self.base_schema, &self.pk_columns);
                 scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
+                if let Some(ref filter) = self.filter {
+                    // See the base arm: `prefilter(true)` makes this a true
+                    // prefilter rather than a lossy post-filter on the top-k.
+                    scanner.filter_expr(filter.clone());
+                    scanner.prefilter(true);
+                }
                 // No `with_row_id/address`: per-source IDs would collide with base.
                 let query_arr = single_query_array(query_vector);
                 scanner.nearest(&self.vector_column, query_arr.as_ref(), k)?;
@@ -427,24 +486,23 @@ impl LsmVectorSearchPlanner {
                 ..
             } => {
                 use crate::dataset::mem_wal::memtable::scanner::MemTableScanner;
-                use arrow_array::Array;
 
                 let mut scanner =
                     MemTableScanner::new(batch_store.clone(), index_store.clone(), schema.clone());
+                // Supply PKs so the memtable scanner can choose HNSW for
+                // append-only data and exact newest-before-top-k search when
+                // PK rewrites or filters make stale suppression necessary.
+                scanner.with_pk_columns(self.pk_columns.clone());
                 // PK auto-included so the staleness filter retains its bloom hash key.
                 let cols =
                     build_scanner_projection(projection, &self.base_schema, &self.pk_columns);
-                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>());
-                // Expose `_rowid` (BatchStore row offset, monotonic with
-                // insert order) so [`WithinSourceDedupExec`] can collapse
-                // duplicate-PK rows to the newest insert. The value is
-                // per-source and NULL'd before reaching the canonical merge.
-                // (VectorIndexExec only plumbs `with_row_id`, not
-                // `with_row_address`, but the two yield identical values
-                // for an active memtable so either would work.)
-                scanner.with_row_id();
-                let query_arr: Arc<dyn Array> = Arc::new(query_vector.clone());
-                scanner.nearest(&self.vector_column, query_arr, k);
+                scanner.project(&cols.iter().map(|s| s.as_str()).collect::<Vec<_>>())?;
+                if let Some(ref filter) = self.filter {
+                    // Routed to filtered brute-force (see `plan_vector_search`):
+                    // the predicate masks rows before the memtable top-k cut.
+                    scanner.filter_expr(filter.clone());
+                }
+                scanner.nearest(&self.vector_column, query_vector, k)?;
                 scanner.nprobes(nprobes);
                 scanner.distance_metric(self.distance_type);
                 scanner.create_plan().await
@@ -461,29 +519,6 @@ impl LsmVectorSearchPlanner {
     }
 }
 
-/// Sort a single-partition plan by `_distance` ascending and cap at `k`.
-///
-/// Used to re-order the active arm after its within-source dedup (which emits
-/// rows unordered) so the cross-source distance merge sees a sorted stream.
-fn sort_by_distance(plan: Arc<dyn ExecutionPlan>, k: usize) -> Result<Arc<dyn ExecutionPlan>> {
-    let idx = plan.schema().index_of(DISTANCE_COLUMN).map_err(|_| {
-        lance_core::Error::invalid_input(format!(
-            "Column '{}' not found in schema",
-            DISTANCE_COLUMN
-        ))
-    })?;
-    let sort_expr = vec![PhysicalSortExpr {
-        expr: Arc::new(Column::new(DISTANCE_COLUMN, idx)),
-        options: SortOptions {
-            descending: false,
-            nulls_first: false,
-        },
-    }];
-    let ordering = LexOrdering::new(sort_expr)
-        .ok_or_else(|| lance_core::Error::internal("Failed to create LexOrdering".to_string()))?;
-    Ok(Arc::new(SortExec::new(ordering, plan).with_fetch(Some(k))))
-}
-
 /// Convert a (typically single-row) FixedSizeList query into the array shape
 /// `Scanner::nearest` expects:
 ///
@@ -505,7 +540,7 @@ mod tests {
     use super::*;
     use crate::dataset::{Dataset, WriteParams};
     use arrow_array::{
-        Int32Array, RecordBatch, RecordBatchIterator, builder::FixedSizeListBuilder,
+        BooleanArray, Int32Array, RecordBatch, RecordBatchIterator, builder::FixedSizeListBuilder,
     };
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use std::collections::HashMap;
@@ -529,6 +564,25 @@ mod tests {
         ]))
     }
 
+    fn create_vector_tombstone_schema() -> Arc<ArrowSchema> {
+        let mut id_metadata = HashMap::new();
+        id_metadata.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_metadata);
+
+        Arc::new(ArrowSchema::new(vec![
+            id_field,
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                true,
+            ),
+            Field::new(crate::dataset::mem_wal::TOMBSTONE, DataType::Boolean, false),
+        ]))
+    }
+
     fn create_query_vector() -> FixedSizeListArray {
         use arrow_array::builder::Float32Builder;
 
@@ -565,12 +619,75 @@ mod tests {
         .unwrap()
     }
 
+    /// One row with an explicit `(id, vector)` — unlike [`create_test_batch`],
+    /// the vector is not derived from `id`, so the same PK can carry different
+    /// vectors across generations (an update / move-out-of-neighborhood).
+    fn create_id_vector_batch(schema: &ArrowSchema, id: i32, vector: [f32; 4]) -> RecordBatch {
+        use arrow_array::builder::Float32Builder;
+        let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for v in vector {
+            builder.values().append_value(v);
+        }
+        builder.append(true);
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from(vec![id])),
+                Arc::new(builder.finish()),
+            ],
+        )
+        .unwrap()
+    }
+
+    fn tombstone_vector_batch(
+        schema: &ArrowSchema,
+        rows: &[(i32, Option<[f32; 4]>, bool)],
+    ) -> RecordBatch {
+        use arrow_array::builder::Float32Builder;
+
+        let mut vector_builder = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for (_, vector, _) in rows {
+            if let Some(vector) = vector {
+                for value in vector {
+                    vector_builder.values().append_value(*value);
+                }
+                vector_builder.append(true);
+            } else {
+                for _ in 0..4 {
+                    vector_builder.values().append_null();
+                }
+                vector_builder.append(false);
+            }
+        }
+        let ids: Vec<i32> = rows.iter().map(|(id, _, _)| *id).collect();
+        let tombstones: Vec<bool> = rows.iter().map(|(_, _, tombstone)| *tombstone).collect();
+
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from(ids)),
+                Arc::new(vector_builder.finish()),
+                Arc::new(BooleanArray::from(tombstones)),
+            ],
+        )
+        .unwrap()
+    }
+
     async fn create_dataset(uri: &str, batches: Vec<RecordBatch>) -> Dataset {
         let schema = batches[0].schema();
-        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
-        Dataset::write(reader, uri, Some(WriteParams::default()))
+        let has_id = schema.column_with_name("id").is_some();
+        let reader = RecordBatchIterator::new(batches.clone().into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, uri, Some(WriteParams::default()))
             .await
-            .unwrap()
+            .unwrap();
+        // Also write the standalone PK sidecar (on `id`) so a flushed-generation
+        // source can be probed by the block-list (harmless for a base table).
+        if has_id {
+            crate::dataset::mem_wal::scanner::block_list::write_pk_sidecar(uri, &batches, &["id"])
+                .await
+                .unwrap();
+        }
+        dataset
     }
 
     #[tokio::test]
@@ -599,6 +716,43 @@ mod tests {
         plan.expect("planner should produce a plan even when memtables are empty");
     }
 
+    #[tokio::test]
+    async fn test_vector_search_validates_k_and_nprobes() {
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let base_batch = create_test_batch(&schema, &[1, 2, 3]);
+        let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await);
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]);
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        );
+
+        let query = create_query_vector();
+        let err = planner
+            .plan_search(&query, 0, 1, None, false, 1.0)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("k must be positive"),
+            "expected k validation error, got {err}"
+        );
+
+        let err = planner
+            .plan_search(&query, 1, 0, None, false, 1.0)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("nprobes must be positive"),
+            "expected nprobes validation error, got {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_projection_includes_pk() {
         let schema = create_vector_schema();
@@ -641,6 +795,7 @@ mod tests {
         // Active memtable with HNSW index over the "vector" column.
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -683,62 +838,849 @@ mod tests {
             .plan_search(&query, 3, 1, None, false, 1.0)
             .await
             .expect("planner should produce a plan");
-
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert!(total > 0, "expected at least one result row");
+
+        let out_schema = batches[0].schema();
+        let out_cols: Vec<String> = out_schema
+            .fields()
+            .iter()
+            .map(|f| f.name().clone())
+            .collect();
+        assert!(
+            out_schema.field_with_name(DISTANCE_COLUMN).is_ok(),
+            "output schema is missing `_distance` column. Got: {:?}",
+            out_cols
+        );
+        // Internal columns must not leak: `_rowid` (added by Lance's fast_search
+        // in the base/flushed arms) and `_memtable_gen` (added by the LSM merge
+        // when bloom filters are present) are bookkeeping, not API.
+        assert!(
+            out_schema.field_with_name("_rowid").is_err(),
+            "`_rowid` leaked into output: {:?}",
+            out_cols
+        );
+        assert!(
+            out_schema
+                .field_with_name(super::super::exec::MEMTABLE_GEN_COLUMN)
+                .is_err(),
+            "`_memtable_gen` leaked into output: {:?}",
+            out_cols
+        );
+
+        // The nearest neighbor to the query vector should be id=1 (exact match),
+        // and its `_distance` must be ~0 — verifying both the column is present
+        // and the values are correct.
+        let id_col = batches[0]
+            .column_by_name("id")
+            .expect("id column missing")
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .expect("id column should be Int32");
+        let dist_col = batches[0]
+            .column_by_name(DISTANCE_COLUMN)
+            .expect("_distance column missing")
+            .as_any()
+            .downcast_ref::<arrow_array::Float32Array>()
+            .expect("_distance column should be Float32");
+        assert_eq!(id_col.value(0), 1, "expected id=1 as nearest neighbor");
+        assert!(
+            dist_col.value(0).abs() < 1e-3,
+            "expected near-zero distance for self-match, got {}",
+            dist_col.value(0)
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_active_tombstone_masks_base_hit() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use crate::index::DatasetIndexExt;
+        use crate::index::vector::VectorIndexParams;
+        use datafusion::prelude::SessionContext;
+        use futures::TryStreamExt;
+        use lance_index::IndexType;
+
+        let base_schema = create_vector_schema();
+        let mem_schema = create_vector_tombstone_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let q = [0.1, 0.2, 0.3, 0.4];
+        let near = [0.12, 0.22, 0.32, 0.42];
+
+        let mut base_dataset = create_dataset(
+            &base_uri,
+            vec![batch_rows(&base_schema, &[(1, q), (2, near)])],
+        )
+        .await;
+        let ivf_flat = VectorIndexParams::ivf_flat(1, lance_linalg::distance::DistanceType::L2);
+        base_dataset
+            .create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+        let base_dataset = Arc::new(base_dataset);
+
+        let active_tombstone = tombstone_vector_batch(&mem_schema, &[(1, None, true)]);
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        let (_, row_offset, batch_position) = batch_store.append(active_tombstone.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&active_tombstone, row_offset, Some(batch_position))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store,
+                    schema: mem_schema,
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            base_schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        );
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 1, 1, None, false, 2.0)
+            .await
+            .unwrap();
+        let stream = plan.execute(0, SessionContext::new().task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let rows = collect_id_dist(&batches);
+
+        assert_eq!(rows.len(), 1, "expected one backfilled hit, got {rows:?}");
+        assert_eq!(
+            rows[0].0, 2,
+            "active tombstone for id=1 must block the older base vector hit; got {rows:?}"
+        );
+    }
+
+    /// A prefilter on a vector search must restrict the KNN to rows matching the
+    /// predicate, even though the nearest (and second-nearest) rows fail it. The
+    /// active memtable has an HNSW index, but a filtered search routes to the
+    /// brute-force arm, which masks rows before the top-k cut.
+    #[tokio::test]
+    async fn test_vector_search_prefilter_restricts_to_matching_rows() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        // Base rows are far from the query and unindexed, so `fast_search`
+        // contributes nothing; the test isolates the memtable prefilter.
+        let base_dataset = Arc::new(
+            create_dataset(&base_uri, vec![create_test_batch(&schema, &[100, 200])]).await,
+        );
+
+        // Active memtable with ids 0..=3 (id=1 is the exact match, id=0 ties id=2).
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        index_store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        let batch = create_test_batch(&schema, &[0, 1, 2, 3]);
+        batch_store.append(batch.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        // `id >= 2` excludes the two nearest rows (id=1 exact, id=0 tie).
+        .with_filter(Some(col("id").gt_eq(lit(2i32))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 10, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a filtered plan");
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+
+        // Only id=2 and id=3 satisfy `id >= 2`; the nearer id=0/id=1 are excluded.
+        assert_eq!(
+            ids.first().copied(),
+            Some(2),
+            "nearest matching row is id=2"
+        );
+        let mut sorted = ids.clone();
+        sorted.sort();
+        assert_eq!(
+            sorted,
+            vec![2, 3],
+            "prefilter must drop id=0 and id=1 (nearer but failing `id >= 2`)"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_search_filtered_active_without_pk_keeps_all_matching_rows() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let base_dataset = Arc::new(
+            create_dataset(&base_uri, vec![create_test_batch(&schema, &[100, 200])]).await,
+        );
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let index_store = IndexStore::new();
+        let batch = create_test_batch(&schema, &[1, 2, 3]);
+        batch_store.append(batch.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec![],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_filter(Some(col("id").gt_eq(lit(1i32))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 3, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a filtered active plan");
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![1, 2, 3],
+            "no-PK filtered active vector search must not collapse all rows into one key"
+        );
+    }
+
+    /// The *base* arm must apply the filter as a true prefilter, not a
+    /// post-filter on the unfiltered top-k. The base is vector-indexed (so
+    /// `fast_search` uses the index); the two rows nearest the query (id=1, id=2)
+    /// fail the predicate while matching rows (id>=3) are farther. Without
+    /// `scanner.prefilter(true)` the base arm runs the ANN unfiltered, takes the
+    /// top-`k`, then post-filters — dropping every row. A true prefilter restricts
+    /// the ANN to matching rows. Regression for a missing base-arm `prefilter(true)`.
+    #[tokio::test]
+    async fn test_vector_search_base_prefilter_is_not_a_lossy_postfilter() {
+        use crate::index::DatasetIndexExt;
+        use crate::index::vector::VectorIndexParams;
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+        use lance_index::IndexType;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+
+        // L2 distance to the query grows monotonically with id (id=1 is the exact
+        // match), so the two nearest rows are id=1, id=2 — both excluded by `id>=3`.
+        let base_batch = create_test_batch(&schema, &[1, 2, 3, 4, 5, 6]);
+        let mut base_dataset = create_dataset(&base_uri, vec![base_batch]).await;
+        let ivf_flat = VectorIndexParams::ivf_flat(1, lance_linalg::distance::DistanceType::L2);
+        base_dataset
+            .create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+        let base_dataset = Arc::new(base_dataset);
+
+        let collector = LsmDataSourceCollector::new(base_dataset.clone(), vec![]);
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_dataset(base_dataset)
+        // Keeps only id>=3, the *farther* matches. A post-filter on the
+        // unfiltered top-2 (id=1, id=2) would drop everything.
+        .with_filter(Some(col("id").gt_eq(lit(3i32))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 2, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a filtered base plan");
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let col = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                ids.push(col.value(i));
+            }
+        }
+        assert_eq!(
+            ids.first().copied(),
+            Some(3),
+            "nearest matching base row is id=3; got {ids:?}"
+        );
+        let mut sorted = ids.clone();
+        sorted.sort();
+        assert_eq!(
+            sorted,
+            vec![3, 4],
+            "base prefilter must return the two nearest matches (id=3, id=4), not \
+             post-filter the unfiltered top-2 (id=1, id=2) down to nothing; got {ids:?}"
+        );
+    }
+
+    /// The flushed arm must also apply the filter as a true prefilter, and that
+    /// prefiltered candidate set must compose with cross-generation block-list
+    /// filtering plus over-fetch. Gen 1's closest predicate-matching row (id=3)
+    /// is superseded by gen 2; with over-fetch, gen 1 should still contribute
+    /// the next live predicate match (id=4).
+    #[tokio::test]
+    async fn test_vector_search_flushed_prefilter_composes_with_block_list() {
+        use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot;
+        use crate::index::DatasetIndexExt;
+        use crate::index::vector::VectorIndexParams;
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+        use lance_index::IndexType;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let shard_id = uuid::Uuid::new_v4();
+        let ivf_flat = VectorIndexParams::ivf_flat(1, lance_linalg::distance::DistanceType::L2);
+
+        let q = [0.1, 0.2, 0.3, 0.4];
+        let near = [0.12, 0.22, 0.32, 0.42];
+        let far = [9.0, 9.0, 9.0, 9.0];
+
+        // Gen 1: id=1 is nearest but fails the predicate. id=3 passes but is
+        // stale (blocked by gen 2). id=4 is the next live predicate match.
+        let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id);
+        let mut gen1 = create_dataset(
+            &gen1_uri,
+            vec![batch_rows(&schema, &[(1, q), (3, q), (4, near)])],
+        )
+        .await;
+        gen1.create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+
+        // Gen 2: newer id=3 shadows gen 1's close copy but is far from query.
+        let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id);
+        let mut gen2 = create_dataset(&gen2_uri, vec![batch_rows(&schema, &[(3, far)])]).await;
+        gen2.create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(3)
+            .with_flushed_generation(1, "gen_1".to_string())
+            .with_flushed_generation(2, "gen_2".to_string());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]);
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_filter(Some(col("id").gt_eq(lit(3i32))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 1, 1, None, false, 2.0)
+            .await
+            .unwrap();
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let rows = collect_id_dist(&batches);
+
+        assert_eq!(rows.len(), 1, "expected one result, got {:?}", rows);
+        assert_eq!(
+            rows[0].0, 4,
+            "flushed prefilter should return live id=4 after stale id=3 is blocked; got {:?}",
+            rows
+        );
+    }
+
+    /// An in-memtable update whose *newest* version fails the prefilter must
+    /// exclude the PK entirely, not leak the stale older version that still
+    /// passes. Regression for filter-before-dedup on the active arm: the filter
+    /// is evaluated against the newest version of each PK.
+    #[tokio::test]
+    async fn test_vector_search_prefilter_excludes_pk_whose_newest_version_fails() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use arrow_array::StringArray;
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_schema::{DataType, Field};
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+
+        let mut id_meta = std::collections::HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                false,
+            ),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        // Same PK id=5 appended twice (an in-memtable update): row0 "active"
+        // (passes), row1 "archived" (the current version, fails the filter). Both
+        // vectors are near the query.
+        let make_batch = |statuses: &[&str]| -> RecordBatch {
+            let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+            for _ in statuses {
+                for d in 0..4 {
+                    vb.values().append_value(0.1 + d as f32 * 0.1);
+                }
+                vb.append(true);
+            }
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(vec![5; statuses.len()])),
+                    Arc::new(vb.finish()),
+                    Arc::new(StringArray::from(statuses.to_vec())),
+                ],
+            )
+            .unwrap()
+        };
+
+        // Base is unindexed, so `fast_search` contributes nothing; the test
+        // isolates the active-memtable dedup-before-filter behavior. Use a
+        // non-conflicting id to avoid any PK confusion.
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let mut fb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+        for d in 0..4 {
+            fb.values().append_value(100.0 + d as f32);
+        }
+        fb.append(true);
+        let far = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![999])),
+                Arc::new(fb.finish()),
+                Arc::new(StringArray::from(vec!["active"])),
+            ],
+        )
+        .unwrap();
+        let base_dataset = Arc::new(create_dataset(&base_uri, vec![far]).await);
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        index_store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+        let batch = make_batch(&["active", "archived"]);
+        batch_store.append(batch.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&batch, 0, Some(0))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]).with_in_memory_memtables(
+            uuid::Uuid::new_v4(),
+            InMemoryMemTables {
+                active: InMemoryMemTableRef {
+                    batch_store,
+                    index_store,
+                    schema: schema.clone(),
+                    generation: 1,
+                },
+                frozen: vec![],
+            },
+        );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_filter(Some(col("status").eq(lit("active"))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 10, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a filtered plan");
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(
+            total, 0,
+            "pk=5's current version is 'archived' and must be excluded; a stale \
+             'active' older version must not leak (filter evaluated on newest)"
+        );
+    }
+
+    /// Cross-arm stale rows must be blocked even if the newer active row fails
+    /// the prefilter. The base copy of pk=5 matches `status = 'active'`, but the
+    /// newer active copy is `archived`; pk=5 must be absent from the result.
+    #[tokio::test]
+    async fn test_vector_search_prefilter_blocks_base_when_active_newest_fails() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use crate::index::DatasetIndexExt;
+        use crate::index::vector::VectorIndexParams;
+        use arrow_array::StringArray;
+        use arrow_array::builder::{FixedSizeListBuilder, Float32Builder};
+        use arrow_schema::{DataType, Field};
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+        use lance_index::IndexType;
+
+        let mut id_meta = std::collections::HashMap::new();
+        id_meta.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false).with_metadata(id_meta),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                false,
+            ),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+        let make_batch = |rows: &[(i32, [f32; 4], &str)]| -> RecordBatch {
+            let mut vectors = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+            for (_, vector, _) in rows {
+                for value in vector {
+                    vectors.values().append_value(*value);
+                }
+                vectors.append(true);
+            }
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(Int32Array::from(
+                        rows.iter().map(|(id, _, _)| *id).collect::<Vec<_>>(),
+                    )),
+                    Arc::new(vectors.finish()),
+                    Arc::new(StringArray::from(
+                        rows.iter()
+                            .map(|(_, _, status)| *status)
+                            .collect::<Vec<_>>(),
+                    )),
+                ],
+            )
+            .unwrap()
+        };
+
+        let query = [0.1, 0.2, 0.3, 0.4];
+        let fallback = [0.2, 0.3, 0.4, 0.5];
+
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let base_batch = make_batch(&[(5, query, "active"), (6, fallback, "active")]);
+        let mut base_dataset = create_dataset(&base_uri, vec![base_batch]).await;
+        let ivf_flat = VectorIndexParams::ivf_flat(1, lance_linalg::distance::DistanceType::L2);
+        base_dataset
+            .create_index(&["vector"], IndexType::Vector, None, &ivf_flat, true)
+            .await
+            .unwrap();
+        let base_dataset = Arc::new(base_dataset);
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        index_store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            16,
+            4,
+        );
+        let active_batch = make_batch(&[(5, query, "archived")]);
+        batch_store.append(active_batch.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&active_batch, 0, Some(0))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let collector = LsmDataSourceCollector::new(base_dataset.clone(), vec![])
+            .with_in_memory_memtables(
+                uuid::Uuid::new_v4(),
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_dataset(base_dataset)
+        .with_filter(Some(col("status").eq(lit("active"))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 10, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a filtered base+active plan");
+
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let mut ids: Vec<i32> = collect_id_dist(&batches)
+            .into_iter()
+            .map(|(id, _)| id)
+            .collect();
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![6],
+            "base pk=5 passes the filter but is superseded by active archived pk=5; got {ids:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn cross_gen_stale_version_blocked_by_newer_memtable() {
+        // A PK updated ACROSS a freeze boundary: its old (near) version lives in
+        // a frozen memtable, its new (far) version in the active one. Both are
+        // in-memory `ActiveMemTable` sources, so each gets only a per-generation
+        // recency filter (`NewestPkFilterExec`) — which can't see the newer gen.
+        // The cross-generation block-list is the only thing that can drop the
+        // stale near copy; if the active arm discards it, the frozen near version
+        // leaks as a phantom near-neighbor. This is the residual wallop fuzz
+        // delete/update phantom: the cluster constantly freezes memtables
+        // (flush_interval ~250ms), so an insert and its later delete/update split
+        // across in-memory generations.
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use datafusion::prelude::SessionContext;
+        use futures::TryStreamExt;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        // Base seed far from the query; id=1 is not in the base.
+        let base_dataset =
+            Arc::new(create_dataset(&base_uri, vec![create_test_batch(&schema, &[99])]).await);
+
+        // One in-memory memtable holding `id` at `vector`, with a PK + HNSW index.
+        let make_gen = |id: i32, vector: [f32; 4], generation: u64| {
+            let batch_store = Arc::new(BatchStore::with_capacity(16));
+            let mut index_store = IndexStore::new();
+            index_store.enable_pk_index(&[("id".to_string(), 0)]);
+            index_store.add_hnsw(
+                "vector_hnsw".to_string(),
+                1,
+                "vector".to_string(),
+                lance_linalg::distance::DistanceType::L2,
+                64,
+                8,
+            );
+            let batch = create_id_vector_batch(&schema, id, vector);
+            batch_store.append(batch.clone()).unwrap();
+            index_store
+                .insert_with_batch_position(&batch, 0, Some(0))
+                .unwrap();
+            InMemoryMemTableRef {
+                batch_store,
+                index_store: Arc::new(index_store),
+                schema: schema.clone(),
+                generation,
+            }
+        };
+
+        // Frozen gen=1: id=1 @ the query vector (distance ~0).
+        let frozen = make_gen(1, [0.1, 0.2, 0.3, 0.4], 1);
+        // Active gen=2: id=1 moved FAR — the newest version of the PK.
+        let active = make_gen(1, [9.0, 9.0, 9.0, 9.0], 2);
+
+        let shard_id = uuid::Uuid::new_v4();
+        let collector = LsmDataSourceCollector::new(base_dataset, vec![]).with_in_memory_memtables(
+            shard_id,
+            InMemoryMemTables {
+                active,
+                frozen: vec![frozen],
+            },
+        );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        );
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 3, 1, None, false, 1.0)
+            .await
+            .expect("planner should produce a plan");
         let ctx = SessionContext::new();
-        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
-        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-
-        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
-        assert!(total > 0, "expected at least one result row");
+        let batches: Vec<RecordBatch> = plan
+            .execute(0, ctx.task_ctx())
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
 
-        let out_schema = batches[0].schema();
-        let out_cols: Vec<String> = out_schema
-            .fields()
-            .iter()
-            .map(|f| f.name().clone())
-            .collect();
-        assert!(
-            out_schema.field_with_name(DISTANCE_COLUMN).is_ok(),
-            "output schema is missing `_distance` column. Got: {:?}",
-            out_cols
-        );
-        // Internal columns must not leak: `_rowid` (added by Lance's fast_search
-        // in the base/flushed arms) and `_memtable_gen` (added by the LSM merge
-        // when bloom filters are present) are bookkeeping, not API.
-        assert!(
-            out_schema.field_with_name("_rowid").is_err(),
-            "`_rowid` leaked into output: {:?}",
-            out_cols
-        );
-        assert!(
-            out_schema
-                .field_with_name(super::super::exec::MEMTABLE_GEN_COLUMN)
-                .is_err(),
-            "`_memtable_gen` leaked into output: {:?}",
-            out_cols
-        );
+        let mut id1_distances = Vec::new();
+        for b in &batches {
+            let ids = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            let dist = b
+                .column_by_name(DISTANCE_COLUMN)
+                .unwrap()
+                .as_any()
+                .downcast_ref::<arrow_array::Float32Array>()
+                .unwrap();
+            for i in 0..b.num_rows() {
+                if ids.value(i) == 1 {
+                    id1_distances.push(dist.value(i));
+                }
+            }
+        }
 
-        // The nearest neighbor to the query vector should be id=1 (exact match),
-        // and its `_distance` must be ~0 — verifying both the column is present
-        // and the values are correct.
-        let id_col = batches[0]
-            .column_by_name("id")
-            .expect("id column missing")
-            .as_any()
-            .downcast_ref::<Int32Array>()
-            .expect("id column should be Int32");
-        let dist_col = batches[0]
-            .column_by_name(DISTANCE_COLUMN)
-            .expect("_distance column missing")
-            .as_any()
-            .downcast_ref::<arrow_array::Float32Array>()
-            .expect("_distance column should be Float32");
-        assert_eq!(id_col.value(0), 1, "expected id=1 as nearest neighbor");
+        // id=1 must surface at most once, and only at its NEWEST (far) distance —
+        // never the stale near ~0 copy the frozen generation still indexes.
         assert!(
-            dist_col.value(0).abs() < 1e-3,
-            "expected near-zero distance for self-match, got {}",
-            dist_col.value(0)
+            id1_distances.len() <= 1,
+            "id=1 leaked its stale frozen copy (appears {} times): {id1_distances:?}",
+            id1_distances.len()
         );
+        if let Some(d) = id1_distances.first() {
+            assert!(
+                *d > 1.0,
+                "id=1 surfaced at the stale near distance {d}; its newest version is far"
+            );
+        }
     }
 
     #[tokio::test]
@@ -759,6 +1701,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -838,6 +1781,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -951,6 +1895,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -994,9 +1939,10 @@ mod tests {
             .await
             .expect("planner should produce a plan");
 
-        // Each arm is independently newest-per-PK (active within-source dedup,
-        // flushed DV) and the block-list handles cross-gen, merged by a
-        // distance SPM. No global PK dedup or source tag node is involved.
+        // Each arm is independently newest-per-PK (active append-only data can
+        // use HNSW directly; rewritten active data falls back to exact search)
+        // and the block-list handles cross-gen, merged by a distance SPM. No
+        // global PK dedup or source tag node is involved.
         let plan_str = format!(
             "{}",
             datafusion::physical_plan::displayable(plan.as_ref()).indent(true)
@@ -1007,9 +1953,8 @@ mod tests {
             plan_str
         );
         assert!(
-            plan_str.contains("WithinSourceDedupExec")
-                && plan_str.contains("SortPreservingMergeExec"),
-            "expected per-arm dedup + distance merge, got:\n{}",
+            plan_str.contains("VectorIndexExec") && plan_str.contains("SortPreservingMergeExec"),
+            "expected append-only active HNSW + distance merge, got:\n{}",
             plan_str
         );
 
@@ -1091,6 +2036,7 @@ mod tests {
         // "right" vector close to the query, plus an unrelated pk=2.
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -1210,6 +2156,7 @@ mod tests {
         // Active memtable: id=3 with HNSW index.
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -1359,6 +2306,32 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_vector_search_rejects_missing_projection_column() {
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]);
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        );
+
+        let projection = vec!["missing".to_string()];
+        let query = create_query_vector();
+        let err = planner
+            .plan_search(&query, 5, 1, Some(&projection), false, 1.0)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("missing"),
+            "unexpected missing-column projection error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_vector_search_without_base_table() {
         use futures::TryStreamExt;
@@ -1436,9 +2409,8 @@ mod tests {
     #[tokio::test]
     async fn test_vector_search_dedup_within_active_memtable() {
         // Regression: same PK inserted twice into one active memtable with
-        // *different* vectors. HNSW indexes each as a distinct node, so
-        // without WithinSourceDedupExec a KNN can return both candidates
-        // for the same PK and pollute top-k. The newer insert must win.
+        // *different* vectors. The exact active search must consider only the
+        // newest version of each PK before top-k. The newer insert must win.
         use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
         use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
         use datafusion::prelude::SessionContext;
@@ -1450,6 +2422,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -1503,25 +2476,23 @@ mod tests {
             lance_linalg::distance::DistanceType::L2,
         );
 
-        // Query is exactly the *newer* vector for pk=1. If the older
-        // vector for pk=1 leaks through, it'd appear in top-k too because
-        // the older row's vector is far from the query but still a graph
-        // node. After dedup we should see pk=1 exactly once.
+        // Query is exactly the *newer* vector for pk=1. The exact active
+        // memtable arm should dedup before top-k so pk=1 appears exactly once.
         let query = create_query_vector();
         let plan = planner
             .plan_search(&query, 5, 1, None, false, 1.0)
             .await
             .unwrap();
 
-        // The active arm collapses duplicate-PK HNSW nodes itself via
-        // WithinSourceDedupExec — there is no cross-source dedup fallback.
+        // The active arm uses exact brute force when PK recency is configured;
+        // there is no cross-source dedup fallback.
         let plan_str = format!(
             "{}",
             datafusion::physical_plan::displayable(plan.as_ref()).indent(true)
         );
         assert!(
-            plan_str.contains("WithinSourceDedupExec"),
-            "active vector arm must self-dedup, got:\n{}",
+            plan_str.contains("MemTableBruteForceVectorExec"),
+            "active vector arm must use exact brute-force PK recency, got:\n{}",
             plan_str
         );
 
@@ -1549,10 +2520,118 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_vector_search_active_stale_update_out_of_neighborhood() {
+        // BUG REPRODUCTION (vector case: a PK update that moves out of the neighborhood).
+        //
+        // Within a *single* active memtable, pk=1 is first inserted ON the query
+        // (distance ~0), then updated to a FAR vector. Exact brute force must
+        // compute newest-per-PK over the whole memtable before top-k, so the
+        // stale near row is dropped even though the fresh row is far away.
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use datafusion::prelude::SessionContext;
+        use futures::TryStreamExt;
+
+        let schema = create_vector_schema();
+        let temp_dir = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap());
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
+        index_store.add_hnsw(
+            "vector_hnsw".to_string(),
+            1,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+            64,
+            8,
+        );
+
+        // First append: stale pk=1 ON the query, plus five filler rows strictly
+        // farther than pk=1 but far nearer than the eventual fresh pk=1.
+        let q = [0.1, 0.2, 0.3, 0.4];
+        let stale_then_fillers = batch_rows(
+            &schema,
+            &[
+                (1, q),
+                (10, [0.11, 0.21, 0.31, 0.41]),
+                (11, [0.13, 0.23, 0.33, 0.43]),
+                (12, [0.15, 0.25, 0.35, 0.45]),
+                (13, [0.17, 0.27, 0.37, 0.47]),
+                (14, [0.19, 0.29, 0.39, 0.49]),
+            ],
+        );
+        let (bp0, off0, _) = batch_store.append(stale_then_fillers.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&stale_then_fillers, off0, Some(bp0))
+            .unwrap();
+
+        // Second append: the UPDATE — pk=1 moved far from the query. This is the
+        // newest version (largest row position) but it sits well outside top-k.
+        let fresh_pk1 = batch_rows(&schema, &[(1, [9.0, 9.0, 9.0, 9.0])]);
+        let (bp1, off1, _) = batch_store.append(fresh_pk1.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&fresh_pk1, off1, Some(bp1))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let shard_id = uuid::Uuid::new_v4();
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                shard_id,
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        );
+
+        // k=3, no over-fetch: exact active search still computes
+        // newest-per-PK across the whole memtable before top-k, so stale
+        // pk1@near cannot leak even though fresh pk1@far ranks 7th by distance.
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 3, 1, None, false, 1.0)
+            .await
+            .unwrap();
+        let ctx = SessionContext::new();
+        let stream = plan.execute(0, ctx.task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        let rows = collect_id_dist(&batches);
+
+        assert_eq!(
+            rows.len(),
+            3,
+            "active PK recency filtering must not underfill k after dropping stale candidates; \
+             results={:?}",
+            rows
+        );
+        assert!(
+            !rows.iter().any(|&(id, d)| id == 1 && d.abs() < 1e-3),
+            "stale near pk=1 leaked: its live vector is far from the query, so it \
+             must not appear at distance ~0. results={:?}",
+            rows
+        );
+    }
+
     #[tokio::test]
     async fn test_vector_search_stale_read_when_fresh_falls_out_of_top_k() {
         // Regression for the cross-generation stale-read gap that the
-        // PkHashFilterExec block-list closes.
+        // PkBlockFilterExec block-list closes.
         //
         // Scenario:
         //   * Base (gen 0): stale pk=1 sitting on the query (distance ~0).
@@ -1587,6 +2666,7 @@ mod tests {
         // active arm surfaces pk=2 and drops fresh pk=1.
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -1682,27 +2762,15 @@ mod tests {
             rows
         );
 
-        // The block-list is now unconditional: a sub-1.0 overfetch_factor is
-        // clamped to 1.0 and the stale base copy of pk=1 stays suppressed (the
-        // factor only tunes the over-fetch multiple, it cannot disable filtering).
-        let still_filtered = planner
+        // The block-list is unconditional and cannot be disabled via
+        // overfetch_factor; invalid sub-1.0 values are rejected instead.
+        let err = planner
             .plan_search(&query, 1, 1, None, false, 0.0)
             .await
-            .unwrap();
-        let still_filtered_rows = {
-            let stream = still_filtered
-                .execute(0, SessionContext::new().task_ctx())
-                .unwrap();
-            let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
-            collect_id_dist(&batches)
-        };
+            .unwrap_err();
         assert!(
-            still_filtered_rows
-                .iter()
-                .all(|&(id, d)| !(id == 1 && d.abs() < 1e-3)),
-            "block-list is unconditional: stale pk=1 must stay suppressed even \
-             with overfetch_factor < 1.0; got {:?}",
-            still_filtered_rows
+            err.to_string().contains("overfetch_factor"),
+            "unexpected error for invalid overfetch factor: {err}"
         );
     }
 
@@ -1783,6 +2851,7 @@ mod tests {
         // Active (gen 1): pk 1,2,3 re-inserted with a far vector (the fresh value).
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -1987,6 +3056,7 @@ mod tests {
         // Active: (1,1) re-inserted far (fresh) + an unrelated nearby (2,2).
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id1".to_string(), 0), ("id2".to_string(), 1)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
@@ -2068,6 +3138,131 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_vector_search_prefilter_excludes_composite_pk_newest_version() {
+        use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables};
+        use crate::dataset::mem_wal::write::{BatchStore, IndexStore};
+        use arrow_array::StringArray;
+        use arrow_array::builder::Float32Builder;
+        use datafusion::prelude::{SessionContext, col, lit};
+        use futures::TryStreamExt;
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id1", DataType::Int32, false),
+            Field::new("id2", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+                false,
+            ),
+            Field::new("status", DataType::Utf8, false),
+        ]));
+
+        fn batch(schema: &ArrowSchema, rows: &[((i32, i32), [f32; 4], &str)]) -> RecordBatch {
+            let mut vb = FixedSizeListBuilder::new(Float32Builder::new(), 4);
+            for (_, vector, _) in rows {
+                for value in vector {
+                    vb.values().append_value(*value);
+                }
+                vb.append(true);
+            }
+            let id1: Vec<i32> = rows.iter().map(|((id1, _), _, _)| *id1).collect();
+            let id2: Vec<i32> = rows.iter().map(|((_, id2), _, _)| *id2).collect();
+            let status: Vec<&str> = rows.iter().map(|(_, _, status)| *status).collect();
+            RecordBatch::try_new(
+                Arc::new(schema.clone()),
+                vec![
+                    Arc::new(Int32Array::from(id1)),
+                    Arc::new(Int32Array::from(id2)),
+                    Arc::new(vb.finish()),
+                    Arc::new(StringArray::from(status)),
+                ],
+            )
+            .unwrap()
+        }
+
+        let q = [0.1, 0.2, 0.3, 0.4];
+        let near = [0.12, 0.22, 0.32, 0.42];
+        let far = [9.0, 9.0, 9.0, 9.0];
+        let batch = batch(
+            &schema,
+            &[
+                ((1, 1), q, "active"),
+                ((1, 1), far, "archived"),
+                ((2, 2), near, "active"),
+            ],
+        );
+
+        let batch_store = Arc::new(BatchStore::with_capacity(16));
+        let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id1".to_string(), 0), ("id2".to_string(), 1)]);
+        let (_, row_offset, batch_position) = batch_store.append(batch.clone()).unwrap();
+        index_store
+            .insert_with_batch_position(&batch, row_offset, Some(batch_position))
+            .unwrap();
+        let index_store = Arc::new(index_store);
+
+        let tmp = tempfile::tempdir().unwrap();
+        let base_uri = format!("{}/base", tmp.path().to_str().unwrap());
+        let shard_id = uuid::Uuid::new_v4();
+        let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![])
+            .with_in_memory_memtables(
+                shard_id,
+                InMemoryMemTables {
+                    active: InMemoryMemTableRef {
+                        batch_store,
+                        index_store,
+                        schema: schema.clone(),
+                        generation: 1,
+                    },
+                    frozen: vec![],
+                },
+            );
+
+        let planner = LsmVectorSearchPlanner::new(
+            collector,
+            vec!["id1".to_string(), "id2".to_string()],
+            schema,
+            "vector".to_string(),
+            lance_linalg::distance::DistanceType::L2,
+        )
+        .with_filter(Some(col("status").eq(lit("active"))));
+
+        let query = create_query_vector();
+        let plan = planner
+            .plan_search(&query, 10, 1, None, false, 1.0)
+            .await
+            .unwrap();
+        let stream = plan.execute(0, SessionContext::new().task_ctx()).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+
+        let mut ids = Vec::new();
+        for batch in &batches {
+            let id1 = batch
+                .column_by_name("id1")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            let id2 = batch
+                .column_by_name("id2")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            for row in 0..batch.num_rows() {
+                ids.push((id1.value(row), id2.value(row)));
+            }
+        }
+
+        assert_eq!(
+            ids,
+            vec![(2, 2)],
+            "composite PK (1,1)'s newest version is archived and must exclude \
+             the older active version; got {ids:?}"
+        );
+    }
+
     #[tokio::test]
     async fn test_vector_search_same_l0_override_newest_wins() {
         // Ported from the #6844 spec. The DANGEROUS within-memtable direction:
@@ -2091,6 +3286,7 @@ mod tests {
 
         let batch_store = Arc::new(BatchStore::with_capacity(16));
         let mut index_store = IndexStore::new();
+        index_store.enable_pk_index(&[("id".to_string(), 0)]);
         index_store.add_hnsw(
             "vector_hnsw".to_string(),
             1,
diff --git a/rust/lance/src/dataset/mem_wal/test_util.rs b/rust/lance/src/dataset/mem_wal/test_util.rs
new file mode 100644
index 00000000000..48e68340d3c
--- /dev/null
+++ b/rust/lance/src/dataset/mem_wal/test_util.rs
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Test-only object store that injects WAL-write failures (for the WAL
+//! persistence-failure fencing path) and records the paths it serves (for
+//! asserting which opens actually resolved through a given `ObjectStoreParams`).
+
+use std::fmt::{Debug, Display, Formatter};
+use std::ops::Range;
+use std::sync::Arc;
+use std::sync::Mutex as StdMutex;
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use bytes::Bytes;
+use futures::stream::BoxStream;
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    ObjectStore as OSObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,
+    RenameOptions, Result as OSResult,
+};
+
+use lance_io::object_store::{
+    ObjectStore, ObjectStoreParams, ObjectStoreRegistry, WrappingObjectStore,
+};
+
+/// Knobs controlling injected WAL-entry write failures, shared with the test.
+#[derive(Debug, Default)]
+pub struct FailControls {
+    /// Remaining WAL-entry `put_opts` calls to fail; saturates at 0. Set high to
+    /// "always fail", set to 0 to "recover".
+    wal_put_failures: AtomicUsize,
+    /// When failing, write to the inner store anyway before reporting the error,
+    /// simulating a lost acknowledgement (the PUT actually landed).
+    simulate_lost_ack: AtomicBool,
+    /// WAL-entry `put_opts` attempts observed, for assertions.
+    wal_put_attempts: AtomicUsize,
+    /// Every location written through this store.
+    put_paths: StdMutex<Vec<String>>,
+    /// Every location read through this store.
+    get_paths: StdMutex<Vec<String>>,
+}
+
+impl FailControls {
+    pub fn fail_wal_puts(&self, n: usize) {
+        self.wal_put_failures.store(n, Ordering::SeqCst);
+    }
+    pub fn recover(&self) {
+        self.wal_put_failures.store(0, Ordering::SeqCst);
+    }
+    pub fn set_lost_ack(&self, v: bool) {
+        self.simulate_lost_ack.store(v, Ordering::SeqCst);
+    }
+    pub fn attempts(&self) -> usize {
+        self.wal_put_attempts.load(Ordering::SeqCst)
+    }
+
+    /// Did any write land on a path containing `needle`? An open that resolved
+    /// its store from other params never reaches this store, so a `false` here
+    /// means the params under test did not reach that open.
+    pub fn wrote_under(&self, needle: &str) -> bool {
+        self.put_paths
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.contains(needle))
+    }
+
+    /// Did any read land on a path containing `needle`? See [`Self::wrote_under`].
+    pub fn read_under(&self, needle: &str) -> bool {
+        self.get_paths
+            .lock()
+            .unwrap()
+            .iter()
+            .any(|p| p.contains(needle))
+    }
+}
+
+/// Wraps the inner store with [`FailingObjectStore`] at construction.
+#[derive(Debug)]
+struct FailingWrapper {
+    controls: Arc<FailControls>,
+}
+
+impl WrappingObjectStore for FailingWrapper {
+    fn wrap(&self, _prefix: &str, original: Arc<dyn OSObjectStore>) -> Arc<dyn OSObjectStore> {
+        Arc::new(FailingObjectStore {
+            inner: original,
+            controls: self.controls.clone(),
+        })
+    }
+}
+
+/// Delegates everything to `inner`, failing WAL-entry PUTs per [`FailControls`].
+#[derive(Debug)]
+struct FailingObjectStore {
+    inner: Arc<dyn OSObjectStore>,
+    controls: Arc<FailControls>,
+}
+
+impl FailingObjectStore {
+    /// WAL entries live under `.../wal/`; manifests (`.../manifest/`) are never
+    /// failed so `ShardWriter::open`/`claim_epoch` still work.
+    fn is_wal_entry(location: &Path) -> bool {
+        location.as_ref().contains("/wal/")
+    }
+
+    fn injected_error() -> object_store::Error {
+        object_store::Error::Generic {
+            store: "failing-test-store",
+            source: "injected transient WAL put failure".to_string().into(),
+        }
+    }
+}
+
+impl Display for FailingObjectStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "FailingObjectStore({})", self.inner)
+    }
+}
+
+#[async_trait::async_trait]
+impl OSObjectStore for FailingObjectStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        payload: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        self.controls
+            .put_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
+        if Self::is_wal_entry(location) {
+            self.controls
+                .wal_put_attempts
+                .fetch_add(1, Ordering::SeqCst);
+            if self.controls.wal_put_failures.load(Ordering::SeqCst) > 0 {
+                self.controls
+                    .wal_put_failures
+                    .fetch_sub(1, Ordering::SeqCst);
+                if self.controls.simulate_lost_ack.load(Ordering::SeqCst) {
+                    // The write lands but we still report failure (lost ack).
+                    let _ = self.inner.put_opts(location, payload, opts).await;
+                }
+                return Err(Self::injected_error());
+            }
+        }
+        self.inner.put_opts(location, payload, opts).await
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        // Data files (`*.lance`) are written multipart, not via `put_opts`.
+        self.controls
+            .put_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
+        self.inner.put_multipart_opts(location, opts).await
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        self.controls
+            .get_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
+        self.inner.get_opts(location, options).await
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        self.controls
+            .get_paths
+            .lock()
+            .unwrap()
+            .push(location.to_string());
+        self.inner.get_ranges(location, ranges).await
+    }
+
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OSResult<Path>>,
+    ) -> BoxStream<'static, OSResult<Path>> {
+        self.inner.delete_stream(locations)
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.inner.list(prefix)
+    }
+
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.inner.list_with_offset(prefix, offset)
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        self.inner.list_with_delimiter(prefix).await
+    }
+
+    async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+        self.inner.copy_opts(from, to, opts).await
+    }
+
+    async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+        self.inner.rename_opts(from, to, opts).await
+    }
+}
+
+/// `ObjectStoreParams` carrying the observable store wrapper, plus the controls
+/// to drive and inspect it. Open a dataset with these and every store resolved
+/// *from these params* — the base and any derived URI they are threaded to —
+/// reports its traffic back through the controls.
+pub fn observable_store_params() -> (ObjectStoreParams, Arc<FailControls>) {
+    let controls = Arc::new(FailControls::default());
+    let params = ObjectStoreParams {
+        object_store_wrapper: Some(Arc::new(FailingWrapper {
+            controls: controls.clone(),
+        })),
+        ..Default::default()
+    };
+    (params, controls)
+}
+
+/// Build an in-memory `ObjectStore` whose WAL-entry writes can be failed on
+/// demand. Returns the store, its base path, and the shared controls.
+pub async fn failing_memory_store() -> (Arc<ObjectStore>, Path, Arc<FailControls>) {
+    let (params, controls) = observable_store_params();
+    let (store, base) = ObjectStore::from_uri_and_params(
+        Arc::new(ObjectStoreRegistry::default()),
+        "memory:///",
+        &params,
+    )
+    .await
+    .unwrap();
+    (store, base, controls)
+}
diff --git a/rust/lance/src/dataset/mem_wal/util.rs b/rust/lance/src/dataset/mem_wal/util.rs
index d1413b84b2a..9dfc019b0b8 100644
--- a/rust/lance/src/dataset/mem_wal/util.rs
+++ b/rust/lance/src/dataset/mem_wal/util.rs
@@ -3,6 +3,7 @@
 
 //! Utility functions for MemWAL operations.
 
+use lance_io::object_store::ObjectStoreParams;
 use object_store::path::Path;
 use uuid::Uuid;
 
@@ -129,6 +130,26 @@ pub fn parse_bit_reversed_filename(filename: &str) -> Option<u64> {
     Some(bit_reverse_u64(reversed))
 }
 
+/// Adapt the store params a base dataset was opened with for use on a URI
+/// *derived* from it (a flushed generation under `_mem_wal/`).
+///
+/// The deprecated `object_store` binding pins a store to one location: given
+/// `Some((store, url))`, both `ObjectStore::from_uri_and_params` and
+/// `DatasetBuilder::build_object_store` take the path from `url` and ignore the
+/// URI they were asked to open. Carried onto a generation URI it would silently
+/// redirect the open — and, on the flush path, the write — at the base table
+/// itself. Drop it so the generation URI resolves its own store; everything
+/// else (storage options, wrapper, credentials, block size) still carries over.
+///
+/// Only the base's *own* URI may reuse the params verbatim.
+pub(crate) fn derived_store_params(params: &ObjectStoreParams) -> ObjectStoreParams {
+    #[allow(deprecated)]
+    ObjectStoreParams {
+        object_store: None,
+        ..params.clone()
+    }
+}
+
 /// Path to the MemWAL root directory.
 ///
 /// Returns: `{base_path}/_mem_wal/`
@@ -169,6 +190,16 @@ pub fn flushed_memtable_path(
     shard_base_path(base_path, shard_id).join(format!("{}_gen_{}", random_hash, generation))
 }
 
+/// Subdirectory of a flushed generation holding its standalone primary-key
+/// dedup index (a sidecar BTree, not registered in the manifest). Both the
+/// flush writer and the block-list probe join this onto the generation path.
+pub const PK_INDEX_DIR: &str = "_pk_index";
+
+/// Path to a flushed generation's standalone primary-key dedup index.
+pub fn pk_index_path(gen_path: &Path) -> Path {
+    gen_path.clone().join(PK_INDEX_DIR)
+}
+
 /// Generate an 8-character random hex string for flushed MemTable directories.
 pub fn generate_random_hash() -> String {
     let bytes: [u8; 4] = rand::random();
@@ -362,4 +393,40 @@ mod tests {
         drop(cell);
         assert_eq!(handle.await.unwrap(), None);
     }
+
+    /// The path-bound store binding is the only thing dropped — credentials and
+    /// storage options must still reach the generation's store.
+    #[test]
+    fn test_derived_store_params_drops_only_the_path_bound_store() {
+        let accessor = lance_io::object_store::StorageOptionsAccessor::with_static_options(
+            std::collections::HashMap::from([("access_key_id".to_string(), "key".to_string())]),
+        );
+        #[allow(deprecated)]
+        let params = ObjectStoreParams {
+            object_store: Some((
+                std::sync::Arc::new(object_store::memory::InMemory::new()),
+                url::Url::parse("memory:///base").unwrap(),
+            )),
+            block_size: Some(1234),
+            storage_options_accessor: Some(std::sync::Arc::new(accessor)),
+            ..Default::default()
+        };
+
+        let derived = derived_store_params(&params);
+
+        #[allow(deprecated)]
+        {
+            assert!(
+                derived.object_store.is_none(),
+                "a store pinned to the base path must not be reused for a generation URI"
+            );
+        }
+        assert_eq!(derived.block_size, Some(1234));
+        assert_eq!(
+            derived
+                .storage_options()
+                .and_then(|o| o.get("access_key_id")),
+            Some(&"key".to_string()),
+        );
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs
index 6232ab0d8ae..b4caff180e1 100644
--- a/rust/lance/src/dataset/mem_wal/wal.rs
+++ b/rust/lance/src/dataset/mem_wal/wal.rs
@@ -12,13 +12,15 @@ use std::sync::Mutex as StdMutex;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::time::Instant;
 
+use std::time::Duration;
+
 use arrow_array::RecordBatch;
 use arrow_ipc::reader::StreamReader;
 use arrow_ipc::writer::StreamWriter;
 use arrow_schema::Schema as ArrowSchema;
 use bytes::Bytes;
 use futures::StreamExt;
-use lance_core::{Error, Result};
+use lance_core::{Error, FenceReason, Result};
 use lance_io::object_store::ObjectStore;
 use object_store::ObjectStoreExt;
 use object_store::path::Path;
@@ -39,6 +41,52 @@ use super::memtable::batch_store::{BatchStore, StoredBatch};
 /// Key for storing writer epoch in Arrow IPC file schema metadata.
 pub const WRITER_EPOCH_KEY: &str = "writer_epoch";
 
+/// Marks a WAL entry as a data-less fence sentinel (observability only;
+/// replay skips sentinels via their empty batch list).
+pub const FENCE_SENTINEL_KEY: &str = "fence_sentinel";
+
+/// True if `error` is a peer fence (a successor claimed a higher epoch).
+#[cfg(test)]
+fn is_fence_error(error: &Error) -> bool {
+    error.fence_reason() == Some(FenceReason::PeerClaimedEpoch)
+}
+
+/// True if `error` is terminal for the writer (either fence kind): the WAL will
+/// never advance, so durability waiters must be woken and later writes rejected.
+fn is_terminal_failure(error: &Error) -> bool {
+    error.fence_reason().is_some()
+}
+
+/// Cloneable carrier that ferries a flush error across the async completion
+/// channels (`WalFlusher::terminal_error` and the per-flush `done` cell), since
+/// `lance_core::Error` is not `Clone`. Preserves the [`FenceReason`] so the typed
+/// [`Error::Fenced`] can be rebuilt rather than flattened to a string.
+#[derive(Clone, Debug)]
+pub struct WalFlushFailure {
+    /// The fence reason if terminal; `None` for an ordinary flush error.
+    pub fence_reason: Option<FenceReason>,
+    /// The error message carrying details about the flush failure.
+    pub message: String,
+}
+
+impl WalFlushFailure {
+    pub(crate) fn from_error(error: &Error) -> Self {
+        Self {
+            fence_reason: error.fence_reason(),
+            message: error.to_string(),
+        }
+    }
+
+    /// Rebuild a typed `Error`, restoring the fence reason when present.
+    pub(crate) fn into_error(self) -> Error {
+        match self.fence_reason {
+            Some(FenceReason::PeerClaimedEpoch) => Error::fenced_by_peer(self.message),
+            Some(FenceReason::PersistenceFailure) => Error::writer_poisoned(self.message),
+            None => Error::io(self.message),
+        }
+    }
+}
+
 /// Watcher for batch durability using watermark-based tracking.
 ///
 /// Uses a shared watch channel that broadcasts the durable watermark.
@@ -49,22 +97,36 @@ pub struct BatchDurableWatcher {
     rx: watch::Receiver<usize>,
     /// Target batch ID to wait for.
     target_batch_position: usize,
+    /// Terminal flush failure shared with the flusher. When set, the watermark
+    /// can never reach the target, so `wait` returns this typed error instead of
+    /// blocking forever.
+    terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
 }
 
 impl BatchDurableWatcher {
     /// Create a new watcher for a specific batch ID.
-    pub fn new(rx: watch::Receiver<usize>, target_batch_position: usize) -> Self {
+    pub fn new(
+        rx: watch::Receiver<usize>,
+        target_batch_position: usize,
+        terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
+    ) -> Self {
         Self {
             rx,
             target_batch_position,
+            terminal_error,
         }
     }
 
     /// Wait until the batch is durable.
     ///
-    /// Returns Ok(()) when `durable_watermark >= target_batch_position`.
+    /// Returns Ok(()) when `durable_watermark >= target_batch_position`, or
+    /// Err if a terminal flush failure (e.g. a fence) means the watermark can
+    /// never reach the target.
     pub async fn wait(&mut self) -> Result<()> {
         loop {
+            if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
+                return Err(failure.into_error());
+            }
             let current = *self.rx.borrow();
             if current >= self.target_batch_position {
                 return Ok(());
@@ -161,8 +223,10 @@ pub struct TriggerWalFlush {
     /// this flush. Use `usize::MAX` to flush all pending batches.
     pub end_batch_position: usize,
     /// Optional cell to write completion result.
-    /// Uses Result<WalFlushResult, String> since Error doesn't implement Clone.
-    pub done: Option<WatchableOnceCell<std::result::Result<WalFlushResult, String>>>,
+    /// Uses `WalFlushFailure` (not `Error`) since `Error` doesn't implement
+    /// `Clone`; the carrier preserves the fence reason so callers waiting on
+    /// this cell still get a typed error.
+    pub done: Option<WatchableOnceCell<std::result::Result<WalFlushResult, WalFlushFailure>>>,
 }
 
 impl std::fmt::Debug for TriggerWalFlush {
@@ -313,6 +377,10 @@ pub struct WalFlusher {
     /// Created at construction and recreated after each flush.
     /// Used by backpressure to wait for WAL flushes.
     wal_flush_cell: std::sync::Mutex<Option<WatchableOnceCell<super::write::DurabilityResult>>>,
+    /// First terminal flush failure, shared with every `BatchDurableWatcher`. It
+    /// wakes durability waiters (the watermark never advances) and is read by
+    /// `check_poisoned` so the write path fails fast.
+    terminal_error: Arc<StdMutex<Option<WalFlushFailure>>>,
 }
 
 impl WalFlusher {
@@ -334,6 +402,7 @@ impl WalFlusher {
             shard_id,
             flush_tx: None,
             wal_flush_cell: std::sync::Mutex::new(Some(wal_flush_cell)),
+            terminal_error: Arc::new(StdMutex::new(None)),
         }
     }
 
@@ -354,7 +423,37 @@ impl WalFlusher {
     pub fn track_batch(&self, batch_position: usize) -> BatchDurableWatcher {
         // Return a watcher that waits for this batch to become durable
         // batch_position is 0-indexed, so we wait for watermark > batch_position (i.e., >= batch_position + 1)
-        BatchDurableWatcher::new(self.durable_watermark_rx.clone(), batch_position + 1)
+        BatchDurableWatcher::new(
+            self.durable_watermark_rx.clone(),
+            batch_position + 1,
+            Arc::clone(&self.terminal_error),
+        )
+    }
+
+    /// Latch a terminal flush failure and wake every durability waiter (the
+    /// watermark never advances, so they must observe the error, not block).
+    /// Idempotent: only the first failure is retained.
+    fn mark_terminal_failure(&self, error: &Error) {
+        {
+            let mut slot = self.terminal_error.lock().unwrap();
+            if slot.is_none() {
+                *slot = Some(WalFlushFailure::from_error(error));
+            }
+        }
+        // Wake `wait`ers without advancing the watermark; each re-checks
+        // `terminal_error` and returns the error.
+        self.durable_watermark_tx.send_modify(|_| {});
+    }
+
+    /// Fail fast with the typed error if this writer has been fenced (by a peer
+    /// or its own persistence failure). The write path calls this before touching
+    /// the memtable so a poisoned writer can't diverge further. Recovery is to
+    /// reopen the shard (replay the WAL).
+    pub fn check_poisoned(&self) -> Result<()> {
+        if let Some(failure) = self.terminal_error.lock().unwrap().clone() {
+            return Err(failure.into_error());
+        }
+        Ok(())
     }
 
     /// Get the current durable watermark.
@@ -401,7 +500,7 @@ impl WalFlusher {
         &self,
         source: WalFlushSource,
         end_batch_position: usize,
-        done: Option<WatchableOnceCell<std::result::Result<WalFlushResult, String>>>,
+        done: Option<WatchableOnceCell<std::result::Result<WalFlushResult, WalFlushFailure>>>,
     ) -> Result<()> {
         if let Some(tx) = &self.flush_tx {
             tx.send(TriggerWalFlush {
@@ -427,7 +526,7 @@ impl WalFlusher {
         source: &WalFlushSource,
         end_batch_position: usize,
     ) -> Result<WalFlushResult> {
-        match source {
+        let result = match source {
             WalFlushSource::BatchStore {
                 batch_store,
                 indexes,
@@ -436,7 +535,15 @@ impl WalFlusher {
                     .await
             }
             WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await,
+        };
+        // A terminal failure means the watermark can never advance; latch the
+        // poison so waiters wake with the typed error and later writes fail fast.
+        if let Err(e) = &result
+            && is_terminal_failure(e)
+        {
+            self.mark_terminal_failure(e);
         }
+        result
     }
 
     async fn flush_from_batch_store(
@@ -674,6 +781,42 @@ const MAX_APPEND_CREATE_CONFLICTS: usize = 1024;
 const APPEND_CONFLICT_REFRESH_INTERVAL: usize = 16;
 const MAX_CURSOR_PROBE: u64 = 4096;
 
+/// Retry policy for transient WAL persistence failures before the writer
+/// self-fences. On a non-conflict object-store error the appender retries the
+/// *same* WAL position up to `max_retries` times with exponential backoff from
+/// `base_delay`; exhausting the budget poisons the writer
+/// ([`Error::writer_poisoned`]).
+#[derive(Debug, Clone, Copy)]
+pub struct WalRetryConfig {
+    /// Maximum number of retries for a transient WAL write failure before the
+    /// writer self-fences.
+    pub max_retries: usize,
+    /// Base duration for exponential backoff between retry attempts.
+    pub base_delay: Duration,
+}
+
+impl Default for WalRetryConfig {
+    fn default() -> Self {
+        Self {
+            max_retries: 3,
+            base_delay: Duration::from_millis(50),
+        }
+    }
+}
+
+impl WalRetryConfig {
+    /// Backoff before retry `attempt` (1-based), capped so a wedged store can't
+    /// block the flush task indefinitely.
+    fn backoff(&self, attempt: usize) -> Duration {
+        const MAX_BACKOFF: Duration = Duration::from_secs(5);
+        let shift = attempt.saturating_sub(1).min(16) as u32;
+        self.base_delay
+            .checked_mul(1u32 << shift)
+            .unwrap_or(MAX_BACKOFF)
+            .min(MAX_BACKOFF)
+    }
+}
+
 /// Result of appending a WAL entry.
 #[derive(Debug, Clone)]
 pub struct WalAppendResult {
@@ -712,6 +855,8 @@ pub struct WalAppender {
     /// so reopened shards report the post-recovery cursor immediately;
     /// updated after each successful append.
     next_entry_position_hint: AtomicU64,
+    /// Retry budget for transient persistence failures before self-fencing.
+    retry: WalRetryConfig,
 }
 
 impl WalAppender {
@@ -740,6 +885,7 @@ impl WalAppender {
             manifest_store,
             writer_epoch,
             position_hint,
+            WalRetryConfig::default(),
         ))
     }
 
@@ -761,6 +907,7 @@ impl WalAppender {
         manifest_store: Arc<ShardManifestStore>,
         writer_epoch: u64,
         next_entry_position_hint_seed: u64,
+        retry: WalRetryConfig,
     ) -> Self {
         Self {
             object_store,
@@ -770,6 +917,7 @@ impl WalAppender {
             writer_epoch,
             next_entry_position: Mutex::new(None),
             next_entry_position_hint: AtomicU64::new(next_entry_position_hint_seed),
+            retry,
         }
     }
 
@@ -822,7 +970,12 @@ impl WalAppender {
             *next_pos = Some(self.discover_next_position().await?);
         }
 
+        // `conflicts` counts position races across the append; `io_attempts` is
+        // the per-position retry budget for transient PUT failures. The latter
+        // only grows while we sit on one position — the sole advancing branch is
+        // `io_attempts == 0`, so it never carries across positions.
         let mut conflicts = 0;
+        let mut io_attempts = 0;
         loop {
             let pos = next_pos.ok_or_else(|| {
                 Error::internal(format!(
@@ -853,7 +1006,20 @@ impl WalAppender {
                     });
                 }
                 Err(AtomicPutError::AlreadyExists) => {
-                    self.check_fenced().await?;
+                    self.check_fenced().await?; // surfaces a peer takeover as a typed fence
+                    // A slot we already failed to PUT is now occupied — ambiguous
+                    // (our lost-ack, or a peer). We can't advance-and-rewrite (would
+                    // duplicate) nor blindly accept it, so poison and let replay
+                    // reconcile on reopen.
+                    if io_attempts > 0 {
+                        return Err(Error::writer_poisoned(format!(
+                            "WAL position {} for shard {} was taken after a failed PUT; \
+                             in-memory state may diverge from the durable WAL, reopen to replay",
+                            pos, self.shard_id
+                        )));
+                    }
+                    // First touch of this slot: an ordinary position conflict
+                    // (stale cursor, our own earlier entries, or contention).
                     conflicts += 1;
                     if conflicts >= MAX_APPEND_CREATE_CONFLICTS {
                         return Err(Error::io(format!(
@@ -870,8 +1036,18 @@ impl WalAppender {
                     }
                 }
                 Err(AtomicPutError::Other(error)) => {
+                    // A successor fence is terminal — don't waste the retry budget.
                     self.check_fenced().await?;
-                    return Err(error);
+                    if io_attempts >= self.retry.max_retries {
+                        return Err(Error::writer_poisoned(format!(
+                            "WAL persistence failed for shard {} at position {} after {} retries; \
+                             in-memory state may diverge from the durable WAL, reopen to replay: {}",
+                            self.shard_id, pos, self.retry.max_retries, error
+                        )));
+                    }
+                    io_attempts += 1;
+                    tokio::time::sleep(self.retry.backoff(io_attempts)).await;
+                    // Retry the same position (next_pos unchanged).
                 }
             }
         }
@@ -882,6 +1058,52 @@ impl WalAppender {
         self.manifest_store.check_fenced(self.writer_epoch).await
     }
 
+    /// Drop a data-less sentinel at the WAL tip so the predecessor's next
+    /// `append` collides on PUT-IF-NOT-EXISTS and learns it is fenced, rather
+    /// than succeeding into the empty next slot. Call *before* replay: any
+    /// predecessor entry below the sentinel is then recovered, not orphaned.
+    /// On a lost slot race, re-probes one past the winner. Seeds next position
+    /// past the sentinel; returns the sentinel position.
+    pub(crate) async fn write_fence_sentinel(&self) -> Result<u64> {
+        let sentinel = Bytes::from(serialize_fence_sentinel(self.writer_epoch)?);
+        let mut next_pos = self.next_entry_position.lock().await;
+        let mut pos = match *next_pos {
+            Some(p) => p,
+            None => self.discover_next_position().await?,
+        };
+        let mut conflicts = 0;
+        loop {
+            match atomic_put(
+                self.object_store.as_ref(),
+                &self.wal_dir,
+                &wal_entry_filename(pos),
+                sentinel.clone(),
+            )
+            .await
+            {
+                Ok(()) => {
+                    let next = pos.checked_add(1).ok_or_else(|| {
+                        Error::io(format!("WAL position overflow for shard {}", self.shard_id))
+                    })?;
+                    *next_pos = Some(next);
+                    self.next_entry_position_hint.store(next, Ordering::SeqCst);
+                    return Ok(pos);
+                }
+                Err(AtomicPutError::AlreadyExists) => {
+                    conflicts += 1;
+                    if conflicts >= MAX_APPEND_CREATE_CONFLICTS {
+                        return Err(Error::io(format!(
+                            "fence sentinel write for shard {} failed after {} conflicts",
+                            self.shard_id, conflicts
+                        )));
+                    }
+                    pos = self.discover_next_position().await?;
+                }
+                Err(AtomicPutError::Other(error)) => return Err(error),
+            }
+        }
+    }
+
     async fn discover_next_position(&self) -> Result<u64> {
         if let Ok(Some(manifest)) = self.manifest_store.read_latest().await {
             let hint = manifest.wal_entry_position_last_seen;
@@ -1053,6 +1275,28 @@ fn serialize_appender_batches(batches: &[RecordBatch], writer_epoch: u64) -> Res
     Ok(buffer)
 }
 
+/// Data-less sentinel: an empty-schema Arrow IPC stream with the writer epoch
+/// and a marker flag, no batches. Reads back as `(epoch, [])` so replay skips
+/// it. See [`WalAppender::write_fence_sentinel`].
+fn serialize_fence_sentinel(writer_epoch: u64) -> Result<Vec<u8>> {
+    let mut metadata = std::collections::HashMap::new();
+    metadata.insert(WRITER_EPOCH_KEY.to_string(), writer_epoch.to_string());
+    metadata.insert(FENCE_SENTINEL_KEY.to_string(), "true".to_string());
+    let ipc_schema = Arc::new(ArrowSchema::new_with_metadata(
+        arrow_schema::Fields::empty(),
+        metadata,
+    ));
+    let mut buffer = Vec::new();
+    {
+        let mut writer = StreamWriter::try_new(&mut buffer, &ipc_schema)
+            .map_err(|e| Error::io(format!("failed to create fence sentinel IPC writer: {}", e)))?;
+        writer
+            .finish()
+            .map_err(|e| Error::io(format!("failed to finish fence sentinel IPC stream: {}", e)))?;
+    }
+    Ok(buffer)
+}
+
 fn deserialize_appender_batches(bytes: Bytes) -> Result<(u64, Vec<RecordBatch>)> {
     let cursor = Cursor::new(bytes);
     let reader = StreamReader::try_new(cursor, None)
@@ -1250,6 +1494,7 @@ async fn best_effort_cursor_update(manifest_store: &ShardManifestStore, entry_po
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::dataset::mem_wal::test_util::failing_memory_store;
     use arrow_array::{Int32Array, StringArray};
     use arrow_schema::{DataType, Field, Schema};
     use std::sync::Arc;
@@ -1302,6 +1547,7 @@ mod tests {
             writer_epoch,
             // Tests start with no entries, so seed the hint at 0.
             0,
+            WalRetryConfig::default(),
         ));
         WalFlusher::new(appender)
     }
@@ -1570,18 +1816,115 @@ mod tests {
         second.append(vec![batch.clone()]).await.unwrap();
 
         let err = first.check_fenced().await.unwrap_err();
-        assert!(
-            err.to_string().contains("Writer fenced"),
-            "expected fence error, got: {err}"
-        );
+        // A peer takeover is a typed peer fence, distinct from a self-poison.
+        assert_eq!(err.fence_reason(), Some(FenceReason::PeerClaimedEpoch));
 
         // Fenced writer's cached next_pos still points at 2; the conflict path
         // must surface the fence error rather than silently advance.
         let err = first.append(vec![batch]).await.unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PeerClaimedEpoch));
+    }
+
+    #[tokio::test]
+    async fn test_fence_sentinel_fences_predecessor_without_successor_write() {
+        // The race the sentinel closes: a successor claims a higher epoch but
+        // has NOT yet written any data batch. Without the sentinel, the
+        // predecessor's next append lands in the empty next slot, succeeds,
+        // and false-acks. With the sentinel, the predecessor collides.
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+
+        let first = WalAppender::open(store.clone(), base_path.clone(), shard_id, 0)
+            .await
+            .unwrap();
+        let schema = create_test_schema();
+        let batch = create_test_batch(&schema, 1);
+        first.append(vec![batch.clone()]).await.unwrap(); // position 1
+
+        // Successor claims epoch 2 and drops a sentinel at the tip (position 2)
+        // — but writes no data of its own.
+        let second = WalAppender::open(store.clone(), base_path.clone(), shard_id, 0)
+            .await
+            .unwrap();
+        assert_eq!(second.writer_epoch(), 2);
+        let sentinel_pos = second.write_fence_sentinel().await.unwrap();
+        assert_eq!(sentinel_pos, 2, "sentinel should land at the tip");
+
+        // Predecessor's next append collides with the sentinel and is fenced.
+        let err = first.append(vec![batch.clone()]).await.unwrap_err();
         assert!(
             err.to_string().contains("Writer fenced"),
             "expected fence error from append, got: {err}"
         );
+
+        // The sentinel is data-less: a tailer reads it back as zero batches so
+        // replay skips it.
+        let tailer = WalTailer::new(store.clone(), base_path.clone(), shard_id);
+        let entry = tailer.read_entry(sentinel_pos).await.unwrap().unwrap();
+        assert_eq!(entry.writer_epoch, 2);
+        assert!(entry.batches.is_empty(), "sentinel must carry no batches");
+
+        // Successor's own writes land after the sentinel (position 3).
+        let res = second.append(vec![batch]).await.unwrap();
+        assert_eq!(res.entry_position, 3);
+    }
+
+    // Regression: a fenced WAL flush never advances the durability watermark.
+    // A `durable_write` put waits on a `BatchDurableWatcher`, so without
+    // terminal-failure propagation the watcher blocks forever (the predecessor
+    // pod's HTTP write hangs until the client times out). The flusher must
+    // surface the fence through the watcher so the caller fails fast with 410.
+    #[tokio::test]
+    async fn test_durable_watcher_aborts_on_fence_instead_of_hanging() {
+        let (store, base_path, _temp_dir) = create_local_store().await;
+        let shard_id = Uuid::new_v4();
+        let schema = create_test_schema();
+
+        // Predecessor claims epoch 1 and writes one entry (position 1), seeding
+        // its cached next position at 2. The flusher shares this appender.
+        let first = Arc::new(
+            WalAppender::open(store.clone(), base_path.clone(), shard_id, 0)
+                .await
+                .unwrap(),
+        );
+        assert_eq!(first.writer_epoch(), 1);
+        first
+            .append(vec![create_test_batch(&schema, 1)])
+            .await
+            .unwrap();
+        let flusher = WalFlusher::new(Arc::clone(&first));
+
+        // Successor claims epoch 2 and drops a sentinel at the predecessor's
+        // next slot (position 2) — a rolling-restart pod replacement.
+        let second = WalAppender::open(store.clone(), base_path.clone(), shard_id, 0)
+            .await
+            .unwrap();
+        assert_eq!(second.writer_epoch(), 2);
+        assert_eq!(second.write_fence_sentinel().await.unwrap(), 2);
+
+        // A durable put on the predecessor: stage a batch and track it.
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+        let mut watcher = flusher.track_batch(0);
+
+        // Flushing collides with the sentinel and fences. Both the flush result
+        // and the watcher must report the fence — and the watcher must resolve
+        // promptly, not block on a watermark that can never advance.
+        let source = batch_store_source(&batch_store);
+        let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err();
+        assert!(
+            is_fence_error(&flush_err),
+            "expected fence error from flush, got: {flush_err}"
+        );
+
+        let waited = tokio::time::timeout(std::time::Duration::from_secs(5), watcher.wait()).await;
+        let err = waited
+            .expect("watcher.wait() hung after a fenced flush")
+            .expect_err("watcher must surface the fence, not report success");
+        assert!(
+            is_fence_error(&err),
+            "watcher must report the fence so the HTTP layer maps 410, got: {err}"
+        );
     }
 
     #[tokio::test]
@@ -1638,4 +1981,137 @@ mod tests {
         // Three entries from a fresh shard land at 1, 2, 3, so next is 4.
         assert_eq!(tailer.next_position().await.unwrap(), 4);
     }
+
+    // A transient PUT failure is retried at the same position and then succeeds.
+    #[tokio::test]
+    async fn test_append_retries_transient_failure_then_succeeds() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.fail_wal_puts(2); // 2 failures, under the default budget of 3
+        let appender = WalAppender::open(store, base, shard_id, 0).await.unwrap();
+
+        let schema = create_test_schema();
+        let res = appender
+            .append(vec![create_test_batch(&schema, 1)])
+            .await
+            .unwrap();
+        assert_eq!(res.entry_position, FIRST_WAL_ENTRY_POSITION);
+        assert_eq!(controls.attempts(), 3); // 2 failed + 1 succeeded
+    }
+
+    // Exhausting the retry budget poisons the writer with a typed persistence
+    // failure (distinct from a peer fence).
+    #[tokio::test]
+    async fn test_append_poisons_after_exhausting_retries() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.fail_wal_puts(usize::MAX);
+        let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2));
+        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
+        let appender = WalAppender::with_claimed_epoch(
+            store,
+            base,
+            shard_id,
+            manifest_store,
+            epoch,
+            0,
+            WalRetryConfig {
+                max_retries: 2,
+                base_delay: Duration::from_millis(1),
+            },
+        );
+
+        let schema = create_test_schema();
+        let err = appender
+            .append(vec![create_test_batch(&schema, 1)])
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert_eq!(controls.attempts(), 3); // io_attempts 0,1,2 then poison
+    }
+
+    // A lost acknowledgement (PUT errored but landed) poisons the writer without
+    // ever writing a duplicate entry at the next position.
+    #[tokio::test]
+    async fn test_append_lost_ack_poisons_without_duplicate() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.set_lost_ack(true);
+        controls.fail_wal_puts(1);
+        let appender = WalAppender::open(store.clone(), base.clone(), shard_id, 0)
+            .await
+            .unwrap();
+
+        let schema = create_test_schema();
+        let err = appender
+            .append(vec![create_test_batch(&schema, 1)])
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // The entry landed exactly once; the next slot stays empty.
+        let tailer = WalTailer::new(store, base, shard_id);
+        assert!(
+            tailer
+                .read_entry(FIRST_WAL_ENTRY_POSITION)
+                .await
+                .unwrap()
+                .is_some()
+        );
+        assert!(
+            tailer
+                .read_entry(FIRST_WAL_ENTRY_POSITION + 1)
+                .await
+                .unwrap()
+                .is_none()
+        );
+    }
+
+    // A persistence failure during flush latches the poison: the flush result,
+    // `check_poisoned`, and the durability watcher all report the typed error
+    // (rather than the watcher hanging on a watermark that never advances).
+    #[tokio::test]
+    async fn test_flush_persistence_failure_poisons_and_wakes_waiter() {
+        let (store, base, controls) = failing_memory_store().await;
+        let shard_id = Uuid::new_v4();
+        controls.fail_wal_puts(usize::MAX);
+        let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2));
+        let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap();
+        let appender = Arc::new(WalAppender::with_claimed_epoch(
+            store,
+            base,
+            shard_id,
+            manifest_store,
+            epoch,
+            0,
+            WalRetryConfig {
+                max_retries: 1,
+                base_delay: Duration::from_millis(1),
+            },
+        ));
+        let flusher = WalFlusher::new(appender);
+
+        let schema = create_test_schema();
+        let batch_store = Arc::new(BatchStore::with_capacity(10));
+        batch_store.append(create_test_batch(&schema, 1)).unwrap();
+        let mut watcher = flusher.track_batch(0);
+
+        let source = batch_store_source(&batch_store);
+        let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err();
+        assert_eq!(
+            flush_err.fence_reason(),
+            Some(FenceReason::PersistenceFailure)
+        );
+
+        assert_eq!(
+            flusher.check_poisoned().unwrap_err().fence_reason(),
+            Some(FenceReason::PersistenceFailure)
+        );
+
+        let waited = tokio::time::timeout(Duration::from_secs(5), watcher.wait())
+            .await
+            .expect("watcher hung after a poisoning flush")
+            .expect_err("watcher must surface the poison");
+        assert_eq!(waited.fence_reason(), Some(FenceReason::PersistenceFailure));
+    }
 }
diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs
index 0788b657366..3cb4586ebb4 100644
--- a/rust/lance/src/dataset/mem_wal/write.rs
+++ b/rust/lance/src/dataset/mem_wal/write.rs
@@ -18,14 +18,14 @@ use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, RwLock as StdRwLock};
 use std::time::{Duration, Instant};
 
-use arrow_array::RecordBatch;
+use arrow_array::{ArrayRef, BooleanArray, RecordBatch, new_null_array};
 use arrow_schema::Schema as ArrowSchema;
 use async_trait::async_trait;
 use lance_core::datatypes::Schema;
 use lance_core::{Error, Result};
 use lance_index::mem_wal::ShardManifest;
 use lance_index::vector::hnsw::builder::HnswBuildParams;
-use lance_io::object_store::ObjectStore;
+use lance_io::object_store::{ObjectStore, ObjectStoreParams};
 use log::{debug, error, info, warn};
 use object_store::path::Path;
 use tokio::sync::{RwLock, mpsc};
@@ -44,12 +44,16 @@ pub use super::memtable::batch_store::{BatchStore, StoreFull, StoredBatch};
 pub use super::memtable::flush::MemTableFlusher;
 pub use super::memtable::scanner::MemTableScanner;
 pub use super::util::{WatchableOnceCell, WatchableOnceCellReader};
-pub use super::wal::{WalEntry, WalEntryData, WalFlushResult, WalFlusher};
+pub use super::wal::{WalEntry, WalEntryData, WalFlushFailure, WalFlushResult, WalFlusher};
 
 use super::memtable::flush::TriggerMemTableFlush;
+use super::scanner::GenerationWarmer;
 use super::wal::{
-    TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState, WalTailer, empty_flush_result,
+    BatchDurableWatcher, TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState,
+    WalRetryConfig, WalTailer, empty_flush_result,
 };
+use super::{TOMBSTONE, schema_with_tombstone};
+use crate::session::Session;
 
 use super::manifest::ShardManifestStore;
 
@@ -106,6 +110,16 @@ pub struct ShardWriterConfig {
     /// Default: 100ms
     pub max_wal_flush_interval: Option<Duration>,
 
+    /// Times a failed WAL PUT is retried (same position, exponential backoff)
+    /// before the writer self-fences. On exhaustion the writer is poisoned
+    /// (`Error::writer_poisoned`) and must be reopened to replay. Absorbs
+    /// transient errors beyond `object_store`'s own retries. Default: 3.
+    pub max_wal_persist_retries: usize,
+
+    /// Base backoff before the first WAL persistence retry; subsequent retries
+    /// back off exponentially (capped). Default: 50ms.
+    pub wal_persist_retry_base_delay: Duration,
+
     /// Maximum MemTable size in bytes before triggering a flush to storage.
     ///
     /// MemTable size is checked every `max_wal_flush_interval` (during WAL flush ticks).
@@ -177,6 +191,21 @@ pub struct ShardWriterConfig {
     /// Default: 60 seconds
     pub stats_log_interval: Option<Duration>,
 
+    /// How long a frozen memtable lingers in memory after its flush commits,
+    /// before it is evicted and served only from the on-disk flushed dataset.
+    ///
+    /// `Duration::ZERO` (the default) disables retention: evict on commit, no
+    /// sweep ticker. Correct for single-shot queries, which can't observe a
+    /// generation evicted mid-read.
+    ///
+    /// A non-zero value is required only for queries split across reads (e.g.
+    /// fresh tier and base table read separately, then deduped): the flushed
+    /// dataset loses the per-batch boundaries that bound as-of membership
+    /// (see [`crate::dataset::mem_wal::scanner::FreshTierWatermark`]), so a
+    /// generation evicted between a query's reads can serve a stale row. Set it
+    /// above the worst-case multi-part query latency, with margin.
+    pub frozen_memtable_grace: Duration,
+
     /// Whether to maintain an in-memory MemTable on top of the WAL.
     ///
     /// When `true` (default), the writer maintains an in-memory `MemTable`,
@@ -216,6 +245,22 @@ pub struct ShardWriterConfig {
     ///
     /// Default: empty.
     pub hnsw_params: HashMap<String, HnswBuildParams>,
+
+    /// Optional warmer fired pre-commit for each new generation (zero cold reads
+    /// on first query). Wired to the flusher; supplied by the consumer (e.g. the
+    /// WAL pod). Default: `None`.
+    pub warmer: Option<Arc<dyn GenerationWarmer>>,
+
+    /// Store params the base dataset was opened with, reused for the flusher's
+    /// opens + writes (base + generations). Injected by `mem_wal_writer`; set
+    /// these to the params of the dataset at `base_uri`, not to params bound to
+    /// some other path — generation URIs are derived from them.
+    /// Default: `None` (open by URI alone).
+    pub store_params: Option<ObjectStoreParams>,
+
+    /// Session for those opens, injected alongside `store_params`.
+    /// Default: `None`.
+    pub session: Option<Arc<Session>>,
 }
 
 impl Default for ShardWriterConfig {
@@ -227,17 +272,23 @@ impl Default for ShardWriterConfig {
             sync_indexed_write: true,
             max_wal_buffer_size: 10 * 1024 * 1024, // 10MB
             max_wal_flush_interval: Some(Duration::from_millis(100)), // 100ms
-            max_memtable_size: 256 * 1024 * 1024,  // 256MB
-            max_memtable_rows: 100_000,            // 100k rows
-            max_memtable_batches: 8_000,           // 8k batches
+            max_wal_persist_retries: 3,
+            wal_persist_retry_base_delay: Duration::from_millis(50),
+            max_memtable_size: 256 * 1024 * 1024, // 256MB
+            max_memtable_rows: 100_000,           // 100k rows
+            max_memtable_batches: 8_000,          // 8k batches
             manifest_scan_batch_size: 2,
             max_unflushed_memtable_bytes: 1024 * 1024 * 1024, // 1GB
             backpressure_log_interval: Duration::from_secs(30),
             async_index_buffer_rows: 10_000,
             async_index_interval: Duration::from_secs(1),
             stats_log_interval: Some(Duration::from_secs(60)), // 1 minute
+            frozen_memtable_grace: Duration::ZERO,
             enable_memtable: true,
             hnsw_params: HashMap::new(),
+            warmer: None,
+            store_params: None,
+            session: None,
         }
     }
 }
@@ -281,6 +332,20 @@ impl ShardWriterConfig {
         self
     }
 
+    /// Set the number of WAL persistence retries before the writer self-fences.
+    /// See [`ShardWriterConfig::max_wal_persist_retries`].
+    pub fn with_max_wal_persist_retries(mut self, retries: usize) -> Self {
+        self.max_wal_persist_retries = retries;
+        self
+    }
+
+    /// Set the base backoff before the first WAL persistence retry.
+    /// See [`ShardWriterConfig::wal_persist_retry_base_delay`].
+    pub fn with_wal_persist_retry_base_delay(mut self, delay: Duration) -> Self {
+        self.wal_persist_retry_base_delay = delay;
+        self
+    }
+
     /// Set maximum MemTable size.
     pub fn with_max_memtable_size(mut self, size: usize) -> Self {
         self.max_memtable_size = size;
@@ -335,6 +400,13 @@ impl ShardWriterConfig {
         self
     }
 
+    /// Set how long a flushed memtable lingers in memory before eviction. MUST
+    /// exceed the maximum query elapsed time — see `frozen_memtable_grace`.
+    pub fn with_frozen_memtable_grace(mut self, grace: Duration) -> Self {
+        self.frozen_memtable_grace = grace;
+        self
+    }
+
     /// Toggle the in-memory MemTable layer. See `enable_memtable` for the
     /// full WAL-only-mode contract. Defaults to `true`.
     pub fn with_enable_memtable(mut self, enable: bool) -> Self {
@@ -708,6 +780,15 @@ pub struct WriteResult {
     pub batch_positions: std::ops::Range<usize>,
 }
 
+/// A sealed memtable kept queryable in memory. `flushed_at_ms` is `None` while
+/// the generation is still awaiting (or retrying) its flush, and `Some(t)` once
+/// the flush commits — after which it lingers for `frozen_memtable_grace` so
+/// in-flight as-of reads keep batch-resolved membership, then is swept.
+struct FrozenMemTable {
+    memtable: Arc<MemTable>,
+    flushed_at_ms: Option<u64>,
+}
+
 /// ShardWriter state shared across tasks.
 struct WriterState {
     memtable: MemTable,
@@ -716,12 +797,13 @@ struct WriterState {
     frozen_memtable_bytes: usize,
     /// Flush watchers for frozen memtables (for backpressure).
     frozen_flush_watchers: VecDeque<(usize, DurabilityWatcher)>,
-    /// Sealed-but-undrained memtables, kept queryable so a concurrent reader
-    /// sees no hole between `freeze_memtable` and the flush task's manifest
-    /// commit. Pushed in `freeze_memtable`; removed by generation in
-    /// `flush_memtable` on commit success only (retained on failure until a
-    /// later flush or WAL replay on reopen).
-    frozen_memtables: VecDeque<Arc<MemTable>>,
+    /// Sealed memtables, kept queryable so a concurrent reader sees no hole
+    /// between `freeze_memtable` and the flush task's manifest commit, and for
+    /// `frozen_memtable_grace` beyond it so as-of reads stay batch-resolved.
+    /// Pushed in `freeze_memtable`; stamped `flushed_at_ms` by `flush_memtable`
+    /// on commit success only (retained un-stamped on failure until a later
+    /// flush or WAL replay on reopen); swept after the grace by `SweepExpired`.
+    frozen_memtables: VecDeque<FrozenMemTable>,
     /// Flag to prevent duplicate memtable flush requests.
     flush_requested: bool,
     /// Counter for WAL flush threshold crossings.
@@ -801,13 +883,24 @@ async fn replay_memtable_from_wal(
             None => break,
             Some(entry) => {
                 if entry.writer_epoch > our_epoch {
-                    return Err(Error::io(format!(
+                    return Err(Error::fenced_by_peer(format!(
                         "WAL replay aborted: entry at position {} has writer_epoch {} > our claimed epoch {} for shard {} (writer was fenced during open)",
                         position, entry.writer_epoch, our_epoch, shard_id
                     )));
                 }
+                // Fence sentinels deserialize to zero batches and are skipped
+                // here — they carry only a position, no rows.
                 if !entry.batches.is_empty() {
-                    memtable.insert_batches_only(entry.batches).await?;
+                    // Entries written before deletes existed lack `_tombstone`;
+                    // inject `false` so they match the extended memtable schema.
+                    // Normal entries already carry it and pass through unchanged.
+                    let target_schema = memtable.schema().clone();
+                    let batches = entry
+                        .batches
+                        .into_iter()
+                        .map(|b| ensure_tombstone_column(b, &target_schema))
+                        .collect::<Result<Vec<_>>>()?;
+                    memtable.insert_batches_only(batches).await?;
                 }
                 position = position.checked_add(1).ok_or_else(|| {
                     Error::io(format!(
@@ -844,6 +937,79 @@ async fn replay_memtable_from_wal(
     Ok(position)
 }
 
+/// Pair each primary-key column name with its field id (both derived from the
+/// schema's primary key, in the same order) for [`IndexStore::enable_pk_index`].
+fn pk_index_columns(pk_columns: &[String], pk_field_ids: &[i32]) -> Vec<(String, i32)> {
+    pk_columns
+        .iter()
+        .cloned()
+        .zip(pk_field_ids.iter().copied())
+        .collect()
+}
+
+/// Ensure `batch` carries the `_tombstone` column required by the extended
+/// memtable schema, injecting `false` for every row when it is absent.
+///
+/// Used on the normal write path ([`ShardWriter::put`]) where callers pass
+/// base-shaped batches, and on WAL replay of entries written before deletes
+/// existed (legacy entries lack the column). A batch that already carries
+/// `_tombstone` (a normal replayed entry) is returned unchanged.
+fn ensure_tombstone_column(
+    batch: RecordBatch,
+    target_schema: &Arc<ArrowSchema>,
+) -> Result<RecordBatch> {
+    if batch.schema().column_with_name(TOMBSTONE).is_some() {
+        return Ok(batch);
+    }
+    let n = batch.num_rows();
+    let mut columns: Vec<ArrayRef> = batch.columns().to_vec();
+    columns.push(Arc::new(BooleanArray::from(vec![false; n])));
+    RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| {
+        Error::invalid_input(format!(
+            "failed to inject _tombstone column (does the batch match the base schema?): {}",
+            e
+        ))
+    })
+}
+
+/// Build a tombstone batch from a key-only `keys` batch: the primary key
+/// columns are carried through, `_tombstone` is set to `true`, and every other
+/// column in the memtable schema is null.
+///
+/// Errors if `keys` is missing a primary key column, or if a non-PK column is
+/// non-nullable (a tombstone must null it) — surfaced via the `RecordBatch`
+/// validation.
+fn build_tombstone_batch(
+    keys: &RecordBatch,
+    target_schema: &Arc<ArrowSchema>,
+    pk_columns: &[String],
+) -> Result<RecordBatch> {
+    let n = keys.num_rows();
+    let mut columns: Vec<ArrayRef> = Vec::with_capacity(target_schema.fields().len());
+    for field in target_schema.fields() {
+        let name = field.name();
+        if name == TOMBSTONE {
+            columns.push(Arc::new(BooleanArray::from(vec![true; n])));
+        } else if pk_columns.iter().any(|c| c == name) {
+            let col = keys.column_by_name(name).ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "delete keys batch is missing primary key column '{}'",
+                    name
+                ))
+            })?;
+            columns.push(col.clone());
+        } else {
+            columns.push(new_null_array(field.data_type(), n));
+        }
+    }
+    RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| {
+        Error::invalid_input(format!(
+            "failed to build tombstone batch (is every non-primary-key column nullable?): {}",
+            e
+        ))
+    })
+}
+
 /// Shared state for writer operations.
 struct SharedWriterState {
     state: Arc<RwLock<WriterState>>,
@@ -853,6 +1019,9 @@ struct SharedWriterState {
     config: ShardWriterConfig,
     schema: Arc<ArrowSchema>,
     pk_field_ids: Vec<i32>,
+    /// Primary-key column names, used to (re)enable the PK-position index on
+    /// each fresh active memtable created at freeze.
+    pk_columns: Vec<String>,
     max_memtable_batches: usize,
     max_memtable_rows: usize,
     index_configs: Vec<MemIndexConfig>,
@@ -868,6 +1037,7 @@ impl SharedWriterState {
         config: ShardWriterConfig,
         schema: Arc<ArrowSchema>,
         pk_field_ids: Vec<i32>,
+        pk_columns: Vec<String>,
         max_memtable_batches: usize,
         max_memtable_rows: usize,
         index_configs: Vec<MemIndexConfig>,
@@ -880,6 +1050,7 @@ impl SharedWriterState {
             config,
             schema,
             pk_field_ids,
+            pk_columns,
             max_memtable_batches,
             max_memtable_rows,
             index_configs,
@@ -905,13 +1076,17 @@ impl SharedWriterState {
             self.max_memtable_batches,
         )?;
 
-        if !self.index_configs.is_empty() {
-            let indexes = Arc::new(IndexStore::from_configs(
+        // Build an IndexStore when there are user indexes *or* a primary key:
+        // the PK dedup index (and its flushed on-disk sidecar) is required for
+        // cross-generation dedup even when no secondary index is configured.
+        if !self.index_configs.is_empty() || !self.pk_columns.is_empty() {
+            let mut indexes = IndexStore::from_configs(
                 &self.index_configs,
                 self.max_memtable_rows,
                 self.max_memtable_batches,
-            )?);
-            new_memtable.set_indexes_arc(indexes);
+            )?;
+            indexes.enable_pk_index(&pk_index_columns(&self.pk_columns, &self.pk_field_ids));
+            new_memtable.set_indexes_arc(Arc::new(indexes));
         }
 
         let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable);
@@ -919,8 +1094,9 @@ impl SharedWriterState {
         let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion();
 
         if pending_wal_range.is_some() {
-            let completion_cell: WatchableOnceCell<std::result::Result<WalFlushResult, String>> =
-                WatchableOnceCell::new();
+            let completion_cell: WatchableOnceCell<
+                std::result::Result<WalFlushResult, WalFlushFailure>,
+            > = WatchableOnceCell::new();
             let completion_reader = completion_cell.reader();
             old_memtable.set_wal_flush_completion(completion_reader);
 
@@ -947,10 +1123,13 @@ impl SharedWriterState {
 
         let frozen_memtable = Arc::new(old_memtable);
 
-        // Keep this generation queryable until its manifest commit lands
-        // (dropped in `flush_memtable`, success only). Arc refcount, not a
-        // copy — the flush task holds it alive for the whole drain anyway.
-        state.frozen_memtables.push_back(frozen_memtable.clone());
+        // Keep this generation queryable past its manifest commit (swept after
+        // the grace by `SweepExpired`). Arc refcount, not a copy — the flush
+        // task holds it alive for the whole drain anyway.
+        state.frozen_memtables.push_back(FrozenMemTable {
+            memtable: frozen_memtable.clone(),
+            flushed_at_ms: None,
+        });
 
         debug!(
             "Frozen memtable generation {}, pending_count = {}",
@@ -958,7 +1137,7 @@ impl SharedWriterState {
             state.frozen_flush_watchers.len()
         );
 
-        let _ = self.memtable_flush_tx.send(TriggerMemTableFlush {
+        let _ = self.memtable_flush_tx.send(TriggerMemTableFlush::Flush {
             memtable: frozen_memtable,
             done: None,
         });
@@ -1169,6 +1348,11 @@ impl ShardWriter {
             ));
         }
 
+        // Callers pass the base schema; lance owns the `_tombstone` column and
+        // appends it here so the memtable/generation schema = base + tombstone.
+        // Idempotent, so a reopen that already extended the schema is a no-op.
+        let schema = schema_with_tombstone(&schema);
+
         let base_uri = base_uri.into();
         let shard_id = config.shard_id;
         let manifest_store = Arc::new(ShardManifestStore::new(
@@ -1206,8 +1390,18 @@ impl ShardWriter {
             manifest_store.clone(),
             epoch,
             position_hint_seed,
+            WalRetryConfig {
+                max_retries: config.max_wal_persist_retries,
+                base_delay: config.wal_persist_retry_base_delay,
+            },
         ));
 
+        // Fence the predecessor before replay (see `write_fence_sentinel`).
+        // Epoch 1 is a fresh shard with no predecessor to fence.
+        if epoch >= 2 {
+            wal_appender.write_fence_sentinel().await?;
+        }
+
         // Create WAL flusher backed by the shared appender.
         let mut wal_flusher = WalFlusher::new(wal_appender);
 
@@ -1279,11 +1473,9 @@ impl ShardWriter {
     ) -> Result<WriterMode> {
         // Create MemTable with primary key field IDs from schema
         let lance_schema = Schema::try_from(schema.as_ref())?;
-        let pk_field_ids: Vec<i32> = lance_schema
-            .unenforced_primary_key()
-            .iter()
-            .map(|f| f.id)
-            .collect();
+        let pk_fields = lance_schema.unenforced_primary_key();
+        let pk_field_ids: Vec<i32> = pk_fields.iter().map(|f| f.id).collect();
+        let pk_columns: Vec<String> = pk_fields.iter().map(|f| f.name.clone()).collect();
         let mut memtable = MemTable::with_capacity(
             schema.clone(),
             manifest.current_generation,
@@ -1292,14 +1484,18 @@ impl ShardWriter {
             config.max_memtable_batches,
         )?;
 
-        // Create indexes if configured and set them on the MemTable.
-        if !index_configs.is_empty() {
-            let indexes = Arc::new(IndexStore::from_configs(
+        // Create indexes if configured and set them on the MemTable. The
+        // PK-position index is enabled before any WAL replay below so replayed
+        // rows are recorded in it. A primary key alone (no secondary index)
+        // still needs the PK index so flush writes its on-disk dedup sidecar.
+        if !index_configs.is_empty() || !pk_columns.is_empty() {
+            let mut indexes = IndexStore::from_configs(
                 index_configs,
                 config.max_memtable_rows,
                 config.max_memtable_batches,
-            )?);
-            memtable.set_indexes_arc(indexes);
+            )?;
+            indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids));
+            memtable.set_indexes_arc(Arc::new(indexes));
         }
 
         // Replay any WAL entries written after the last successfully-flushed
@@ -1349,13 +1545,11 @@ impl ShardWriter {
 
         let (memtable_flush_tx, memtable_flush_rx) = mpsc::unbounded_channel();
 
-        let flusher = Arc::new(MemTableFlusher::new(
-            object_store,
-            base_path,
-            base_uri,
-            shard_id,
-            manifest_store,
-        ));
+        let flusher = Arc::new(
+            MemTableFlusher::new(object_store, base_path, base_uri, shard_id, manifest_store)
+                .with_warmer(config.warmer.clone())
+                .with_storage_context(config.store_params.clone(), config.session.clone()),
+        );
 
         let backpressure = BackpressureController::new(config.clone());
 
@@ -1370,8 +1564,14 @@ impl ShardWriter {
 
         // Background MemTable flush handler — frozen memtable to Lance file.
         // It rebuilds the same secondary indexes on each flushed generation.
-        let memtable_handler =
-            MemTableFlushHandler::new(state.clone(), flusher, epoch, index_configs.to_vec(), stats);
+        let memtable_handler = MemTableFlushHandler::new(
+            state.clone(),
+            flusher,
+            epoch,
+            index_configs.to_vec(),
+            stats,
+            config.frozen_memtable_grace,
+        );
         task_executor.add_handler(
             "memtable_flusher".to_string(),
             Box::new(memtable_handler),
@@ -1387,6 +1587,7 @@ impl ShardWriter {
             config.clone(),
             schema.clone(),
             pk_field_ids,
+            pk_columns,
             config.max_memtable_batches,
             config.max_memtable_rows,
             index_configs.to_vec(),
@@ -1452,14 +1653,7 @@ impl ShardWriter {
     /// `AlreadyExists`, indicating this writer has been fenced.
     #[instrument(name = "sw_put", level = "info", skip_all, fields(batch_count = batches.len(), shard_id = %self.config.shard_id))]
     pub async fn put(&self, batches: Vec<RecordBatch>) -> Result<WriteResult> {
-        if batches.is_empty() {
-            return Err(Error::invalid_input("Cannot write empty batch list"));
-        }
-        for (i, batch) in batches.iter().enumerate() {
-            if batch.num_rows() == 0 {
-                return Err(Error::invalid_input(format!("Batch {} is empty", i)));
-            }
-        }
+        Self::validate_non_empty(&batches)?;
 
         match &self.mode {
             WriterMode::MemTable {
@@ -1467,6 +1661,13 @@ impl ShardWriter {
                 writer_state,
                 backpressure,
             } => {
+                // Inject `_tombstone = false` so the batch matches the
+                // extended memtable schema; callers only ever pass base-shaped
+                // batches and never name the column.
+                let batches = batches
+                    .into_iter()
+                    .map(|b| ensure_tombstone_column(b, &writer_state.schema))
+                    .collect::<Result<Vec<_>>>()?;
                 self.put_memtable(batches, state, writer_state, backpressure)
                     .await
             }
@@ -1482,6 +1683,138 @@ impl ShardWriter {
         }
     }
 
+    /// Delete rows by primary key.
+    ///
+    /// Each batch in `keys` must carry the shard's primary key column(s); other
+    /// columns are ignored. lance builds a tombstone row per key — the primary
+    /// key plus `_tombstone = true` and null in every other column — and
+    /// appends it like an ordinary write. The tombstone is the newest value for
+    /// its key: it wins newest-per-PK resolution (suppressing the older real
+    /// row) and is then dropped from query results.
+    ///
+    /// Only supported in memtable mode. Because a tombstone nulls every non-PK
+    /// column, those columns must be nullable in the base schema; a delete
+    /// against a schema with a non-nullable non-PK column errors.
+    ///
+    /// ```
+    /// # use lance::Result;
+    /// # use lance::dataset::mem_wal::ShardWriter;
+    /// # use arrow_array::RecordBatch;
+    /// # async fn doc(writer: &ShardWriter, keys: Vec<RecordBatch>) -> Result<()> {
+    /// writer.delete(keys).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    #[instrument(name = "sw_delete", level = "info", skip_all, fields(batch_count = keys.len(), shard_id = %self.config.shard_id))]
+    pub async fn delete(&self, keys: Vec<RecordBatch>) -> Result<WriteResult> {
+        let (result, watcher) = self.delete_no_wait(keys).await?;
+        // Wait for durability if configured (mirrors `put` → `put_memtable`).
+        if let Some(mut watcher) = watcher {
+            watcher.wait().await?;
+        }
+        Ok(result)
+    }
+
+    /// Like [`Self::delete`], but returns the durability watcher *without*
+    /// awaiting it — the tombstone lands in the in-memory tier the instant this
+    /// returns, but the index-driven LSM read only folds it once the watcher
+    /// resolves the flush (which advances the visibility watermark and updates
+    /// the PK index). The delete analog of [`Self::put_no_wait`], so a caller
+    /// can hold an external lock across only the in-memory insert and await
+    /// durability after releasing it. MemTable mode only.
+    #[instrument(name = "sw_delete_no_wait", level = "info", skip_all, fields(batch_count = keys.len(), shard_id = %self.config.shard_id))]
+    pub async fn delete_no_wait(
+        &self,
+        keys: Vec<RecordBatch>,
+    ) -> Result<(WriteResult, Option<BatchDurableWatcher>)> {
+        if keys.is_empty() {
+            return Err(Error::invalid_input("Cannot delete with empty key list"));
+        }
+        for (i, batch) in keys.iter().enumerate() {
+            if batch.num_rows() == 0 {
+                return Err(Error::invalid_input(format!("Key batch {} is empty", i)));
+            }
+        }
+
+        match &self.mode {
+            WriterMode::MemTable {
+                state,
+                writer_state,
+                backpressure,
+            } => {
+                if writer_state.pk_columns.is_empty() {
+                    return Err(Error::invalid_input(
+                        "delete requires a primary key, but this shard has no primary key columns",
+                    ));
+                }
+                let tombstones = keys
+                    .into_iter()
+                    .map(|k| {
+                        build_tombstone_batch(&k, &writer_state.schema, &writer_state.pk_columns)
+                    })
+                    .collect::<Result<Vec<_>>>()?;
+                self.put_memtable_no_wait(tombstones, state, writer_state, backpressure)
+                    .await
+            }
+            WriterMode::WalOnly { .. } => Err(Error::invalid_input(
+                "delete is only supported in memtable mode (enable_memtable = true)",
+            )),
+        }
+    }
+
+    /// Like [`Self::put`], but returns the durability watcher *without* awaiting
+    /// it. The row lands in the in-memory tier the instant this returns, but the
+    /// index-driven LSM read only surfaces it once the watcher resolves the
+    /// flush (which advances the visibility watermark and updates the indexes);
+    /// the caller awaits durability via the watcher (`None` when `durable_write`
+    /// is off).
+    ///
+    /// This lets a caller hold an *external* lock across only the in-memory
+    /// read-merge-insert and await durability after releasing it, so concurrent
+    /// flushes still coalesce. The insert stays guarded by the internal
+    /// `state_lock`, so `BatchStore`'s single-writer invariant holds regardless.
+    ///
+    /// MemTable mode only; errors in WAL-only mode (no in-memory tier).
+    #[instrument(name = "sw_put_no_wait", level = "info", skip_all, fields(batch_count = batches.len(), shard_id = %self.config.shard_id))]
+    pub async fn put_no_wait(
+        &self,
+        batches: Vec<RecordBatch>,
+    ) -> Result<(WriteResult, Option<BatchDurableWatcher>)> {
+        Self::validate_non_empty(&batches)?;
+
+        match &self.mode {
+            WriterMode::MemTable {
+                state,
+                writer_state,
+                backpressure,
+            } => {
+                // Inject `_tombstone = false` to match the extended memtable
+                // schema, mirroring `put`.
+                let batches = batches
+                    .into_iter()
+                    .map(|b| ensure_tombstone_column(b, &writer_state.schema))
+                    .collect::<Result<Vec<_>>>()?;
+                self.put_memtable_no_wait(batches, state, writer_state, backpressure)
+                    .await
+            }
+            WriterMode::WalOnly { .. } => Err(Error::invalid_input(
+                "put_no_wait is only supported in MemTable mode",
+            )),
+        }
+    }
+
+    fn validate_non_empty(batches: &[RecordBatch]) -> Result<()> {
+        if batches.is_empty() {
+            return Err(Error::invalid_input("Cannot write empty batch list"));
+        }
+        for (i, batch) in batches.iter().enumerate() {
+            if batch.num_rows() == 0 {
+                return Err(Error::invalid_input(format!("Batch {} is empty", i)));
+            }
+        }
+        Ok(())
+    }
+
     async fn put_memtable(
         &self,
         batches: Vec<RecordBatch>,
@@ -1489,6 +1822,30 @@ impl ShardWriter {
         writer_state: &Arc<SharedWriterState>,
         backpressure: &BackpressureController,
     ) -> Result<WriteResult> {
+        let (result, watcher) = self
+            .put_memtable_no_wait(batches, state_lock, writer_state, backpressure)
+            .await?;
+        // Wait for durability if configured (outside the lock).
+        if let Some(mut watcher) = watcher {
+            watcher.wait().await?;
+        }
+        Ok(result)
+    }
+
+    /// In-memory half of [`Self::put_memtable`]: insert under `state_lock`,
+    /// trigger the WAL flush, and return the watcher un-awaited for the caller
+    /// to wait on. `None` when `durable_write` is off. See [`Self::put_no_wait`].
+    async fn put_memtable_no_wait(
+        &self,
+        batches: Vec<RecordBatch>,
+        state_lock: &Arc<RwLock<WriterState>>,
+        writer_state: &Arc<SharedWriterState>,
+        backpressure: &BackpressureController,
+    ) -> Result<(WriteResult, Option<BatchDurableWatcher>)> {
+        // Reject writes on a fenced writer before mutating the memtable, so a
+        // poisoned writer can't drift further from the durable WAL.
+        self.wal_flusher.check_poisoned()?;
+
         // Apply backpressure if needed (before acquiring main lock)
         backpressure
             .maybe_apply_backpressure(|| {
@@ -1502,7 +1859,7 @@ impl ShardWriter {
         let start = std::time::Instant::now();
 
         // Acquire write lock for entire operation (atomic approach)
-        let (batch_positions, mut durable_watcher, batch_store, indexes) = {
+        let (batch_positions, durable_watcher, batch_store, indexes) = {
             let mut state = state_lock.write().await;
 
             // 1. Insert all batches into memtable atomically
@@ -1533,8 +1890,9 @@ impl ShardWriter {
 
         self.stats.record_put(start.elapsed());
 
-        // Wait for durability if configured (outside the lock)
-        if self.config.durable_write {
+        // Trigger the flush here (outside the lock) so the watcher can resolve;
+        // only the `wait()` is the caller's to schedule.
+        let watcher = if self.config.durable_write {
             self.wal_flusher.trigger_flush(
                 WalFlushSource::BatchStore {
                     batch_store,
@@ -1543,10 +1901,12 @@ impl ShardWriter {
                 batch_positions.end,
                 None,
             )?;
-            durable_watcher.wait().await?;
-        }
+            Some(durable_watcher)
+        } else {
+            None
+        };
 
-        Ok(WriteResult { batch_positions })
+        Ok((WriteResult { batch_positions }, watcher))
     }
 
     async fn put_wal_only(
@@ -1557,6 +1917,10 @@ impl ShardWriter {
         trigger: &StdRwLock<WalOnlyTriggerState>,
         backpressure: &BackpressureController,
     ) -> Result<WriteResult> {
+        // Reject writes on a fenced writer before enqueuing — see
+        // `put_memtable_no_wait`.
+        self.wal_flusher.check_poisoned()?;
+
         // Apply backpressure against the pending queue before pushing. The
         // budget reuses `max_unflushed_memtable_bytes` since WAL-only mode
         // shares the same "in-memory bytes waiting for durable storage"
@@ -1610,7 +1974,9 @@ impl ShardWriter {
             let mut reader = reader;
             match reader.await_value().await {
                 Some(Ok(_)) => {}
-                Some(Err(msg)) => return Err(Error::io(msg)),
+                // Rebuild the typed error (peer fence vs. persistence-failure
+                // self-fence) so a WAL-only durable caller can tell them apart.
+                Some(Err(failure)) => return Err(failure.into_error()),
                 None => {
                     return Err(Error::io(
                         "WAL flush handler exited before reporting durability",
@@ -1781,7 +2147,7 @@ impl ShardWriter {
             frozen: state
                 .frozen_memtables
                 .iter()
-                .map(|m| in_memory_ref(m))
+                .map(|m| in_memory_ref(&m.memtable))
                 .collect(),
         })
     }
@@ -1825,6 +2191,7 @@ impl ShardWriter {
                 ..
             } => {
                 self.check_fenced().await?;
+                self.wal_flusher.check_poisoned()?;
                 let mut state = state.write().await;
                 if state.memtable.batch_count() == 0 {
                     return Ok(());
@@ -1885,12 +2252,96 @@ impl ShardWriter {
         }
     }
 
+    /// Abort the writer without flushing.
+    ///
+    /// Shuts down the background flush tasks and leaves all buffered
+    /// memtable state to be dropped with the writer. Unlike
+    /// [`Self::close`], no WAL/MemTable flush is issued: pending in-memory
+    /// rows are discarded, not made durable, and no object-store IO is
+    /// performed. Used on drop-table, where the dataset directory is about
+    /// to be removed and a flush would only race fresh files back into a
+    /// doomed path.
+    ///
+    /// Caller-quiesce contract: `abort` takes `&self` (so it can be called
+    /// through the `Arc<ShardWriter>` callers hold) and therefore cannot
+    /// structurally bar a concurrent or subsequent `put` the way consuming
+    /// `close(self)` does. After abort the dispatchers are gone, so a later
+    /// `put` would buffer data that never flushes. Callers MUST stop
+    /// issuing writes before calling abort.
+    ///
+    /// Blocks until any flush already mid-`handle()` settles —
+    /// cancellation only fires between messages — so no flush task lingers
+    /// after abort returns. Idempotent: a second call re-cancels an
+    /// already-cancelled token and joins an already-emptied task set.
+    #[instrument(name = "sw_abort", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))]
+    pub async fn abort(&self) -> Result<()> {
+        info!(
+            "Aborting ShardWriter for shard {} (no flush)",
+            self.config.shard_id
+        );
+        self.task_executor.shutdown_all().await?;
+        Ok(())
+    }
+
+    /// Send the close-time final WAL flush and await its completion.
+    ///
+    /// Sends directly on the flush channel rather than via
+    /// [`WalFlusher::trigger_flush`]: the latter silently returns `Ok` when the
+    /// flusher's `flush_tx` is unset, which would let close report success
+    /// without ever persisting the final WAL entry. A closed send channel must
+    /// surface as an error here so close never acknowledges durability it did
+    /// not achieve.
+    async fn flush_final_wal(
+        wal_flush_tx: &mpsc::UnboundedSender<TriggerWalFlush>,
+        source: WalFlushSource,
+        end_batch_position: usize,
+    ) -> Result<()> {
+        let done = WatchableOnceCell::new();
+        let mut reader = done.reader();
+        if wal_flush_tx
+            .send(TriggerWalFlush {
+                source,
+                end_batch_position,
+                done: Some(done),
+            })
+            .is_err()
+        {
+            return Err(Error::io("WAL flush channel closed during close"));
+        }
+
+        match reader.await_value().await {
+            Some(Ok(_)) => Ok(()),
+            Some(Err(failure)) => Err(failure.into_error()),
+            None => Err(Error::io(
+                "WAL flush handler exited before reporting durability during close",
+            )),
+        }
+    }
+
+    fn merge_close_stage(
+        close_result: Result<()>,
+        stage: &str,
+        stage_result: Result<()>,
+    ) -> Result<()> {
+        if let (Err(_), Err(stage_error)) = (&close_result, &stage_result) {
+            warn!("Close stage '{stage}' also failed: {stage_error}");
+        }
+        close_result.and(stage_result)
+    }
+
     /// Close the writer gracefully.
     ///
     /// Flushes pending data and shuts down background tasks.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if pending WAL data cannot be persisted, an active or
+    /// frozen MemTable cannot be flushed, a flush handler exits before reporting
+    /// completion, or background tasks cannot be shut down.
     #[instrument(name = "sw_close", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))]
     pub async fn close(self) -> Result<()> {
         info!("Closing ShardWriter for shard {}", self.config.shard_id);
+        let mut close_result: Result<()> = Ok(());
 
         match &self.mode {
             WriterMode::MemTable {
@@ -1906,23 +2357,17 @@ impl ShardWriter {
                 drop(st);
 
                 if batch_count > 0 {
-                    let done = WatchableOnceCell::new();
-                    let reader = done.reader();
-                    if writer_state
-                        .wal_flush_tx
-                        .send(TriggerWalFlush {
-                            source: WalFlushSource::BatchStore {
-                                batch_store,
-                                indexes,
-                            },
-                            end_batch_position: batch_count,
-                            done: Some(done),
-                        })
-                        .is_ok()
-                    {
-                        let mut reader = reader;
-                        let _ = reader.await_value().await;
-                    }
+                    let stage_result = Self::flush_final_wal(
+                        &writer_state.wal_flush_tx,
+                        WalFlushSource::BatchStore {
+                            batch_store,
+                            indexes,
+                        },
+                        batch_count,
+                    )
+                    .await;
+                    close_result =
+                        Self::merge_close_stage(close_result, "final WAL flush", stage_result);
                 }
 
                 // Freeze the active memtable (if any rows) so it joins the
@@ -1938,22 +2383,37 @@ impl ShardWriter {
                 // Propagate any freeze error: at close time the caller
                 // has explicitly asked for full durability, so silently
                 // dropping a freeze failure would lose data without any
-                // signal. If freeze fails, surface the error rather than
-                // continuing on to drain only the pre-existing frozen
-                // memtables (whose flushes can still be waited on, but
-                // the caller now knows the close was incomplete).
+                // signal. If freeze fails, its error is recorded as the
+                // first causal failure, but close still drains any
+                // pre-existing frozen MemTable watchers so a successor
+                // failure is logged without replacing the first error.
                 let watchers: Vec<_> = {
                     let mut st = state.write().await;
                     if st.memtable.row_count() > 0 {
-                        writer_state.freeze_memtable(&mut st)?;
+                        let freeze_result = writer_state.freeze_memtable(&mut st).map(|_| ());
+                        close_result = Self::merge_close_stage(
+                            close_result,
+                            "active MemTable freeze",
+                            freeze_result,
+                        );
                     }
                     st.frozen_flush_watchers
                         .iter()
                         .map(|(_, w)| w.clone())
                         .collect()
                 };
-                for mut w in watchers {
-                    let _ = w.await_value().await;
+                for mut watcher in watchers {
+                    let stage_result = match watcher.await_value().await {
+                        Some(durability) => durability.into_result(),
+                        None => Err(Error::io(
+                            "MemTable flush handler exited before reporting completion during close",
+                        )),
+                    };
+                    close_result = Self::merge_close_stage(
+                        close_result,
+                        "frozen MemTable flush watcher",
+                        stage_result,
+                    );
                 }
             }
             WriterMode::WalOnly {
@@ -1966,30 +2426,32 @@ impl ShardWriter {
                 let pending = state.batch_count();
                 let end_position = state.next_batch_position();
                 if pending > 0 {
-                    let done = WatchableOnceCell::new();
-                    let reader = done.reader();
-                    if wal_flush_tx
-                        .send(TriggerWalFlush {
-                            source: WalFlushSource::WalOnly {
-                                state: state.clone(),
-                            },
-                            end_batch_position: end_position,
-                            done: Some(done),
-                        })
-                        .is_ok()
-                    {
-                        let mut reader = reader;
-                        let _ = reader.await_value().await;
-                    }
+                    let stage_result = Self::flush_final_wal(
+                        wal_flush_tx,
+                        WalFlushSource::WalOnly {
+                            state: state.clone(),
+                        },
+                        end_position,
+                    )
+                    .await;
+                    close_result =
+                        Self::merge_close_stage(close_result, "final WAL flush", stage_result);
                 }
             }
         }
 
         // Shutdown background tasks
-        self.task_executor.shutdown_all().await?;
-
-        info!("ShardWriter closed for shard {}", self.config.shard_id);
-        Ok(())
+        let shutdown_result = self.task_executor.shutdown_all().await;
+        let close_result = Self::merge_close_stage(close_result, "task shutdown", shutdown_result);
+
+        match &close_result {
+            Ok(()) => info!("ShardWriter closed for shard {}", self.config.shard_id),
+            Err(error) => warn!(
+                "ShardWriter close for shard {} failed: {error}",
+                self.config.shard_id
+            ),
+        }
+        close_result
     }
 }
 
@@ -2071,9 +2533,10 @@ impl MessageHandler<TriggerWalFlush> for WalFlushHandler {
                 state.last_flushed_wal_entry_position.max(entry.position);
         }
 
-        // Notify completion if requested
+        // Notify completion if requested. Carry the typed fence reason through
+        // the cell (not just a string) so a waiter rebuilds the right error.
         if let Some(cell) = done {
-            cell.write(result.map_err(|e| e.to_string()));
+            cell.write(result.map_err(|e| WalFlushFailure::from_error(&e)));
         }
 
         Ok(())
@@ -2174,6 +2637,9 @@ struct MemTableFlushHandler {
     /// at all.
     index_configs: Vec<MemIndexConfig>,
     stats: SharedWriteStats,
+    /// How long a frozen memtable lingers in memory after its flush commits
+    /// before `SweepExpired` evicts it. See `ShardWriterConfig::frozen_memtable_grace`.
+    grace: Duration,
 }
 
 impl MemTableFlushHandler {
@@ -2183,6 +2649,7 @@ impl MemTableFlushHandler {
         epoch: u64,
         index_configs: Vec<MemIndexConfig>,
         stats: SharedWriteStats,
+        grace: Duration,
     ) -> Self {
         Self {
             state,
@@ -2190,22 +2657,51 @@ impl MemTableFlushHandler {
             epoch,
             index_configs,
             stats,
+            grace,
         }
     }
+
+    /// Evict frozen memtables whose post-flush grace has elapsed. Un-stamped
+    /// (not-yet-flushed) entries are always kept.
+    async fn sweep_expired_frozen(&self) {
+        let now = now_millis();
+        let grace_ms = self.grace.as_millis() as u64;
+        let mut state = self.state.write().await;
+        state
+            .frozen_memtables
+            .retain(|frozen| match frozen.flushed_at_ms {
+                Some(flushed_at) => now.saturating_sub(flushed_at) < grace_ms,
+                None => true,
+            });
+    }
 }
 
 #[async_trait]
 impl MessageHandler<TriggerMemTableFlush> for MemTableFlushHandler {
-    async fn handle(&mut self, message: TriggerMemTableFlush) -> Result<()> {
-        let TriggerMemTableFlush { memtable, done } = message;
+    fn tickers(&mut self) -> Vec<(Duration, MessageFactory<TriggerMemTableFlush>)> {
+        // Zero grace evicts on commit, so no sweeper is needed.
+        if self.grace.is_zero() {
+            return vec![];
+        }
+        // Sweep often enough that eviction lags the grace by at most ~1/3, so a
+        // generation lives no more than ~grace * 4/3 past its flush commit.
+        let tick = (self.grace / 3).max(Duration::from_millis(100));
+        vec![(tick, Box::new(|| TriggerMemTableFlush::SweepExpired))]
+    }
 
-        let result = self.flush_memtable(memtable).await;
-        if let Some(tx) = done {
-            // Send result through the channel - caller is waiting for it
-            let _ = tx.send(result);
-        } else {
-            // No done channel, propagate errors
-            result?;
+    async fn handle(&mut self, message: TriggerMemTableFlush) -> Result<()> {
+        match message {
+            TriggerMemTableFlush::Flush { memtable, done } => {
+                let result = self.flush_memtable(memtable).await;
+                if let Some(tx) = done {
+                    // Send result through the channel - caller is waiting for it
+                    let _ = tx.send(result);
+                } else {
+                    // No done channel, propagate errors
+                    result?;
+                }
+            }
+            TriggerMemTableFlush::SweepExpired => self.sweep_expired_frozen().await,
         }
         Ok(())
     }
@@ -2240,7 +2736,9 @@ impl MemTableFlushHandler {
                 if let Some(mut completion_reader) = memtable.take_wal_flush_completion() {
                     match completion_reader.await_value().await {
                         Some(Ok(flush_result)) => flush_result.entry.map(|e| e.position),
-                        Some(Err(e)) => return Err(Error::io(format!("WAL flush failed: {}", e))),
+                        // Rebuild the typed error so a fence/poison reason
+                        // propagates to the memtable-flush caller too.
+                        Some(Err(e)) => return Err(e.into_error()),
                         None => {
                             return Err(Error::io(
                                 "WAL flush handler exited before reporting completion",
@@ -2299,15 +2797,26 @@ impl MemTableFlushHandler {
                 state.frozen_memtable_bytes =
                     state.frozen_memtable_bytes.saturating_sub(memtable_size);
             }
-            // Drop the queryable handle ONLY on commit success. On failure
-            // keep it: rows must stay in the read union until a later flush
-            // or WAL replay, else a transient flush error reopens the hole.
-            // Keyed by generation, so non-FIFO completion is fine.
+            // Retire the frozen handle on commit success, keyed by generation
+            // (non-FIFO completion is fine). Zero grace evicts here; otherwise
+            // stamp the grace clock so it lingers for multi-part as-of reads
+            // until `SweepExpired`. On failure leave it un-stamped: rows stay in
+            // the read union until a later flush or WAL replay, else a transient
+            // error reopens the hole.
             if flush_result.is_ok() {
                 let flushed_generation = memtable.generation();
-                state
-                    .frozen_memtables
-                    .retain(|m| m.generation() != flushed_generation);
+                if self.grace.is_zero() {
+                    state
+                        .frozen_memtables
+                        .retain(|frozen| frozen.memtable.generation() != flushed_generation);
+                } else {
+                    let now = now_millis();
+                    for frozen in state.frozen_memtables.iter_mut() {
+                        if frozen.memtable.generation() == flushed_generation {
+                            frozen.flushed_at_ms = Some(now);
+                        }
+                    }
+                }
             }
         }
 
@@ -2505,11 +3014,7 @@ impl WriteStatsSnapshot {
 
     /// Get average WAL flush size in bytes.
     pub fn avg_wal_flush_bytes(&self) -> Option<u64> {
-        if self.wal_flush_count > 0 {
-            Some(self.wal_flush_bytes / self.wal_flush_count)
-        } else {
-            None
-        }
+        self.wal_flush_bytes.checked_div(self.wal_flush_count)
     }
 
     /// Get WAL write throughput (bytes per second based on WAL flush time).
@@ -2541,11 +3046,7 @@ impl WriteStatsSnapshot {
 
     /// Get average rows per index update.
     pub fn avg_index_update_rows(&self) -> Option<u64> {
-        if self.index_update_count > 0 {
-            Some(self.index_update_rows / self.index_update_count)
-        } else {
-            None
-        }
+        self.index_update_rows.checked_div(self.index_update_count)
     }
 
     /// Get average MemTable flush latency.
@@ -2559,11 +3060,8 @@ impl WriteStatsSnapshot {
 
     /// Get average MemTable flush size in rows.
     pub fn avg_memtable_flush_rows(&self) -> Option<u64> {
-        if self.memtable_flush_count > 0 {
-            Some(self.memtable_flush_rows / self.memtable_flush_count)
-        } else {
-            None
-        }
+        self.memtable_flush_rows
+            .checked_div(self.memtable_flush_count)
     }
 
     /// Log stats summary using tracing (for structured telemetry).
@@ -2618,8 +3116,11 @@ pub fn new_shared_stats() -> SharedWriteStats {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::dataset::mem_wal::test_util::failing_memory_store;
     use arrow_array::{Int32Array, StringArray};
     use arrow_schema::{DataType, Field};
+    use lance_core::FenceReason;
+    use rstest::rstest;
     use tempfile::TempDir;
 
     async fn create_local_store() -> (Arc<ObjectStore>, Path, String, TempDir) {
@@ -2629,49 +3130,708 @@ mod tests {
         (store, path, uri, temp_dir)
     }
 
-    fn create_test_schema() -> Arc<ArrowSchema> {
+    #[test]
+    fn test_merge_close_stage_preserves_first_error() {
+        let result = ShardWriter::merge_close_stage(
+            Err(Error::io("primary close error")),
+            "secondary close stage",
+            Err(Error::io("secondary close error")),
+        );
+
+        let error = result.expect_err("close must preserve the first error");
+        assert!(matches!(&error, Error::IO { .. }));
+        assert!(
+            error.to_string().contains("primary close error"),
+            "unexpected error: {error}"
+        );
+        assert!(
+            !error.to_string().contains("secondary close error"),
+            "secondary error replaced the primary error: {error}"
+        );
+    }
+
+    /// Base schema with `id` marked as the unenforced primary key (delete needs
+    /// a PK). `name` is nullable so a tombstone can null it.
+    fn create_pk_test_schema() -> Arc<ArrowSchema> {
+        let mut id_metadata = std::collections::HashMap::new();
+        id_metadata.insert(
+            "lance-schema:unenforced-primary-key".to_string(),
+            "true".to_string(),
+        );
+        let id = Field::new("id", DataType::Int32, false).with_metadata(id_metadata);
         Arc::new(ArrowSchema::new(vec![
-            Field::new("id", DataType::Int32, false),
+            id,
             Field::new("name", DataType::Utf8, true),
         ]))
     }
 
-    fn create_test_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch {
+    fn id_only_keys(ids: &[i32]) -> RecordBatch {
         RecordBatch::try_new(
-            Arc::new(schema.clone()),
-            vec![
-                Arc::new(Int32Array::from_iter_values(
-                    start_id..start_id + num_rows as i32,
-                )),
-                Arc::new(StringArray::from_iter_values(
-                    (0..num_rows).map(|i| format!("name_{}", start_id as usize + i)),
-                )),
-            ],
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "id",
+                DataType::Int32,
+                false,
+            )])),
+            vec![Arc::new(Int32Array::from(ids.to_vec()))],
         )
         .unwrap()
     }
 
-    #[tokio::test]
-    async fn test_shard_writer_basic_write() {
-        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
-        let schema = create_test_schema();
-
-        let config = ShardWriterConfig {
-            shard_id: Uuid::new_v4(),
-            shard_spec_id: 0,
-            durable_write: false,
-            sync_indexed_write: false,
-            max_wal_buffer_size: 1024 * 1024,
-            max_wal_flush_interval: None,
-            max_memtable_size: 64 * 1024 * 1024,
-            manifest_scan_batch_size: 2,
-            ..Default::default()
-        };
+    #[test]
+    fn test_ensure_tombstone_column_injects_false() {
+        let base = create_test_schema();
+        let target = schema_with_tombstone(&base);
+        let out = ensure_tombstone_column(create_test_batch(&base, 0, 3), &target).unwrap();
+        assert_eq!(out.schema(), target);
+        let ts = out
+            .column_by_name(TOMBSTONE)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .unwrap();
+        assert!(
+            (0..3).all(|i| !ts.value(i)),
+            "put injects _tombstone = false"
+        );
+        // Idempotent: a batch already carrying the column passes through.
+        let again = ensure_tombstone_column(out.clone(), &target).unwrap();
+        assert_eq!(again.schema(), out.schema());
+    }
 
-        let writer = ShardWriter::open(
-            store,
-            base_path,
-            base_uri,
+    #[test]
+    fn test_build_tombstone_batch_shape() {
+        let target = schema_with_tombstone(&create_test_schema());
+        let tomb =
+            build_tombstone_batch(&id_only_keys(&[5, 7]), &target, &["id".to_string()]).unwrap();
+        assert_eq!(tomb.schema(), target);
+        assert_eq!(tomb.num_rows(), 2);
+        let ids = tomb
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(ids.values(), &[5, 7]);
+        let name = tomb.column_by_name("name").unwrap();
+        assert!(
+            name.is_null(0) && name.is_null(1),
+            "non-PK columns are null in a tombstone"
+        );
+        let ts = tomb
+            .column_by_name(TOMBSTONE)
+            .unwrap()
+            .as_any()
+            .downcast_ref::<BooleanArray>()
+            .unwrap();
+        assert!(ts.value(0) && ts.value(1), "_tombstone = true");
+    }
+
+    #[test]
+    fn test_build_tombstone_batch_missing_pk_errors() {
+        let target = schema_with_tombstone(&create_test_schema());
+        let keys = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "other",
+                DataType::Int32,
+                false,
+            )])),
+            vec![Arc::new(Int32Array::from(vec![1]))],
+        )
+        .unwrap();
+        assert!(build_tombstone_batch(&keys, &target, &["id".to_string()]).is_err());
+    }
+
+    #[test]
+    fn test_build_tombstone_batch_non_nullable_nonpk_errors() {
+        // A tombstone must null every non-PK column; a non-nullable one fails.
+        let base = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("v", DataType::Int32, false),
+        ]));
+        let target = schema_with_tombstone(&base);
+        assert!(build_tombstone_batch(&id_only_keys(&[1]), &target, &["id".to_string()]).is_err());
+    }
+
+    #[tokio::test]
+    async fn test_shard_writer_delete_round_trip() {
+        use crate::dataset::mem_wal::scanner::LsmScanner;
+        use futures::TryStreamExt;
+
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            durable_write: true,
+            ..Default::default()
+        };
+        let shard_id = config.shard_id;
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri.clone(),
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // ids 0..5, then delete id=2.
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+        writer.delete(vec![id_only_keys(&[2])]).await.unwrap();
+
+        let refs = writer.in_memory_memtable_refs().await.unwrap();
+        let scanner = LsmScanner::without_base_table(
+            schema.clone(),
+            base_uri,
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_in_memory_memtables(shard_id, refs);
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let arr = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            ids.extend((0..arr.len()).map(|i| arr.value(i)));
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![0, 1, 3, 4],
+            "id=2 deleted; tombstone not surfaced"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// `delete_no_wait` lands the tombstone in the in-memory tier (visible at
+    /// the batch-store level the instant it returns) and hands back the
+    /// durability watcher *without* awaiting it. Index-driven LSM read
+    /// visibility — folding the tombstone out — follows once the watcher
+    /// resolves the flush that advances the visibility watermark and updates
+    /// the PK index. The delete analog of
+    /// `test_put_no_wait_durable_visible_then_durable`.
+    #[tokio::test]
+    async fn test_shard_writer_delete_no_wait_durable_visible_after_watcher() {
+        use crate::dataset::mem_wal::scanner::LsmScanner;
+        use futures::TryStreamExt;
+
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            durable_write: true,
+            ..Default::default()
+        };
+        let shard_id = config.shard_id;
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri.clone(),
+            config,
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+
+        // Tombstone id=2 without blocking on durability.
+        let (_result, watcher) = writer
+            .delete_no_wait(vec![id_only_keys(&[2])])
+            .await
+            .unwrap();
+
+        // The tombstone is in the in-memory tier immediately (5 rows + 1
+        // tombstone), even though the index-driven read can't fold it until the
+        // flush behind the watcher lands. `durable_write` is on, so a watcher is
+        // returned to await.
+        assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6);
+        let mut watcher = watcher.expect("durable_write returns a watcher");
+
+        // Awaiting the watcher waits for the flush, which advances the
+        // visibility watermark and updates the PK index — only then does the LSM
+        // read fold the delete.
+        watcher.wait().await.unwrap();
+
+        let refs = writer.in_memory_memtable_refs().await.unwrap();
+        let scanner = LsmScanner::without_base_table(
+            schema.clone(),
+            base_uri,
+            vec![],
+            vec!["id".to_string()],
+        )
+        .with_in_memory_memtables(shard_id, refs);
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let arr = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            ids.extend((0..arr.len()).map(|i| arr.value(i)));
+        }
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![0, 1, 3, 4],
+            "delete folded by the LSM read once the watcher resolves"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// With `durable_write` off, `delete_no_wait` returns no watcher (nothing to
+    /// await), but the tombstone still lands in the in-memory tier. The delete
+    /// analog of `test_put_no_wait_non_durable_returns_no_watcher`.
+    #[tokio::test]
+    async fn test_shard_writer_delete_no_wait_non_durable_returns_no_watcher() {
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            durable_write: false,
+            ..Default::default()
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+
+        let (_result, watcher) = writer
+            .delete_no_wait(vec![id_only_keys(&[2])])
+            .await
+            .unwrap();
+        assert!(watcher.is_none(), "non-durable delete has nothing to await");
+
+        // Tombstone landed in the in-memory tier (5 rows + 1 tombstone).
+        assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6);
+
+        writer.close().await.unwrap();
+    }
+
+    /// Read every surviving `id` through the full LSM read path (which folds
+    /// `NOT _tombstone` per source) over the writer's *flushed* generations,
+    /// with an optional filter. Mirrors how a query reads a WAL table after a
+    /// flush — the path the wallop fuzz exercised when it caught a deleted row
+    /// resurfacing.
+    async fn read_flushed_ids_via_lsm(
+        writer: &ShardWriter,
+        schema: Arc<ArrowSchema>,
+        base_uri: &str,
+        shard_id: Uuid,
+        filter: Option<&str>,
+    ) -> Vec<i32> {
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use futures::TryStreamExt;
+
+        let manifest = writer.manifest().await.unwrap().unwrap();
+        let mut snapshot =
+            ShardSnapshot::new(shard_id).with_current_generation(manifest.current_generation);
+        for fg in &manifest.flushed_generations {
+            snapshot = snapshot.with_flushed_generation(fg.generation, fg.path.clone());
+        }
+        let mut scanner = LsmScanner::without_base_table(
+            schema,
+            base_uri.to_string(),
+            vec![snapshot],
+            vec!["id".to_string()],
+        );
+        if let Some(predicate) = filter {
+            scanner = scanner.filter(predicate).unwrap();
+        }
+        let batches: Vec<RecordBatch> = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let mut ids: Vec<i32> = Vec::new();
+        for b in &batches {
+            let arr = b
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int32Array>()
+                .unwrap();
+            ids.extend((0..arr.len()).map(|i| arr.value(i)));
+        }
+        ids.sort_unstable();
+        ids
+    }
+
+    fn flush_test_config(shard_id: Uuid) -> ShardWriterConfig {
+        ShardWriterConfig {
+            shard_id,
+            durable_write: false,
+            sync_indexed_write: true,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        }
+    }
+
+    /// Delete a key, then flush: the tombstone and the live row land in the
+    /// *same* flushed generation, so flush-time dedup must keep the tombstone
+    /// (newest) and the read must fold it away. Regression for the wallop
+    /// phantom (deleted row resurfacing in a filtered read after flush).
+    #[tokio::test]
+    async fn test_shard_writer_delete_then_flush_round_trip() {
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let shard_id = Uuid::new_v4();
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri.clone(),
+            flush_test_config(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // ids 0..5, delete id=2, THEN flush (insert + tombstone in one gen).
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+        writer.delete(vec![id_only_keys(&[2])]).await.unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            vec![0, 1, 3, 4],
+            "id=2 deleted before flush; tombstone must not surface in a flushed-gen scan"
+        );
+        // The filtered read path (folds NOT _tombstone into the predicate) must
+        // also drop it — this is the exact wallop failure shape (`id < 3`).
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3"))
+                .await,
+            vec![0, 1],
+            "filtered read after flush must not resurface deleted id=2"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// Insert+flush, then delete+flush: the tombstone lands in a *newer*
+    /// generation than the live row, so the cross-generation block-list must
+    /// mask the older row by PK. This is the wallop scenario (seed flushed,
+    /// then delete, then flush).
+    #[tokio::test]
+    async fn test_shard_writer_delete_across_flushed_generations() {
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let shard_id = Uuid::new_v4();
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri.clone(),
+            flush_test_config(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // gen 1: ids 0..5.
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        // gen 2: tombstone for id=0 (a later generation than the live row).
+        writer.delete(vec![id_only_keys(&[0])]).await.unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            vec![1, 2, 3, 4],
+            "id=0 tombstoned in a newer gen must mask the older gen's live row"
+        );
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
+                .await,
+            Vec::<i32>::new(),
+            "filtered read 'id < 1' must not resurface cross-gen deleted id=0"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// Same as the cross-generation case, but the flushed generations carry a
+    /// BTree index on `id` (as every wallop table does). A filtered read
+    /// `id < 1` resolves through the scalar index; the `NOT _tombstone` residual
+    /// must still be applied or the deleted row leaks. This is the exact wallop
+    /// failure (BTree id + `FilteredRead 'id < 1'` resurfacing deleted id=0).
+    #[tokio::test]
+    async fn test_shard_writer_delete_across_flushed_generations_indexed() {
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+        let schema = create_pk_test_schema();
+        let shard_id = Uuid::new_v4();
+        let index_configs = vec![MemIndexConfig::BTree(BTreeIndexConfig {
+            name: "id_idx".to_string(),
+            field_id: 0,
+            column: "id".to_string(),
+        })];
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri.clone(),
+            flush_test_config(shard_id),
+            schema.clone(),
+            index_configs,
+        )
+        .await
+        .unwrap();
+
+        // gen 1: ids 0..5 (indexed). gen 2: tombstone for id=0.
+        writer
+            .put(vec![create_test_batch(&schema, 0, 5)])
+            .await
+            .unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+        writer.delete(vec![id_only_keys(&[0])]).await.unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await,
+            vec![1, 2, 3, 4],
+            "indexed cross-gen: full scan must mask deleted id=0"
+        );
+        assert_eq!(
+            read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1"))
+                .await,
+            Vec::<i32>::new(),
+            "indexed filtered read 'id < 1' must not resurface deleted id=0 (wallop repro)"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_delete_validation_errors() {
+        let (store, base_path, base_uri, _temp) = create_local_store().await;
+
+        // No primary key on the schema → delete is rejected.
+        let no_pk = create_test_schema();
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            ShardWriterConfig {
+                shard_id: Uuid::new_v4(),
+                durable_write: false,
+                ..Default::default()
+            },
+            no_pk,
+            vec![],
+        )
+        .await
+        .unwrap();
+        assert!(
+            writer.delete(vec![id_only_keys(&[1])]).await.is_err(),
+            "delete without a primary key must error"
+        );
+        // Empty key list is rejected.
+        assert!(writer.delete(vec![]).await.is_err());
+        writer.close().await.unwrap();
+    }
+
+    /// The compaction Phase-1 hard-delete primitive: a key-only `merge_insert`
+    /// with `WhenMatched::Delete` + `use_index` over a COMPOSITE join key
+    /// `(id, shard_key)` where only `id` carries a scalar index. The partial
+    /// index makes `indexed_join_keys` non-empty, so the scalar-index merge path
+    /// engages — and it must still delete exactly the `(0, 0)` row. If it
+    /// no-ops (or mismatches on the partial key), the compactor trims the WAL
+    /// tombstone while the base row survives → the wallop resurface phantom.
+    #[tokio::test]
+    async fn test_indexed_composite_key_delete_removes_row() {
+        use crate::Dataset;
+        use crate::dataset::{WhenMatched, WhenNotMatched, WhenNotMatchedBySource};
+        use crate::index::DatasetIndexExt;
+        use arrow_array::{Int64Array, RecordBatchIterator};
+        use lance_index::IndexType;
+        use lance_index::scalar::ScalarIndexParams;
+
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("shard_key", DataType::Int64, false),
+            Field::new("value", DataType::Int64, true),
+        ]));
+        // ids 0..5, shard_key = id % 8 (the wallop composite PK), value = id*100.
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from_iter_values(0..5)),
+                Arc::new(Int64Array::from_iter_values((0..5).map(|i| i % 8))),
+                Arc::new(Int64Array::from_iter_values((0..5).map(|i| i * 100))),
+            ],
+        )
+        .unwrap();
+
+        let uri = format!("shared-memory://phase1-delete-{}/", Uuid::new_v4().simple());
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new([Ok(batch)], schema.clone()),
+            &uri,
+            None,
+        )
+        .await
+        .unwrap();
+        // Scalar index on `id` only — `shard_key` is unindexed, mirroring the
+        // WAL base table (BTree on `id`, composite PK `(id, shard_key)`).
+        dataset
+            .create_index(
+                &["id"],
+                IndexType::BTree,
+                Some("id_btree".to_string()),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Phase-1 call: key-only source `(id=0, shard_key=0)`, delete-when-matched.
+        let keys = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![
+                Field::new("id", DataType::Int64, false),
+                Field::new("shard_key", DataType::Int64, false),
+            ])),
+            vec![
+                Arc::new(Int64Array::from(vec![0_i64])),
+                Arc::new(Int64Array::from(vec![0_i64])),
+            ],
+        )
+        .unwrap();
+        let mut builder = crate::dataset::MergeInsertBuilder::try_new(
+            Arc::new(dataset),
+            vec!["id".to_string(), "shard_key".to_string()],
+        )
+        .unwrap();
+        builder
+            .when_matched(WhenMatched::Delete)
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .when_not_matched_by_source(WhenNotMatchedBySource::Keep)
+            .use_index(true);
+        let job = builder.try_build().unwrap();
+        let keys_schema = keys.schema();
+        let (dataset, _stats) = job
+            .execute_reader(RecordBatchIterator::new([Ok(keys)], keys_schema))
+            .await
+            .unwrap();
+
+        // id=0 must be gone — both from a full scan and from an indexed filter.
+        let all = dataset.scan().try_into_batch().await.unwrap();
+        let mut ids: Vec<i64> = all
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap()
+            .values()
+            .to_vec();
+        ids.sort_unstable();
+        assert_eq!(
+            ids,
+            vec![1, 2, 3, 4],
+            "Phase-1 must hard-delete (0, 0) from base"
+        );
+
+        let filtered = dataset
+            .scan()
+            .filter("id < 1")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(
+            filtered.num_rows(),
+            0,
+            "indexed filter must not resurface the deleted composite key"
+        );
+    }
+
+    fn create_test_schema() -> Arc<ArrowSchema> {
+        Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, true),
+        ]))
+    }
+
+    fn create_test_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch {
+        RecordBatch::try_new(
+            Arc::new(schema.clone()),
+            vec![
+                Arc::new(Int32Array::from_iter_values(
+                    start_id..start_id + num_rows as i32,
+                )),
+                Arc::new(StringArray::from_iter_values(
+                    (0..num_rows).map(|i| format!("name_{}", start_id as usize + i)),
+                )),
+            ],
+        )
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_shard_writer_basic_write() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            sync_indexed_write: false,
+            max_wal_buffer_size: 1024 * 1024,
+            max_wal_flush_interval: None,
+            max_memtable_size: 64 * 1024 * 1024,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+
+        let writer = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
             config.clone(),
             schema.clone(),
             vec![],
@@ -2679,18 +3839,87 @@ mod tests {
         .await
         .unwrap();
 
-        // Write a batch
+        // Write a batch
+        let batch = create_test_batch(&schema, 0, 10);
+        let result = writer.put(vec![batch]).await.unwrap();
+
+        assert_eq!(result.batch_positions, 0..1);
+
+        // Check stats
+        let stats = writer.memtable_stats().await.unwrap();
+        assert_eq!(stats.row_count, 10);
+        assert_eq!(stats.batch_count, 1);
+
+        // Close writer
+        writer.close().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_put_no_wait_durable_visible_then_durable() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: true,
+            sync_indexed_write: false,
+            max_wal_buffer_size: 1024 * 1024,
+            max_wal_flush_interval: None,
+            max_memtable_size: 64 * 1024 * 1024,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
         let batch = create_test_batch(&schema, 0, 10);
-        let result = writer.put(vec![batch]).await.unwrap();
+        let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap();
+        assert_eq!(result.batch_positions, 0..1);
+
+        // Row is visible in memory before durability is awaited.
+        let stats = writer.memtable_stats().await.unwrap();
+        assert_eq!(stats.row_count, 10);
+
+        // durable_write is on, so a watcher is returned and resolves once the
+        // triggered flush lands.
+        let mut watcher = watcher.expect("durable_write returns a watcher");
+        watcher.wait().await.unwrap();
+
+        writer.close().await.unwrap();
+    }
 
+    #[tokio::test]
+    async fn test_put_no_wait_non_durable_returns_no_watcher() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            sync_indexed_write: false,
+            max_wal_buffer_size: 1024 * 1024,
+            max_wal_flush_interval: None,
+            max_memtable_size: 64 * 1024 * 1024,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        let batch = create_test_batch(&schema, 0, 10);
+        let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap();
         assert_eq!(result.batch_positions, 0..1);
+        assert!(watcher.is_none(), "non-durable put has nothing to await");
 
-        // Check stats
         let stats = writer.memtable_stats().await.unwrap();
         assert_eq!(stats.row_count, 10);
-        assert_eq!(stats.batch_count, 1);
 
-        // Close writer
         writer.close().await.unwrap();
     }
 
@@ -3165,6 +4394,56 @@ mod tests {
         reopened.close().await.unwrap();
     }
 
+    #[rstest]
+    #[case::memtable(true)]
+    #[case::wal_only(false)]
+    #[tokio::test]
+    async fn test_close_propagates_final_wal_persistence_failure(#[case] enable_memtable: bool) {
+        let (store, base_path, controls) = failing_memory_store().await;
+        let base_uri = "memory:///";
+        let schema = create_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            enable_memtable,
+            sync_indexed_write: false,
+            max_wal_buffer_size: usize::MAX,
+            max_wal_flush_interval: None,
+            max_wal_persist_retries: 0,
+            max_memtable_size: usize::MAX,
+            max_unflushed_memtable_bytes: usize::MAX,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+        let task_executor = writer.task_executor.clone();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        controls.fail_wal_puts(usize::MAX);
+
+        let error = writer
+            .close()
+            .await
+            .expect_err("close must propagate the final WAL persistence failure");
+        assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure));
+        assert!(
+            error
+                .to_string()
+                .contains("injected transient WAL put failure"),
+            "unexpected error: {error}"
+        );
+        assert!(
+            task_executor.tasks.read().unwrap().is_empty(),
+            "close must join background tasks before returning an error"
+        );
+    }
+
     /// Regression: the memtable flush should successfully fire many
     /// times in a row. A bug where every flush wrote the same path was
     /// caught by lance-format/lance#6713.
@@ -3638,6 +4917,60 @@ mod tests {
         ]))
     }
 
+    // A durable write whose WAL PUT keeps failing poisons the writer with a
+    // typed persistence failure; the next write fails fast with the same reason;
+    // and once storage heals, reopening replays the WAL and writes resume.
+    #[tokio::test]
+    async fn test_writer_poisons_on_persistence_failure_and_recovers_on_reopen() {
+        let (store, base_path, controls) = failing_memory_store().await;
+        let base_uri = "memory:///";
+        let shard_id = Uuid::new_v4();
+        let schema = schema_with_pk();
+        controls.fail_wal_puts(usize::MAX);
+
+        let config = ShardWriterConfig {
+            max_wal_persist_retries: 1,
+            wal_persist_retry_base_delay: Duration::from_millis(1),
+            ..memtable_config_with_pk(shard_id)
+        };
+
+        let writer = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri,
+            config.clone(),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+
+        // The durable put waits on the flush, which poisons after its retries.
+        let err = writer
+            .put(vec![create_test_batch(&schema, 0, 1)])
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+
+        // A poisoned writer rejects further writes fast, same typed reason.
+        let err = writer
+            .put(vec![create_test_batch(&schema, 1, 1)])
+            .await
+            .unwrap_err();
+        assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure));
+        drop(writer);
+
+        // Storage heals: reopening replays the WAL and accepts writes again.
+        controls.recover();
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+        writer
+            .put(vec![create_test_batch(&schema, 2, 1)])
+            .await
+            .unwrap();
+    }
+
     /// Replay-on-open recovers durable WAL entries that were never flushed
     /// to a Lance generation. Setup: writer A durably writes batches, drops
     /// without close (so MemTable freeze never runs); writer B reopens and
@@ -3876,6 +5209,7 @@ mod tests {
             // hint seed irrelevant; the real position counter is discovered
             // lazily on the first append.
             0,
+            WalRetryConfig::default(),
         );
         high_epoch_appender
             .append(vec![create_test_batch(&schema, 999, 1)])
@@ -3897,6 +5231,15 @@ mod tests {
         let Err(err) = result else {
             panic!("expected open to fail with fence error during replay");
         };
+        // Assert the *typed* fence reason, not just the message: a regression
+        // reverting this to `Error::io` would still carry a message containing
+        // "fenced" and slip past a string check, but must not report a
+        // `FenceReason`.
+        assert_eq!(
+            err.fence_reason(),
+            Some(FenceReason::PeerClaimedEpoch),
+            "replay must abort with a typed peer-fence error, got: {err}"
+        );
         let msg = err.to_string();
         assert!(
             msg.contains("WAL replay aborted") && msg.contains("fenced"),
@@ -4190,10 +5533,69 @@ mod tests {
         writer.close().await.unwrap();
     }
 
-    /// On a successful flush commit the sealed generation is dropped from
-    /// the queryable set (no leak), and its rows land in the manifest.
+    /// `abort` tears down the background flush tasks WITHOUT flushing —
+    /// buffered memtable rows are discarded, not sealed into an L0
+    /// generation the way `close` would. Idempotent on a second call.
+    #[tokio::test]
+    async fn test_abort_discards_without_flushing_and_is_idempotent() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+
+        // Thresholds high enough that nothing auto-flushes; the rows stay
+        // in the active memtable until abort discards them.
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            sync_indexed_write: false,
+            max_wal_buffer_size: 64 * 1024 * 1024,
+            max_wal_flush_interval: None,
+            max_memtable_size: 64 * 1024 * 1024,
+            manifest_scan_batch_size: 2,
+            ..Default::default()
+        };
+
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        let flushed_before = writer
+            .manifest()
+            .await
+            .unwrap()
+            .map(|m| m.flushed_generations.len())
+            .unwrap_or(0);
+
+        writer.abort().await.unwrap();
+
+        // No generation was sealed — contrast with `close`, which flushes
+        // the 10 buffered rows into a new L0 generation.
+        let flushed_after = writer
+            .manifest()
+            .await
+            .unwrap()
+            .map(|m| m.flushed_generations.len())
+            .unwrap_or(0);
+        assert_eq!(
+            flushed_after, flushed_before,
+            "abort must not flush a new L0 generation"
+        );
+
+        // Idempotent: re-cancels the already-cancelled token, joins an
+        // already-emptied task set.
+        writer.abort().await.unwrap();
+    }
+
+    /// On a successful flush commit the sealed generation's rows land in the
+    /// manifest immediately, but the in-memory handle is NOT dropped — it
+    /// lingers for `frozen_memtable_grace` (so in-flight as-of reads keep
+    /// batch-resolved membership), then is swept by the `SweepExpired` ticker.
     #[tokio::test]
-    async fn test_frozen_dropped_after_successful_flush() {
+    async fn test_frozen_retained_during_grace_then_swept() {
         let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
         let schema = create_test_schema();
         let config = ShardWriterConfig {
@@ -4205,6 +5607,8 @@ mod tests {
             max_wal_flush_interval: None,
             max_memtable_size: 64 * 1024 * 1024,
             manifest_scan_batch_size: 2,
+            // Short grace so the sweep is observable without a slow test.
+            frozen_memtable_grace: Duration::from_secs(1),
             ..Default::default()
         };
         let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
@@ -4219,23 +5623,131 @@ mod tests {
         writer.force_seal_active().await.unwrap();
         writer.wait_for_flush_drain().await.unwrap();
 
+        // Recorded in the manifest at commit time.
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert!(
+            manifest
+                .flushed_generations
+                .iter()
+                .any(|g| g.generation == initial_gen),
+            "flushed generation must be recorded in the manifest"
+        );
+
+        // Still queryable in memory immediately after commit (within grace).
+        let refs = writer.in_memory_memtable_refs().await.unwrap();
+        assert_eq!(refs.active.generation, initial_gen + 1);
+        assert!(
+            refs.frozen.iter().any(|f| f.generation == initial_gen),
+            "flushed generation must stay queryable during the grace window"
+        );
+
+        // After the grace elapses (plus a sweep tick) the handle is evicted.
+        tokio::time::sleep(Duration::from_millis(1_500)).await;
         let refs = writer.in_memory_memtable_refs().await.unwrap();
         assert!(
             refs.frozen.is_empty(),
-            "frozen handle must be dropped once the flush commit lands"
+            "frozen handle must be swept once the grace elapses"
         );
-        assert_eq!(refs.active.generation, initial_gen + 1);
 
+        writer.close().await.unwrap();
+    }
+
+    /// With zero grace (the default) a frozen handle is evicted synchronously on
+    /// flush commit — no sweep tick, no lingering window.
+    #[tokio::test]
+    async fn test_frozen_evicted_immediately_with_zero_grace() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = create_test_schema();
+        let config = ShardWriterConfig {
+            shard_id: Uuid::new_v4(),
+            shard_spec_id: 0,
+            durable_write: false,
+            sync_indexed_write: false,
+            max_wal_buffer_size: 64 * 1024 * 1024,
+            max_wal_flush_interval: None,
+            max_memtable_size: 64 * 1024 * 1024,
+            manifest_scan_batch_size: 2,
+            frozen_memtable_grace: Duration::ZERO,
+            ..Default::default()
+        };
+        let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![])
+            .await
+            .unwrap();
+
+        let initial_gen = writer.memtable_stats().await.unwrap().generation;
+        writer
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        // Rows are durably in the manifest...
         let manifest = writer.manifest().await.unwrap().expect("manifest exists");
         assert!(
-            manifest
-                .flushed_generations
-                .iter()
-                .any(|g| g.generation == initial_gen),
-            "flushed generation must be recorded in the manifest"
+            manifest
+                .flushed_generations
+                .iter()
+                .any(|g| g.generation == initial_gen),
+            "flushed generation must be recorded in the manifest"
+        );
+
+        // ...and the in-memory handle is already gone, no sweep tick needed.
+        let refs = writer.in_memory_memtable_refs().await.unwrap();
+        assert!(
+            refs.frozen.is_empty(),
+            "frozen handle must be evicted on commit when grace is zero"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_close_propagates_frozen_memtable_flush_failure() {
+        let (store, base_path, base_uri, _temp_dir) = create_local_store().await;
+        let schema = schema_with_pk();
+        let shard_id = Uuid::new_v4();
+        let writer_a = ShardWriter::open(
+            store.clone(),
+            base_path.clone(),
+            base_uri.clone(),
+            memtable_config_with_pk(shard_id),
+            schema.clone(),
+            vec![],
+        )
+        .await
+        .unwrap();
+        writer_a
+            .put(vec![create_test_batch(&schema, 0, 10)])
+            .await
+            .unwrap();
+
+        let writer_b = ShardWriter::open(
+            store,
+            base_path,
+            base_uri,
+            memtable_config_with_pk(shard_id),
+            schema,
+            vec![],
+        )
+        .await
+        .unwrap();
+        assert!(writer_b.epoch() > writer_a.epoch());
+
+        let error = writer_a
+            .close()
+            .await
+            .expect_err("close must propagate the fenced MemTable flush");
+        assert!(
+            matches!(error, Error::IO { .. }),
+            "unexpected error: {error}"
+        );
+        assert!(
+            error.to_string().contains("Writer fenced"),
+            "unexpected error: {error}"
         );
 
-        writer.close().await.unwrap();
+        writer_b.close().await.unwrap();
     }
 
     /// Regression: a transient flush failure must NOT reopen the
@@ -4326,8 +5838,8 @@ mod shard_writer_tests {
     use arrow_schema::{DataType, Field, Schema as ArrowSchema};
     use lance_arrow::FixedSizeListArrayExt;
     use lance_index::IndexType;
-    use lance_index::scalar::ScalarIndexParams;
-    use lance_index::scalar::inverted::InvertedIndexParams;
+    use lance_index::scalar::inverted::{InvertedIndexParams, InvertedListFormatVersion};
+    use lance_index::scalar::{FullTextSearchQuery, ScalarIndexParams};
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::pq::builder::PQBuildParams;
     use lance_linalg::distance::MetricType;
@@ -4486,7 +5998,12 @@ mod shard_writer_tests {
 
         let vector_dim = 32;
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_multi_segment_index_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://multi-segment-index-{}/",
+            Uuid::new_v4().simple()
+        );
 
         // Initial fragment + an IVF vector index covering it.
         let initial = create_test_batch(&schema, 0, 256, vector_dim);
@@ -4557,13 +6074,179 @@ mod shard_writer_tests {
         writer.close().await.unwrap();
     }
 
+    /// A generation of only tombstones (delete nulls the vector, and the HNSW
+    /// skips nulls) has an empty vector index. Flushing it must succeed —
+    /// writing the data with no HNSW index for that generation — not fail the
+    /// drain with "HnswMemIndex is empty". Regression for the wallop fuzz
+    /// `flush drain failed: HnswMemIndex is empty` on a delete-only memtable.
+    #[tokio::test]
+    async fn test_flush_all_tombstone_generation_skips_empty_hnsw() {
+        let vector_dim = 32;
+        let schema = create_test_schema(vector_dim);
+        // `shared-memory` (not `memory`): the flush writes the generation dataset
+        // and the drain reopens it by URI, which a plain in-memory store wouldn't
+        // share across opens. A unique authority isolates this test.
+        let uri = format!("shared-memory://tombstone-{}/", Uuid::new_v4().simple());
+
+        // Base dataset + maintained vector index, so the writer carries an HNSW
+        // mem index.
+        let initial = create_test_batch(&schema, 0, 256, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("vector_idx".to_string()),
+                &VectorIndexParams::ivf_flat(1, MetricType::L2),
+                true,
+            )
+            .await
+            .unwrap();
+        dataset
+            .initialize_mem_wal()
+            .maintained_indexes(["vector_idx"])
+            .execute()
+            .await
+            .unwrap();
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(
+                shard_id,
+                ShardWriterConfig::new(shard_id).with_durable_write(false),
+            )
+            .await
+            .unwrap();
+
+        // Delete only — the memtable holds a single tombstone with a null vector,
+        // so its HNSW index is empty.
+        let keys = RecordBatch::try_new(
+            Arc::new(ArrowSchema::new(vec![Field::new(
+                "id",
+                DataType::Int64,
+                false,
+            )])),
+            vec![Arc::new(Int64Array::from(vec![0_i64]))],
+        )
+        .unwrap();
+        writer.delete(vec![keys]).await.unwrap();
+
+        // The drain must not error on the empty HNSW.
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.unwrap();
+
+        // The tombstone-only generation still flushed (data without an HNSW index).
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert_eq!(
+            manifest.flushed_generations.len(),
+            1,
+            "the all-tombstone generation must still flush"
+        );
+        writer.close().await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_mem_wal_maintained_fts_v1_flush_preserves_format() {
+        use tempfile::TempDir;
+
+        let vector_dim = 32;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().expect("Failed to create temp dir");
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+
+        let fts_params =
+            InvertedIndexParams::default().format_version(InvertedListFormatVersion::V1);
+        dataset
+            .create_index(
+                &["text"],
+                IndexType::Inverted,
+                Some("text_fts".to_string()),
+                &fts_params,
+                false,
+            )
+            .await
+            .expect("Failed to create v1 FTS index");
+        let base_indices = dataset.load_indices().await.unwrap();
+        assert_eq!(base_indices.len(), 1);
+        assert_eq!(base_indices[0].index_version, 1);
+
+        dataset
+            .initialize_mem_wal()
+            .maintained_indexes(["text_fts"])
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        let shard_id = Uuid::new_v4();
+        let config = ShardWriterConfig::new(shard_id)
+            .with_durable_write(true)
+            .with_sync_indexed_write(true);
+        let writer = dataset
+            .mem_wal_writer(shard_id, config)
+            .await
+            .expect("Failed to create MemWAL writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 3, vector_dim)])
+            .await
+            .expect("Failed to write MemWAL batch");
+        writer.close().await.expect("Failed to close writer");
+
+        let (store, base_path) = lance_io::object_store::ObjectStore::from_uri(&uri)
+            .await
+            .expect("Failed to open store");
+        let manifest_store =
+            super::super::manifest::ShardManifestStore::new(store, &base_path, shard_id, 2);
+        let manifest = manifest_store
+            .read_latest()
+            .await
+            .expect("Failed to read manifest")
+            .expect("Manifest should exist");
+        assert_eq!(manifest.flushed_generations.len(), 1);
+
+        let flushed = &manifest.flushed_generations[0];
+        let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, flushed.path);
+        let flushed_dataset = Dataset::open(&gen_uri)
+            .await
+            .expect("Failed to open flushed generation");
+        let flushed_indices = flushed_dataset.load_indices().await.unwrap();
+        assert_eq!(flushed_indices.len(), 1);
+        assert_eq!(flushed_indices[0].name, "text_fts");
+        assert_eq!(
+            flushed_indices[0].index_version, 1,
+            "maintained v1 FTS index must flush as v1"
+        );
+
+        let results = flushed_dataset
+            .scan()
+            .full_text_search(FullTextSearchQuery::new("Sample".to_owned()))
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(results.num_rows(), 3);
+    }
+
     #[tokio::test]
     async fn test_writer_hnsw_params_override() {
         use lance_index::vector::hnsw::builder::HnswBuildParams;
 
         let vector_dim = 32;
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_writer_hnsw_params_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://writer-hnsw-params-{}/",
+            Uuid::new_v4().simple()
+        );
 
         let initial = create_test_batch(&schema, 0, 256, vector_dim);
         let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
@@ -4871,7 +6554,12 @@ mod shard_writer_tests {
         let target_id = 1_000i64 + 37;
 
         let schema = create_test_schema(vector_dim);
-        let uri = format!("memory://test_shard_writer_hnsw_{}", Uuid::new_v4());
+        // The generation flusher reopens by URI, so this independent open must
+        // resolve to the same in-memory backend. The unique authority isolates the test.
+        let uri = format!(
+            "shared-memory://shard-writer-hnsw-{}/",
+            Uuid::new_v4().simple()
+        );
 
         let initial_batch = create_test_batch(&schema, 0, 256, vector_dim);
         let batches = RecordBatchIterator::new([Ok(initial_batch)], schema.clone());
@@ -4924,7 +6612,7 @@ mod shard_writer_tests {
             (0..vector_dim as usize).map(|d| (target_id as f32 * 0.1 + d as f32 * 0.01).sin()),
         );
         let mut scanner = writer.scan().await.unwrap();
-        scanner.nearest("vector", Arc::new(query), 80);
+        scanner.nearest("vector", &query, 80).unwrap();
         let result = scanner.try_into_batch().await.expect("Failed to scan");
 
         assert!(result.num_rows() > 0, "vector query returned no rows");
@@ -5268,4 +6956,309 @@ mod shard_writer_tests {
             .await
             .expect("Failed to close new writer");
     }
+
+    /// Regression: a base opened with a *path-bound* store binding (the
+    /// deprecated `ObjectStoreParams::object_store`) must still flush and read
+    /// generations at their own paths.
+    ///
+    /// The binding pins a store to one location, and both
+    /// `ObjectStore::from_uri_and_params` and `DatasetBuilder::build_object_store`
+    /// take the path from it while ignoring the URI they were handed. Reusing the
+    /// base's params verbatim therefore aimed every generation write and open at
+    /// the base table itself: the flush failed ("dataset already exists") and any
+    /// derived open returned base rows as generation rows.
+    #[tokio::test]
+    async fn test_flush_and_read_with_path_bound_object_store() {
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use futures::TryStreamExt;
+        use lance_io::object_store::ObjectStoreParams;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        // Re-bind the base to a store pinned at the base's own path — what
+        // `DatasetBuilder::with_object_store` leaves on an opened dataset.
+        #[allow(deprecated)]
+        let store_params = ObjectStoreParams {
+            object_store: Some((
+                Arc::new(object_store::local::LocalFileSystem::new()),
+                url::Url::parse(&uri).unwrap(),
+            )),
+            ..Default::default()
+        };
+        let dataset = dataset.with_object_store(dataset.object_store.clone(), Some(store_params));
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer
+            .wait_for_flush_drain()
+            .await
+            .expect("flush must not be redirected at the base table");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert_eq!(manifest.flushed_generations.len(), 1);
+        let flushed = manifest.flushed_generations[0].clone();
+
+        // The generation landed under `_mem_wal/`, and the base table is untouched.
+        let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, flushed.path);
+        let generation = Dataset::open(&gen_uri)
+            .await
+            .expect("generation must exist at its own path");
+        assert_eq!(generation.count_rows(None).await.unwrap(), 8);
+        let base = Dataset::open(&uri).await.unwrap();
+        assert_eq!(
+            base.count_rows(None).await.unwrap(),
+            16,
+            "the generation write must not land in the base table"
+        );
+
+        // The read path resolves the generation, not the base: 16 base + 8 flushed.
+        // Opening the base instead would dedup back down to 16 rows.
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_flushed_generation(flushed.generation, flushed.path.clone());
+        let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]);
+        let rows: usize = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .expect("scan must open the generation, not the base")
+            .iter()
+            .map(|batch| batch.num_rows())
+            .sum();
+        assert_eq!(rows, 24);
+
+        writer.close().await.unwrap();
+    }
+
+    /// The store params a base was opened with must reach every *derived* open:
+    /// the flush that writes a generation and the scan that reads it back.
+    ///
+    /// This is the point of threading them at all. A namespace-vended store
+    /// exists only on the params (credentials, endpoint, wrapper), so a
+    /// generation resolved by URI alone would silently sign with the ambient
+    /// identity instead — succeeding against a local store and failing against
+    /// the vended one. Asserting on the *generation folder* rather than
+    /// `_mem_wal/` is what makes this bite: WAL entries are written through the
+    /// base dataset's own store, so they would show up here either way.
+    #[tokio::test]
+    async fn test_store_params_reach_generation_write_and_read() {
+        use crate::dataset::builder::DatasetBuilder;
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use crate::dataset::mem_wal::test_util::observable_store_params;
+        use futures::TryStreamExt;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+
+        // Open the base through an observable store, exactly as a namespace
+        // client would hand in a vended-credential store.
+        let (store_params, controls) = observable_store_params();
+        let mut dataset = DatasetBuilder::from_uri(&uri)
+            .with_store_params(store_params)
+            .load()
+            .await
+            .expect("Failed to open dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.expect("flush failed");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        assert_eq!(manifest.flushed_generations.len(), 1);
+        let flushed = manifest.flushed_generations[0].clone();
+
+        // The generation's own Lance manifest is the signal to key on. Keying on
+        // the generation folder alone would pass vacuously: sidecars like
+        // `{gen}/bloom_filter.bin` are written through the *base* dataset's
+        // store, which is observable no matter what the params do. And the
+        // fragments can't be used either — `ObjectStore::create` writes local
+        // files through `tokio::fs`, bypassing the object store entirely, so
+        // `{gen}/data/` never reaches a wrapper under `file://`. The manifest
+        // goes through `put_opts`, and only the flusher's `Dataset::write` /
+        // `open_generation` writes it — both of which must carry the params.
+        let gen_manifest = format!("{}/_versions", flushed.path);
+
+        assert!(
+            controls.wrote_under(&gen_manifest),
+            "the flush must write the generation through the base's store params, \
+             not a store resolved from the generation URI alone"
+        );
+
+        // And the read path must resolve the generation through them too.
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_flushed_generation(flushed.generation, flushed.path.clone());
+        let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]);
+        let rows: usize = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .expect("scan failed")
+            .iter()
+            .map(|batch| batch.num_rows())
+            .sum();
+        assert_eq!(rows, 24);
+
+        // Reads key on the data files, not the manifest: the flusher already
+        // pulled the generation's manifest into the shared session cache, so the
+        // scan's open serves it from memory and never touches the store. The
+        // fragments are read through it (reads have no local bypass), as is the
+        // generation's standalone PK index.
+        assert!(
+            controls.read_under(&format!("{}/data/", flushed.path)),
+            "the scan must read the generation through the base's store params"
+        );
+
+        writer.close().await.unwrap();
+    }
+
+    /// A fresh-tier-only scanner reaches its store params through
+    /// `with_store_params`, not `new()`, so the setter must strip the path-bound
+    /// store binding too. Left raw, it redirects the generation open at the base
+    /// table and the scan silently returns base rows as WAL rows.
+    #[tokio::test]
+    async fn test_fresh_tier_scan_with_path_bound_object_store() {
+        use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot};
+        use futures::TryStreamExt;
+        use lance_io::object_store::ObjectStoreParams;
+        use tempfile::TempDir;
+
+        let vector_dim = 8;
+        let schema = create_test_schema(vector_dim);
+        let temp_dir = TempDir::new().unwrap();
+        let uri = format!("file://{}", temp_dir.path().display());
+
+        // 16 base rows with ids 0..16; the WAL gets 8 rows with ids 1000..1008,
+        // so a redirected generation open is unambiguous in the output.
+        let initial = create_test_batch(&schema, 0, 16, vector_dim);
+        let batches = RecordBatchIterator::new([Ok(initial)], schema.clone());
+        let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default()))
+            .await
+            .expect("Failed to create dataset");
+        dataset
+            .initialize_mem_wal()
+            .execute()
+            .await
+            .expect("Failed to initialize MemWAL");
+
+        let shard_id = Uuid::new_v4();
+        let writer = dataset
+            .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id))
+            .await
+            .expect("Failed to create writer");
+        writer
+            .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)])
+            .await
+            .expect("Failed to write");
+        writer.force_seal_active().await.unwrap();
+        writer.wait_for_flush_drain().await.expect("flush failed");
+
+        let manifest = writer.manifest().await.unwrap().expect("manifest exists");
+        let flushed = manifest.flushed_generations[0].clone();
+        let snapshot = ShardSnapshot::new(shard_id)
+            .with_current_generation(manifest.current_generation)
+            .with_flushed_generation(flushed.generation, flushed.path.clone());
+
+        // What `DatasetBuilder::with_object_store` leaves on an opened dataset:
+        // a store pinned at the base's own path.
+        #[allow(deprecated)]
+        let store_params = ObjectStoreParams {
+            object_store: Some((
+                Arc::new(object_store::local::LocalFileSystem::new()),
+                url::Url::parse(&uri).unwrap(),
+            )),
+            ..Default::default()
+        };
+
+        let arrow_schema: Arc<ArrowSchema> = schema.clone();
+        let batches = LsmScanner::without_base_table(
+            arrow_schema,
+            uri.clone(),
+            vec![snapshot],
+            vec!["id".to_string()],
+        )
+        .with_session(dataset.session())
+        .with_store_params(store_params)
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .expect("scan must open the generation, not the base");
+
+        let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(
+            rows, 8,
+            "fresh tier holds only the 8 WAL rows; 16 means the generation open \
+             was redirected at the base table"
+        );
+        let ids: Vec<i64> = batches
+            .iter()
+            .flat_map(|batch| {
+                batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int64Array>()
+                    .unwrap()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        assert!(
+            ids.iter().all(|id| (1_000..1_008).contains(id)),
+            "expected the WAL's own rows, got {ids:?}"
+        );
+
+        writer.close().await.unwrap();
+    }
 }
diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs
index d591e42cc73..345d10f71db 100644
--- a/rust/lance/src/dataset/optimize.rs
+++ b/rust/lance/src/dataset/optimize.rs
@@ -80,6 +80,7 @@
 //! the successful tasks can be committed. You can also commit in batches if
 //! you wish. As long as the tasks don't rewrite any of the same fragments,
 //! they can be committed in any order.
+use lance_core::utils::row_addr_remap::{GroupInput, RowAddrRemap};
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::io::Cursor;
@@ -113,7 +114,8 @@ use lance_core::Error;
 use lance_core::datatypes::{BlobHandling, BlobKind};
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::{DATASET_COMPACTING_EVENT, TRACE_DATASET_EVENTS};
-use lance_index::frag_reuse::FragReuseGroup;
+use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseGroup};
+use lance_index::is_system_index;
 use lance_table::format::{Fragment, RowIdMeta};
 use roaring::{RoaringBitmap, RoaringTreemap};
 use serde::{Deserialize, Serialize};
@@ -154,6 +156,39 @@ impl TryFrom<&str> for CompactionMode {
     }
 }
 
+/// Controls how the old-to-new row-address mapping is built when remapping
+/// indices during compaction.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+pub enum IndexRemapMode {
+    /// Store a compact remap and compute row-address mappings during lookup.
+    ///
+    /// Best for large compactions where peak memory is the constraint. Uses
+    /// less memory, but each lookup does extra bitmap/range computation.
+    Compact,
+    /// Store the full row-address remap in memory for fast direct lookups.
+    ///
+    /// Best when the remap fits comfortably in memory and remap speed is the
+    /// priority. Uses more peak memory because every rewritten/deleted row has
+    /// a materialized mapping entry.
+    #[default]
+    Direct,
+}
+
+impl TryFrom<&str> for IndexRemapMode {
+    type Error = Error;
+
+    fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
+        match value.to_lowercase().as_str() {
+            "compact" => Ok(Self::Compact),
+            "direct" => Ok(Self::Direct),
+            _ => Err(Error::invalid_input(format!(
+                "Invalid index remap mode \"{}\". Valid values: \"compact\", \"direct\"",
+                value
+            ))),
+        }
+    }
+}
+
 /// Options to be passed to [compact_files].
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct CompactionOptions {
@@ -191,10 +226,21 @@ pub struct CompactionOptions {
     /// specified then the default (see
     /// [`crate::dataset::Scanner::batch_size`]) will be used.
     pub batch_size: Option<usize>,
+    /// The number of bytes to allow to queue up in the I/O buffer when scanning
+    /// the input fragments.  If not specified then the default (see
+    /// [`crate::dataset::Scanner::io_buffer_size`]) will be used.
+    ///
+    /// Increasing this can avoid a deadlock that occurs when a single batch of
+    /// data is larger than the I/O buffer size.
+    pub io_buffer_size: Option<u64>,
     /// Whether to defer remapping indices during compaction. If true, indices will
     /// not be remapped during this compaction operation. Instead, the fragment reuse index
     /// is updated and will be used to perform remapping later.
     pub defer_index_remap: bool,
+    /// How the old-to-new row-address mapping used to remap indices is built.
+    /// Defaults to [`IndexRemapMode::Direct`].
+    #[serde(default)]
+    pub index_remap_mode: IndexRemapMode,
     /// The compaction mode to use. When set, this takes priority over the
     /// deprecated `enable_binary_copy` and `enable_binary_copy_force` fields.
     ///
@@ -216,6 +262,15 @@ pub struct CompactionOptions {
     /// fragments at a time).
     /// Defaults to `None` (no limit, all eligible fragments are compacted).
     pub max_source_fragments: Option<usize>,
+    /// Maximum number of data overlay files a fragment may carry before it is
+    /// fully compacted. When set, any fragment with more than this many overlays
+    /// is rewritten into a fresh fragment with its overlays (and deletions)
+    /// materialized into the base data, dropping the fragment from any index
+    /// left stale by those overlays.
+    /// Defaults to `Some(10)`. Set to `Some(0)` to compact every fragment that
+    /// carries any overlay, or `None` to disable the overlay-count trigger
+    /// entirely.
+    pub max_overlays_per_fragment: Option<usize>,
     /// Transaction properties to store with this commit.
     ///
     /// These key-value pairs are stored in the transaction file
@@ -237,12 +292,15 @@ impl Default for CompactionOptions {
             num_threads: None,
             max_bytes_per_file: None,
             batch_size: None,
+            io_buffer_size: None,
             defer_index_remap: false,
+            index_remap_mode: IndexRemapMode::Direct,
             compaction_mode: None,
             enable_binary_copy: false,
             enable_binary_copy_force: false,
             binary_copy_read_batch_bytes: Some(16 * 1024 * 1024),
             max_source_fragments: None,
+            max_overlays_per_fragment: Some(10),
             transaction_properties: None,
         }
     }
@@ -263,10 +321,13 @@ impl CompactionOptions {
     /// - `lance.compaction.materialize_deletions`
     /// - `lance.compaction.materialize_deletions_threshold`
     /// - `lance.compaction.defer_index_remap`
+    /// - `lance.compaction.index_remap_mode`
     /// - `lance.compaction.batch_size`
+    /// - `lance.compaction.io_buffer_size`
     /// - `lance.compaction.compaction_mode`
     /// - `lance.compaction.binary_copy_read_batch_bytes`
     /// - `lance.compaction.max_source_fragments`
+    /// - `lance.compaction.max_overlays_per_fragment`
     pub fn from_dataset_config(config: &HashMap<String, String>) -> Result<Self> {
         let mut opts = Self::default();
         opts.apply_dataset_config(config)?;
@@ -339,6 +400,9 @@ impl CompactionOptions {
                         }
                     };
                 }
+                "index_remap_mode" => {
+                    self.index_remap_mode = IndexRemapMode::try_from(value.as_str())?;
+                }
                 "batch_size" => {
                     self.batch_size = Some(value.parse().map_err(|_| {
                         Error::invalid_input(format!(
@@ -347,6 +411,14 @@ impl CompactionOptions {
                         ))
                     })?);
                 }
+                "io_buffer_size" => {
+                    self.io_buffer_size = Some(value.parse().map_err(|_| {
+                        Error::invalid_input(format!(
+                            "Invalid value for {}: '{}' (expected a non-negative integer)",
+                            key, value
+                        ))
+                    })?);
+                }
                 "compaction_mode" => {
                     self.compaction_mode = Some(CompactionMode::try_from(value.as_str())?);
                 }
@@ -366,6 +438,19 @@ impl CompactionOptions {
                         ))
                     })?);
                 }
+                "max_overlays_per_fragment" => {
+                    // The default is `Some(10)`, so an explicit "none" is the only
+                    // way to disable the trigger through the manifest config.
+                    self.max_overlays_per_fragment = match value.to_ascii_lowercase().as_str() {
+                        "none" => None,
+                        _ => Some(value.parse().map_err(|_| {
+                            Error::invalid_input(format!(
+                                "Invalid value for {}: '{}' (expected a non-negative integer or 'none')",
+                                key, value
+                            ))
+                        })?),
+                    };
+                }
                 _ => {
                     warn!("Ignoring unknown compaction config key: {}", key);
                 }
@@ -610,6 +695,15 @@ impl DefaultCompactionPlanner {
 #[async_trait::async_trait]
 impl CompactionPlanner for DefaultCompactionPlanner {
     async fn plan(&self, dataset: &Dataset) -> Result<CompactionPlan> {
+        if self.options.defer_index_remap && dataset.manifest.uses_stable_row_ids() {
+            return Err(Error::invalid_input(
+                "defer_index_remap=true is not supported on datasets with stable row IDs: \
+                 stable row IDs do not require index remapping during compaction, so there \
+                 is nothing to defer."
+                    .to_string(),
+            ));
+        }
+
         // get_fragments should be returning fragments in sorted order (by id)
         // and fragment ids should be unique
         let fragments = dataset.get_fragments();
@@ -644,7 +738,16 @@ impl CompactionPlanner for DefaultCompactionPlanner {
         while let Some(res) = fragment_metrics.next().await {
             let (fragment, metrics) = res?;
 
-            let candidacy = if self.options.materialize_deletions
+            let over_overlay_limit = self
+                .options
+                .max_overlays_per_fragment
+                .is_some_and(|max| fragment.overlays.len() > max);
+
+            let candidacy = if over_overlay_limit {
+                // Too many overlays: fully compact this fragment on its own,
+                // regardless of its size or deletion count.
+                Some(CompactionCandidacy::CompactItself)
+            } else if self.options.materialize_deletions
                 && metrics.deletion_percentage() > self.options.materialize_deletions_threshold
             {
                 Some(CompactionCandidacy::CompactItself)
@@ -770,7 +873,7 @@ pub async fn compact_files_with_planner(
 
     let dataset_ref = &dataset.clone();
 
-    let result_stream = futures::stream::iter(compaction_plan.tasks.into_iter())
+    let result_stream = futures::stream::iter(compaction_plan.tasks)
         .map(|task| rewrite_files(Cow::Borrowed(dataset_ref), task, &compaction_plan.options))
         .buffer_unordered(
             compaction_plan
@@ -1194,6 +1297,8 @@ async fn transform_blob_v2_batch(
 ///   and preserve insertion order.
 /// - `batch_size`: Optional batch size; if provided, set it on the scanner to control
 ///   read batching.
+/// - `io_buffer_size`: Optional I/O buffer size in bytes; if provided, set it on the
+///   scanner to control how much data is queued during reads.
 /// - `with_frags`: Whether to scan only the specified old fragments and force
 ///   in-order reading.
 /// - `capture_row_ids`: When index remapping is needed, include and capture the
@@ -1209,6 +1314,7 @@ async fn prepare_reader(
     dataset: &Dataset,
     fragments: &[Fragment],
     batch_size: Option<usize>,
+    io_buffer_size: Option<u64>,
     with_frags: bool,
     capture_row_ids: bool,
 ) -> Result<(
@@ -1234,6 +1340,9 @@ async fn prepare_reader(
     if let Some(bs) = batch_size {
         scanner.batch_size(bs);
     }
+    if let Some(io_buffer_size) = io_buffer_size {
+        scanner.io_buffer_size(io_buffer_size);
+    }
     if with_frags {
         scanner
             .with_fragments(fragments.to_vec())
@@ -1378,12 +1487,21 @@ impl CandidateBin {
 async fn load_index_fragmaps(dataset: &Dataset) -> Result<Vec<RoaringBitmap>> {
     let indices = dataset.load_indices().await?;
     let mut index_fragmaps = Vec::with_capacity(indices.len());
-    for index in indices.iter() {
+    // System indices (fragment-reuse, mem-wal) don't define data coverage and
+    // aren't remapped per rewrite group, so they must not constrain compaction
+    // bins -- otherwise deferred compaction's fragment-reuse index repeatedly
+    // splits the small-fragment run and they never coalesce.
+    for index in indices.iter().filter(|idx| !is_system_index(idx)) {
         if let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() {
             index_fragmaps.push(fragment_bitmap.clone());
         } else {
             let dataset_at_index = dataset.checkout_version(index.dataset_version).await?;
-            let frags = 0..dataset_at_index.manifest.max_fragment_id.unwrap_or(0);
+            // max_fragment_id is inclusive (the highest id); +1 for an exclusive
+            // upper bound so the last fragment is covered (None => empty range).
+            let frags = 0..dataset_at_index
+                .manifest
+                .max_fragment_id
+                .map_or(0, |m| m + 1);
             index_fragmaps.push(RoaringBitmap::from_sorted_iter(frags).unwrap());
         }
     }
@@ -1515,6 +1633,7 @@ async fn rewrite_files(
             dataset.as_ref(),
             &fragments,
             options.batch_size,
+            options.io_buffer_size,
             true,
             needs_remapping,
         )
@@ -1684,7 +1803,8 @@ async fn rewrite_files(
     let row_addrs = match row_addrs_result {
         Ok(v) => v,
         Err(e) => {
-            cleanup_data_fragments(&dataset.object_store, &dataset.base, &new_fragments).await;
+            cleanup_data_fragments(&dataset.object_store, &dataset.base, None, &new_fragments)
+                .await;
             return Err(e);
         }
     };
@@ -1859,8 +1979,8 @@ async fn recalc_versions_for_rewritten_fragments(
     // Set both version metadata on new fragments
     for ((fragment, last_updated_seq), created_at_seq) in new_fragments
         .iter_mut()
-        .zip(new_last_updated_sequences.into_iter())
-        .zip(new_created_at_sequences.into_iter())
+        .zip(new_last_updated_sequences)
+        .zip(new_created_at_sequences)
     {
         fragment.last_updated_at_version_meta = Some(
             lance_table::format::RowDatasetVersionMeta::from_sequence(&last_updated_seq).unwrap(),
@@ -1927,10 +2047,36 @@ pub async fn commit_compaction(
     let mut rewrite_groups = Vec::with_capacity(completed_tasks.len());
     let mut metrics = CompactionMetrics::default();
 
-    let mut row_id_map: HashMap<u64, Option<u64>> = HashMap::default();
+    let mut remap_group_inputs: Vec<GroupInput> = Vec::new();
+    let mut direct_row_id_map: HashMap<u64, Option<u64>> = HashMap::default();
     let mut frag_reuse_groups: Vec<FragReuseGroup> = Vec::new();
     let mut new_fragment_bitmap: RoaringBitmap = RoaringBitmap::new();
 
+    // Write an FRI only when the compaction touches data an index must later
+    // remap: a rewrite group covered by a data index, or by the existing FRI's new
+    // fragments (the composed remap chain). Compacting only not-yet-indexed data
+    // needs no FRI (one written for it is un-drainable). Decide all-or-nothing per
+    // compaction, never per group -- a partial FRI is unsound: a concurrent reindex
+    // can make a skipped fragment indexed and the conflict resolver's FRI-present
+    // path won't re-check it.
+    let indexed_frags: RoaringBitmap = if options.defer_index_remap {
+        let mut covered = RoaringBitmap::new();
+        for bm in load_index_fragmaps(dataset).await? {
+            covered |= bm;
+        }
+        if let Some(bm) = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await?
+            .and_then(|fri| fri.fragment_bitmap)
+        {
+            covered |= bm;
+        }
+        covered
+    } else {
+        RoaringBitmap::new()
+    };
+    let mut any_group_indexed = false;
+
     for task in completed_tasks {
         metrics += task.metrics;
         let rewrite_group = RewriteGroup {
@@ -1942,14 +2088,51 @@ pub async fn commit_compaction(
             if let Some(row_addrs_bytes) = task.row_addrs {
                 let row_addrs =
                     RoaringTreemap::deserialize_from(&mut Cursor::new(&row_addrs_bytes))?;
-                let transposed = remapping::transpose_row_addrs(
-                    row_addrs,
-                    &task.original_fragments,
-                    &task.new_fragments,
-                );
-                row_id_map.extend(transposed);
+                match options.index_remap_mode {
+                    IndexRemapMode::Direct => {
+                        let transposed = remapping::transpose_row_addrs(
+                            row_addrs,
+                            &task.original_fragments,
+                            &task.new_fragments,
+                        );
+                        direct_row_id_map.extend(transposed);
+                    }
+                    IndexRemapMode::Compact => {
+                        let new_frags = task
+                            .new_fragments
+                            .iter()
+                            .map(|f| {
+                                let physical_rows = f.physical_rows.ok_or_else(|| {
+                                    Error::invalid_input(format!(
+                                        "compacted fragment {} is missing physical_rows",
+                                        f.id
+                                    ))
+                                })?;
+                                Ok((f.id as u32, physical_rows as u32))
+                            })
+                            .collect::<Result<Vec<_>>>()?;
+
+                        remap_group_inputs.push(GroupInput {
+                            rewritten_old_row_addrs: row_addrs,
+                            old_frag_ids: task
+                                .original_fragments
+                                .iter()
+                                .map(|f| f.id as u32)
+                                .collect(),
+                            new_frags,
+                        });
+                    }
+                }
             }
         } else if options.defer_index_remap {
+            // Record every group; track whether any touches indexed/chain data.
+            if task
+                .original_fragments
+                .iter()
+                .any(|f| indexed_frags.contains(f.id as u32))
+            {
+                any_group_indexed = true;
+            }
             let changed_row_addrs = task.row_addrs.ok_or_else(|| {
                 Error::internal(
                     "defer_index_remap requires row_addrs but none were provided".to_string(),
@@ -1975,9 +2158,11 @@ pub async fn commit_compaction(
             .flat_map(|group| group.old_fragments.iter().map(|frag| frag.id))
             .collect::<Vec<_>>();
 
-        let remapped_indices = index_remapper
-            .remap_indices(row_id_map, &affected_ids)
-            .await?;
+        let remap = match options.index_remap_mode {
+            IndexRemapMode::Direct => RowAddrRemap::direct(direct_row_id_map),
+            IndexRemapMode::Compact => RowAddrRemap::compact(remap_group_inputs)?,
+        };
+        let remapped_indices = index_remapper.remap_indices(remap, &affected_ids).await?;
         remapped_indices
             .into_iter()
             .map(|rewritten| RewrittenIndex {
@@ -2002,9 +2187,15 @@ pub async fn commit_compaction(
         Vec::new()
     };
 
-    let frag_reuse_index = if options.defer_index_remap {
+    // No indexed/chain data touched -> no FRI (all-or-nothing, see above).
+    let frag_reuse_index = if options.defer_index_remap && any_group_indexed {
         Some(build_new_frag_reuse_index(dataset, frag_reuse_groups, new_fragment_bitmap).await?)
     } else {
+        if options.defer_index_remap {
+            log::debug!(
+                "skipping fragment-reuse index: no rewritten fragments were covered by an index"
+            );
+        }
         None
     };
 
@@ -2037,7 +2228,13 @@ pub async fn commit_compaction(
         .apply_commit(transaction, &Default::default(), &Default::default())
         .await
     {
-        cleanup_data_fragments(&dataset.object_store, &dataset.base, &all_new_fragments).await;
+        cleanup_data_fragments(
+            &dataset.object_store,
+            &dataset.base,
+            None,
+            &all_new_fragments,
+        )
+        .await;
         return Err(e);
     }
 
@@ -2072,6 +2269,7 @@ mod tests {
     use lance_datagen::Dimension;
     use lance_file::version::LanceFileVersion;
     use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
+    use lance_index::frag_reuse::FragReuseIndexHandle;
     use lance_index::scalar::{
         BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams,
     };
@@ -2101,6 +2299,7 @@ mod tests {
         let fragment = Fragment {
             id: 0,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(0),
@@ -2153,6 +2352,20 @@ mod tests {
         .unwrap()
     }
 
+    /// Build (or, with `replace`, rebuild) a scalar index named "scalar" on `col`.
+    async fn create_scalar_index(dataset: &mut Dataset, col: &str, replace: bool) {
+        dataset
+            .create_index(
+                &[col],
+                IndexType::Scalar,
+                Some("scalar".into()),
+                &ScalarIndexParams::default(),
+                replace,
+            )
+            .await
+            .unwrap();
+    }
+
     #[derive(Debug, Default, Clone, PartialEq)]
     struct MockIndexRemapperExpectation {
         expected: HashMap<u64, Option<u64>>,
@@ -2201,18 +2414,31 @@ mod tests {
     impl IndexRemapper for MockIndexRemapper {
         async fn remap_indices(
             &self,
-            index_map: HashMap<u64, Option<u64>>,
+            index_map: RowAddrRemap,
             _: &[u64],
         ) -> Result<Vec<RemappedIndex>> {
             for expectation in &self.expectations {
-                if expectation.expected == index_map {
+                let matches = match &index_map {
+                    RowAddrRemap::Direct(map) => map == &expectation.expected,
+                    RowAddrRemap::Compact(_) => {
+                        let expected_frags: RoaringBitmap = expectation
+                            .expected
+                            .keys()
+                            .map(|addr| (addr >> 32) as u32)
+                            .collect();
+                        index_map.affected_fragments() == expected_frags
+                            && expectation
+                                .expected
+                                .iter()
+                                .all(|(k, v)| index_map.get(*k) == Some(*v))
+                    }
+                };
+                if matches {
                     return Ok(expectation.answer.clone());
                 }
             }
             panic!(
-                "Unexpected index map (len={}): {}\n  Options: {}",
-                index_map.len(),
-                Self::stringify_map(&index_map),
+                "Unexpected index map; expected one of:\n  {}",
                 self.expectations
                     .iter()
                     .map(|expectation| Self::stringify_map(&expectation.expected))
@@ -2636,6 +2862,57 @@ mod tests {
         assert_eq!(scanned_data, data);
     }
 
+    #[rstest]
+    #[tokio::test]
+    async fn test_compact_with_io_buffer_size(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) {
+        // Compaction should succeed and produce correct results when an
+        // explicit io_buffer_size is provided via CompactionOptions.
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let data = sample_data();
+
+        // Create a table with 2 small fragments so there is something to compact.
+        let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema());
+        let write_params = WriteParams {
+            max_rows_per_file: 5_000,
+            max_rows_per_group: 1_000,
+            data_storage_version: Some(data_storage_version),
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write(reader, test_uri, Some(write_params))
+            .await
+            .unwrap();
+        assert_eq!(dataset.get_fragments().len(), 2);
+
+        let options = CompactionOptions {
+            // A generous buffer so the read does not deadlock on large batches.
+            io_buffer_size: Some(256 * 1024 * 1024),
+            ..Default::default()
+        };
+        let plan = plan_compaction(&dataset, &options).await.unwrap();
+        assert_eq!(plan.tasks().len(), 1);
+
+        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
+        assert_eq!(metrics.fragments_removed, 2);
+        assert_eq!(metrics.fragments_added, 1);
+
+        // All rows are preserved after compaction.
+        let scanner = dataset.scan();
+        let batches = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let scanned_data = concat_batches(&batches[0].schema(), &batches).unwrap();
+        assert_eq!(scanned_data.num_rows(), data.num_rows());
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_compact_deletions(
@@ -2697,11 +2974,7 @@ mod tests {
 
     #[async_trait]
     impl IndexRemapper for IgnoreRemap {
-        async fn remap_indices(
-            &self,
-            _: HashMap<u64, Option<u64>>,
-            _: &[u64],
-        ) -> Result<Vec<RemappedIndex>> {
+        async fn remap_indices(&self, _: RowAddrRemap, _: &[u64]) -> Result<Vec<RemappedIndex>> {
             Ok(Vec::new())
         }
     }
@@ -2934,6 +3207,10 @@ mod tests {
 
         assert_eq!(dataset.get_fragments().len(), num_fragments);
 
+        // An FRI is only written for compactions that touch indexed data, so
+        // index the column being compacted.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         // Delete a few rows from each fragment so compaction has something to do.
         dataset.delete("i % 1000 = 0").await.unwrap();
 
@@ -2961,6 +3238,49 @@ mod tests {
             .expect("loading large frag reuse index details must not fail");
     }
 
+    #[tokio::test]
+    async fn test_defer_index_remap_rejected_with_stable_row_ids() {
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let data = sample_data();
+        let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 9000))], data.schema());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 1000, // 9 fragments
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert!(dataset.manifest.uses_stable_row_ids());
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 3_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+
+        // Fails at planning time, before any fragment is rewritten.
+        let plan_err = plan_compaction(&dataset, &options).await.unwrap_err();
+        assert!(matches!(plan_err, Error::InvalidInput { .. }));
+        let msg = plan_err.to_string();
+        assert!(msg.contains("defer_index_remap"));
+        assert!(msg.contains("stable row IDs"));
+
+        // The full compact_files entry point fails the same way and leaves the
+        // dataset untouched (no new manifest version, no orphaned data files).
+        let version_before = dataset.manifest.version;
+        let compact_err = compact_files(&mut dataset, options, None)
+            .await
+            .unwrap_err();
+        assert!(matches!(compact_err, Error::InvalidInput { .. }));
+        assert_eq!(dataset.manifest.version, version_before);
+    }
+
     #[tokio::test]
     async fn test_defer_index_remap() {
         let mut data_gen = BatchGenerator::new()
@@ -3146,7 +3466,9 @@ mod tests {
             open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref())
                 .await
                 .unwrap();
-        let stats = frag_reuse_index.statistics().unwrap();
+        let stats = FragReuseIndexHandle(Arc::new(frag_reuse_index.clone()))
+            .statistics()
+            .unwrap();
         assert_eq!(
             serde_json::to_string(&stats).unwrap(),
             dataset
@@ -3204,6 +3526,52 @@ mod tests {
         assert_eq!(current_scalar_index.uuid, original_scalar_uuid);
     }
 
+    #[tokio::test]
+    async fn test_defer_index_remap_skips_fri_when_no_indexed_data() {
+        // A deferred compaction touching no indexed data must write no FRI --
+        // such a version is un-drainable (remap no-ops, trim retains it forever).
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned())));
+
+        let mut dataset = Dataset::write(
+            data_gen.batch(600),
+            "memory://test/noindex",
+            Some(WriteParams {
+                max_rows_per_file: 100, // 6 small files -> compaction has work
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // No index at all: nothing covers any fragment.
+        assert!(dataset.load_indices().await.unwrap().is_empty());
+        let fragments_before = dataset.get_fragments().len();
+        assert!(fragments_before > 1, "need multiple fragments to compact");
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 100_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+        compact_files(&mut dataset, options, None).await.unwrap();
+
+        // Compaction actually ran...
+        assert!(
+            dataset.get_fragments().len() < fragments_before,
+            "compaction should have merged fragments"
+        );
+        // ...but no fragment-reuse index was created.
+        assert!(
+            dataset
+                .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+                .await
+                .unwrap()
+                .is_none(),
+            "deferred compaction with no indexed data must not create an FRI"
+        );
+    }
+
     #[tokio::test]
     async fn test_defer_index_remap_multiple_compactions() {
         let mut data_gen = BatchGenerator::new()
@@ -3223,6 +3591,10 @@ mod tests {
         .await
         .unwrap();
 
+        // FRI is written only for compactions touching indexed data; index "i" so
+        // the successive deferred compactions build a chained fragment-reuse index.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -3276,49 +3648,209 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_remap_index_after_compaction() {
-        let mut data_gen = BatchGenerator::new()
-            .col(Box::new(
-                RandomVector::new().vec_width(128).named("vec".to_owned()),
-            ))
-            .col(Box::new(IncrementingInt32::new().named("i".to_owned())));
-
+    async fn test_defer_index_remap_mixed_records_all_groups() {
+        // All-or-nothing: a compaction touching any indexed data records the full
+        // FRI, including the unindexed group (a per-group filter would drop it).
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned())));
         let mut dataset = Dataset::write(
-            data_gen.batch(6_000),
-            "memory://test/table",
+            data_gen.batch(300),
+            "memory://test/mixed",
             Some(WriteParams {
-                max_rows_per_file: 1_000, // 6 files
+                max_rows_per_file: 100, // 3 fragments
                 ..Default::default()
             }),
         )
         .await
         .unwrap();
 
-        // Create a index to be remapped
-        let index_name = Some("scalar".into());
-        dataset
-            .create_index(
-                &["i"],
-                IndexType::Scalar,
-                index_name.clone(),
-                &ScalarIndexParams::default(),
-                false,
-            )
+        // Index the initial fragments, then append more that stay unindexed.
+        create_scalar_index(&mut dataset, "i", false).await;
+        Dataset::write(
+            data_gen.batch(300),
+            WriteDestination::Dataset(Arc::new(dataset.clone())),
+            Some(WriteParams {
+                max_rows_per_file: 100, // 3 more, unindexed
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        dataset.checkout_latest().await.unwrap();
+
+        // Fragments not covered by the scalar index are the "unindexed" ones.
+        let indexed: HashSet<u32> = dataset
+            .load_index_by_name("scalar")
+            .await
+            .unwrap()
+            .unwrap()
+            .fragment_bitmap
+            .unwrap()
+            .iter()
+            .collect();
+        let unindexed_frags: Vec<u64> = dataset
+            .fragments()
+            .iter()
+            .map(|f| f.id)
+            .filter(|id| !indexed.contains(&(*id as u32)))
+            .collect();
+        assert!(
+            !unindexed_frags.is_empty(),
+            "expected some unindexed fragments"
+        );
+
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 100_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        // All-or-nothing: because indexed fragments were compacted, the FRI is
+        // written AND records the unindexed group too (a per-group filter would
+        // have dropped it).
+        let fri_meta = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+            .expect("mixed compaction must write an FRI");
+        let details = load_frag_reuse_index_details(&dataset, &fri_meta)
             .await
             .unwrap();
+        let recorded_old: HashSet<u64> = details
+            .versions
+            .iter()
+            .flat_map(|v| v.old_frag_ids())
+            .collect();
+        for f in &unindexed_frags {
+            assert!(
+                recorded_old.contains(f),
+                "unindexed fragment {f} must be recorded in the FRI (all-or-nothing)"
+            );
+        }
+    }
 
+    #[tokio::test]
+    async fn test_deferred_compaction_not_split_by_frag_reuse_index() {
+        // The fragment-reuse index is a system index and must be excluded from
+        // compaction bin planning; otherwise its covered fragment is isolated and
+        // the small fragments never coalesce back to one.
+        let data = sample_data();
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
         let options = CompactionOptions {
-            target_rows_per_fragment: 2_000,
             defer_index_remap: true,
             ..Default::default()
         };
 
-        // Remap without a frag reuse index should yield unsupported
-        let Some(scalar_index) = dataset.load_index_by_name("scalar").await.unwrap() else {
-            panic!("scalar index must be available");
-        };
+        // Two small fragments -> deferred compaction folds them into one,
+        // creating the fragment-reuse index.
+        let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 400))], data.schema());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 200,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
 
-        let result = remapping::remap_column_index(&mut dataset, &["i"], index_name.clone()).await;
+        // Index "a" so the deferred compaction records an FRI (only written for
+        // compactions touching indexed data). The FRI is a system index and must
+        // still not split later compaction bins -- the property this test guards.
+        create_scalar_index(&mut dataset, "a", false).await;
+        compact_files(&mut dataset, options.clone(), None)
+            .await
+            .unwrap();
+        assert_eq!(dataset.get_fragments().len(), 1);
+        assert!(
+            dataset
+                .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+                .await
+                .unwrap()
+                .is_some()
+        );
+
+        // Append two more small fragments, then compact again.
+        let reader = RecordBatchIterator::new(vec![Ok(data.slice(400, 400))], data.schema());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 200,
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(dataset.get_fragments().len(), 3);
+
+        // Reindex so every fragment is data-indexed -- then the FRI (a system
+        // index, correctly excluded from bin planning) is the only thing that
+        // could split the bin.
+        create_scalar_index(&mut dataset, "a", true).await;
+
+        compact_files(&mut dataset, options, None).await.unwrap();
+        assert_eq!(
+            dataset.get_fragments().len(),
+            1,
+            "FRI (a system index) must not split the compaction bin; all fragments coalesce"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_remap_index_after_compaction() {
+        let mut data_gen = BatchGenerator::new()
+            .col(Box::new(
+                RandomVector::new().vec_width(128).named("vec".to_owned()),
+            ))
+            .col(Box::new(IncrementingInt32::new().named("i".to_owned())));
+
+        let mut dataset = Dataset::write(
+            data_gen.batch(6_000),
+            "memory://test/table",
+            Some(WriteParams {
+                max_rows_per_file: 1_000, // 6 files
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Create a index to be remapped
+        let index_name = Some("scalar".into());
+        dataset
+            .create_index(
+                &["i"],
+                IndexType::Scalar,
+                index_name.clone(),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 2_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+
+        // Remap without a frag reuse index should yield unsupported
+        let Some(scalar_index) = dataset.load_index_by_name("scalar").await.unwrap() else {
+            panic!("scalar index must be available");
+        };
+
+        let result = remapping::remap_column_index(&mut dataset, &["i"], index_name.clone()).await;
         assert!(matches!(result, Err(Error::NotSupported { .. })));
 
         let plan = plan_compaction(&dataset, &options).await.unwrap();
@@ -3595,6 +4127,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -3691,9 +4226,13 @@ mod tests {
             .unwrap();
         let new_frags3 = frag_reuse_details3.versions.last().unwrap().new_frag_ids();
 
-        // Concurrently commit a frag_reuse_index cleanup operation.
-        // Because there is no index, it should remove the first version.
-        // but after rebase it should contain the new compaction versions.
+        // Concurrently commit a frag_reuse_index cleanup operation. dataset_clone
+        // only knows the first reuse version; catch its index up so the cleanup
+        // removes that version. After rebase onto the other compactions it should
+        // contain the new compaction versions.
+        remapping::remap_column_index(&mut dataset_clone, &["i"], Some("scalar".into()))
+            .await
+            .unwrap();
         cleanup_frag_reuse_index(&mut dataset_clone).await.unwrap();
 
         // Load and verify the fragment reuse index content
@@ -3733,6 +4272,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -3771,8 +4313,12 @@ mod tests {
             .unwrap();
         assert_eq!(frag_reuse_details.versions.len(), 1);
 
-        // First commit the frag_reuse_index cleanup
-        // Because there is no index, it should remove the first version.
+        // Catch the index up to the compaction (on `dataset` only; `dataset_clone`
+        // keeps the un-caught-up index for the concurrent rewrite below), then
+        // clean up: with the index caught up the trim removes the first version.
+        remapping::remap_column_index(&mut dataset, &["i"], Some("scalar".into()))
+            .await
+            .unwrap();
         cleanup_frag_reuse_index(&mut dataset).await.unwrap();
 
         // Load and verify the fragment reuse index content
@@ -3843,6 +4389,9 @@ mod tests {
             .await
             .unwrap();
 
+        // Index "i" so the deferred compaction touches indexed data and writes an FRI.
+        create_scalar_index(&mut dataset, "i", false).await;
+
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
             defer_index_remap: true,
@@ -4090,150 +4639,358 @@ mod tests {
         );
     }
 
+    #[rstest]
+    #[case(IndexRemapMode::Compact)]
+    #[case(IndexRemapMode::Direct)]
     #[tokio::test]
-    async fn test_read_inverted_index_with_defer_index_remap() {
-        // Generate random words using lance-datagen
-        let mut words_gen = lance_datagen::array::random_sentence(1, 100, true);
-        let doc_col = words_gen
-            .generate_default(lance_datagen::RowCount::from(6000))
+    async fn test_btree_index_remap_after_compaction(#[case] index_remap_mode: IndexRemapMode) {
+        let mut dataset = lance_datagen::gen_batch()
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(32)),
+            )
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
             .unwrap();
 
-        let batch = RecordBatch::try_new(
-            Schema::new(vec![Field::new("doc", DataType::LargeUtf8, false)]).into(),
-            vec![doc_col.clone()],
-        )
-        .unwrap();
-        let schema_ref = batch.schema();
-        let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema_ref);
-        let mut dataset = Dataset::write(
-            stream,
-            "memory://test/table",
-            Some(WriteParams {
-                max_rows_per_file: 1_000, // 6 files
-                ..Default::default()
-            }),
-        )
-        .await
-        .unwrap();
-
-        // Get initial counts for some word searches
-        // Extract some test words from the generated documents
-        let large_string_array = doc_col.as_any().downcast_ref::<LargeStringArray>().unwrap();
-        let sample_words: Vec<String> = large_string_array
-            .value(0)
-            .split_whitespace()
-            .take(10)
-            .map(|s| s.to_string())
-            .collect();
-        let test_word1 = &sample_words[0];
-        let test_word2 = &sample_words[1];
-        let test_word3 = &sample_words[2];
+        // Delete rows scattered across fragments so the remap must drop some old
+        // addresses and shift the survivors.
+        dataset.delete("id % 10 == 0").await.unwrap();
 
-        // Create an inverted index on the doc column
-        let index_name = Some("doc_idx".into());
         dataset
             .create_index(
-                &["doc"],
-                IndexType::Inverted,
-                index_name.clone(),
-                &InvertedIndexParams::default(),
+                &["id"],
+                IndexType::BTree,
+                Some("id_idx".into()),
+                &ScalarIndexParams::default(),
                 false,
             )
             .await
             .unwrap();
-        let indices = dataset.load_indices().await.unwrap();
-        let original_index = indices.iter().find(|idx| idx.name == "doc_idx").unwrap();
 
-        // Run compaction with deferred index remapping
+        let count_low = dataset
+            .count_rows(Some("id < 1000".to_owned()))
+            .await
+            .unwrap();
+        let count_mid = dataset
+            .count_rows(Some("id >= 2000 and id < 3000".to_owned()))
+            .await
+            .unwrap();
+        let count_high = dataset
+            .count_rows(Some("id >= 5000".to_owned()))
+            .await
+            .unwrap();
+
         let options = CompactionOptions {
-            target_rows_per_fragment: 2_000,
-            defer_index_remap: true,
+            target_rows_per_fragment: 50_000,
+            index_remap_mode,
             ..Default::default()
         };
-
         let metrics = compact_files(&mut dataset, options, None).await.unwrap();
         assert!(metrics.fragments_removed > 0);
         assert!(metrics.fragments_added > 0);
 
-        // Verify the index UUID is unchanged (it should not be remapped yet)
-        let Some(current_index) = dataset.load_index_by_name("doc_idx").await.unwrap() else {
-            panic!("doc index must be available");
-        };
-        assert_eq!(current_index.uuid, original_index.uuid);
-
-        // Initial scan
-        let mut scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
-            .unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let count1 = scanner.count_rows().await.unwrap();
-        scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word2.clone()))
-            .unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let count2 = scanner.count_rows().await.unwrap();
-        scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word3.clone()))
-            .unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let count3 = scanner.count_rows().await.unwrap();
-
-        // Verify that after index creation and compaction, scan uses inverted index scan
+        // The index was remapped inline and must still drive scans.
         let mut scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
-            .unwrap();
+        scanner.filter("id >= 2000 and id < 3000").unwrap();
         scanner.project::<String>(&[]).unwrap().with_row_id();
-        let plan = scanner.explain_plan(true).await.unwrap();
+        let plan = scanner.explain_plan(false).await.unwrap();
         assert!(
-            plan.contains("MatchQuery"),
-            "Expected inverted index scan in plan: {}",
+            plan.contains("ScalarIndexQuery: query=[id >= 2000 && id < 3000]@id_idx(BTree)"),
+            "Expected scalar index query in plan: {}",
             plan
         );
-        assert!(
-            !plan.contains("LanceScan"),
-            "Expected no fragment scan in plan: {}",
-            plan
+
+        // Counts resolved through the remapped index match the pre-compaction
+        // values in both remap modes.
+        assert_eq!(
+            dataset
+                .count_rows(Some("id < 1000".to_owned()))
+                .await
+                .unwrap(),
+            count_low
+        );
+        assert_eq!(
+            dataset
+                .count_rows(Some("id >= 2000 and id < 3000".to_owned()))
+                .await
+                .unwrap(),
+            count_mid
+        );
+        assert_eq!(
+            dataset
+                .count_rows(Some("id >= 5000".to_owned()))
+                .await
+                .unwrap(),
+            count_high
         );
+    }
 
-        // Reindex to the latest
+    #[rstest]
+    #[case(IndexRemapMode::Compact)]
+    #[case(IndexRemapMode::Direct)]
+    #[tokio::test]
+    async fn test_ivf_pq_index_remap_after_compaction(#[case] index_remap_mode: IndexRemapMode) {
+        use arrow_array::cast::AsArray;
+        use lance_index::vector::pq::PQBuildParams;
+
+        const DIM: u32 = 32;
+        let mut dataset = lance_datagen::gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(DIM)),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            small_ivf(),
+            PQBuildParams {
+                max_iters: 2,
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
         dataset
             .create_index(
-                &["doc"],
-                IndexType::Inverted,
-                index_name.clone(),
-                &InvertedIndexParams::default(),
-                true,
+                &["vec"],
+                IndexType::Vector,
+                Some("vec_idx".into()),
+                &params,
+                false,
             )
             .await
             .unwrap();
+        let original_uuid = dataset
+            .load_index_by_name("vec_idx")
+            .await
+            .unwrap()
+            .unwrap()
+            .uuid;
 
-        // Verify that scans still work correctly and return the same counts
-        let mut scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
-            .unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        assert_eq!(scanner.count_rows().await.unwrap(), count1);
-        scanner = dataset.scan();
-        scanner
-            .full_text_search(FullTextSearchQuery::new(test_word2.clone()))
-            .unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        assert_eq!(scanner.count_rows().await.unwrap(), count2);
-        scanner = dataset.scan();
+        // Delete rows scattered across fragments so the remap must drop some old
+        // addresses and shift the survivors.
+        dataset.delete("id % 10 == 0").await.unwrap();
+
+        // Sample queries from surviving vectors and capture the pre-compaction
+        // KNN answer and the surviving id set.
+        let mut survivors: Vec<(i32, Vec<f32>)> = Vec::new();
+        {
+            let mut scanner = dataset.scan();
+            scanner.project(&["id", "vec"]).unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            for batch in &batches {
+                let ids = batch["id"].as_primitive::<Int32Type>();
+                let vecs = batch["vec"].as_fixed_size_list();
+                for i in 0..batch.num_rows() {
+                    let v = vecs.value(i);
+                    survivors.push((
+                        ids.value(i),
+                        v.as_primitive::<Float32Type>().values().to_vec(),
+                    ));
+                }
+            }
+        }
+        let surviving_ids: std::collections::HashSet<i32> =
+            survivors.iter().map(|(id, _)| *id).collect();
+        let step = (survivors.len() / 16).max(1);
+        let queries: Vec<Vec<f32>> = survivors
+            .iter()
+            .step_by(step)
+            .map(|(_, v)| v.clone())
+            .collect();
+        let k = 10;
+        let mut baseline: Vec<Vec<i32>> = Vec::new();
+        for q in &queries {
+            baseline.push(vector_knn_ids(&dataset, q, k).await);
+        }
+
+        // Inline remap (defer_index_remap = false): compaction physically
+        // rebuilds the vector index through the configured remap mode.
+        let metrics = compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 50_000,
+                index_remap_mode,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(metrics.fragments_added > 0);
+
+        // The index was physically remapped inline, so its uuid must change.
+        assert_ne!(
+            dataset
+                .load_index_by_name("vec_idx")
+                .await
+                .unwrap()
+                .unwrap()
+                .uuid,
+            original_uuid,
+            "vector index must be physically remapped inline"
+        );
+
+        // The remap only relabels row addresses; it must not resurrect deleted
+        // rows, and KNN must stay close to the pre-compaction answer in both
+        // remap modes.
+        for (i, q) in queries.iter().enumerate() {
+            let after = vector_knn_ids(&dataset, q, k).await;
+            for id in &after {
+                assert!(
+                    surviving_ids.contains(id),
+                    "KNN returned id {id} that is not a surviving row (query #{i}, mode {index_remap_mode:?})"
+                );
+            }
+            let overlap = after.iter().filter(|id| baseline[i].contains(id)).count();
+            assert!(
+                overlap >= 8,
+                "KNN top-{k} diverged after compaction: overlap {overlap} < 8 (query #{i}, mode {index_remap_mode:?})"
+            );
+        }
+    }
+
+    #[rstest]
+    #[case(IndexRemapMode::Compact)]
+    #[case(IndexRemapMode::Direct)]
+    #[tokio::test]
+    async fn test_inverted_index_remap_after_compaction(#[case] index_remap_mode: IndexRemapMode) {
+        use arrow_array::cast::AsArray;
+
+        let mut dataset = lance_datagen::gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col("doc", lance_datagen::array::random_sentence(1, 100, false))
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        dataset
+            .create_index(
+                &["doc"],
+                IndexType::Inverted,
+                Some("doc_idx".into()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let original_uuid = dataset
+            .load_index_by_name("doc_idx")
+            .await
+            .unwrap()
+            .unwrap()
+            .uuid;
+
+        // Sample a few words from a real document to drive full-text searches.
+        let words: Vec<String> = {
+            let mut scanner = dataset.scan();
+            scanner
+                .project(&["doc"])
+                .unwrap()
+                .limit(Some(1), None)
+                .unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            let mut words: Vec<String> = batches[0]["doc"]
+                .as_string::<i32>()
+                .value(0)
+                .split_whitespace()
+                .map(|s| s.to_string())
+                .collect();
+            words.sort();
+            words.dedup();
+            words.truncate(3);
+            words
+        };
+        assert!(!words.is_empty(), "sampled document must contain words");
+
+        // Delete rows scattered across fragments so the remap must drop some old
+        // addresses and shift the survivors.
+        dataset.delete("id % 10 == 0").await.unwrap();
+
+        // Capture the post-deletion full-text-search counts (resolved through the
+        // index + deletion vectors) before compaction physically remaps.
+        let mut before = Vec::new();
+        for word in &words {
+            let mut scanner = dataset.scan();
+            scanner
+                .full_text_search(FullTextSearchQuery::new(word.clone()))
+                .unwrap();
+            scanner.project::<String>(&[]).unwrap().with_row_id();
+            before.push(scanner.count_rows().await.unwrap());
+        }
+
+        // Inline remap (defer_index_remap = false): compaction physically rebuilds
+        // the inverted index through the configured remap mode.
+        let options = CompactionOptions {
+            target_rows_per_fragment: 50_000,
+            index_remap_mode,
+            ..Default::default()
+        };
+        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(metrics.fragments_added > 0);
+
+        // The index was physically remapped inline, so its uuid must change.
+        assert_ne!(
+            dataset
+                .load_index_by_name("doc_idx")
+                .await
+                .unwrap()
+                .unwrap()
+                .uuid,
+            original_uuid,
+            "inverted index must be physically remapped inline (mode {index_remap_mode:?})"
+        );
+
+        // The remapped index must still drive full-text search.
+        let mut scanner = dataset.scan();
         scanner
-            .full_text_search(FullTextSearchQuery::new(test_word3.clone()))
+            .full_text_search(FullTextSearchQuery::new(words[0].clone()))
             .unwrap();
         scanner.project::<String>(&[]).unwrap().with_row_id();
-        assert_eq!(scanner.count_rows().await.unwrap(), count3);
+        let plan = scanner.explain_plan(true).await.unwrap();
+        assert!(
+            plan.contains("MatchQuery"),
+            "Expected inverted index scan in plan: {}",
+            plan
+        );
+
+        // Counts resolved through the remapped index match the pre-compaction
+        // values in both remap modes.
+        for (word, expected) in words.iter().zip(before) {
+            let mut scanner = dataset.scan();
+            scanner
+                .full_text_search(FullTextSearchQuery::new(word.clone()))
+                .unwrap();
+            scanner.project::<String>(&[]).unwrap().with_row_id();
+            assert_eq!(
+                scanner.count_rows().await.unwrap(),
+                expected,
+                "full-text count for {word:?} changed after compaction (mode {index_remap_mode:?})"
+            );
+        }
     }
 
     #[tokio::test]
-    async fn test_read_ngram_index_with_defer_index_remap() {
+    async fn test_read_inverted_index_with_defer_index_remap() {
         // Generate random words using lance-datagen
         let mut words_gen = lance_datagen::array::random_sentence(1, 100, true);
         let doc_col = words_gen
@@ -4276,9 +5033,9 @@ mod tests {
         dataset
             .create_index(
                 &["doc"],
-                IndexType::NGram,
+                IndexType::Inverted,
                 index_name.clone(),
-                &ScalarIndexParams::default(),
+                &InvertedIndexParams::default(),
                 false,
             )
             .await
@@ -4286,20 +5043,6 @@ mod tests {
         let indices = dataset.load_indices().await.unwrap();
         let original_index = indices.iter().find(|idx| idx.name == "doc_idx").unwrap();
 
-        // Initial scan
-        let count1 = dataset
-            .count_rows(Some(format!("contains(doc, '{}')", test_word1)))
-            .await
-            .unwrap();
-        let count2 = dataset
-            .count_rows(Some(format!("contains(doc, '{}')", test_word2)))
-            .await
-            .unwrap();
-        let count3 = dataset
-            .count_rows(Some(format!("contains(doc, '{}')", test_word3)))
-            .await
-            .unwrap();
-
         // Run compaction with deferred index remapping
         let options = CompactionOptions {
             target_rows_per_fragment: 2_000,
@@ -4317,84 +5060,367 @@ mod tests {
         };
         assert_eq!(current_index.uuid, original_index.uuid);
 
-        // Verify that scans still work correctly and return the same counts
-        assert_eq!(
-            dataset
-                .count_rows(Some(format!("contains(doc, '{}')", test_word1)))
-                .await
-                .unwrap(),
-            count1
-        );
-        assert_eq!(
-            dataset
-                .count_rows(Some(format!("contains(doc, '{}')", test_word2)))
-                .await
-                .unwrap(),
-            count2
-        );
-        assert_eq!(
-            dataset
-                .count_rows(Some(format!("contains(doc, '{}')", test_word3)))
-                .await
-                .unwrap(),
-            count3
-        );
+        // Initial scan
+        let mut scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
+            .unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let count1 = scanner.count_rows().await.unwrap();
+        scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word2.clone()))
+            .unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let count2 = scanner.count_rows().await.unwrap();
+        scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word3.clone()))
+            .unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let count3 = scanner.count_rows().await.unwrap();
 
         // Verify that after index creation and compaction, scan uses inverted index scan
         let mut scanner = dataset.scan();
         scanner
-            .filter(&format!("contains(doc, '{}')", test_word1))
+            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
             .unwrap();
         scanner.project::<String>(&[]).unwrap().with_row_id();
-        let plan = scanner.explain_plan(false).await.unwrap();
+        let plan = scanner.explain_plan(true).await.unwrap();
         assert!(
-            plan.contains("ScalarIndexQuery: query=[contains(doc, Utf8"),
-            "Expected scalar index query in plan: {}",
+            plan.contains("MatchQuery"),
+            "Expected inverted index scan in plan: {}",
+            plan
+        );
+        assert!(
+            !plan.contains("LanceScan"),
+            "Expected no fragment scan in plan: {}",
             plan
         );
-    }
 
-    #[tokio::test]
-    async fn test_read_label_list_index_with_defer_index_remap() {
-        // Create a dataset with list data for labels
-        let mut dataset = lance_datagen::gen_batch()
-            .col(
-                "vec",
-                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(128)),
-            )
-            .col(
-                "labels",
-                lance_datagen::array::rand_list_any(
-                    lance_datagen::array::cycle::<Int64Type>(vec![1, 2, 3, 4, 5]),
-                    false,
-                ),
+        // Reindex to the latest
+        dataset
+            .create_index(
+                &["doc"],
+                IndexType::Inverted,
+                index_name.clone(),
+                &InvertedIndexParams::default(),
+                true,
             )
-            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
             .await
             .unwrap();
 
-        // Get initial counts for different label values
-        let count1 = dataset
-            .count_rows(Some("array_has_any(labels, [1])".to_owned()))
-            .await
+        // Verify that scans still work correctly and return the same counts
+        let mut scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word1.clone()))
             .unwrap();
-        let count2 = dataset
-            .count_rows(Some("array_has_any(labels, [5])".to_owned()))
-            .await
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        assert_eq!(scanner.count_rows().await.unwrap(), count1);
+        scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word2.clone()))
             .unwrap();
-        let count3 = dataset
-            .count_rows(Some("array_has_any(labels, [10])".to_owned()))
-            .await
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        assert_eq!(scanner.count_rows().await.unwrap(), count2);
+        scanner = dataset.scan();
+        scanner
+            .full_text_search(FullTextSearchQuery::new(test_word3.clone()))
             .unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        assert_eq!(scanner.count_rows().await.unwrap(), count3);
+    }
+
+    /// Deferred compaction that materializes deletions must not corrupt an
+    /// inverted (FTS) index read through the fragment-reuse index. The index's
+    /// posting lists reference doc_ids positionally; if the load-time remap
+    /// dropped the deleted rows it would renumber the doc_ids and desync the
+    /// posting lists (out-of-bounds `num_tokens`, wrong/stale row ids). The
+    /// tombstone-preserve-positions load path must keep results correct in the
+    /// FRI window and after the physical remap + trim.
+    #[tokio::test]
+    async fn test_read_inverted_index_with_defer_index_remap_and_deletions() {
+        // Enough surviving docs for several compressed posting-list blocks
+        // (BLOCK_SIZE = 128), split across several fragments so compaction has
+        // real work — but no larger.
+        const ROWS: i32 = 1200;
+        const DELETED: i32 = 400;
+
+        // Every row contains "lance", so the term matches all live rows; `id`
+        // tells us exactly which rows survive.
+        let ids = Int32Array::from_iter_values(0..ROWS);
+        let docs = LargeStringArray::from_iter_values((0..ROWS).map(|_| "lance apple orange"));
+        let batch = RecordBatch::try_new(
+            Schema::new(vec![
+                Field::new("id", DataType::Int32, false),
+                Field::new("doc", DataType::LargeUtf8, false),
+            ])
+            .into(),
+            vec![Arc::new(ids) as ArrayRef, Arc::new(docs) as ArrayRef],
+        )
+        .unwrap();
+        let schema_ref = batch.schema();
+        let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema_ref);
+        let mut dataset = Dataset::write(
+            stream,
+            "memory://test/table",
+            Some(WriteParams {
+                max_rows_per_file: 200, // 6 fragments
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
 
-        // Create a label list index on the labels column
-        let index_name = Some("labels_idx".into());
         dataset
             .create_index(
-                &["labels"],
-                IndexType::LabelList,
-                index_name.clone(),
-                &ScalarIndexParams::default(),
+                &["doc"],
+                IndexType::Inverted,
+                Some("doc_idx".into()),
+                &InvertedIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Delete a prefix, then deferred-compact so the deletions are
+        // materialized into the fragment-reuse index the index is read through.
+        dataset.delete(&format!("id < {DELETED}")).await.unwrap();
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 2_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(
+            dataset
+                .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+                .await
+                .unwrap()
+                .is_some(),
+            "deferred compaction must leave a fragment-reuse index"
+        );
+
+        // FTS "lance" → sorted surviving ids. Projecting `id` forces a take, so
+        // a stale row address would error or return a wrong/dead row.
+        async fn search_ids(dataset: &Dataset) -> Vec<i32> {
+            let mut scanner = dataset.scan();
+            scanner
+                .full_text_search(FullTextSearchQuery::new("lance".to_owned()))
+                .unwrap();
+            scanner.project::<&str>(&["id"]).unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            let mut ids: Vec<i32> = batches
+                .iter()
+                .flat_map(|b| {
+                    b.column_by_name("id")
+                        .unwrap()
+                        .as_any()
+                        .downcast_ref::<Int32Array>()
+                        .unwrap()
+                        .values()
+                        .to_vec()
+                })
+                .collect();
+            ids.sort_unstable();
+            ids
+        }
+
+        let expected = (DELETED..ROWS).collect::<Vec<_>>();
+
+        // FRI window: index read through the reuse index.
+        let during = search_ids(&dataset).await;
+        assert_eq!(
+            during, expected,
+            "FRI-window FTS must return exactly the surviving rows (no resurrection, no loss, no stale rows)"
+        );
+
+        // Physical remap + trim: must still be correct.
+        remapping::remap_column_index(&mut dataset, &["doc"], Some("doc_idx".into()))
+            .await
+            .unwrap();
+        cleanup_frag_reuse_index(&mut dataset).await.unwrap();
+        let after = search_ids(&dataset).await;
+        assert_eq!(
+            after, expected,
+            "FTS must stay correct after physical remap + fragment-reuse trim"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_ngram_index_with_defer_index_remap() {
+        // Generate random words using lance-datagen
+        let mut words_gen = lance_datagen::array::random_sentence(1, 100, true);
+        let doc_col = words_gen
+            .generate_default(lance_datagen::RowCount::from(6000))
+            .unwrap();
+
+        let batch = RecordBatch::try_new(
+            Schema::new(vec![Field::new("doc", DataType::LargeUtf8, false)]).into(),
+            vec![doc_col.clone()],
+        )
+        .unwrap();
+        let schema_ref = batch.schema();
+        let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema_ref);
+        let mut dataset = Dataset::write(
+            stream,
+            "memory://test/table",
+            Some(WriteParams {
+                max_rows_per_file: 1_000, // 6 files
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Get initial counts for some word searches
+        // Extract some test words from the generated documents
+        let large_string_array = doc_col.as_any().downcast_ref::<LargeStringArray>().unwrap();
+        let sample_words: Vec<String> = large_string_array
+            .value(0)
+            .split_whitespace()
+            .take(10)
+            .map(|s| s.to_string())
+            .collect();
+        let test_word1 = &sample_words[0];
+        let test_word2 = &sample_words[1];
+        let test_word3 = &sample_words[2];
+
+        // Create an inverted index on the doc column
+        let index_name = Some("doc_idx".into());
+        dataset
+            .create_index(
+                &["doc"],
+                IndexType::NGram,
+                index_name.clone(),
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+        let indices = dataset.load_indices().await.unwrap();
+        let original_index = indices.iter().find(|idx| idx.name == "doc_idx").unwrap();
+
+        // Initial scan
+        let count1 = dataset
+            .count_rows(Some(format!("contains(doc, '{}')", test_word1)))
+            .await
+            .unwrap();
+        let count2 = dataset
+            .count_rows(Some(format!("contains(doc, '{}')", test_word2)))
+            .await
+            .unwrap();
+        let count3 = dataset
+            .count_rows(Some(format!("contains(doc, '{}')", test_word3)))
+            .await
+            .unwrap();
+
+        // Run compaction with deferred index remapping
+        let options = CompactionOptions {
+            target_rows_per_fragment: 2_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+
+        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(metrics.fragments_added > 0);
+
+        // Verify the index UUID is unchanged (it should not be remapped yet)
+        let Some(current_index) = dataset.load_index_by_name("doc_idx").await.unwrap() else {
+            panic!("doc index must be available");
+        };
+        assert_eq!(current_index.uuid, original_index.uuid);
+
+        // Verify that scans still work correctly and return the same counts
+        assert_eq!(
+            dataset
+                .count_rows(Some(format!("contains(doc, '{}')", test_word1)))
+                .await
+                .unwrap(),
+            count1
+        );
+        assert_eq!(
+            dataset
+                .count_rows(Some(format!("contains(doc, '{}')", test_word2)))
+                .await
+                .unwrap(),
+            count2
+        );
+        assert_eq!(
+            dataset
+                .count_rows(Some(format!("contains(doc, '{}')", test_word3)))
+                .await
+                .unwrap(),
+            count3
+        );
+
+        // Verify that after index creation and compaction, scan uses inverted index scan
+        let mut scanner = dataset.scan();
+        scanner
+            .filter(&format!("contains(doc, '{}')", test_word1))
+            .unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let plan = scanner.explain_plan(false).await.unwrap();
+        assert!(
+            plan.contains("ScalarIndexQuery: query=[contains(doc, Utf8"),
+            "Expected scalar index query in plan: {}",
+            plan
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_label_list_index_with_defer_index_remap() {
+        // Create a dataset with list data for labels
+        let mut dataset = lance_datagen::gen_batch()
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(128)),
+            )
+            .col(
+                "labels",
+                lance_datagen::array::rand_list_any(
+                    lance_datagen::array::cycle::<Int64Type>(vec![1, 2, 3, 4, 5]),
+                    false,
+                ),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        // Get initial counts for different label values
+        let count1 = dataset
+            .count_rows(Some("array_has_any(labels, [1])".to_owned()))
+            .await
+            .unwrap();
+        let count2 = dataset
+            .count_rows(Some("array_has_any(labels, [5])".to_owned()))
+            .await
+            .unwrap();
+        let count3 = dataset
+            .count_rows(Some("array_has_any(labels, [10])".to_owned()))
+            .await
+            .unwrap();
+
+        // Create a label list index on the labels column
+        let index_name = Some("labels_idx".into());
+        dataset
+            .create_index(
+                &["labels"],
+                IndexType::LabelList,
+                index_name.clone(),
+                &ScalarIndexParams::default(),
                 false,
             )
             .await
@@ -4544,77 +5570,739 @@ mod tests {
         let indices = dataset.load_indices().await.unwrap();
         let original_index = indices.iter().find(|idx| idx.name == "vec_idx").unwrap();
 
-        // Run compaction with deferred index remapping
-        let options = CompactionOptions {
-            target_rows_per_fragment: 2_000,
-            defer_index_remap: true,
-            ..Default::default()
-        };
+        // Run compaction with deferred index remapping
+        let options = CompactionOptions {
+            target_rows_per_fragment: 2_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+
+        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(metrics.fragments_added > 0);
+
+        // Verify the index UUID is unchanged (it should not be remapped yet)
+        let Some(current_index) = dataset.load_index_by_name("vec_idx").await.unwrap() else {
+            panic!("vec index must be available");
+        };
+        assert_eq!(current_index.uuid, original_index.uuid);
+
+        // Verify that KNN searches still work correctly and return the same counts
+        let mut scanner = dataset.scan();
+        scanner.nearest("vec", &query_vec1, 10).unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let new_results1 = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(new_results1.len(), count1);
+
+        scanner = dataset.scan();
+        scanner.nearest("vec", &query_vec2, 10).unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let new_results2 = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(new_results2.len(), count2);
+
+        scanner = dataset.scan();
+        scanner.nearest("vec", &query_vec3, 10).unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let new_results3 = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(new_results3.len(), count3);
+
+        // Verify that after index creation and compaction, scan uses vector index scan
+        let mut scanner = dataset.scan();
+        scanner.nearest("vec", &query_vec1, 10).unwrap();
+        scanner.project::<String>(&[]).unwrap().with_row_id();
+        let plan = scanner.explain_plan(false).await.unwrap();
+        assert!(
+            plan.contains("ANNSubIndex"),
+            "Expected vector index scan in plan: {}",
+            plan
+        );
+        assert!(
+            !plan.contains("LanceScan"),
+            "Expected no fragment scan in plan: {}",
+            plan
+        );
+    }
+
+    #[tokio::test]
+    async fn test_read_ivf_rq_index_v3_with_defer_index_remap() {
+        use arrow_array::cast::AsArray;
+        use lance_index::vector::bq::RQBuildParams;
+
+        let mut dataset = lance_datagen::gen_batch()
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(128)),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        let stored: Vec<Vec<f32>> = {
+            let mut scanner = dataset.scan();
+            scanner.project(&["vec"]).unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            let mut out = Vec::new();
+            for batch in &batches {
+                let vecs = batch["vec"].as_fixed_size_list();
+                for i in 0..batch.num_rows() {
+                    let values = vecs.value(i);
+                    let values = values.as_primitive::<Float32Type>();
+                    out.push(values.values().to_vec());
+                }
+            }
+            out
+        };
+
+        let index_name = Some("vec_idx".into());
+        dataset
+            .create_index(
+                &["vec"],
+                IndexType::Vector,
+                index_name.clone(),
+                &VectorIndexParams {
+                    metric_type: DistanceType::L2,
+                    stages: vec![
+                        StageParams::Ivf(IvfBuildParams {
+                            max_iters: 2,
+                            num_partitions: Some(2),
+                            sample_rate: 2,
+                            ..Default::default()
+                        }),
+                        StageParams::RQ(RQBuildParams::new(1)),
+                    ],
+                    version: crate::index::vector::IndexFileVersion::V3,
+                    skip_transpose: false,
+                    runtime_hints: Default::default(),
+                },
+                false,
+            )
+            .await
+            .unwrap();
+        let indices = dataset.load_indices().await.unwrap();
+        let original_index = indices.iter().find(|idx| idx.name == "vec_idx").unwrap();
+
+        let options = CompactionOptions {
+            target_rows_per_fragment: 2_000,
+            defer_index_remap: true,
+            ..Default::default()
+        };
+        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(metrics.fragments_added > 0);
+
+        let Some(current_index) = dataset.load_index_by_name("vec_idx").await.unwrap() else {
+            panic!("vec index must be available");
+        };
+        assert_eq!(current_index.uuid, original_index.uuid);
+
+        let frag_reuse_present = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME);
+        assert!(
+            frag_reuse_present,
+            "defer_index_remap must record a {} index",
+            FRAG_REUSE_INDEX_NAME
+        );
+
+        let sample_step = (stored.len() / 8).max(1);
+        let mut checked = 0;
+        for query in stored.iter().step_by(sample_step) {
+            let query_vec = PrimitiveArray::<Float32Type>::from_iter_values(query.iter().copied());
+            let mut scanner = dataset.scan();
+            scanner.nearest("vec", &query_vec, 5).unwrap();
+            scanner.project(&["vec"]).unwrap().with_row_id();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            assert!(!batches.is_empty(), "query returned no batches");
+            let top = &batches[0];
+            assert!(top.num_rows() > 0, "query returned empty top batch");
+            let top_vec = top["vec"].as_fixed_size_list().value(0);
+            let top_vec = top_vec.as_primitive::<Float32Type>();
+            assert_eq!(
+                top_vec.values(),
+                query.as_slice(),
+                "top-1 self-recall returned a different vector than the query"
+            );
+            checked += 1;
+        }
+        assert!(checked > 0, "expected to check at least one stored vector");
+    }
+
+    /// Build an `id` + `vec` dataset, create the given IVF vector index,
+    /// optionally delete rows, then run deferred compaction (which materializes
+    /// the deletions into the fragment-reuse index) and assert that KNN over
+    /// surviving vectors during the FRI window (a) never returns a deleted row
+    /// and (b) stays consistent with the pre-compaction answer.
+    ///
+    /// The deletion path is the interesting one: materialized deletions drop
+    /// rows from the quantization storage at load time, which shifts storage
+    /// positions. Flat storage (FLAT/PQ/SQ/RQ) is scanned linearly so this is
+    /// fine, but the HNSW graph addresses storage positionally and is not
+    /// frag-reuse aware, so a desync would surface here as recall collapse or a
+    /// resurrected/again-deleted row.
+    /// Top-k `id`s for a KNN query against the `vec` column.
+    async fn vector_knn_ids(dataset: &Dataset, query: &[f32], k: usize) -> Vec<i32> {
+        use arrow_array::cast::AsArray;
+        use arrow_array::types::{Float32Type, Int32Type};
+        let qa = PrimitiveArray::<Float32Type>::from_iter_values(query.iter().copied());
+        let mut scanner = dataset.scan();
+        scanner.nearest("vec", &qa, k).unwrap();
+        scanner.project(&["id"]).unwrap();
+        let batches = scanner
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let mut ids = Vec::new();
+        for b in &batches {
+            ids.extend(b["id"].as_primitive::<Int32Type>().values().iter().copied());
+        }
+        ids
+    }
+
+    async fn check_vector_defer_compaction(
+        params: VectorIndexParams,
+        delete_predicate: Option<&str>,
+        k: usize,
+        min_overlap: usize,
+    ) {
+        use arrow_array::cast::AsArray;
+        use arrow_array::types::{Float32Type, Int32Type};
+        use lance_datagen::Dimension;
+
+        const DIM: u32 = 32;
+        let mut dataset = lance_datagen::gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(DIM)),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+
+        dataset
+            .create_index(
+                &["vec"],
+                IndexType::Vector,
+                Some("vec_idx".into()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+        let original_uuid = dataset
+            .load_index_by_name("vec_idx")
+            .await
+            .unwrap()
+            .unwrap()
+            .uuid;
+
+        if let Some(pred) = delete_predicate {
+            dataset.delete(pred).await.unwrap();
+        }
+
+        // Collect surviving (id, vec) pairs and the set of surviving ids.
+        let mut survivors: Vec<(i32, Vec<f32>)> = Vec::new();
+        {
+            let mut scanner = dataset.scan();
+            scanner.project(&["id", "vec"]).unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            for batch in &batches {
+                let ids = batch["id"].as_primitive::<Int32Type>();
+                let vecs = batch["vec"].as_fixed_size_list();
+                for i in 0..batch.num_rows() {
+                    let v = vecs.value(i);
+                    let v = v.as_primitive::<Float32Type>().values().to_vec();
+                    survivors.push((ids.value(i), v));
+                }
+            }
+        }
+        assert!(!survivors.is_empty());
+        let surviving_ids: std::collections::HashSet<i32> =
+            survivors.iter().map(|(id, _)| *id).collect();
+
+        // Sample queries from survivors and capture the pre-compaction answer.
+        let step = (survivors.len() / 16).max(1);
+        let queries: Vec<(i32, Vec<f32>)> = survivors.iter().step_by(step).cloned().collect();
+        let mut baseline: Vec<Vec<i32>> = Vec::new();
+        for (_, q) in &queries {
+            baseline.push(vector_knn_ids(&dataset, q, k).await);
+        }
+
+        // Deferred compaction materializes the deletions into the frag-reuse index.
+        let metrics = compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 2_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(
+            dataset
+                .load_indices()
+                .await
+                .unwrap()
+                .iter()
+                .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME),
+            "deferred compaction must record a frag-reuse index"
+        );
+        assert_eq!(
+            dataset
+                .load_index_by_name("vec_idx")
+                .await
+                .unwrap()
+                .unwrap()
+                .uuid,
+            original_uuid,
+            "index must not be physically remapped yet (FRI window)"
+        );
+
+        // During the FRI window: no deleted rows, and stable vs the baseline.
+        for (i, (_, q)) in queries.iter().enumerate() {
+            let after = vector_knn_ids(&dataset, q, k).await;
+            for id in &after {
+                assert!(
+                    surviving_ids.contains(id),
+                    "KNN returned id {id} that is not a surviving row (query #{i})"
+                );
+            }
+            let overlap = after.iter().filter(|id| baseline[i].contains(id)).count();
+            assert!(
+                overlap >= min_overlap,
+                "KNN top-{k} diverged after deferred compaction: overlap {overlap} < {min_overlap} (query #{i})"
+            );
+        }
+    }
+
+    fn small_ivf() -> lance_index::vector::ivf::IvfBuildParams {
+        lance_index::vector::ivf::IvfBuildParams {
+            max_iters: 2,
+            num_partitions: Some(2),
+            sample_rate: 2,
+            ..Default::default()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_ivf_flat_defer_compaction_with_deletions() {
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, small_ivf());
+        // Flat storage is scanned linearly; dropping deleted rows is exact.
+        check_vector_defer_compaction(params, Some("id < 1500"), 10, 10).await;
+    }
+
+    #[tokio::test]
+    async fn test_ivf_hnsw_sq_defer_compaction_merge_only() {
+        use lance_index::vector::{hnsw::builder::HnswBuildParams, sq::builder::SQBuildParams};
+        let params = VectorIndexParams::with_ivf_hnsw_sq_params(
+            DistanceType::L2,
+            small_ivf(),
+            HnswBuildParams::default(),
+            SQBuildParams::default(),
+        );
+        // No deletions: storage positions are stable, so the graph stays aligned.
+        check_vector_defer_compaction(params, None, 10, 9).await;
+    }
+
+    // NOTE: IVF_HNSW_* under materialized deletions is a known gap (lance#3993,
+    // HNSW auto-remap not implemented) — the HNSW graph isn't realigned after the
+    // frag-reuse drop. Deferred remap is gated off for HNSW tables, so there is
+    // no lance-level reproducer here; the gate is tested in the data plane.
+    // Merge-only HNSW is covered (see the *_remap_and_trim tests).
+
+    #[tokio::test]
+    async fn test_ivf_pq_defer_compaction_with_deletions() {
+        use lance_index::vector::pq::PQBuildParams;
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            small_ivf(),
+            PQBuildParams {
+                max_iters: 2,
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
+        check_vector_defer_compaction(params, Some("id < 1500"), 10, 8).await;
+    }
+
+    #[tokio::test]
+    async fn test_ivf_sq_defer_compaction_with_deletions() {
+        use lance_index::vector::sq::builder::SQBuildParams;
+        let params = VectorIndexParams::with_ivf_sq_params(
+            DistanceType::L2,
+            small_ivf(),
+            SQBuildParams::default(),
+        );
+        check_vector_defer_compaction(params, Some("id < 1500"), 10, 8).await;
+    }
+
+    #[tokio::test]
+    async fn test_ivf_rq_defer_compaction_with_deletions() {
+        use lance_index::vector::bq::RQBuildParams;
+        let params = VectorIndexParams::with_ivf_rq_params(
+            DistanceType::L2,
+            small_ivf(),
+            RQBuildParams::new(1),
+        );
+        check_vector_defer_compaction(params, Some("id < 1500"), 10, 8).await;
+    }
+
+    /// Merge-only deferred compaction, then a PHYSICAL remap + FRI trim. Asserts
+    /// the index is rebuilt, the fragment-reuse index trims to zero versions,
+    /// and KNN stays consistent with the pre-compaction answer through both the
+    /// FRI window and the physical remap. (HNSW rebuilds its graph on physical
+    /// remap, so the overlap is recall-tolerant.)
+    async fn check_vector_remap_and_trim(
+        params: VectorIndexParams,
+        k: usize,
+        window_overlap: usize,
+        post_remap_overlap: Option<usize>,
+    ) {
+        use arrow_array::cast::AsArray;
+        use arrow_array::types::{Float32Type, Int32Type};
+        use lance_datagen::Dimension;
+
+        const DIM: u32 = 32;
+        let mut dataset = lance_datagen::gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vec",
+                lance_datagen::array::rand_vec::<Float32Type>(Dimension::from(DIM)),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["vec"],
+                IndexType::Vector,
+                Some("vec_idx".into()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+        let original_uuid = dataset
+            .load_index_by_name("vec_idx")
+            .await
+            .unwrap()
+            .unwrap()
+            .uuid;
+
+        // Sample queries from stored vectors + capture the pre-compaction answer.
+        let mut rows: Vec<Vec<f32>> = Vec::new();
+        {
+            let mut scanner = dataset.scan();
+            scanner.project(&["vec"]).unwrap();
+            let batches = scanner
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            for batch in &batches {
+                let vecs = batch["vec"].as_fixed_size_list();
+                for i in 0..batch.num_rows() {
+                    let v = vecs.value(i);
+                    rows.push(v.as_primitive::<Float32Type>().values().to_vec());
+                }
+            }
+        }
+        let step = (rows.len() / 16).max(1);
+        let queries: Vec<Vec<f32>> = rows.iter().step_by(step).cloned().collect();
+        let mut baseline: Vec<Vec<i32>> = Vec::new();
+        for q in &queries {
+            baseline.push(vector_knn_ids(&dataset, q, k).await);
+        }
+
+        // Merge-only deferred compaction.
+        let metrics = compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 2_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert_eq!(
+            dataset
+                .load_index_by_name("vec_idx")
+                .await
+                .unwrap()
+                .unwrap()
+                .uuid,
+            original_uuid,
+            "index must not be physically remapped yet (FRI window)"
+        );
+        for (i, q) in queries.iter().enumerate() {
+            let window = vector_knn_ids(&dataset, q, k).await;
+            let overlap = window.iter().filter(|id| baseline[i].contains(id)).count();
+            assert!(
+                overlap >= window_overlap,
+                "FRI-window KNN diverged: overlap {overlap} < {window_overlap} (query #{i})"
+            );
+        }
+
+        // Physical remap + trim the fragment-reuse index.
+        remapping::remap_column_index(&mut dataset, &["vec"], Some("vec_idx".into()))
+            .await
+            .unwrap();
+        cleanup_frag_reuse_index(&mut dataset).await.unwrap();
+
+        let remapped_uuid = dataset
+            .load_index_by_name("vec_idx")
+            .await
+            .unwrap()
+            .unwrap()
+            .uuid;
+        assert_ne!(
+            remapped_uuid, original_uuid,
+            "index should have been physically remapped"
+        );
+        if let Some(meta) = dataset
+            .load_index_by_name(FRAG_REUSE_INDEX_NAME)
+            .await
+            .unwrap()
+        {
+            let versions = load_frag_reuse_index_details(&dataset, &meta)
+                .await
+                .unwrap()
+                .versions
+                .len();
+            assert_eq!(versions, 0, "frag-reuse index must trim to zero versions");
+        }
+
+        for (i, q) in queries.iter().enumerate() {
+            let after = vector_knn_ids(&dataset, q, k).await;
+            // No stale/desynced addresses (a bad address fails the take above).
+            assert!(
+                !after.is_empty(),
+                "post-remap KNN returned no rows (query #{i})"
+            );
+            // Physical remap rebuilds the HNSW graph, so recall is only compared
+            // for the exact (non-HNSW) types.
+            if let Some(min_overlap) = post_remap_overlap {
+                let overlap = after.iter().filter(|id| baseline[i].contains(id)).count();
+                assert!(
+                    overlap >= min_overlap,
+                    "post-remap KNN diverged: overlap {overlap} < {min_overlap} (query #{i})"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_ivf_flat_remap_and_trim() {
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, small_ivf());
+        check_vector_remap_and_trim(params, 10, 8, Some(8)).await;
+    }
+
+    // Regression: PQ storage used to remap its codes through the frag-reuse
+    // index but keep the pre-remap `row_ids` field, so search returned stale
+    // (compacted-away) addresses and the take failed with "fragment ... does
+    // not exist" — even merge-only, and only observable when the query fetches
+    // row content (the existing `test_read_ivf_pq_index_v3_with_defer_index_remap`
+    // projects no columns, so it never takes and missed this).
+    #[tokio::test]
+    async fn test_ivf_pq_remap_and_trim() {
+        use lance_index::vector::pq::PQBuildParams;
+        let params = VectorIndexParams::with_ivf_pq_params(
+            DistanceType::L2,
+            small_ivf(),
+            PQBuildParams {
+                max_iters: 2,
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
+        check_vector_remap_and_trim(params, 10, 8, Some(8)).await;
+    }
+
+    #[tokio::test]
+    async fn test_ivf_sq_remap_and_trim() {
+        use lance_index::vector::sq::builder::SQBuildParams;
+        let params = VectorIndexParams::with_ivf_sq_params(
+            DistanceType::L2,
+            small_ivf(),
+            SQBuildParams::default(),
+        );
+        check_vector_remap_and_trim(params, 10, 8, Some(8)).await;
+    }
 
-        let metrics = compact_files(&mut dataset, options, None).await.unwrap();
-        assert!(metrics.fragments_removed > 0);
-        assert!(metrics.fragments_added > 0);
+    #[tokio::test]
+    async fn test_ivf_rq_remap_and_trim() {
+        use lance_index::vector::bq::RQBuildParams;
+        let params = VectorIndexParams::with_ivf_rq_params(
+            DistanceType::L2,
+            small_ivf(),
+            RQBuildParams::new(1),
+        );
+        check_vector_remap_and_trim(params, 10, 8, Some(8)).await;
+    }
 
-        // Verify the index UUID is unchanged (it should not be remapped yet)
-        let Some(current_index) = dataset.load_index_by_name("vec_idx").await.unwrap() else {
-            panic!("vec index must be available");
-        };
-        assert_eq!(current_index.uuid, original_index.uuid);
+    #[tokio::test]
+    async fn test_ivf_hnsw_sq_remap_and_trim() {
+        use lance_index::vector::{hnsw::builder::HnswBuildParams, sq::builder::SQBuildParams};
+        let params = VectorIndexParams::with_ivf_hnsw_sq_params(
+            DistanceType::L2,
+            small_ivf(),
+            HnswBuildParams::default(),
+            SQBuildParams::default(),
+        );
+        // Physical remap rebuilds the HNSW graph, so use a recall-tolerant overlap.
+        check_vector_remap_and_trim(params, 10, 7, None).await;
+    }
 
-        // Verify that KNN searches still work correctly and return the same counts
-        let mut scanner = dataset.scan();
-        scanner.nearest("vec", &query_vec1, 10).unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let new_results1 = scanner
-            .try_into_stream()
-            .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-        assert_eq!(new_results1.len(), count1);
+    #[tokio::test]
+    async fn test_ivf_hnsw_pq_remap_and_trim() {
+        use lance_index::vector::{hnsw::builder::HnswBuildParams, pq::PQBuildParams};
+        let params = VectorIndexParams::with_ivf_hnsw_pq_params(
+            DistanceType::L2,
+            small_ivf(),
+            HnswBuildParams::default(),
+            PQBuildParams {
+                max_iters: 2,
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
+        check_vector_remap_and_trim(params, 10, 7, None).await;
+    }
 
-        scanner = dataset.scan();
-        scanner.nearest("vec", &query_vec2, 10).unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let new_results2 = scanner
-            .try_into_stream()
+    // Scalar index correctness across deferred compaction WITH materialized
+    // deletions. The existing test_read_*_index_with_defer_index_remap tests are
+    // merge-only and project no columns (count-only), so they never take and
+    // never exercise the deletion drop path. These add an `id` column, delete a
+    // prefix, defer-compact, then run the indexed query *projecting id* (a take)
+    // and assert no deleted row is returned. Bitmap/BTree have no positional
+    // internal structure so the drop path is exact; the Inverted (FTS) index
+    // does (see its test below), and currently desyncs under deletions.
+
+    #[tokio::test]
+    async fn test_bitmap_index_defer_compaction_with_deletions() {
+        use arrow_array::cast::AsArray;
+        use arrow_array::types::Int32Type;
+        let mut dataset = lance_datagen::gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "category",
+                lance_datagen::array::cycle::<Int32Type>(vec![1, 2, 3]),
+            )
+            .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000))
             .await
-            .unwrap()
-            .try_collect::<Vec<_>>()
+            .unwrap();
+        dataset
+            .create_index(
+                &["category"],
+                IndexType::Bitmap,
+                Some("category_idx".into()),
+                &ScalarIndexParams::default(),
+                false,
+            )
             .await
             .unwrap();
-        assert_eq!(new_results2.len(), count2);
+        dataset.delete("id < 1500").await.unwrap();
+        let metrics = compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 2_000,
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(metrics.fragments_removed > 0);
+        assert!(
+            dataset
+                .load_indices()
+                .await
+                .unwrap()
+                .iter()
+                .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME),
+            "deferred compaction must record a frag-reuse index"
+        );
 
-        scanner = dataset.scan();
-        scanner.nearest("vec", &query_vec3, 10).unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let new_results3 = scanner
+        let mut scanner = dataset.scan();
+        scanner.filter("category = 3").unwrap();
+        scanner.project(&["id"]).unwrap();
+        let batches = scanner
             .try_into_stream()
             .await
             .unwrap()
             .try_collect::<Vec<_>>()
             .await
             .unwrap();
-        assert_eq!(new_results3.len(), count3);
-
-        // Verify that after index creation and compaction, scan uses vector index scan
-        let mut scanner = dataset.scan();
-        scanner.nearest("vec", &query_vec1, 10).unwrap();
-        scanner.project::<String>(&[]).unwrap().with_row_id();
-        let plan = scanner.explain_plan(false).await.unwrap();
-        assert!(
-            plan.contains("ANNSubIndex"),
-            "Expected vector index scan in plan: {}",
-            plan
-        );
-        assert!(
-            !plan.contains("LanceScan"),
-            "Expected no fragment scan in plan: {}",
-            plan
-        );
+        let mut returned = 0;
+        for b in &batches {
+            for id in b["id"].as_primitive::<Int32Type>().values() {
+                assert!(
+                    *id >= 1500,
+                    "bitmap returned deleted id {id} in the FRI window"
+                );
+                returned += 1;
+            }
+        }
+        assert!(returned > 0, "expected surviving category=3 rows");
     }
 
+    // NOTE: Inverted/FTS under materialized deletions is broken (BM25 scores
+    // via positional num_tokens[doc_id]; the frag-reuse drop shifts doc_id
+    // positions -> out-of-bounds). It is gated off defer in the data plane
+    // until fixed, so there is no lance-level reproducer here. Merge-only FTS
+    // is covered by test_read_inverted_index_with_defer_index_remap.
+
     #[tokio::test]
     async fn test_default_compaction_planner() {
         let test_dir = TempStrDir::default();
@@ -4683,6 +6371,10 @@ mod tests {
                 "lance.compaction.batch_size".to_string(),
                 "4096".to_string(),
             ),
+            (
+                "lance.compaction.io_buffer_size".to_string(),
+                "1073741824".to_string(),
+            ),
             (
                 "lance.compaction.compaction_mode".to_string(),
                 "try_binary_copy".to_string(),
@@ -4691,6 +6383,10 @@ mod tests {
                 "lance.compaction.binary_copy_read_batch_bytes".to_string(),
                 "8388608".to_string(),
             ),
+            (
+                "lance.compaction.index_remap_mode".to_string(),
+                "compact".to_string(),
+            ),
         ]);
 
         let opts = CompactionOptions::from_dataset_config(&config).unwrap();
@@ -4701,8 +6397,11 @@ mod tests {
         assert!((opts.materialize_deletions_threshold - 0.25).abs() < f32::EPSILON);
         assert!(opts.defer_index_remap);
         assert_eq!(opts.batch_size, Some(4096));
+        assert_eq!(opts.io_buffer_size, Some(1_073_741_824));
         assert_eq!(opts.compaction_mode, Some(CompactionMode::TryBinaryCopy));
         assert_eq!(opts.binary_copy_read_batch_bytes, Some(8_388_608));
+        // A non-default value proves the config string was actually parsed.
+        assert_eq!(opts.index_remap_mode, IndexRemapMode::Compact);
     }
 
     #[test]
@@ -4722,6 +6421,8 @@ mod tests {
             defaults.materialize_deletions_threshold
         );
         assert_eq!(opts.defer_index_remap, defaults.defer_index_remap);
+        assert_eq!(opts.index_remap_mode, defaults.index_remap_mode);
+        assert_eq!(opts.index_remap_mode, IndexRemapMode::Direct);
         assert_eq!(opts.batch_size, defaults.batch_size);
         assert_eq!(opts.compaction_mode, defaults.compaction_mode);
         assert_eq!(
@@ -4826,6 +6527,29 @@ mod tests {
         assert!(err_msg.contains("invalid_mode"));
     }
 
+    #[test]
+    fn test_from_dataset_config_max_overlays_per_fragment() {
+        let key = "lance.compaction.max_overlays_per_fragment".to_string();
+
+        // An integer sets the threshold.
+        let config = HashMap::from([(key.clone(), "3".to_string())]);
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.max_overlays_per_fragment, Some(3));
+
+        // "none" (case-insensitive) disables the trigger, overriding the Some(10) default.
+        let config = HashMap::from([(key.clone(), "None".to_string())]);
+        let opts = CompactionOptions::from_dataset_config(&config).unwrap();
+        assert_eq!(opts.max_overlays_per_fragment, None);
+
+        // Anything else is rejected.
+        let config = HashMap::from([(key, "not_a_number".to_string())]);
+        let err_msg = CompactionOptions::from_dataset_config(&config)
+            .unwrap_err()
+            .to_string();
+        assert!(err_msg.contains("max_overlays_per_fragment"));
+        assert!(err_msg.contains("not_a_number"));
+    }
+
     #[test]
     fn test_apply_dataset_config_overrides() {
         let config = HashMap::from([(
@@ -6447,4 +8171,339 @@ mod tests {
             ]
         );
     }
+    // ---- `max_overlays_per_fragment` compaction trigger ----
+    //
+    // Tests for the trigger that fully compacts a fragment carrying too many data
+    // overlay files into a fresh fragment with the overlays (and deletions)
+    // materialized into the base data.
+    use arrow_array::record_batch;
+    use lance_file::writer::{FileWriter, FileWriterOptions};
+    use lance_io::utils::CachedFileSize;
+    use lance_table::format::DataFile;
+    use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+    use std::collections::BTreeMap;
+
+    use crate::dataset::DATA_DIR;
+    use crate::dataset::transaction::DataOverlayGroup;
+
+    /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) =
+    /// id * 10, six rows per fragment (fragments 0 and 1).
+    async fn create_base_dataset(uri: &str) -> Dataset {
+        let batch = record_batch!(
+            ("id", Int32, (0..12).collect::<Vec<_>>()),
+            ("val", Int32, (0..12).map(|v| v * 10).collect::<Vec<_>>())
+        )
+        .unwrap();
+        let schema = batch.schema();
+        let write_params = WriteParams {
+            max_rows_per_file: 6,
+            max_rows_per_group: 6,
+            data_storage_version: Some(LanceFileVersion::Stable),
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        Dataset::write(reader, uri, Some(write_params))
+            .await
+            .unwrap()
+    }
+
+    fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+        Arc::new(Int32Array::from_iter(values))
+    }
+
+    fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+        RoaringBitmap::from_iter(offsets)
+    }
+
+    /// Write a dense overlay covering `fields` of `fragment_id` with `columns`
+    /// as the per-field value columns, then commit it as a `DataOverlay`.
+    async fn commit_overlay(
+        dataset: Dataset,
+        fragment_id: u64,
+        fields: &[i32],
+        coverage: OverlayCoverage,
+        columns: Vec<ArrayRef>,
+    ) -> Dataset {
+        let read_version = dataset.version().version;
+        let overlay_schema = dataset.schema().project_by_ids(fields, true);
+        let filename = format!("{}.lance", Uuid::new_v4());
+        let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str());
+        let obj_writer = dataset.object_store.create(&path).await.unwrap();
+        let mut writer = FileWriter::try_new(
+            obj_writer,
+            overlay_schema,
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        let (major, minor) = writer.version().to_numbers();
+        for (column_index, array) in columns.into_iter().enumerate() {
+            writer.write_column(column_index, array).await.unwrap();
+        }
+        let summary = writer.finish().await.unwrap();
+
+        let mut data_file = DataFile::new_unstarted(filename, major, minor);
+        data_file.fields = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(f, _)| *f as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.column_indices = writer
+            .field_id_to_column_indices()
+            .iter()
+            .map(|(_, c)| *c as i32)
+            .collect::<Vec<_>>()
+            .into();
+        data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes);
+
+        Dataset::commit(
+            WriteDestination::Dataset(Arc::new(dataset)),
+            Operation::DataOverlay {
+                groups: vec![DataOverlayGroup {
+                    fragment_id,
+                    overlays: vec![DataOverlayFile {
+                        data_file,
+                        coverage,
+                        committed_version: 0,
+                    }],
+                }],
+            },
+            Some(read_version),
+            None,
+            None,
+            Arc::new(Default::default()),
+            false,
+        )
+        .await
+        .unwrap()
+    }
+
+    /// Commit `n` distinct single-cell overlays to fragment 0 (offset `i`, val
+    /// column set to `1000 + i`), so the fragment ends up with `n` overlays. The
+    /// `1000 +` offset keeps overlaid values clear of the base `id * 10` values.
+    async fn commit_n_overlays(mut dataset: Dataset, n: u32) -> Dataset {
+        for i in 0..n {
+            dataset = commit_overlay(
+                dataset,
+                0,
+                &[1],
+                OverlayCoverage::dense(bitmap([i])),
+                vec![i32_array([Some(1000 + i as i32)])],
+            )
+            .await;
+        }
+        dataset
+    }
+
+    /// Options whose only compaction trigger is the overlay limit: base
+    /// fragments here are far below the default 1M-row target, which would
+    /// otherwise make them size-based compaction candidates on their own.
+    fn overlay_only_options(max_overlays_per_fragment: usize) -> CompactionOptions {
+        CompactionOptions {
+            max_overlays_per_fragment: Some(max_overlays_per_fragment),
+            target_rows_per_fragment: 6,
+            ..Default::default()
+        }
+    }
+
+    /// Scan `id` and `val` and return an `id -> val` map (order-independent).
+    async fn id_val_map(dataset: &Dataset) -> BTreeMap<i32, Option<i32>> {
+        let mut scanner = dataset.scan();
+        scanner.project(&["id", "val"]).unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let mut out = BTreeMap::new();
+        let ids = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        let vals = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        for i in 0..batch.num_rows() {
+            let v = if vals.is_null(i) {
+                None
+            } else {
+                Some(vals.value(i))
+            };
+            out.insert(ids.value(i), v);
+        }
+        out
+    }
+
+    #[tokio::test]
+    async fn test_max_overlays_triggers_full_compaction() {
+        // Fragment 0 gets 3 overlays; fragment 1 stays clean.
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+        assert_eq!(
+            dataset.get_fragment(0).unwrap().metadata().overlays.len(),
+            3
+        );
+
+        // Threshold 2: only fragment 0 (3 > 2) is compacted.
+        let metrics = compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+        assert_eq!(metrics.fragments_removed, 1);
+        assert_eq!(metrics.fragments_added, 1);
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+        // The compacted fragment is a fresh single-data-file fragment with no
+        // overlays; fragment 1 is untouched.
+        let compacted = fragments
+            .iter()
+            .find(|f| f.id() != 1)
+            .expect("a new fragment id was assigned");
+        assert!(compacted.metadata().overlays.is_empty());
+        assert_eq!(compacted.metadata().files.len(), 1);
+
+        // The overlaid values were materialized: id i in 0..3 -> 1000 + i.
+        let values = id_val_map(&dataset).await;
+        let expected: BTreeMap<i32, Option<i32>> = (0..12)
+            .map(|id| {
+                let v = if id < 3 { 1000 + id } else { id * 10 };
+                (id, Some(v))
+            })
+            .collect();
+        assert_eq!(values, expected);
+    }
+
+    #[tokio::test]
+    async fn test_below_threshold_is_a_noop() {
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 2).await;
+
+        // 2 overlays, threshold 2: `overlays > max` is false, so no compaction.
+        let metrics = compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+        assert_eq!(metrics.fragments_removed, 0);
+        assert_eq!(metrics.fragments_added, 0);
+        assert_eq!(
+            dataset.get_fragment(0).unwrap().metadata().overlays.len(),
+            2
+        );
+    }
+
+    #[tokio::test]
+    async fn test_overlay_compaction_materializes_deletions() {
+        let dataset = create_base_dataset("memory://").await;
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+        // Delete a row from the overlaid fragment (id 2 is at offset 2).
+        dataset.delete("id = 2").await.unwrap();
+        assert!(
+            dataset
+                .get_fragment(0)
+                .unwrap()
+                .metadata()
+                .deletion_file
+                .is_some()
+        );
+
+        compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+
+        // The deletion was materialized: no deletion file remains and id 2 is gone.
+        for fragment in dataset.get_fragments() {
+            assert!(fragment.metadata().deletion_file.is_none());
+            assert!(fragment.metadata().overlays.is_empty());
+        }
+        let values = id_val_map(&dataset).await;
+        assert!(!values.contains_key(&2));
+        // Surviving overlaid cells still carry their materialized values.
+        assert_eq!(values.get(&0), Some(&Some(1000)));
+        assert_eq!(values.get(&1), Some(&Some(1001)));
+    }
+
+    #[tokio::test]
+    async fn test_overlay_compaction_reconciles_stale_index() {
+        let mut dataset = create_base_dataset("memory://").await;
+        // Index `val` before any overlay -> the index is stale once val is overlaid.
+        dataset
+            .create_index(
+                &["val"],
+                IndexType::Scalar,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Overlay val[0] 0 -> 100 (committed after the index) and push fragment 0
+        // over the overlay limit.
+        let mut dataset = commit_n_overlays(dataset, 3).await;
+
+        let val_index_before = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .find(|i| i.fields == vec![1])
+            .expect("val index present")
+            .clone();
+        assert!(
+            val_index_before
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .contains(0)
+        );
+
+        compact_files(&mut dataset, overlay_only_options(2), None)
+            .await
+            .unwrap();
+
+        // The stale val index no longer covers the compacted fragment, so its
+        // rows fall back to a flat scan instead of serving stale values.
+        let indices = dataset.load_indices().await.unwrap();
+        let val_index = indices
+            .iter()
+            .find(|i| i.fields == vec![1])
+            .expect("val index present");
+        let compacted_id = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .find(|id| *id != 1)
+            .unwrap();
+        assert!(
+            !val_index
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .contains(compacted_id),
+            "stale index must drop the compacted fragment from its coverage"
+        );
+
+        // The indexed query is correct: the materialized value is found and the
+        // stale pre-overlay value is gone.
+        let mut scanner = dataset.scan();
+        scanner
+            .filter("val = 1000")
+            .unwrap()
+            .project(&["id"])
+            .unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        let ids = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        assert_eq!(ids.len(), 1);
+        assert_eq!(ids.value(0), 0);
+
+        let mut scanner = dataset.scan();
+        scanner.filter("val = 0").unwrap().project(&["id"]).unwrap();
+        let batch = scanner.try_into_batch().await.unwrap();
+        assert_eq!(batch.num_rows(), 0, "stale value 0 must no longer match");
+    }
 }
diff --git a/rust/lance/src/dataset/optimize/remapping.rs b/rust/lance/src/dataset/optimize/remapping.rs
index dab62bf6166..aef2cd231fc 100644
--- a/rust/lance/src/dataset/optimize/remapping.rs
+++ b/rust/lance/src/dataset/optimize/remapping.rs
@@ -12,6 +12,7 @@ use crate::{Dataset, index};
 use async_trait::async_trait;
 use lance_core::Error;
 use lance_core::utils::address::RowAddress;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragDigest};
 use lance_table::format::{Fragment, IndexFile, IndexMetadata};
 use lance_table::io::manifest::read_manifest_indexes;
@@ -51,7 +52,7 @@ pub struct RemappedIndex {
 pub trait IndexRemapper: Send + Sync {
     async fn remap_indices(
         &self,
-        index_map: HashMap<u64, Option<u64>>,
+        index_map: RowAddrRemap,
         affected_fragment_ids: &[u64],
     ) -> Result<Vec<RemappedIndex>>;
 }
@@ -69,11 +70,7 @@ pub struct IgnoreRemap {}
 
 #[async_trait]
 impl IndexRemapper for IgnoreRemap {
-    async fn remap_indices(
-        &self,
-        _: HashMap<u64, Option<u64>>,
-        _: &[u64],
-    ) -> Result<Vec<RemappedIndex>> {
+    async fn remap_indices(&self, _: RowAddrRemap, _: &[u64]) -> Result<Vec<RemappedIndex>> {
         Ok(Vec::new())
     }
 }
@@ -220,25 +217,37 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> {
         return Ok(());
     }
 
-    // Sequentially apply the row addr maps from oldest to latest
-    let mut curr_index_id = *index_id;
-    for (i, row_id_map) in frag_reuse_index.row_id_maps.iter().enumerate() {
-        let version = &frag_reuse_index.details.versions[i];
-        // load on-disk index metadata before auto-remap
-        let curr_index_meta = read_manifest_indexes(
-            &dataset.object_store,
-            &dataset.manifest_location,
-            &dataset.manifest,
-        )
-        .await?
-        .into_iter()
-        .find(|idx| idx.uuid == curr_index_id)
-        .unwrap();
-
-        let maybe_index_bitmap = curr_index_meta.fragment_bitmap.clone();
-        let (should_remap, bitmap_after_remap) = match maybe_index_bitmap {
-            Some(mut index_frag_bitmap) => {
-                let mut should_remap = false;
+    // Read the index's on-disk metadata once. Its stored row addresses are at
+    // this baseline; we compose all reuse versions into a single remap so the
+    // index file is rebuilt and committed exactly once, rather than once per
+    // version (the reuse index can accumulate many versions before remap runs).
+    let curr_index_meta = read_manifest_indexes(
+        &dataset.object_store,
+        &dataset.manifest_location,
+        &dataset.manifest,
+    )
+    .await?
+    .into_iter()
+    .find(|idx| idx.uuid == *index_id)
+    .ok_or_else(|| {
+        Error::index(format!(
+            "index {index_id} not found in manifest; it may have been concurrently dropped"
+        ))
+    })?;
+
+    // Compose the coverage (fragment bitmap) remap across every reuse version in
+    // one pass. Chaining is automatic: a version inserts its new fragments,
+    // which a later version then sees as its old fragments. `data_predates_version`
+    // is evaluated against the fixed baseline (there are no intermediate
+    // commits), and the new-fragment branch handles a bitmap that was already
+    // coverage-remapped + persisted before the data was remapped (e.g. while
+    // remapping a *sibling* index).
+    let baseline_version = curr_index_meta.dataset_version;
+    let (should_remap, bitmap_after_remap) = match curr_index_meta.fragment_bitmap.clone() {
+        Some(mut index_frag_bitmap) => {
+            let mut should_remap = false;
+            for version in frag_reuse_index.details.versions.iter() {
+                let data_predates_version = baseline_version < version.dataset_version;
                 for group in version.groups.iter() {
                     let mut old_frag_in_index = 0;
                     for old_frag in group.old_frags.iter() {
@@ -258,67 +267,98 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> {
                                 group.old_frags
                             )));
                         }
-                        index_frag_bitmap
-                            .extend(group.new_frags.clone().into_iter().map(|f| f.id as u32));
+                        index_frag_bitmap.extend(group.new_frags.iter().map(|f| f.id as u32));
+                        should_remap = true;
+                    } else if data_predates_version
+                        && group
+                            .new_frags
+                            .iter()
+                            .any(|new_frag| index_frag_bitmap.contains(new_frag.id as u32))
+                    {
+                        // The bitmap was already coverage-remapped onto this
+                        // group's new fragments and persisted before the data was
+                        // remapped, so the old fragments are gone from the bitmap
+                        // but the index data still needs remapping.
                         should_remap = true;
                     }
                 }
-                (should_remap, Some(index_frag_bitmap))
             }
-            // if there is no fragment bitmap for the index,
-            // we attempt remapping but will not update the fragment bitmap.
-            None => (true, None),
-        };
-
-        if should_remap {
-            let remap_result = index::remap_index(dataset, &curr_index_id, row_id_map).await?;
-
-            let new_index_meta = match remap_result {
-                RemapResult::Drop => continue,
-                RemapResult::Keep(new_id) => IndexMetadata {
-                    uuid: new_id,
-                    name: curr_index_meta.name.clone(),
-                    fields: curr_index_meta.fields.clone(),
-                    dataset_version: dataset.manifest.version,
-                    fragment_bitmap: bitmap_after_remap,
-                    index_details: curr_index_meta.index_details.clone(),
-                    index_version: curr_index_meta.index_version,
-                    created_at: curr_index_meta.created_at,
-                    base_id: None,
-                    files: curr_index_meta.files.clone(),
-                },
-                RemapResult::Remapped(remapped_index) => IndexMetadata {
-                    uuid: remapped_index.new_id,
-                    name: curr_index_meta.name.clone(),
-                    fields: curr_index_meta.fields.clone(),
-                    dataset_version: dataset.manifest.version,
-                    fragment_bitmap: bitmap_after_remap,
-                    index_details: Some(Arc::new(remapped_index.index_details)),
-                    index_version: remapped_index.index_version as i32,
-                    created_at: curr_index_meta.created_at,
-                    base_id: None,
-                    files: remapped_index.files,
-                },
-            };
+            (should_remap, Some(index_frag_bitmap))
+        }
+        // if there is no fragment bitmap for the index,
+        // we attempt remapping but will not update the fragment bitmap.
+        None => (true, None),
+    };
 
-            let new_id = new_index_meta.uuid;
+    if !should_remap {
+        return Ok(());
+    }
 
-            let transaction = Transaction::new(
-                dataset.manifest.version,
-                Operation::CreateIndex {
-                    new_indices: vec![new_index_meta],
-                    removed_indices: vec![curr_index_meta.clone()],
-                },
-                None,
-            );
+    // Compose the row-address remap across all versions. `remap_row_id` already
+    // chains every version (and passes through addresses a version does not
+    // touch), so mapping the union of all versions' keys yields a single
+    // baseline -> final address map applied in one rebuild.
+    //
+    // Map every old address; do NOT filter by the current `fragment_bitmap`. In
+    // the sibling-coverage-remap case the bitmap was already advanced onto the
+    // new fragments while the index data still holds old addresses, so filtering
+    // by it would drop exactly the keys this index needs and leave its data
+    // stale (an empty map makes `index::remap_index` return `Keep`). The map is
+    // bounded by the rows the reuse index touched; addresses this index does not
+    // store are simply never looked up.
+    let composed_row_id_map: HashMap<u64, Option<u64>> = frag_reuse_index
+        .row_id_maps
+        .iter()
+        .flat_map(|row_id_map| row_id_map.keys().copied())
+        .map(|old_addr| (old_addr, frag_reuse_index.remap_row_id(old_addr)))
+        .collect();
+
+    let remapper = RowAddrRemap::direct(composed_row_id_map);
+    let remap_result = index::remap_index(dataset, index_id, &remapper).await?;
+
+    let new_index_meta = match remap_result {
+        // The composed remap emptied the index (every row deleted). Matching the
+        // prior per-version behavior, leave the existing index untouched and
+        // commit nothing -- there is no remap to apply.
+        RemapResult::Drop => return Ok(()),
+        RemapResult::Keep(new_id) => IndexMetadata {
+            uuid: new_id,
+            name: curr_index_meta.name.clone(),
+            fields: curr_index_meta.fields.clone(),
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: bitmap_after_remap,
+            index_details: curr_index_meta.index_details.clone(),
+            index_version: curr_index_meta.index_version,
+            created_at: curr_index_meta.created_at,
+            base_id: None,
+            files: curr_index_meta.files.clone(),
+        },
+        RemapResult::Remapped(remapped_index) => IndexMetadata {
+            uuid: remapped_index.new_id,
+            name: curr_index_meta.name.clone(),
+            fields: curr_index_meta.fields.clone(),
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: bitmap_after_remap,
+            index_details: Some(Arc::new(remapped_index.index_details)),
+            index_version: remapped_index.index_version as i32,
+            created_at: curr_index_meta.created_at,
+            base_id: None,
+            files: remapped_index.files,
+        },
+    };
 
-            dataset
-                .apply_commit(transaction, &Default::default(), &Default::default())
-                .await?;
+    let transaction = Transaction::new(
+        dataset.manifest.version,
+        Operation::CreateIndex {
+            new_indices: vec![new_index_meta],
+            removed_indices: vec![curr_index_meta],
+        },
+        None,
+    );
 
-            curr_index_id = new_id;
-        }
-    }
+    dataset
+        .apply_commit(transaction, &Default::default(), &Default::default())
+        .await?;
 
     Ok(())
 }
@@ -369,6 +409,87 @@ pub async fn remap_column_index(
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_compact_matches_transpose() {
+        use lance_core::utils::row_addr_remap::GroupInput;
+        // Ascending old fragments (compaction's scan order), with deletions.
+        let old = vec![
+            FragDigest {
+                id: 0,
+                physical_rows: 5,
+                num_deleted_rows: 2,
+            },
+            FragDigest {
+                id: 1,
+                physical_rows: 4,
+                num_deleted_rows: 1,
+            },
+            FragDigest {
+                id: 3,
+                physical_rows: 3,
+                num_deleted_rows: 0,
+            },
+        ];
+        // 9 rewritten rows (offsets that survived in each old fragment).
+        let rewritten = [
+            (0, 1),
+            (0, 2),
+            (0, 4),
+            (1, 0),
+            (1, 1),
+            (1, 3),
+            (3, 0),
+            (3, 1),
+            (3, 2),
+        ];
+        let addrs = RoaringTreemap::from_iter(
+            rewritten
+                .iter()
+                .map(|(f, o)| u64::from(RowAddress::new_from_parts(*f, *o))),
+        );
+        // 9 rewritten rows split across two new fragments.
+        let new = vec![
+            FragDigest {
+                id: 10,
+                physical_rows: 4,
+                num_deleted_rows: 0,
+            },
+            FragDigest {
+                id: 11,
+                physical_rows: 5,
+                num_deleted_rows: 0,
+            },
+        ];
+
+        let expected = transpose_row_ids_from_digest(addrs.clone(), &old, &new);
+        let compact = RowAddrRemap::compact([GroupInput {
+            rewritten_old_row_addrs: addrs,
+            old_frag_ids: old.iter().map(|f| f.id as u32).collect(),
+            new_frags: new
+                .iter()
+                .map(|f| (f.id as u32, f.physical_rows as u32))
+                .collect(),
+        }])
+        .unwrap();
+
+        // Every real address in the old fragments must map identically.
+        for f in &old {
+            for o in 0..f.physical_rows as u32 {
+                let a = u64::from(RowAddress::new_from_parts(f.id as u32, o));
+                assert_eq!(
+                    compact.get(a),
+                    expected.get(&a).copied(),
+                    "mismatch at ({}, {})",
+                    f.id,
+                    o
+                );
+            }
+        }
+        // A fragment outside the group is unaffected by both.
+        let outside = u64::from(RowAddress::new_from_parts(99, 0));
+        assert_eq!(compact.get(outside), expected.get(&outside).copied());
+    }
+
     #[test]
     fn test_missing_indices() {
         // Sanity test to make sure MissingIds works.  Does not test actual functionality so
diff --git a/rust/lance/src/dataset/overlay.rs b/rust/lance/src/dataset/overlay.rs
new file mode 100644
index 00000000000..d73329d8a41
--- /dev/null
+++ b/rust/lance/src/dataset/overlay.rs
@@ -0,0 +1,1064 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Resolution of data overlay files on read.
+//!
+//! An overlay supplies replacement values for some `(row, field)` cells without
+//! rewriting the base data. Resolving a read means, for each row we return,
+//! deciding whether its value comes from the base column or from an overlay.
+//!
+//! Three coordinate spaces show up throughout this module; keeping them straight
+//! is most of the work:
+//!
+//! - `offset_in_frag`: a row's physical position in the fragment (0-based over all
+//!   physical rows, ignoring deletions). This is how a cell is addressed on disk
+//!   and in an overlay's coverage bitmap.
+//! - `offset_in_batch`: a row's position within the batch we are currently
+//!   assembling (0-based). The output column is indexed by this.
+//! - `offset_in_overlay`: the position of a value in an overlay's value column.
+//!   An overlay stores its values densely — one per covered cell, in ascending
+//!   `offset_in_frag` order — so a covered cell's value is found by counting how
+//!   many covered cells come before it. (That count is what a roaring bitmap calls
+//!   the cell's "rank".)
+//!
+//! For a given field, the overlays covering it are consulted newest to oldest: the
+//! first overlay that covers a row wins, and its value is read at that row's
+//! `offset_in_overlay`. A row that no overlay covers keeps its base value.
+//!
+//! The rows to resolve are passed in as a list of `offset_in_frag` (one per output
+//! row), so a single code path serves both scans (a contiguous range of offsets)
+//! and `take` (arbitrary offsets).
+//!
+//! Deletions win over overlays, but nothing here handles that: the merge runs on
+//! physical rows *before* deletions are applied, so an overlay value computed for a
+//! deleted row is simply dropped along with the row. This matches the spec with no
+//! special casing.
+
+use std::collections::{BTreeSet, HashMap};
+use std::sync::Arc;
+
+use arrow_array::{Array, ArrayRef, RecordBatch, StructArray};
+use arrow_select::interleave::interleave;
+use futures::StreamExt;
+use lance_core::datatypes::{Field, Schema};
+use lance_core::{Error, Result};
+use roaring::RoaringBitmap;
+
+use lance_table::format::DataFile;
+use lance_table::utils::stream::ReadBatchFut;
+
+use crate::dataset::fragment::{FileFragment, FragReadConfig, GenericFileReader};
+
+/// The plan for merging one field's overlays into one batch: which source (base or
+/// a particular overlay) supplies each output row, and which overlay values must be
+/// fetched to do it.
+///
+/// Built by [`route_overlays`] from the coverage bitmaps alone — before any value
+/// column is read — so the caller can fetch only the overlay values it will
+/// actually use (its `offsets_in_overlay`) rather than whole columns, then build the
+/// merged column with [`assemble_overlay_column`].
+struct OverlayRouting {
+    /// One `(source, position)` pair per output row, ready to hand to `interleave`.
+    /// Source `0` is the base column, with `position` = the row's `offset_in_batch`;
+    /// source `k + 1` is overlay `k`'s fetched values, with `position` = the row's
+    /// index into those fetched values.
+    indices: Vec<(usize, usize)>,
+    /// Per overlay (newest-first): the sorted, deduplicated `offset_in_overlay`
+    /// values this batch needs from that overlay — i.e. exactly which entries of its
+    /// value column to fetch.
+    offsets_in_overlay: Vec<Vec<u32>>,
+    /// Whether any row is covered by an overlay at all (false ⇒ every row falls
+    /// through to the base column, so the base is already the answer and no overlay
+    /// values need to be read).
+    any_overlay: bool,
+}
+
+/// For each row in `offsets_in_frag`, decide whether its value comes from the base
+/// column or from an overlay — and if from an overlay, at which `offset_in_overlay`.
+///
+/// Only the coverage bitmaps are consulted (newest-first), so this runs before any
+/// value column is read and reports exactly which overlay values the caller must
+/// fetch.
+///
+/// A scan asks for a contiguous, ascending range of offsets, which enables a faster
+/// bitmap-driven path ([`route_contiguous`]); `take` asks for arbitrary offsets and
+/// uses the general path ([`route_arbitrary`]). Both produce identical routing.
+fn route_overlays(
+    offsets_in_frag: &[u32],
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    match contiguous_frag_start(offsets_in_frag) {
+        Some(frag_start) => {
+            route_contiguous(frag_start, offsets_in_frag.len(), coverages_newest_first)
+        }
+        None => route_arbitrary(offsets_in_frag, coverages_newest_first),
+    }
+}
+
+/// If `offsets_in_frag` is a contiguous ascending run `[start, start + 1, ...]`,
+/// return `start`; otherwise `None` (including when empty).
+fn contiguous_frag_start(offsets_in_frag: &[u32]) -> Option<u32> {
+    let start = *offsets_in_frag.first()?;
+    offsets_in_frag
+        .iter()
+        .enumerate()
+        .all(|(i, &offset)| offset as u64 == start as u64 + i as u64)
+        .then_some(start)
+}
+
+/// Fast path for a scan, where the batch is a contiguous run of offsets starting at
+/// `frag_start`. Because the offsets are contiguous, a row's `offset_in_batch` is
+/// just `offset_in_frag - frag_start`, so a coverage's set bits map straight to
+/// output rows — no need to test each row against each coverage.
+///
+/// For each coverage we intersect it with the batch's offset range. Roaring does
+/// this a block at a time, so a coverage that does not overlap the batch (e.g. a
+/// scan batch past the last cell this overlay touches) is skipped cheaply without
+/// inspecting individual bits.
+///
+/// Within the batch a coverage's cells appear in ascending order, so their
+/// `offset_in_overlay` values are consecutive: the first in-batch cell sits at
+/// `offset_in_overlay = <cells this coverage has before the batch>` (a single
+/// `rank` lookup), and each following cell is one more. Coverages are applied
+/// newest-first, and the first overlay to claim a row wins.
+fn route_contiguous(
+    frag_start: u32,
+    len: usize,
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    let mut offsets_in_overlay: Vec<Vec<u32>> = vec![Vec::new(); coverages_newest_first.len()];
+    // Indexed by offset_in_batch: which (overlay, fetch position) supplies the row.
+    let mut routed: Vec<Option<(usize, usize)>> = vec![None; len];
+    let range_end = (frag_start as u64 + len as u64).min(u32::MAX as u64) as u32;
+    let mut batch_range = RoaringBitmap::new();
+    batch_range.insert_range(frag_start..range_end);
+
+    for (k, coverage) in coverages_newest_first.iter().enumerate() {
+        let covered_in_batch = *coverage & &batch_range;
+        if covered_in_batch.is_empty() {
+            continue;
+        }
+        // offset_in_overlay of this coverage's first in-batch cell = the number of
+        // its cells that lie before the batch.
+        let first_offset_in_overlay = if frag_start == 0 {
+            0
+        } else {
+            coverage.rank(frag_start - 1) as u32
+        };
+        for (nth_in_batch, offset_in_frag) in covered_in_batch.iter().enumerate() {
+            let offset_in_batch = (offset_in_frag - frag_start) as usize;
+            if routed[offset_in_batch].is_none() {
+                routed[offset_in_batch] = Some((k, offsets_in_overlay[k].len()));
+                offsets_in_overlay[k].push(first_offset_in_overlay + nth_in_batch as u32);
+            }
+        }
+    }
+
+    let mut any_overlay = false;
+    let indices = routed
+        .into_iter()
+        .enumerate()
+        .map(|(offset_in_batch, routed)| match routed {
+            None => (0, offset_in_batch),
+            Some((k, fetch_pos)) => {
+                any_overlay = true;
+                (k + 1, fetch_pos)
+            }
+        })
+        .collect();
+
+    OverlayRouting {
+        indices,
+        offsets_in_overlay,
+        any_overlay,
+    }
+}
+
+/// General path for arbitrary offsets (e.g. `take`): test each row's
+/// `offset_in_frag` against the coverages newest-first. `take` batches are small,
+/// so this `O(rows * overlays)` probing is not a concern.
+fn route_arbitrary(
+    offsets_in_frag: &[u32],
+    coverages_newest_first: &[&RoaringBitmap],
+) -> OverlayRouting {
+    // Per overlay: the distinct offset_in_overlay values this batch needs, sorted.
+    let mut offset_sets: Vec<BTreeSet<u32>> = vec![BTreeSet::new(); coverages_newest_first.len()];
+    // Per output row: the (overlay, offset_in_overlay) that supplies it, if any.
+    let mut routed_per_row: Vec<Option<(usize, u32)>> = Vec::with_capacity(offsets_in_frag.len());
+    for &offset_in_frag in offsets_in_frag {
+        let mut routed = None;
+        for (k, coverage) in coverages_newest_first.iter().enumerate() {
+            if coverage.contains(offset_in_frag) {
+                // offset_in_overlay = number of covered cells before this one.
+                let offset_in_overlay = coverage.rank(offset_in_frag) as u32 - 1;
+                offset_sets[k].insert(offset_in_overlay);
+                routed = Some((k, offset_in_overlay));
+                break;
+            }
+        }
+        routed_per_row.push(routed);
+    }
+
+    let offsets_in_overlay: Vec<Vec<u32>> = offset_sets
+        .iter()
+        .map(|offsets| offsets.iter().copied().collect())
+        .collect();
+    // For each overlay, map an offset_in_overlay to its position in the fetched
+    // (sorted, deduplicated) value list.
+    let fetch_positions: Vec<HashMap<u32, usize>> = offsets_in_overlay
+        .iter()
+        .map(|offsets| {
+            offsets
+                .iter()
+                .enumerate()
+                .map(|(pos, &o)| (o, pos))
+                .collect()
+        })
+        .collect();
+
+    let mut any_overlay = false;
+    let indices = routed_per_row
+        .into_iter()
+        .enumerate()
+        .map(|(offset_in_batch, routed)| match routed {
+            None => (0, offset_in_batch),
+            Some((k, offset_in_overlay)) => {
+                any_overlay = true;
+                (k + 1, fetch_positions[k][&offset_in_overlay])
+            }
+        })
+        .collect();
+
+    OverlayRouting {
+        indices,
+        offsets_in_overlay,
+        any_overlay,
+    }
+}
+
+/// Build the merged column from `base` and the overlay values fetched for the
+/// `offset_in_overlay` values [`route_overlays`] asked for.
+///
+/// `fetched_newest_first[k]` holds overlay `k`'s values for `routing`'s
+/// `offsets_in_overlay[k]`, in that order. The result has the same length and
+/// type as `base`. A covered row whose overlay value is NULL resolves **to** NULL
+/// (distinct from a fall-through, which keeps the base value).
+fn assemble_overlay_column(
+    base: &ArrayRef,
+    routing: &OverlayRouting,
+    fetched_newest_first: &[ArrayRef],
+) -> Result<ArrayRef> {
+    if !routing.any_overlay {
+        return Ok(base.clone());
+    }
+    if fetched_newest_first.len() != routing.offsets_in_overlay.len() {
+        return Err(Error::invalid_input(format!(
+            "overlay assembly got {} value columns but routing expects {}",
+            fetched_newest_first.len(),
+            routing.offsets_in_overlay.len()
+        )));
+    }
+    for (k, values) in fetched_newest_first.iter().enumerate() {
+        if values.len() != routing.offsets_in_overlay[k].len() {
+            return Err(Error::invalid_input(format!(
+                "overlay value column {} has {} values but {} were requested",
+                k,
+                values.len(),
+                routing.offsets_in_overlay[k].len()
+            )));
+        }
+    }
+
+    let mut sources: Vec<&dyn Array> = Vec::with_capacity(fetched_newest_first.len() + 1);
+    sources.push(base.as_ref());
+    for values in fetched_newest_first {
+        sources.push(values.as_ref());
+    }
+    interleave(&sources, &routing.indices).map_err(Error::from)
+}
+
+/// One overlay's contribution to one projected atomic field, with its file reader opened.
+#[derive(Debug, Clone)]
+struct LoadedAtomicFieldOverlay {
+    /// The `offset_in_frag` cells this overlay covers for the atomic field.
+    coverage: Arc<RoaringBitmap>,
+    /// Reader over the overlay data file, projected to the covered atomic fields; shared
+    /// across the atomic fields that the same file covers.
+    reader: Arc<dyn GenericFileReader>,
+}
+
+/// The overlays that apply to a single projected atomic field — a per-row field an overlay
+/// can replace as a unit (a primitive leaf, or a whole list/map field; structs are
+/// recursed through, not treated as atomic fields). Ordered newest-first, with readers opened
+/// and pruned to a specific read. Produced by [`resolve_overlays`] and consumed by
+/// [`merge_overlay_batch`].
+#[derive(Debug, Clone)]
+pub struct LoadedAtomicField {
+    /// The top-level output column the atomic field lives in (its name locates the batch
+    /// column; its field tree drives the descend/splice into that column).
+    top_field: Arc<Field>,
+    /// Child field ids from `top_field` down to the atomic field (empty when the atomic
+    /// field *is* the top-level column). Drives descending to, and splicing back, the
+    /// atomic field.
+    ancestor_ids: Vec<i32>,
+    /// Projection of exactly the atomic field (its ancestor path pruned to the atomic
+    /// field subtree), used to fetch the atomic field's values from the overlay file.
+    fetch_projection: Arc<Schema>,
+    overlays_newest_first: Vec<LoadedAtomicFieldOverlay>,
+}
+
+/// One overlay file that may contribute to a read, before it is opened. Opened
+/// lazily by [`resolve_overlays`], and only if the read actually touches it.
+#[derive(Debug, Clone)]
+struct PlannedOverlayFile {
+    data_file: DataFile,
+    /// The covered ∩ projected atomic fields to project when the file is opened, so a single
+    /// reader serves every atomic field the file contributes to.
+    open_projection: Arc<Schema>,
+}
+
+/// One overlay's contribution to one projected atomic field, before the file is opened.
+#[derive(Debug, Clone)]
+struct PlannedAtomicFieldOverlay {
+    /// Index into [`OverlayReadPlanner::files`] of the file that supplies the value.
+    file: usize,
+    coverage: Arc<RoaringBitmap>,
+}
+
+/// The overlays that apply to a single projected atomic field, ordered newest-first, before
+/// any file is opened.
+#[derive(Debug, Clone)]
+struct PlannedAtomicField {
+    top_field: Arc<Field>,
+    ancestor_ids: Vec<i32>,
+    fetch_projection: Arc<Schema>,
+    overlays_newest_first: Vec<PlannedAtomicFieldOverlay>,
+}
+
+/// A fragment's overlay-resolution plan for a projection, derived from coverage
+/// metadata alone — no file opened, no IO. [`resolve_overlays`] turns it into opened
+/// [`LoadedAtomicField`]s for one specific read, opening only the files whose cells
+/// the read's rows actually touch.
+#[derive(Debug, Clone)]
+pub struct OverlayReadPlanner {
+    files: Vec<PlannedOverlayFile>,
+    atomic_fields: Vec<PlannedAtomicField>,
+}
+
+impl OverlayReadPlanner {
+    /// True when no projected atomic field has any overlay, so there is nothing to resolve.
+    pub fn is_empty(&self) -> bool {
+        self.atomic_fields.is_empty()
+    }
+}
+
+/// Plan `fragment`'s overlay resolution for a projection from coverage metadata
+/// alone. No files are opened here (see [`resolve_overlays`]) — this only reads the
+/// already-parsed coverage bitmaps, so it is cheap enough to run on every open.
+///
+/// Overlays are stored oldest-first (sorted newest-last on load, see
+/// `sort_overlays_newest_last`), so walking them in reverse gives newest-first
+/// precedence.
+///
+/// Resolution is per *atomic field* — a per-row field that an overlay replaces as a unit: a
+/// primitive leaf, or a whole list/map field. Structs are internal nodes, so each
+/// leaf of a struct is its own atomic field and can be overlaid independently of its
+/// siblings. An overlay is written against the leaf ids it stores (the V2_1
+/// structural encoding records only leaves), so an overlay contributes to a projected
+/// atomic field when any id in its `data_file.fields` falls in that atomic field's leaf
+/// set. At merge time the atomic field's value is fetched and spliced into its output
+/// column, so an overlay on a sub-field never disturbs the column's other leaves. Each
+/// contributing overlay *file* appears once in `files`, shared by every atomic field it
+/// covers.
+pub fn plan_overlays(fragment: &FileFragment, projection: &Schema) -> Result<OverlayReadPlanner> {
+    let overlays = &fragment.metadata.overlays;
+    debug_assert!(
+        overlays
+            .windows(2)
+            .all(|w| w[0].committed_version <= w[1].committed_version),
+        "overlays must be sorted newest-last (see sort_overlays_newest_last)"
+    );
+
+    // The projection's atomic fields, and a leaf-id -> atomic-field-index map so an
+    // overlay's stored leaf ids resolve to the atomic field they belong to in O(1).
+    struct AtomicFieldInfo<'a> {
+        top_field: &'a Field,
+        ancestor_ids: Vec<i32>,
+        atomic_field_id: i32,
+    }
+    let mut atomic_field_infos: Vec<AtomicFieldInfo> = Vec::new();
+    let mut leaf_to_atomic_field: HashMap<i32, usize> = HashMap::new();
+    for top in &projection.fields {
+        for (atomic_field, ancestor_ids) in enumerate_atomic_fields(top) {
+            let idx = atomic_field_infos.len();
+            let mut value_leaf_ids = Vec::new();
+            collect_leaf_ids(atomic_field, &mut value_leaf_ids);
+            for leaf in value_leaf_ids {
+                leaf_to_atomic_field.insert(leaf, idx);
+            }
+            atomic_field_infos.push(AtomicFieldInfo {
+                top_field: top,
+                ancestor_ids,
+                atomic_field_id: atomic_field.id,
+            });
+        }
+    }
+
+    // Walk overlays newest-first. For each overlay, find the atomic fields it covers and push
+    // (newest-first, for free) into their per-atomic field overlay lists.
+    let mut files = Vec::new();
+    let mut atomic_field_overlays: Vec<Vec<PlannedAtomicFieldOverlay>> =
+        vec![Vec::new(); atomic_field_infos.len()];
+    for overlay in overlays.iter().rev() {
+        // atomic field index -> the `data_file.fields` position whose coverage to read. An
+        // overlay writes one value per row per atomic field, so its leaves share a coverage;
+        // the first leaf of each atomic field to appear wins.
+        let mut covered: HashMap<usize, usize> = HashMap::new();
+        for (field_pos, &field_id) in overlay.data_file.fields.iter().enumerate() {
+            if let Some(&atomic_field_idx) = leaf_to_atomic_field.get(&field_id) {
+                covered.entry(atomic_field_idx).or_insert(field_pos);
+            }
+        }
+        if covered.is_empty() {
+            continue;
+        }
+        let file = files.len();
+        let covered_ids: Vec<i32> = covered
+            .keys()
+            .map(|&i| atomic_field_infos[i].atomic_field_id)
+            .collect();
+        files.push(PlannedOverlayFile {
+            data_file: overlay.data_file.clone(),
+            open_projection: Arc::new(projection.project_by_ids(&covered_ids, true)),
+        });
+        for (atomic_field_idx, field_pos) in covered {
+            atomic_field_overlays[atomic_field_idx].push(PlannedAtomicFieldOverlay {
+                file,
+                coverage: overlay.coverage_for_field(field_pos)?,
+            });
+        }
+    }
+
+    // Emit one PlannedAtomicField per projected atomic field that has overlays, in
+    // atomic field order.
+    let mut atomic_fields = Vec::new();
+    for (idx, info) in atomic_field_infos.iter().enumerate() {
+        let overlays_newest_first = std::mem::take(&mut atomic_field_overlays[idx]);
+        if overlays_newest_first.is_empty() {
+            continue;
+        }
+        atomic_fields.push(PlannedAtomicField {
+            top_field: Arc::new(info.top_field.clone()),
+            ancestor_ids: info.ancestor_ids.clone(),
+            fetch_projection: Arc::new(projection.project_by_ids(&[info.atomic_field_id], true)),
+            overlays_newest_first,
+        });
+    }
+    Ok(OverlayReadPlanner {
+        files,
+        atomic_fields,
+    })
+}
+
+/// The per-row atomic fields of a projected top-level field, each with the child-id path from
+/// the top-level field down to it. Structs are recursed through; a primitive leaf or a
+/// whole list/map field is an atomic field (values are one-per-row). A top-level primitive or
+/// list yields a single atomic field with an empty path.
+fn enumerate_atomic_fields(top: &Field) -> Vec<(&Field, Vec<i32>)> {
+    fn recurse<'a>(field: &'a Field, path: &mut Vec<i32>, out: &mut Vec<(&'a Field, Vec<i32>)>) {
+        if field.logical_type.is_struct() {
+            for child in &field.children {
+                path.push(child.id);
+                recurse(child, path, out);
+                path.pop();
+            }
+        } else {
+            out.push((field, path.clone()));
+        }
+    }
+    let mut out = Vec::new();
+    let mut path = Vec::new();
+    recurse(top, &mut path, &mut out);
+    out
+}
+
+/// Collect the leaf field ids in `field`'s subtree — the ids an overlay stores for
+/// this atomic field (its own id if primitive; its item leaves if a list/map).
+fn collect_leaf_ids(field: &Field, out: &mut Vec<i32>) {
+    if field.children.is_empty() {
+        out.push(field.id);
+    } else {
+        for child in &field.children {
+            collect_leaf_ids(child, out);
+        }
+    }
+}
+
+/// Follow a path of child field ids from `field` down through nested structs, taking
+/// the corresponding child array at each step. Returns the array at the end of the
+/// path (the whole `array` when `ancestor_ids` is empty).
+fn descend_by_ids(array: &ArrayRef, field: &Field, ancestor_ids: &[i32]) -> Result<ArrayRef> {
+    let mut arr = array.clone();
+    let mut fld = field;
+    for &id in ancestor_ids {
+        let child_pos = fld
+            .children
+            .iter()
+            .position(|c| c.id == id)
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "overlay descend: field id {id} not found under '{}'",
+                    fld.name
+                ))
+            })?;
+        let structs = arr.as_any().downcast_ref::<StructArray>().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay descend: expected a struct at '{}'",
+                fld.name
+            ))
+        })?;
+        arr = structs.column(child_pos).clone();
+        fld = &fld.children[child_pos];
+    }
+    Ok(arr)
+}
+
+/// Rebuild `array` with the array at `ancestor_ids` replaced by `new_atomic_field`, cloning
+/// the struct spine along the path and preserving each struct's null buffer and other
+/// children. With an empty path this is just `new_atomic_field` (whole-column replacement).
+fn splice_by_ids(
+    array: &ArrayRef,
+    field: &Field,
+    ancestor_ids: &[i32],
+    new_atomic_field: ArrayRef,
+) -> Result<ArrayRef> {
+    let Some((&id, rest)) = ancestor_ids.split_first() else {
+        return Ok(new_atomic_field);
+    };
+    let child_pos = field
+        .children
+        .iter()
+        .position(|c| c.id == id)
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay splice: field id {id} not found under '{}'",
+                field.name
+            ))
+        })?;
+    let structs = array
+        .as_any()
+        .downcast_ref::<StructArray>()
+        .ok_or_else(|| {
+            Error::invalid_input(format!(
+                "overlay splice: expected a struct at '{}'",
+                field.name
+            ))
+        })?;
+    let len = structs.len();
+    let (fields, mut children, nulls) = structs.clone().into_parts();
+    children[child_pos] = splice_by_ids(
+        &children[child_pos],
+        &field.children[child_pos],
+        rest,
+        new_atomic_field,
+    )?;
+    Ok(Arc::new(StructArray::try_new_with_length(
+        fields, children, nulls, len,
+    )?))
+}
+
+/// Open the overlay readers a specific read needs and return the per-field plans to
+/// merge, pruned to that read.
+///
+/// `offsets_in_frag` are the rows the read will return. An overlay whose coverage is
+/// disjoint from those rows contributes nothing, so it is dropped and its file is
+/// never opened — a `take` that misses an overlay's cells pays no IO for it. Each
+/// surviving file is opened once, concurrently, projected to the covered fields; the
+/// value bytes are still not read here (the per-batch [`merge_overlay_batch`] fetches
+/// only the values it needs).
+pub async fn resolve_overlays(
+    planner: &OverlayReadPlanner,
+    offsets_in_frag: &[u32],
+    fragment: &FileFragment,
+    read_config: &FragReadConfig,
+) -> Result<Vec<LoadedAtomicField>> {
+    let read_offsets = read_offsets_bitmap(offsets_in_frag);
+
+    // A file is opened only if some atomic field it covers has cells among the requested rows.
+    // This is the row-selection pruning: overlays outside the read are skipped.
+    let mut file_needed = vec![false; planner.files.len()];
+    for atomic_field in &planner.atomic_fields {
+        for overlay in &atomic_field.overlays_newest_first {
+            if !overlay.coverage.is_disjoint(&read_offsets) {
+                file_needed[overlay.file] = true;
+            }
+        }
+    }
+
+    // Open each needed file once, concurrently. The reader is shared (via `Arc`) by
+    // every atomic field that file covers.
+    //
+    // These reads use priority 0 (highest): they are issued only when a ready
+    // consumer polls the batch task (see `merge_overlay_batch`), so we have already
+    // committed to reading this batch and the overlay reads cannot clog the
+    // backpressure queue ahead of work we are not ready for. (A future optimization
+    // could start the overlay fetches earlier to fill compute bubbles, which would
+    // want a priority tied to the base read.)
+    let opened: Vec<Option<Arc<dyn GenericFileReader>>> =
+        futures::future::try_join_all(planner.files.iter().enumerate().map(|(i, file)| {
+            let needed = file_needed[i];
+            async move {
+                if !needed {
+                    return Ok::<_, Error>(None);
+                }
+                Ok(fragment
+                    .open_reader(&file.data_file, Some(&file.open_projection), read_config)
+                    .await?
+                    .map(Arc::from))
+            }
+        }))
+        .await?;
+
+    let mut plans = Vec::new();
+    for atomic_field in &planner.atomic_fields {
+        let mut overlays_newest_first = Vec::new();
+        for overlay in &atomic_field.overlays_newest_first {
+            let Some(reader) = &opened[overlay.file] else {
+                continue; // pruned: coverage disjoint from the read
+            };
+            overlays_newest_first.push(LoadedAtomicFieldOverlay {
+                coverage: overlay.coverage.clone(),
+                reader: reader.clone(),
+            });
+        }
+        if !overlays_newest_first.is_empty() {
+            plans.push(LoadedAtomicField {
+                top_field: atomic_field.top_field.clone(),
+                ancestor_ids: atomic_field.ancestor_ids.clone(),
+                fetch_projection: atomic_field.fetch_projection.clone(),
+                overlays_newest_first,
+            });
+        }
+    }
+    Ok(plans)
+}
+
+/// The set of `offset_in_frag` a read will return, as a bitmap for cheap
+/// intersection against overlay coverages. Contiguous scans build a single range;
+/// arbitrary `take` offsets (small batches) are inserted individually.
+fn read_offsets_bitmap(offsets_in_frag: &[u32]) -> RoaringBitmap {
+    let mut bitmap = RoaringBitmap::new();
+    match contiguous_frag_start(offsets_in_frag) {
+        Some(start) => {
+            let end = (start as u64 + offsets_in_frag.len() as u64).min(u32::MAX as u64) as u32;
+            bitmap.insert_range(start..end);
+        }
+        None => bitmap.extend(offsets_in_frag.iter().copied()),
+    }
+    bitmap
+}
+
+/// Resolve overlays for one base batch: route each projected atomic field against the batch's
+/// `offsets_in_frag`, fetch only the overlay values the batch needs (concurrently with
+/// the base read), assemble the merged atomic field, and splice it into its output column.
+/// AtomicFields with no covered rows, and columns with no plan, pass through.
+pub async fn merge_overlay_batch(
+    base: ReadBatchFut,
+    offsets_in_frag: &[u32],
+    plans: &[LoadedAtomicField],
+) -> Result<RecordBatch> {
+    let atomic_field_work = futures::future::try_join_all(plans.iter().map(|plan| async move {
+        let coverages: Vec<&RoaringBitmap> = plan
+            .overlays_newest_first
+            .iter()
+            .map(|overlay| overlay.coverage.as_ref())
+            .collect();
+        let routing = route_overlays(offsets_in_frag, &coverages);
+        if !routing.any_overlay {
+            return Ok::<_, Error>((plan, None));
+        }
+        // Fetch each overlay's values and descend to the atomic field array. The fetch is
+        // projected to the atomic field's ancestor path, so the fetched column is the pruned
+        // top-level column; `descend_by_ids` walks it down to the atomic field.
+        let atomic_field = &plan.fetch_projection.fields[0];
+        let fetched = futures::future::try_join_all(
+            plan.overlays_newest_first
+                .iter()
+                .zip(&routing.offsets_in_overlay)
+                .map(|(overlay, offsets_in_overlay)| async move {
+                    let column = fetch_overlay_values(
+                        overlay.reader.as_ref(),
+                        plan.fetch_projection.clone(),
+                        offsets_in_overlay,
+                    )
+                    .await?;
+                    descend_by_ids(&column, atomic_field, &plan.ancestor_ids)
+                }),
+        )
+        .await?;
+        Ok((plan, Some((routing, fetched))))
+    }));
+
+    // The base read and every overlay value read proceed concurrently.
+    let (batch, resolved) = futures::future::try_join(base, atomic_field_work).await?;
+
+    let schema = batch.schema();
+    let mut columns = batch.columns().to_vec();
+    for (plan, work) in resolved {
+        let Some((routing, fetched)) = work else {
+            continue;
+        };
+        let Some(idx) = schema.index_of(&plan.top_field.name).ok() else {
+            // The plan's column is not in this batch's projection; skip it.
+            continue;
+        };
+        let base_atomic_field = descend_by_ids(&columns[idx], &plan.top_field, &plan.ancestor_ids)?;
+        let merged_atomic_field = assemble_overlay_column(&base_atomic_field, &routing, &fetched)?;
+        columns[idx] = splice_by_ids(
+            &columns[idx],
+            &plan.top_field,
+            &plan.ancestor_ids,
+            merged_atomic_field,
+        )?;
+    }
+    Ok(RecordBatch::try_new(schema, columns)?)
+}
+
+/// Fetch one overlay's values at the given `offsets_in_overlay` (sorted, unique):
+/// the corresponding entries of its value column, as the top-level column pruned to
+/// `projection`. Returns `offsets_in_overlay.len()` rows in the same order; empty
+/// input reads nothing and returns an empty column.
+async fn fetch_overlay_values(
+    reader: &dyn GenericFileReader,
+    projection: Arc<Schema>,
+    offsets_in_overlay: &[u32],
+) -> Result<ArrayRef> {
+    if offsets_in_overlay.is_empty() {
+        return Ok(arrow_array::new_empty_array(
+            &projection.fields[0].data_type(),
+        ));
+    }
+    let mut tasks = reader
+        .take_all_tasks(
+            offsets_in_overlay,
+            offsets_in_overlay.len() as u32,
+            projection,
+            None,
+        )
+        .await?;
+    let mut chunks: Vec<ArrayRef> = Vec::new();
+    while let Some(task) = tasks.next().await {
+        let batch = task.task.await?;
+        chunks.push(batch.column(0).clone());
+    }
+    let chunk_refs: Vec<&dyn arrow_array::Array> = chunks.iter().map(|a| a.as_ref()).collect();
+    Ok(arrow_select::concat::concat(&chunk_refs)?)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::{Int32Array, StringArray, UInt32Array};
+    use std::sync::Arc;
+
+    fn i32_array(values: impl IntoIterator<Item = Option<i32>>) -> ArrayRef {
+        Arc::new(Int32Array::from_iter(values))
+    }
+
+    fn bitmap(offsets: impl IntoIterator<Item = u32>) -> RoaringBitmap {
+        RoaringBitmap::from_iter(offsets)
+    }
+
+    /// Physical offsets for a contiguous range `[start, start + len)`.
+    fn offsets(start: u32, len: usize) -> Vec<u32> {
+        (start..start + len as u32).collect()
+    }
+
+    /// Drive the production flow purely in memory: route against the coverage
+    /// bitmaps, then fetch just the requested `offset_in_overlay` entries from each
+    /// overlay's *full* value column (exactly what the value-pushdown `take` does on
+    /// disk), then assemble. `overlays_newest_first` holds each overlay's
+    /// `(coverage, full value column indexed by offset_in_overlay)`.
+    fn resolve(
+        base: &ArrayRef,
+        offsets: &[u32],
+        overlays_newest_first: &[(RoaringBitmap, ArrayRef)],
+    ) -> ArrayRef {
+        let coverages: Vec<&RoaringBitmap> = overlays_newest_first.iter().map(|(c, _)| c).collect();
+        let routing = route_overlays(offsets, &coverages);
+        let fetched: Vec<ArrayRef> = overlays_newest_first
+            .iter()
+            .zip(&routing.offsets_in_overlay)
+            .map(|((_, full), offsets_in_overlay)| {
+                let indices = UInt32Array::from(offsets_in_overlay.clone());
+                arrow_select::take::take(full.as_ref(), &indices, None).unwrap()
+            })
+            .collect();
+        assemble_overlay_column(base, &routing, &fetched).unwrap()
+    }
+
+    fn assert_i32_eq(actual: &ArrayRef, expected: impl IntoIterator<Item = Option<i32>>) {
+        let actual = actual.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(actual, &Int32Array::from_iter(expected));
+    }
+
+    #[test]
+    fn test_no_overlays_returns_base() {
+        let base = i32_array([Some(1), Some(2), Some(3)]);
+        let resolved = resolve(&base, &offsets(0, 3), &[]);
+        assert_i32_eq(&resolved, [Some(1), Some(2), Some(3)]);
+    }
+
+    #[test]
+    fn test_single_overlay_value_offset() {
+        // Base ages [30, 25, 40, 22]; overlay sets offset_in_frag 1 -> 26, whose
+        // value sits at offset_in_overlay 0.
+        let base = i32_array([Some(30), Some(25), Some(40), Some(22)]);
+        let overlay = (bitmap([1]), i32_array([Some(26)]));
+        let resolved = resolve(&base, &offsets(0, 4), &[overlay]);
+        assert_i32_eq(&resolved, [Some(30), Some(26), Some(40), Some(22)]);
+    }
+
+    #[test]
+    fn test_value_offsets_multiple_cells() {
+        // Coverage {0, 2, 3} -> values at offset_in_overlay 0, 1, 2.
+        let base = i32_array([Some(10), Some(11), Some(12), Some(13)]);
+        let overlay = (
+            bitmap([0, 2, 3]),
+            i32_array([Some(100), Some(120), Some(130)]),
+        );
+        let resolved = resolve(&base, &offsets(0, 4), &[overlay]);
+        assert_i32_eq(&resolved, [Some(100), Some(11), Some(120), Some(130)]);
+    }
+
+    #[test]
+    fn test_newest_overlay_wins() {
+        // Two overlays both cover offset_in_frag 1; the newest (first in the slice)
+        // wins.
+        let base = i32_array([Some(0), Some(1), Some(2)]);
+        let newest = (bitmap([1]), i32_array([Some(999)]));
+        let older = (bitmap([1, 2]), i32_array([Some(111), Some(222)]));
+        let resolved = resolve(&base, &offsets(0, 3), &[newest, older]);
+        // offset 1 -> newest (999); offset 2 -> only older covers it (222).
+        assert_i32_eq(&resolved, [Some(0), Some(999), Some(222)]);
+    }
+
+    #[test]
+    fn test_null_override_vs_fall_through() {
+        // A covered offset with a NULL value overrides the cell to NULL; an
+        // absent offset falls through to the base.
+        let base = i32_array([Some(1), Some(2), Some(3)]);
+        let overlay = (bitmap([0]), i32_array([None]));
+        let resolved = resolve(&base, &offsets(0, 3), &[overlay]);
+        assert_i32_eq(&resolved, [None, Some(2), Some(3)]);
+    }
+
+    #[test]
+    fn test_physical_start_offset() {
+        // The batch covers physical rows [10, 13); the overlay covers offset 11.
+        let base = i32_array([Some(0), Some(0), Some(0)]);
+        let overlay = (bitmap([11]), i32_array([Some(7)]));
+        let resolved = resolve(&base, &offsets(10, 3), &[overlay]);
+        assert_i32_eq(&resolved, [Some(0), Some(7), Some(0)]);
+    }
+
+    #[test]
+    fn test_string_column_merge() {
+        let base: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"]));
+        let overlay = (
+            bitmap([0, 2]),
+            Arc::new(StringArray::from(vec!["A", "C"])) as ArrayRef,
+        );
+        let resolved = resolve(&base, &offsets(0, 3), &[overlay]);
+        let expected: ArrayRef = Arc::new(StringArray::from(vec!["A", "b", "C"]));
+        assert_eq!(&resolved, &expected);
+    }
+
+    #[test]
+    fn test_non_contiguous_offsets() {
+        // `take` supplies arbitrary, non-contiguous offsets_in_frag. The base rows
+        // correspond to offsets 5, 1, 8 (in that order); the overlay covers offsets
+        // {1, 8}, whose values sit at offset_in_overlay 0, 1.
+        let base = i32_array([Some(50), Some(10), Some(80)]);
+        let overlay = (bitmap([1, 8]), i32_array([Some(11), Some(88)]));
+        let resolved = resolve(&base, &[5, 1, 8], &[overlay]);
+        // offset 5 uncovered -> base 50; offset 1 -> offset_in_overlay 0 (11);
+        // offset 8 -> offset_in_overlay 1 (88).
+        assert_i32_eq(&resolved, [Some(50), Some(11), Some(88)]);
+    }
+
+    #[test]
+    fn test_routing_dedups_repeated_offsets() {
+        // A `take` may request the same offset twice; both rows must route to the
+        // same overlay value, and that value is fetched only once.
+        let coverage = bitmap([2, 5]);
+        let routing = route_overlays(&[5, 2, 5], &[&coverage]);
+        // offset_in_frag 5 is offset_in_overlay 1, offset_in_frag 2 is
+        // offset_in_overlay 0: distinct values {0, 1}, sorted.
+        assert_eq!(routing.offsets_in_overlay, vec![vec![0, 1]]);
+        let full = i32_array([Some(20), Some(50)]); // values at offset_in_overlay 0, 1
+        let fetched = vec![
+            arrow_select::take::take(
+                full.as_ref(),
+                &UInt32Array::from(routing.offsets_in_overlay[0].clone()),
+                None,
+            )
+            .unwrap(),
+        ];
+        let base = i32_array([Some(0), Some(0), Some(0)]);
+        let resolved = assemble_overlay_column(&base, &routing, &fetched).unwrap();
+        assert_i32_eq(&resolved, [Some(50), Some(20), Some(50)]);
+    }
+
+    #[test]
+    fn test_assemble_value_count_mismatch_errors() {
+        let coverage = bitmap([0, 1]);
+        let routing = route_overlays(&[0, 1], &[&coverage]);
+        let base = i32_array([Some(1), Some(2)]);
+        // One value supplied for two requested offsets is a caller bug.
+        let fetched = vec![i32_array([Some(9)])];
+        assert!(assemble_overlay_column(&base, &routing, &fetched).is_err());
+    }
+
+    #[test]
+    fn test_contiguous_fast_path_matches_general() {
+        // The contiguous fast path must produce byte-for-byte identical routing to
+        // the general offset-major path for any contiguous batch. Fuzz a range of
+        // fragment starts, lengths, overlay counts, and coverage densities —
+        // including bits outside the batch range — and compare both paths.
+        let mut state = 0x9e3779b97f4a7c15u64;
+        let mut next = || {
+            state = state
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407);
+            (state >> 33) as u32
+        };
+        for _ in 0..500 {
+            let frag_start = next() % 64;
+            let len = (next() % 48 + 1) as usize;
+            let num_overlays = (next() % 5) as usize;
+            let coverages: Vec<RoaringBitmap> = (0..num_overlays)
+                .map(|_| {
+                    let density = next() % 101;
+                    let mut b = RoaringBitmap::new();
+                    for off in frag_start.saturating_sub(3)..frag_start + len as u32 + 3 {
+                        if next() % 100 < density {
+                            b.insert(off);
+                        }
+                    }
+                    b
+                })
+                .collect();
+            let refs: Vec<&RoaringBitmap> = coverages.iter().collect();
+            let contiguous_offsets: Vec<u32> = (frag_start..frag_start + len as u32).collect();
+
+            let fast = route_contiguous(frag_start, len, &refs);
+            let general = route_arbitrary(&contiguous_offsets, &refs);
+            assert_eq!(fast.indices, general.indices, "indices differ");
+            assert_eq!(
+                fast.offsets_in_overlay, general.offsets_in_overlay,
+                "offsets_in_overlay differ"
+            );
+            assert_eq!(fast.any_overlay, general.any_overlay, "any_overlay differs");
+        }
+    }
+
+    /// `outer { middle { a, b } }` for exercising the descend/splice helpers.
+    fn nested_struct() -> (Schema, ArrayRef) {
+        use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
+        let mid = Fields::from(vec![
+            ArrowField::new("a", DataType::Int32, true),
+            ArrowField::new("b", DataType::Int32, true),
+        ]);
+        let outer_fields =
+            Fields::from(vec![ArrowField::new("middle", DataType::Struct(mid), true)]);
+        let arrow_schema = ArrowSchema::new(vec![ArrowField::new(
+            "outer",
+            DataType::Struct(outer_fields),
+            true,
+        )]);
+        let schema = Schema::try_from(&arrow_schema).unwrap();
+        let middle = StructArray::from(vec![
+            (
+                Arc::new(ArrowField::new("a", DataType::Int32, true)),
+                i32_array([Some(1), Some(2), Some(3)]),
+            ),
+            (
+                Arc::new(ArrowField::new("b", DataType::Int32, true)),
+                i32_array([Some(10), Some(20), Some(30)]),
+            ),
+        ]);
+        let outer: ArrayRef = Arc::new(StructArray::from(vec![(
+            Arc::new(ArrowField::new("middle", middle.data_type().clone(), true)),
+            Arc::new(middle) as ArrayRef,
+        )]));
+        (schema, outer)
+    }
+
+    #[test]
+    fn test_descend_and_splice_roundtrip() {
+        let (schema, outer_arr) = nested_struct();
+        let outer_field = &schema.fields[0];
+        let middle_id = outer_field.children[0].id;
+        let a_id = outer_field.children[0].children[0].id;
+        let path = [middle_id, a_id];
+
+        // Descend to the deep leaf `outer.middle.a`.
+        let a = descend_by_ids(&outer_arr, outer_field, &path).unwrap();
+        assert_i32_eq(&a, [Some(1), Some(2), Some(3)]);
+
+        // Splice a replacement in; only `a` changes, `b` is preserved.
+        let spliced = splice_by_ids(
+            &outer_arr,
+            outer_field,
+            &path,
+            i32_array([Some(7), Some(8), Some(9)]),
+        )
+        .unwrap();
+        let middle = spliced
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap()
+            .column(0)
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap()
+            .clone();
+        assert_i32_eq(&middle.column(0).clone(), [Some(7), Some(8), Some(9)]);
+        assert_i32_eq(&middle.column(1).clone(), [Some(10), Some(20), Some(30)]);
+    }
+
+    #[test]
+    fn test_splice_preserves_struct_nulls() {
+        use arrow_buffer::NullBuffer;
+        let (schema, base) = nested_struct();
+        let outer_field = &schema.fields[0];
+        // Rebuild `outer` with a null at row 1 (a null struct value).
+        let base = base.as_any().downcast_ref::<StructArray>().unwrap();
+        let (fields, children, _) = base.clone().into_parts();
+        let outer_arr: ArrayRef = Arc::new(
+            StructArray::try_new(
+                fields,
+                children,
+                Some(NullBuffer::from(vec![true, false, true])),
+            )
+            .unwrap(),
+        );
+        let path = [
+            outer_field.children[0].id,
+            outer_field.children[0].children[0].id,
+        ];
+        let spliced = splice_by_ids(
+            &outer_arr,
+            outer_field,
+            &path,
+            i32_array([Some(7), Some(8), Some(9)]),
+        )
+        .unwrap();
+        let spliced = spliced.as_any().downcast_ref::<StructArray>().unwrap();
+        // The outer struct's null buffer survives the splice.
+        assert!(!spliced.is_null(0));
+        assert!(spliced.is_null(1));
+        assert!(!spliced.is_null(2));
+    }
+}
diff --git a/rust/lance/src/dataset/refs.rs b/rust/lance/src/dataset/refs.rs
index 98b4f0cbc0a..0d3f65f7959 100644
--- a/rust/lance/src/dataset/refs.rs
+++ b/rust/lance/src/dataset/refs.rs
@@ -21,7 +21,6 @@ use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::fmt;
 use std::fmt::Formatter;
-use std::io::ErrorKind;
 use uuid::Uuid;
 
 pub const MAIN_BRANCH: &str = "main";
@@ -476,7 +475,7 @@ impl Branches<'_> {
 
         if !self.object_store().exists(&manifest_file.path).await? {
             return Err(Error::VersionNotFound {
-                message: format!("Manifest file {} does not exist", &manifest_file.path),
+                message: format!("Manifest file {} does not exist", manifest_file.path),
             });
         };
 
@@ -611,16 +610,8 @@ impl Branches<'_> {
             && let Err(e) = self.refs.object_store.remove_dir_all(delete_path).await
         {
             match &e {
-                Error::IO { source, .. } => {
-                    if let Some(io_err) = source.downcast_ref::<std::io::Error>() {
-                        if io_err.kind() == ErrorKind::NotFound {
-                            log::debug!("Branch directory already deleted: {}", io_err);
-                        } else {
-                            return Err(e);
-                        }
-                    } else {
-                        return Err(e);
-                    }
+                Error::NotFound { .. } => {
+                    log::debug!("Branch directory already deleted");
                 }
                 _ => return Err(e),
             }
diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs
index 9a5cd94dd09..da8348d57cc 100644
--- a/rust/lance/src/dataset/scanner.rs
+++ b/rust/lance/src/dataset/scanner.rs
@@ -72,7 +72,7 @@ use lance_index::scalar::inverted::query::{
     FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, PhraseQuery, fill_fts_query_column,
 };
 use lance_index::scalar::inverted::{SCORE_COL, SCORE_FIELD};
-use lance_index::vector::{DEFAULT_QUERY_PARALLELISM, DIST_COL, Query};
+use lance_index::vector::{ApproxMode, DEFAULT_QUERY_PARALLELISM, DIST_COL, Query};
 use lance_index::{metrics::NoOpMetricsCollector, scalar::inverted::FTS_SCHEMA};
 use lance_io::stream::RecordBatchStream;
 use lance_linalg::distance::MetricType;
@@ -91,7 +91,9 @@ use crate::index::scalar_logical::scalar_index_fragment_bitmap;
 use crate::index::vector::utils::{
     default_distance_type_for, get_vector_dim, get_vector_type, validate_distance_type_for,
 };
-use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions};
+use crate::io::exec::filtered_read::{
+    FilteredReadExec, FilteredReadOptions, FilteredReadThreadingMode,
+};
 use crate::io::exec::fts::{
     BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec,
 };
@@ -829,6 +831,15 @@ pub struct Scanner {
     /// Which version of the relational algebra to use when generating the physical plan
     relational_algebra_version: u32,
 
+    /// Target degree of parallelism for the physical optimizer.
+    ///
+    /// This is passed as `ConfigOptions::execution::target_partitions` to the
+    /// physical optimizer (e.g. `EnforceDistribution`), which uses it to decide
+    /// how many parallel partitions to target when inserting exchange nodes.
+    ///
+    /// Defaults to `get_num_compute_intensive_cpus()`.
+    target_parallelism: Option<usize>,
+
     // Legacy fields to help migrate some old projection behavior to new behavior
     //
     // There are two behaviors we are moving away from:
@@ -1053,6 +1064,7 @@ impl Scanner {
             explicit_projection: false,
             autoproject_scoring_columns: true,
             relational_algebra_version: LANCE_RELATIONAL_ALGEBRA_VERSION,
+            target_parallelism: None,
         };
         scanner.apply_blob_handling();
         scanner
@@ -1348,24 +1360,25 @@ impl Scanner {
     /// used by the scanner.  If the buffer is full then the scanner will block until
     /// the buffer is processed.
     ///
-    /// Generally this should scale with the number of concurrent I/O threads.  The
-    /// default is 2GiB which comfortably provides enough space for somewhere between
-    /// 32 and 256 concurrent I/O threads.
+    /// Generally this should scale with the number of concurrent I/O threads.  If
+    /// unset, v2 scans choose a default based on the object store and
+    /// `LANCE_DEFAULT_IO_BUFFER_SIZE` can override that default.
     ///
     /// This value is not a hard cap on the amount of RAM the scanner will use.  Some
     /// space is used for the compute (which can be controlled by the batch size) and
     /// Lance does not keep track of memory after it is returned to the user.
     ///
-    /// Currently, if there is a single batch of data which is larger than the io buffer
-    /// size then the scanner will deadlock.  This is a known issue and will be fixed in
-    /// a future release.
     pub fn io_buffer_size(&mut self, size: u64) -> &mut Self {
         self.io_buffer_size = Some(size);
         self
     }
 
-    /// Set the prefetch size.
-    /// Ignored in v2 and newer format
+    /// Set the number of batches to decode concurrently.
+    ///
+    /// This bounds the decode fan-out of the scan: at most this many batch-decode
+    /// tasks run in flight at once. Defaults to `get_num_compute_intensive_cpus()`.
+    ///
+    /// `nbatches` must be greater than zero.
     pub fn batch_readahead(&mut self, nbatches: usize) -> &mut Self {
         self.batch_readahead = nbatches;
         self
@@ -1379,6 +1392,16 @@ impl Scanner {
         self
     }
 
+    /// Set the target number of partitions for the physical optimizer.
+    ///
+    /// Overrides the default (`get_num_compute_intensive_cpus()`). Used by
+    /// `EnforceDistribution` and similar rules to decide how many parallel
+    /// partitions to use. Set to 1 in tests that assert specific plan shapes.
+    pub fn target_parallelism(&mut self, n: usize) -> &mut Self {
+        self.target_parallelism = Some(n);
+        self
+    }
+
     /// Set whether to read data in order (default: true)
     ///
     /// A scan will always read from the disk concurrently.  If this property
@@ -1572,6 +1595,7 @@ impl Scanner {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: Default::default(),
         });
         self.nearest_query_count = query_count;
         self.is_batch_nearest = is_batch_nearest;
@@ -1724,7 +1748,7 @@ impl Scanner {
                     .field(&column.column_name)
                     .ok_or(Error::invalid_input(format!(
                         "Column {} not found",
-                        &column.column_name
+                        column.column_name
                     )))?;
             }
         }
@@ -1740,6 +1764,19 @@ impl Scanner {
         self
     }
 
+    /// Configure the speed / accuracy tradeoff for approximate vector search.
+    ///
+    /// This setting is currently only used by RQ-quantized indexes, such as
+    /// IVF_RQ. Other index types ignore this setting.
+    pub fn approx_mode(&mut self, approx_mode: ApproxMode) -> &mut Self {
+        if let Some(q) = self.nearest.as_mut() {
+            q.approx_mode = approx_mode;
+        } else {
+            log::warn!("approx_mode is not set because nearest has not been called yet");
+        }
+        self
+    }
+
     /// Configure partition-search concurrency for each vector query.
     ///
     /// The default is 0.
@@ -1868,6 +1905,38 @@ impl Scanner {
         }
     }
 
+    /// Ensure `input` exposes `column_name` as a top-level column.
+    ///
+    /// Nested FTS flat-search paths read the projected struct column from storage
+    /// but the FTS executor consumes a single document column by name.
+    fn ensure_column_alias(
+        &self,
+        input: Arc<dyn ExecutionPlan>,
+        column_name: &str,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let input_schema = input.schema();
+        if input_schema.column_with_name(column_name).is_some() {
+            return Ok(input);
+        }
+
+        let mut projection_exprs = Vec::with_capacity(input_schema.fields().len() + 1);
+        for field in input_schema.fields() {
+            projection_exprs.push((
+                Arc::new(Column::new_with_schema(
+                    field.name(),
+                    input_schema.as_ref(),
+                )?) as Arc<dyn PhysicalExpr>,
+                field.name().clone(),
+            ));
+        }
+        projection_exprs.push((
+            Self::create_column_expr(column_name, self.dataset.as_ref(), input_schema.as_ref())?,
+            column_name.to_string(),
+        ));
+
+        Ok(Arc::new(ProjectionExec::try_new(projection_exprs, input)?))
+    }
+
     /// Set whether to use statistics to optimize the scan (default: true)
     ///
     /// This is used for debugging or benchmarking purposes.
@@ -2155,6 +2224,9 @@ impl Scanner {
     }
 
     #[allow(clippy::type_complexity)]
+    // TODO(datafusion-54): migrate off the deprecated
+    // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder.
+    #[allow(deprecated)]
     fn build_physical_aggregate_expr(
         &self,
         expr: &Expr,
@@ -2279,11 +2351,12 @@ impl Scanner {
             MaterializationStyle::AllLate => false,
             MaterializationStyle::AllEarlyExcept(ref cols) => !cols.contains(&(field.id as u32)),
             MaterializationStyle::Heuristic => {
-                if field.is_blob() {
-                    // By default, blobs are loaded as descriptions, and so should be early
-                    //
-                    // TODO: Once we make blob handling configurable, we should use the blob
-                    // handling setting here.
+                if field.is_blob() && self.blob_handling.returns_description(field) {
+                    // A blob returned as a description (offset + size) is tiny, so it is
+                    // cheaper to read eagerly. When blob_handling materializes the full
+                    // binary value instead (e.g. `all_binary`), fall through to the
+                    // width-based heuristic so a selective filter can late-materialize it
+                    // rather than reading the whole column.
                     return true;
                 }
 
@@ -2331,6 +2404,12 @@ impl Scanner {
     }
 
     fn validate_options(&self) -> Result<()> {
+        if self.batch_readahead == 0 {
+            return Err(Error::invalid_input_source(
+                "batch_readahead must be greater than 0, got 0".into(),
+            ));
+        }
+
         if self.include_deleted_rows && !self.projection_plan.physical_projection.with_row_id {
             return Err(Error::invalid_input_source(
                 "include_deleted_rows is set but with_row_id is false".into(),
@@ -2617,7 +2696,10 @@ impl Scanner {
             plan = self.apply_aggregate(plan, agg).await?;
 
             let optimizer = get_physical_optimizer();
-            let options = Default::default();
+            let mut options = ConfigOptions::default();
+            options.execution.target_partitions = self
+                .target_parallelism
+                .unwrap_or_else(get_num_compute_intensive_cpus);
             for rule in optimizer.rules {
                 plan = rule.optimize(plan, &options)?;
             }
@@ -2681,7 +2763,10 @@ impl Scanner {
         }
 
         let optimizer = get_physical_optimizer();
-        let options: ConfigOptions = Default::default();
+        let mut options = ConfigOptions::default();
+        options.execution.target_partitions = self
+            .target_parallelism
+            .unwrap_or_else(get_num_compute_intensive_cpus);
         for rule in optimizer.rules {
             plan = rule.optimize(plan, &options)?;
         }
@@ -2837,6 +2922,11 @@ impl Scanner {
             read_options = read_options.with_batch_size(batch_size as u32);
         }
 
+        // Bound the decode fan-out by `batch_readahead`.
+        read_options = read_options.with_threading_mode(
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(self.batch_readahead),
+        );
+
         if let Some(file_reader_options) = self.resolved_file_reader_options() {
             read_options = read_options.with_file_reader_options(file_reader_options);
         }
@@ -3550,34 +3640,37 @@ impl Scanner {
             ))?
             .clone();
 
-        let mut columns = vec![column];
-        if let Some(expr) = filter_plan.full_expr.as_ref() {
-            let filter_columns = Planner::column_names_in_expr(expr);
-            columns.extend(filter_columns);
+        let mut columns = vec![column.clone()];
+        if let Some(refine_expr) = filter_plan.refine_expr.as_ref() {
+            columns.extend(Planner::column_names_in_expr(refine_expr));
         }
-        let flat_fts_scan_schema = Arc::new(self.dataset.schema().project(&columns).unwrap());
-        let mut scan_node = self.scan_fragments(
-            true,
-            false,
-            false,
-            false,
-            false,
-            flat_fts_scan_schema,
-            Arc::new(fragments),
-            None,
-            false,
-        );
+        let scan_projection = self
+            .dataset
+            .empty_projection()
+            .with_row_id()
+            .union_columns(&columns, OnMissing::Error)?;
+
+        let PlannedFilteredScan { mut plan, .. } = self
+            .filtered_read(
+                filter_plan,
+                scan_projection,
+                /*make_deletions_null=*/ false,
+                Some(Arc::new(fragments)),
+                None,
+                /*is_prefilter=*/ true,
+            )
+            .await?;
 
-        if let Some(expr) = filter_plan.full_expr.as_ref() {
-            // If there is a prefilter we need to manually apply it to the new data
-            scan_node = Arc::new(LanceFilterExec::try_new(expr.clone(), scan_node)?);
+        if let Some(refine_expr) = filter_plan.refine_expr.as_ref() {
+            plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?);
         }
+        plan = self.ensure_column_alias(plan, &column)?;
 
         let flat_match_plan = Arc::new(FlatMatchQueryExec::new(
             self.dataset.clone(),
             query.clone(),
             params.clone(),
-            scan_node,
+            plan,
         ));
         Ok(flat_match_plan)
     }
@@ -3655,7 +3748,7 @@ impl Scanner {
                         .dataset
                         .open_vector_index(
                             q.column.as_str(),
-                            &selected_index_segments[0].uuid.to_string(),
+                            &selected_index_segments[0].uuid,
                             &NoOpMetricsCollector,
                         )
                         .await?;
@@ -3689,11 +3782,7 @@ impl Scanner {
                     // Fall back to opening the index for legacy indices without details
                     let idx = self
                         .dataset
-                        .open_vector_index(
-                            q.column.as_str(),
-                            &index.uuid.to_string(),
-                            &NoOpMetricsCollector,
-                        )
+                        .open_vector_index(q.column.as_str(), &index.uuid, &NoOpMetricsCollector)
                         .await?;
                     idx.metric_type()
                 };
@@ -4248,6 +4337,7 @@ impl Scanner {
             with_make_deletions_null,
             ordered_output: ordered,
             file_reader_options: self.resolved_file_reader_options(),
+            parallelism_cap: None,
         };
         Arc::new(LanceScanExec::new(
             self.dataset.clone(),
@@ -4324,7 +4414,8 @@ impl Scanner {
                         .dataset
                         .empty_projection()
                         .union_column(&column, OnMissing::Error)?;
-                    self.take(input, projection)?
+                    let input = self.take(input, projection)?;
+                    self.ensure_column_alias(input, &column)?
                 } else {
                     input
                 };
@@ -4374,7 +4465,8 @@ impl Scanner {
                         .dataset
                         .empty_projection()
                         .union_column(&column, OnMissing::Error)?;
-                    self.take(input, projection)?
+                    let input = self.take(input, projection)?;
+                    self.ensure_column_alias(input, &column)?
                 } else {
                     input
                 };
@@ -4442,6 +4534,14 @@ impl Scanner {
         } else {
             input
         };
+        let retain_vector = if self.is_batch_nearest {
+            let vector_field_id = self.dataset.schema().field_id(q.column.as_str())?;
+            self.projection_plan
+                .physical_projection
+                .contains_field_id(vector_field_id)
+        } else {
+            false
+        };
         let flat_dist = Arc::new(KNNVectorDistanceExec::try_new_batch(
             input,
             &q.column,
@@ -4453,6 +4553,7 @@ impl Scanner {
                 lower_bound: q.lower_bound,
                 upper_bound: q.upper_bound,
                 distance_type: metric_type,
+                retain_vector,
             },
         )?);
 
@@ -4828,6 +4929,39 @@ impl Scanner {
 
         Ok(format!("{}", display.indent(verbose)))
     }
+
+    /// Run [`Self::count_rows`]'s underlying plan and return it formatted with
+    /// runtime metrics. Equivalent to [`Self::analyze_plan`] but with a
+    /// `COUNT(*)` aggregate auto-applied first — the only way for callers
+    /// without a hand-built `AggregateExpr` (e.g. the Python bindings) to
+    /// inspect the plan that `count_rows` actually executed.
+    #[instrument(level = "info", skip(self))]
+    pub async fn analyze_count_plan(&self) -> Result<String> {
+        let mut scanner = self.clone();
+        scanner.aggregate(AggregateExpr::builder().count_star().build())?;
+        let plan = scanner.create_plan().await?;
+        analyze_plan(
+            plan,
+            LanceExecutionOptions {
+                batch_size: self.batch_size,
+                ..Default::default()
+            },
+        )
+        .await
+    }
+}
+
+fn is_fts_indexable_field(field: &Field) -> bool {
+    match field.data_type() {
+        DataType::Utf8 | DataType::LargeUtf8 => true,
+        DataType::List(inner_field) | DataType::LargeList(inner_field) => {
+            matches!(
+                inner_field.data_type(),
+                DataType::Utf8 | DataType::LargeUtf8
+            )
+        }
+        _ => false,
+    }
 }
 
 // Search over all indexed fields including nested ones, collecting columns that have an
@@ -4835,19 +4969,7 @@ impl Scanner {
 async fn fts_indexed_columns(dataset: Arc<Dataset>) -> Result<Vec<String>> {
     let mut indexed_columns = Vec::new();
     for field in dataset.schema().fields_pre_order() {
-        // Check if this field is a string type that could have an inverted index
-        let is_string_field = match field.data_type() {
-            DataType::Utf8 | DataType::LargeUtf8 => true,
-            DataType::List(inner_field) | DataType::LargeList(inner_field) => {
-                matches!(
-                    inner_field.data_type(),
-                    DataType::Utf8 | DataType::LargeUtf8
-                )
-            }
-            _ => false,
-        };
-
-        if is_string_field {
+        if is_fts_indexable_field(field) {
             // Build the full field path for nested fields
             let column_path =
                 if let Some(ancestors) = dataset.schema().field_ancestry_by_id(field.id) {
@@ -5131,7 +5253,13 @@ pub mod test_dataset {
         }
 
         pub async fn make_fts_index(&mut self) -> Result<()> {
-            let params = InvertedIndexParams::default().with_position(true);
+            // These scanner tests search for the token "s" (from the `s-{N}`
+            // column values) to exercise fragment/append coverage, and "s" is
+            // in the full English stop-word list. Keep the token searchable;
+            // stop-word behavior itself is covered by the tokenizer tests.
+            let params = InvertedIndexParams::default()
+                .with_position(true)
+                .remove_stop_words(false);
             self.dataset
                 .create_index(&["s"], IndexType::Inverted, None, &params, true)
                 .await?;
@@ -5563,6 +5691,149 @@ mod test {
         Ok(())
     }
 
+    // Regression for #6580: a scan with `filter` + `project` of a
+    // `(Large)List<Struct>` column used to panic in `merge_with_schema`
+    // (called from `TakeStream::map_batch`) because the filtered batch arrived
+    // as a sliced view of a larger batch and the cloned list offsets did not
+    // start at zero. The trigger requires (a) a `(Large)List<Struct>`
+    // projection where the struct is split across `filtered_read` and
+    // `TakeExec` and (b) a sparse-tail selectivity pattern so the trailing
+    // filter result lands deep inside the values buffer of its source batch.
+    // Parametrized over `List`/`LargeList` since the fix touches both offset
+    // widths in `merge_with_schema`.
+    #[rstest]
+    #[tokio::test]
+    async fn test_filter_project_list_struct_sparse_tail(
+        // The panic is specific to v2.x storage; the legacy reader takes a
+        // different code path. V2_0 and V2_2 are the versions called out in
+        // the original report.
+        #[values(
+            LanceFileVersion::V2_0,
+            LanceFileVersion::Stable,
+            LanceFileVersion::V2_2
+        )]
+        data_storage_version: LanceFileVersion,
+        #[values(false, true)] large_list: bool,
+    ) {
+        use arrow_array::{LargeListArray, ListArray, UInt16Array};
+        use arrow_buffer::{OffsetBuffer, ScalarBuffer};
+
+        let struct_fields = Fields::from(vec![
+            Arc::new(ArrowField::new("a", DataType::Int32, true)),
+            Arc::new(ArrowField::new("b", DataType::Int32, true)),
+        ]);
+        let item_field = Arc::new(ArrowField::new(
+            "item",
+            DataType::Struct(struct_fields.clone()),
+            true,
+        ));
+        let items_dtype = if large_list {
+            DataType::LargeList(item_field.clone())
+        } else {
+            DataType::List(item_field.clone())
+        };
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("grp", DataType::UInt16, false),
+            ArrowField::new("items", items_dtype, false),
+        ]));
+
+        let make_batch = |start: i32, n: usize, group: u16| -> RecordBatch {
+            let ids = Int32Array::from_iter_values(start..start + n as i32);
+            let groups = UInt16Array::from(vec![group; n]);
+
+            let mut offsets = Vec::with_capacity(n + 1);
+            let mut a_vals: Vec<i32> = Vec::new();
+            let mut b_vals: Vec<i32> = Vec::new();
+            offsets.push(0i64);
+            for i in 0..n {
+                // Variable-length lists (1..=18) so offsets don't land on
+                // batch-row boundaries.
+                let len = 1 + (i % 18);
+                for j in 0..len {
+                    a_vals.push(j as i32);
+                    b_vals.push(-(j as i32));
+                }
+                offsets.push(a_vals.len() as i64);
+            }
+            let struct_arr = Arc::new(StructArray::new(
+                struct_fields.clone(),
+                vec![
+                    Arc::new(Int32Array::from(a_vals)) as ArrayRef,
+                    Arc::new(Int32Array::from(b_vals)) as ArrayRef,
+                ],
+                None,
+            ));
+            let items: ArrayRef = if large_list {
+                Arc::new(LargeListArray::new(
+                    item_field.clone(),
+                    OffsetBuffer::new(ScalarBuffer::from(offsets)),
+                    struct_arr,
+                    None,
+                ))
+            } else {
+                let offsets_i32: Vec<i32> = offsets.iter().map(|&o| o as i32).collect();
+                Arc::new(ListArray::new(
+                    item_field.clone(),
+                    OffsetBuffer::new(ScalarBuffer::from(offsets_i32)),
+                    struct_arr,
+                    None,
+                ))
+            };
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(ids) as ArrayRef,
+                    Arc::new(groups) as ArrayRef,
+                    items,
+                ],
+            )
+            .unwrap()
+        };
+
+        // Sparse-tail selectivity (matching the original report's shape at a
+        // smaller scale): a large leading block of matches, a large gap of
+        // non-matches, then a small trailing match. Single fragment.
+        let batches = vec![
+            make_batch(0, 100_000, 7),
+            make_batch(100_000, 400_000, 1),
+            make_batch(500_000, 7_300, 7),
+        ];
+
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write(
+            reader,
+            "memory://",
+            Some(WriteParams {
+                max_rows_per_file: 1_000_000,
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Force a column split inside the `items` struct by marking `items.b`
+        // as a late-materialized field: `filtered_read` returns the batch with
+        // `items.a`, and `TakeExec` adds `items.b`. `merge_with_schema` then
+        // takes its `List<Struct>` branch, which is where the panic was.
+        let items_b_field_id = dataset
+            .schema()
+            .field("items")
+            .unwrap()
+            .child("item")
+            .unwrap()
+            .child("b")
+            .unwrap()
+            .id as u32;
+        let mut scan = dataset.scan();
+        scan.filter("grp = 7").unwrap();
+        scan.project(&["id", "items"]).unwrap();
+        scan.materialization_style(MaterializationStyle::AllEarlyExcept(vec![items_b_field_id]));
+        let result = scan.try_into_batch().await.unwrap();
+        assert_eq!(result.num_rows(), 107_300);
+    }
+
     #[tokio::test]
     async fn test_scan_regexp_match_and_non_empty_captions() {
         // Build a small dataset with three Utf8 columns and verify the full
@@ -5883,6 +6154,114 @@ mod test {
         (queries, query_values)
     }
 
+    async fn nested_vector_test_dataset(dim: u32) -> (TempStrDir, Dataset) {
+        let path = TempStrDir::default();
+        let vec_field = ArrowField::new(
+            "vec",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                dim as i32,
+            ),
+            true,
+        );
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(vec![vec_field.clone()].into()),
+            true,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            payload_field.clone(),
+        ]));
+
+        let batches: Vec<RecordBatch> = (0..5)
+            .map(|batch_idx| {
+                let vector_values: Float32Array = (0..dim * 80).map(|v| v as f32).collect();
+                let vectors =
+                    FixedSizeListArray::try_new_from_values(vector_values, dim as i32).unwrap();
+                let payload = StructArray::from(vec![(
+                    Arc::new(vec_field.clone()),
+                    Arc::new(vectors) as ArrayRef,
+                )]);
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(Int32Array::from_iter_values(
+                            batch_idx * 80..(batch_idx + 1) * 80,
+                        )),
+                        Arc::new(payload),
+                    ],
+                )
+                .unwrap()
+            })
+            .collect();
+
+        let params = WriteParams {
+            max_rows_per_group: 10,
+            max_rows_per_file: 200,
+            data_storage_version: Some(LanceFileVersion::Stable),
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, &path, Some(params)).await.unwrap();
+        (path, dataset)
+    }
+
+    async fn escaped_nested_vector_test_dataset(dim: u32) -> (TempStrDir, Dataset) {
+        let path = TempStrDir::default();
+        let vec_field = ArrowField::new(
+            "vec.with.dot",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                dim as i32,
+            ),
+            true,
+        );
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(vec![vec_field.clone()].into()),
+            true,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            payload_field.clone(),
+        ]));
+
+        let batches: Vec<RecordBatch> = (0..5)
+            .map(|batch_idx| {
+                let vector_values: Float32Array = (0..dim * 80).map(|v| v as f32).collect();
+                let vectors =
+                    FixedSizeListArray::try_new_from_values(vector_values, dim as i32).unwrap();
+                let payload = StructArray::from(vec![(
+                    Arc::new(vec_field.clone()),
+                    Arc::new(vectors) as ArrayRef,
+                )]);
+                RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(Int32Array::from_iter_values(
+                            batch_idx * 80..(batch_idx + 1) * 80,
+                        )),
+                        Arc::new(payload),
+                    ],
+                )
+                .unwrap()
+            })
+            .collect();
+
+        let params = WriteParams {
+            max_rows_per_group: 10,
+            max_rows_per_file: 200,
+            data_storage_version: Some(LanceFileVersion::Stable),
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
+        let dataset = Dataset::write(reader, &path, Some(params)).await.unwrap();
+        (path, dataset)
+    }
+
     fn assert_query_index_field(batch: &RecordBatch) {
         let schema = batch.schema();
         let field = schema.field(0);
@@ -5891,6 +6270,14 @@ mod test {
         assert!(!field.is_nullable());
     }
 
+    fn assert_batch_knn_output_has_no_vector(batch: &RecordBatch, vector_column: &str) {
+        assert!(
+            batch.schema().column_with_name(vector_column).is_none(),
+            "batch flat KNN output must not include vector column '{vector_column}' when it is not projected; columns: {:?}",
+            batch.schema().field_names()
+        );
+    }
+
     async fn assert_batch_matches_single_queries(
         dataset: &Dataset,
         batch: &RecordBatch,
@@ -5965,6 +6352,7 @@ mod test {
 
         let batch = scan.try_into_batch().await.unwrap();
         assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, "vec");
         assert_eq!(
             batch.num_rows(),
             2 * k,
@@ -5987,6 +6375,25 @@ mod test {
         }
         assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await;
 
+        let mut scan_with_vec = dataset.scan();
+        scan_with_vec.nearest("vec", &queries, k).unwrap();
+        scan_with_vec.use_index(false);
+        scan_with_vec.project(&["i", "vec"]).unwrap();
+        let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap();
+        assert!(
+            batch_with_vec.schema().column_with_name("vec").is_some(),
+            "batch flat KNN should return vector column when projected"
+        );
+        assert_batch_matches_single_queries(
+            dataset,
+            &batch_with_vec,
+            &query_values,
+            k,
+            false,
+            None,
+        )
+        .await;
+
         let query_values_one = (32..64).map(|v| v as f32).collect::<Vec<_>>();
         let queries_one = FixedSizeListArray::try_new_from_values(
             Float32Array::from(query_values_one.clone()),
@@ -6012,6 +6419,7 @@ mod test {
 
         let batch = scan.try_into_batch().await.unwrap();
         assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, "vec");
         assert_eq!(
             batch[QUERY_INDEX_COL].as_primitive::<Int32Type>().values(),
             &[0, 0]
@@ -6019,46 +6427,223 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_primitive_query_length_multiple_of_dim_is_rejected() {
+    async fn test_batch_knn_flat_omits_vector_without_projection() {
         let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
             .await
             .unwrap();
         let dataset = &test_ds.dataset;
-        let q: Float32Array = (32..96).map(|v| v as f32).collect();
+        let k = 2;
+        let (queries, query_values) = batch_knn_two_queries();
 
-        let err = match dataset.scan().nearest("vec", &q, 2) {
-            Err(err) => err.to_string(),
-            Ok(_) => panic!("expected primitive query length mismatch error"),
-        };
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.use_index(false);
+        scan.project(&["i"]).unwrap();
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_batch_knn_output_has_no_vector(&batch, "vec");
+        assert_query_index_field(&batch);
+        assert!(batch.schema().column_with_name("i").is_some());
+        assert!(batch.schema().column_with_name(DIST_COL).is_some());
+        assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await;
+
+        let mut scan_rowid_only = dataset.scan();
+        scan_rowid_only.nearest("vec", &queries, k).unwrap();
+        scan_rowid_only.use_index(false);
+        scan_rowid_only.project(&[ROW_ID]).unwrap();
+        let batch_rowid_only = scan_rowid_only.try_into_batch().await.unwrap();
+        assert_batch_knn_output_has_no_vector(&batch_rowid_only, "vec");
+        assert!(batch_rowid_only.schema().column_with_name(ROW_ID).is_some());
+        assert!(batch_rowid_only.schema().column_with_name("i").is_none());
+
+        let mut scan_with_vec = dataset.scan();
+        scan_with_vec.nearest("vec", &queries, k).unwrap();
+        scan_with_vec.use_index(false);
+        scan_with_vec.project(&["vec"]).unwrap();
+        let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap();
         assert!(
-            err.contains("query dim(64) doesn't match the column vec vector dim(32)"),
-            "unexpected error: {err}"
+            batch_with_vec.schema().column_with_name("vec").is_some(),
+            "batch flat KNN must include vector column when vec is projected"
         );
     }
 
-    async fn dataset_with_query_index_column() -> (TempStrDir, Dataset) {
-        let path = TempStrDir::default();
-        let schema = Arc::new(ArrowSchema::new(vec![
-            ArrowField::new("i", DataType::Int32, true),
-            ArrowField::new(
-                "vec",
-                DataType::FixedSizeList(
-                    Arc::new(ArrowField::new("item", DataType::Float32, true)),
-                    32,
-                ),
-                true,
-            ),
-            ArrowField::new(QUERY_INDEX_COL, DataType::UInt32, true),
-        ]));
-        let vector_values: Float32Array = (0..32 * 80).map(|v| v as f32).collect();
-        let vectors = FixedSizeListArray::try_new_from_values(vector_values, 32).unwrap();
-        let batch = RecordBatch::try_new(
-            schema.clone(),
-            vec![
-                Arc::new(Int32Array::from_iter_values(0..80)),
-                Arc::new(vectors),
-                Arc::new(UInt32Array::from_iter((0..80).map(|v| v as u32))),
-            ],
+    #[tokio::test]
+    async fn test_batch_knn_flat_filter_keeps_non_vector_columns() {
+        let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        let dataset = &test_ds.dataset;
+        let k = 2;
+        let (queries, query_values) = batch_knn_two_queries();
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.use_index(false);
+        scan.filter("i >= 0").unwrap();
+        scan.project(&["i"]).unwrap();
+        let batch = scan.try_into_batch().await.unwrap();
+
+        assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, "vec");
+        assert!(batch.schema().column_with_name("i").is_some());
+
+        let query_indices = batch[QUERY_INDEX_COL].as_primitive::<Int32Type>();
+        for query_index in 0..2 {
+            let query =
+                Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec());
+            let mut single = dataset.scan();
+            single.nearest("vec", &query, k).unwrap();
+            single.use_index(false);
+            single.filter("i >= 0").unwrap();
+            single.project(&["i"]).unwrap();
+            let single_batch = single.try_into_batch().await.unwrap();
+
+            let mask = BooleanArray::from_iter(
+                query_indices
+                    .iter()
+                    .map(|value| value.map(|value| value == query_index as i32)),
+            );
+            let batch_slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap();
+            assert_eq!(
+                batch_slice["i"].as_primitive::<Int32Type>().values(),
+                single_batch["i"].as_primitive::<Int32Type>().values()
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_batch_knn_flat_nested_vector_projection() {
+        const VECTOR_COLUMN: &str = "payload.vec";
+        let (_tmp, dataset) = nested_vector_test_dataset(32).await;
+        let k = 2;
+        let (queries, _query_values) = batch_knn_two_queries();
+
+        let mut scan = dataset.scan();
+        scan.nearest(VECTOR_COLUMN, &queries, k).unwrap();
+        scan.use_index(false);
+        scan.project(&["i"]).unwrap();
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN);
+        assert_eq!(batch.num_rows(), 2 * k);
+        assert!(batch.schema().column_with_name("i").is_some());
+
+        let mut scan_with_vec = dataset.scan();
+        scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap();
+        scan_with_vec.use_index(false);
+        scan_with_vec.project(&[VECTOR_COLUMN]).unwrap();
+        let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap();
+        assert!(
+            batch_with_vec
+                .schema()
+                .column_with_name(VECTOR_COLUMN)
+                .is_some(),
+            "batch flat KNN must include nested vector column when projected; columns: {:?}",
+            batch_with_vec.schema().field_names()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_batch_knn_flat_escaped_nested_vector_projection() {
+        const VECTOR_COLUMN: &str = "payload.`vec.with.dot`";
+        let (_tmp, dataset) = escaped_nested_vector_test_dataset(32).await;
+        let k = 2;
+        let (queries, _) = batch_knn_two_queries();
+
+        let mut scan = dataset.scan();
+        scan.nearest(VECTOR_COLUMN, &queries, k).unwrap();
+        scan.use_index(false);
+        scan.project(&["i"]).unwrap();
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN);
+        assert_eq!(batch.num_rows(), 2 * k);
+        assert!(batch.schema().column_with_name("i").is_some());
+
+        let mut scan_with_vec = dataset.scan();
+        scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap();
+        scan_with_vec.use_index(false);
+        scan_with_vec.project(&[VECTOR_COLUMN]).unwrap();
+        let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap();
+        assert!(
+            batch_with_vec
+                .schema()
+                .column_with_name(VECTOR_COLUMN)
+                .is_some(),
+            "batch flat KNN must include escaped nested vector column when projected; columns: {:?}",
+            batch_with_vec.schema().field_names()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_batch_knn_flat_projects_row_id_and_row_addr_without_vector() {
+        let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        let dataset = &test_ds.dataset;
+        let k = 2;
+        let (queries, _) = batch_knn_two_queries();
+
+        let mut scan = dataset.scan();
+        scan.nearest("vec", &queries, k).unwrap();
+        scan.use_index(false);
+        scan.project(&[ROW_ID]).unwrap();
+        scan.with_row_address();
+
+        let batch = scan.try_into_batch().await.unwrap();
+        assert_query_index_field(&batch);
+        assert_batch_knn_output_has_no_vector(&batch, "vec");
+        assert_eq!(batch.num_rows(), 2 * k);
+        assert!(batch.schema().column_with_name(ROW_ID).is_some());
+        assert!(batch.schema().column_with_name(ROW_ADDR).is_some());
+        assert!(batch.schema().column_with_name(DIST_COL).is_some());
+        assert_eq!(
+            batch[ROW_ADDR].as_primitive::<UInt64Type>().null_count(),
+            0,
+            "row addresses should be materialized for all top-k rows"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_primitive_query_length_multiple_of_dim_is_rejected() {
+        let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true)
+            .await
+            .unwrap();
+        let dataset = &test_ds.dataset;
+        let q: Float32Array = (32..96).map(|v| v as f32).collect();
+
+        let err = match dataset.scan().nearest("vec", &q, 2) {
+            Err(err) => err.to_string(),
+            Ok(_) => panic!("expected primitive query length mismatch error"),
+        };
+        assert!(
+            err.contains("query dim(64) doesn't match the column vec vector dim(32)"),
+            "unexpected error: {err}"
+        );
+    }
+
+    async fn dataset_with_query_index_column() -> (TempStrDir, Dataset) {
+        let path = TempStrDir::default();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            ArrowField::new(
+                "vec",
+                DataType::FixedSizeList(
+                    Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                    32,
+                ),
+                true,
+            ),
+            ArrowField::new(QUERY_INDEX_COL, DataType::UInt32, true),
+        ]));
+        let vector_values: Float32Array = (0..32 * 80).map(|v| v as f32).collect();
+        let vectors = FixedSizeListArray::try_new_from_values(vector_values, 32).unwrap();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..80)),
+                Arc::new(vectors),
+                Arc::new(UInt32Array::from_iter((0..80).map(|v| v as u32))),
+            ],
         )
         .unwrap();
         let dataset = Dataset::write(
@@ -8276,6 +8861,9 @@ mod test {
         expected: &str,
     ) -> Result<()> {
         let mut scan = dataset.scan();
+        // Pin target_parallelism=1 so EnforceDistribution produces deterministic plans
+        // regardless of the machine's CPU count.
+        scan.target_parallelism(1);
         plan(&mut scan)?;
         let exec_plan = scan.create_plan().await?;
         assert_plan_node_equals(exec_plan, expected).await
@@ -8352,6 +8940,198 @@ mod test {
         .unwrap();
     }
 
+    #[tokio::test]
+    async fn test_ngram_regex_index_scan() {
+        use arrow::array::AsArray;
+
+        // A small, fixed corpus written across multiple fragments so the ngram
+        // index spans fragment boundaries.
+        let values = [
+            "rhino",       // 0
+            "rhinos nose", // 1
+            "cat",         // 2
+            "dog",         // 3
+            "cat dog",     // 4
+            "elephant",    // 5
+            "catalog",     // 6
+            "scatter",     // 7
+            "rhino horn",  // 8
+            "mouse",       // 9
+            "category",    // 10
+            "dogma",       // 11
+        ];
+        let array = StringArray::from_iter_values(values);
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Utf8,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let write_params = WriteParams {
+            max_rows_per_file: 4, // 12 rows -> 3 fragments
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write(reader, "memory://test_ngram_regex", Some(write_params))
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["s"],
+                IndexType::NGram,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        assert!(
+            dataset.get_fragments().len() > 1,
+            "expected a multi-fragment dataset"
+        );
+
+        // Scan with `filter` and return the matched `s` values, sorted.
+        async fn matched(dataset: &Dataset, filter: &str) -> Vec<String> {
+            let mut scan = dataset.scan();
+            scan.filter(filter).unwrap();
+            let batches = scan
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            let mut out = Vec::new();
+            for batch in batches {
+                let col = batch.column_by_name("s").unwrap().as_string::<i32>();
+                out.extend(col.iter().flatten().map(|s| s.to_string()));
+            }
+            out.sort();
+            out
+        }
+
+        // `regexp_like`: a plain literal substring.
+        assert_eq!(
+            matched(&dataset, "regexp_like(s, 'rhino')").await,
+            ["rhino", "rhino horn", "rhinos nose"]
+        );
+        // `regexp_match` (coerced to `IsNotNull(regexp_match(...))`) accelerates too.
+        assert_eq!(
+            matched(&dataset, "regexp_match(s, 'rhino')").await,
+            ["rhino", "rhino horn", "rhinos nose"]
+        );
+        // Anchored: recheck must drop trigram false positives -- the `cat`
+        // trigram also occurs in cat dog / catalog / scatter / category.
+        assert_eq!(matched(&dataset, "regexp_like(s, 'cat$')").await, ["cat"]);
+        // AND across `.*`: row 8 ("rhino horn") shares the rhino trigrams but
+        // lacks the nose trigrams, so only "rhinos nose" survives.
+        assert_eq!(
+            matched(&dataset, "regexp_like(s, 'rhino.*nose')").await,
+            ["rhinos nose"]
+        );
+        // Alternation.
+        assert_eq!(
+            matched(&dataset, "regexp_like(s, '(catalog|elephant)')").await,
+            ["catalog", "elephant"]
+        );
+        // A non-accelerable pattern (no trigram derivable) still returns correct
+        // results via a full recheck.
+        assert_eq!(matched(&dataset, "regexp_like(s, 'o.m')").await, ["dogma"]);
+        // A case-insensitive flag is not accelerated (the index normalization
+        // disagrees with Unicode case folding) but must still return correct
+        // results via a full recheck -- here matching despite the upper-case
+        // pattern. This exercises the three-argument `regexp_like` flags path.
+        assert_eq!(
+            matched(&dataset, "regexp_like(s, 'RHINO', 'i')").await,
+            ["rhino", "rhino horn", "rhinos nose"]
+        );
+
+        // Infix LIKE is accelerated through the same machinery (a plain-literal
+        // `regexp_like` is rewritten to LIKE before it reaches the index).
+        assert_eq!(
+            matched(&dataset, "s LIKE '%rhino%'").await,
+            ["rhino", "rhino horn", "rhinos nose"]
+        );
+        // Prefix LIKE: recheck drops "scatter", which contains the `cat` trigram
+        // but does not start with "cat".
+        assert_eq!(
+            matched(&dataset, "s LIKE 'cat%'").await,
+            ["cat", "cat dog", "catalog", "category"]
+        );
+
+        // The ngram index is actually engaged for every accelerated form.
+        for filter in [
+            "regexp_like(s, 'rhino')",
+            "regexp_match(s, 'rhino')",
+            "s LIKE '%rhino%'",
+        ] {
+            let mut scan = dataset.scan();
+            scan.filter(filter).unwrap();
+            let plan = scan.create_plan().await.unwrap();
+            let plan_str = format!(
+                "{}",
+                datafusion::physical_plan::displayable(plan.as_ref()).indent(true)
+            );
+            assert!(
+                plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"),
+                "expected ngram index usage for `{filter}`, got plan:\n{plan_str}"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_ngram_regex_non_accelerable_recheck() {
+        // `a.b` yields no trigram, so the index returns "recheck everything".
+        // This must still produce ALL correct matches across fragments, not an
+        // empty set (a regression test for the AtLeast recheck path, which a
+        // single-match case would not catch).
+        let unit = ["acb", "dog", "axb", "cat", "qqq", "rhino"];
+        let values: Vec<&str> = unit.iter().copied().cycle().take(60).collect();
+        let array = StringArray::from_iter_values(values);
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "text",
+            DataType::Utf8,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let write_params = WriteParams {
+            max_rows_per_file: 20, // 60 rows -> 3 fragments
+            ..Default::default()
+        };
+        let mut dataset =
+            Dataset::write(reader, "memory://test_ngram_regex_ne", Some(write_params))
+                .await
+                .unwrap();
+        dataset
+            .create_index(
+                &["text"],
+                IndexType::NGram,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        async fn count(dataset: &Dataset, filter: &str) -> usize {
+            let mut scan = dataset.scan();
+            scan.filter(filter).unwrap();
+            let batches = scan
+                .try_into_stream()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+            batches.iter().map(|b| b.num_rows()).sum()
+        }
+
+        // "acb" and "axb" each appear 10 times in the 60 rows -> 20 matches.
+        assert_eq!(count(&dataset, "regexp_match(text, 'a.b')").await, 20);
+        assert_eq!(count(&dataset, "regexp_like(text, 'a.b')").await, 20);
+    }
+
     #[tokio::test]
     async fn test_like_prefix_with_btree_index() {
         // Create dataset with string data that has various prefixes
@@ -8783,48 +9563,320 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         );
     }
 
+    /// Regression for over-matching on the zone-map recheck path: a literal prefix
+    /// containing LIKE metacharacters (`_`, `%`) must be matched literally, not as
+    /// wildcards. The indexed (recheck) result must equal the unindexed ground truth.
     #[tokio::test]
-    async fn test_like_prefix_with_segmented_zone_map() {
+    async fn test_like_prefix_zone_map_escapes_metacharacters() {
         use lance_index::scalar::BuiltinIndexType;
 
-        let data = gen_batch()
-            .col(
-                "name",
-                array::cycle_utf8_literals(&[
-                    "apple",
-                    "application",
-                    "app",
-                    "banana",
-                    "band",
-                    "testns1",
-                    "testns2",
-                    "test",
-                    "testing",
-                    "zoo",
-                ]),
-            )
-            .col("id", array::step::<Int32Type>())
-            .into_reader_rows(RowCount::from(150), BatchCount::from(6));
-
-        let write_params = WriteParams {
-            max_rows_per_file: 25,
-            max_rows_per_group: 10,
-            ..Default::default()
-        };
-
-        let mut dataset = Dataset::write(
-            data,
-            "memory://test_like_segmented_zonemap",
-            Some(write_params),
+        let names: Vec<&str> = vec!["a_b", "a_c", "axb", "a1c", "b%c", "bxc", "bcc", "zoo"];
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("name", DataType::Utf8, false),
+            ArrowField::new("id", DataType::Int32, false),
+        ]));
+        let data = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(names.clone())),
+                Arc::new(Int32Array::from_iter_values(0..names.len() as i32)),
+            ],
         )
-        .await
         .unwrap();
-
-        let fragments = dataset.get_fragments();
-        assert!(fragments.len() > 1, "expected multiple fragments");
+        let reader = RecordBatchIterator::new(vec![Ok(data)], schema.clone());
+        let mut dataset = Dataset::write(reader, "memory://test_like_zonemap_escape", None)
+            .await
+            .unwrap();
 
         let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
-        let mut segments = Vec::with_capacity(fragments.len());
+        dataset
+            .create_index(
+                &["name"],
+                IndexType::Scalar,
+                Some("name_zonemap".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let collect_names = |batch: &RecordBatch| -> BTreeSet<String> {
+            batch
+                .column_by_name("name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap()
+                .iter()
+                .map(|s| s.unwrap().to_string())
+                .collect()
+        };
+
+        // Ensure the predicate actually exercises the zone-map LikePrefix recheck path.
+        let mut scanner = dataset.scan();
+        scanner.filter("starts_with(name, 'a_')").unwrap();
+        let plan_str = format!("{:?}", scanner.create_plan().await.unwrap());
+        assert!(
+            plan_str.contains("LikePrefix"),
+            "expected a zone-map LikePrefix plan, got: {plan_str}"
+        );
+
+        // `_` and `%` in the prefix are literal characters; the indexed result must match
+        // the unindexed evaluation for each predicate (no wildcard over-match).
+        for predicate in ["starts_with(name, 'a_')", "starts_with(name, 'b%')"] {
+            let with_index = dataset
+                .scan()
+                .filter(predicate)
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let without_index = dataset
+                .scan()
+                .use_scalar_index(false)
+                .filter(predicate)
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            assert_eq!(
+                collect_names(&with_index),
+                collect_names(&without_index),
+                "indexed result over-matched for predicate `{predicate}`"
+            );
+        }
+
+        // Explicit expectation so the intended (non-over-matching) result is obvious.
+        let result = dataset
+            .scan()
+            .filter("starts_with(name, 'a_')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(
+            collect_names(&result),
+            BTreeSet::from(["a_b".to_string(), "a_c".to_string()])
+        );
+    }
+
+    /// A bitmap index cannot answer prefix queries, so `LIKE 'prefix%'` / `starts_with`
+    /// must fall back to ordinary filtering (returning correct results) instead of being
+    /// planned as a `LikePrefix` index scan that bitmap search would reject.
+    #[tokio::test]
+    async fn test_like_prefix_bitmap_falls_back_to_filter() {
+        use lance_index::scalar::BuiltinIndexType;
+
+        let names: Vec<&str> = vec!["apple", "app", "application", "banana", "band", "zoo"];
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("name", DataType::Utf8, false),
+            ArrowField::new("id", DataType::Int32, false),
+        ]));
+        let data = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(names.clone())),
+                Arc::new(Int32Array::from_iter_values(0..names.len() as i32)),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(data)], schema.clone());
+        let mut dataset = Dataset::write(reader, "memory://test_like_bitmap_fallback", None)
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap);
+        dataset
+            .create_index(
+                &["name"],
+                IndexType::Scalar,
+                Some("name_bitmap".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // The predicate must not be planned as a LikePrefix index scan (bitmap rejects it).
+        let mut scanner = dataset.scan();
+        scanner.filter("name LIKE 'app%'").unwrap();
+        let plan_str = format!("{:?}", scanner.create_plan().await.unwrap());
+        assert!(
+            !plan_str.contains("LikePrefix"),
+            "bitmap LIKE must not use a LikePrefix index scan, got: {plan_str}"
+        );
+
+        let collect_names = |batch: &RecordBatch| -> BTreeSet<String> {
+            batch
+                .column_by_name("name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap()
+                .iter()
+                .map(|s| s.unwrap().to_string())
+                .collect()
+        };
+
+        // And it must execute successfully with correct results (previously errored).
+        let result = dataset
+            .scan()
+            .filter("name LIKE 'app%'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(
+            collect_names(&result),
+            BTreeSet::from([
+                "apple".to_string(),
+                "app".to_string(),
+                "application".to_string(),
+            ])
+        );
+
+        let result = dataset
+            .scan()
+            .filter("starts_with(name, 'ban')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert_eq!(
+            collect_names(&result),
+            BTreeSet::from(["banana".to_string(), "band".to_string()])
+        );
+    }
+
+    /// Build an in-memory dataset with a single `Dictionary(Int16, Utf8)` column.
+    /// The dictionary cycles through "a", "b", "c" so each value appears in a
+    /// predictable, repeated pattern.
+    async fn dictionary_string_dataset() -> Dataset {
+        use arrow_array::{Int16Array, Int16DictionaryArray};
+
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "etld",
+            DataType::Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8)),
+            false,
+        )]));
+
+        let dictionary = Arc::new(StringArray::from(vec!["a", "b", "c"]));
+        let indices = Int16Array::from((0..30).map(|i| i % 3).collect::<Vec<_>>());
+        let dict_array = Int16DictionaryArray::try_new(indices, dictionary).unwrap();
+
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(dict_array)]).unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        Dataset::write(reader, "memory://test_dict_filter", None)
+            .await
+            .unwrap()
+    }
+
+    /// Regression test for filtering a dictionary-encoded string column via the
+    /// SQL string path (`Scanner::filter`). This used to fail to plan with
+    /// "could not convert to literal of type 'Dictionary(Int16, Utf8)'".
+    #[tokio::test]
+    async fn test_filter_on_dictionary_string_column() {
+        let dataset = dictionary_string_dataset().await;
+
+        // Equality predicate.
+        let count = dataset
+            .scan()
+            .filter("etld = 'a'")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(count, 10);
+
+        // IN-list predicate.
+        let count = dataset
+            .scan()
+            .filter("etld IN ('a', 'b')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(count, 20);
+    }
+
+    /// An `IN`/`=` predicate on a dictionary column with a scalar index should be
+    /// pushed down to the index rather than falling back to a full scan.
+    #[tokio::test]
+    async fn test_dictionary_string_column_uses_scalar_index() {
+        use lance_index::scalar::BuiltinIndexType;
+
+        let mut dataset = dictionary_string_dataset().await;
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap);
+        dataset
+            .create_index(&["etld"], IndexType::Scalar, None, &params, true)
+            .await
+            .unwrap();
+
+        let mut scanner = dataset.scan();
+        scanner.filter("etld IN ('a', 'b')").unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let plan_str = format!("{:?}", plan);
+        assert!(
+            plan_str.contains("ScalarIndexExec") || plan_str.contains("MaterializeIndex"),
+            "IN on a dictionary column should use the scalar index, but got: {}",
+            plan_str
+        );
+
+        let count = dataset
+            .scan()
+            .filter("etld IN ('a', 'b')")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(count, 20);
+    }
+
+    #[tokio::test]
+    async fn test_like_prefix_with_segmented_zone_map() {
+        use lance_index::scalar::BuiltinIndexType;
+
+        let data = gen_batch()
+            .col(
+                "name",
+                array::cycle_utf8_literals(&[
+                    "apple",
+                    "application",
+                    "app",
+                    "banana",
+                    "band",
+                    "testns1",
+                    "testns2",
+                    "test",
+                    "testing",
+                    "zoo",
+                ]),
+            )
+            .col("id", array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(150), BatchCount::from(6));
+
+        let write_params = WriteParams {
+            max_rows_per_file: 25,
+            max_rows_per_group: 10,
+            ..Default::default()
+        };
+
+        let mut dataset = Dataset::write(
+            data,
+            "memory://test_like_segmented_zonemap",
+            Some(write_params),
+        )
+        .await
+        .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() > 1, "expected multiple fragments");
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        let mut segments = Vec::with_capacity(fragments.len());
         for fragment in &fragments {
             let mut builder = dataset.create_index_builder(&["name"], IndexType::Scalar, &params);
             builder = builder
@@ -9233,6 +10285,160 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_io_lt!(io_stats, read_bytes, index_scan_bytes);
     }
 
+    #[tokio::test]
+    async fn test_blob_all_binary_late_materialization() {
+        // A selective filter that projects a blob column with `blob_handling=all_binary`
+        // must late-materialize the blob (take only the matched rows) rather than eagerly
+        // reading the whole column. Blobs returned as descriptions stay eager (they are
+        // tiny), but full binary values should follow the width-based heuristic like any
+        // other wide column.
+        use lance_io::assert_io_lt;
+        use lance_table::io::commit::RenameCommitHandler;
+
+        // 8KB stays under the 64KB inline threshold, so the blob is a normal column in
+        // the data file rather than a dedicated blob file.
+        let blob_meta = std::collections::HashMap::from([(
+            "lance-encoding:blob".to_string(),
+            "true".to_string(),
+        )]);
+        let blobs = array::rand_fixedbin(ByteCount::from(8 * 1024), true).with_metadata(blob_meta);
+        let data = gen_batch()
+            .col("filterme", array::step::<UInt64Type>())
+            .col("blobs", blobs)
+            .into_reader_rows(RowCount::from(500), BatchCount::from(8));
+
+        let dataset = Dataset::write(
+            data,
+            "memory://test",
+            Some(WriteParams {
+                commit_handler: Some(Arc::new(RenameCommitHandler)),
+                data_storage_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Baseline: read the blob column as binary for the whole table.
+        let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset
+        dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .try_into_batch()
+            .await
+            .unwrap();
+        let full_scan_bytes = dataset
+            .object_store
+            .as_ref()
+            .io_stats_incremental()
+            .read_bytes;
+
+        // A filter matching a single row out of 4000 should read far less than the whole
+        // column: only the filter leaf plus the one materialized blob.
+        dataset
+            .scan()
+            .project(&["blobs"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .filter("filterme = 100")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store.as_ref().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+    }
+
+    #[tokio::test]
+    async fn test_nested_blob_all_binary_late_materialization() {
+        // Same as above, but the blob is a leaf *inside* a struct and the filter is on a
+        // sibling leaf. Materialization is decided per leaf (fields_pre_order), so the
+        // nested blob must late-materialize under `all_binary` just like a top-level one.
+        use lance_io::assert_io_lt;
+        use lance_table::io::commit::RenameCommitHandler;
+
+        let blob_meta = std::collections::HashMap::from([(
+            "lance-encoding:blob".to_string(),
+            "true".to_string(),
+        )]);
+        let a_field = ArrowField::new("a", DataType::Int32, false);
+        let blob_field =
+            ArrowField::new("blob", DataType::LargeBinary, false).with_metadata(blob_meta);
+        let struct_fields: Fields = vec![a_field, blob_field].into();
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Struct(struct_fields.clone()),
+            false,
+        )]));
+
+        let rows_per_batch = 500usize;
+        let batches: Vec<RecordBatch> = (0..8)
+            .map(|b| {
+                let base = (b * rows_per_batch) as i32;
+                let a = Arc::new(Int32Array::from_iter_values(
+                    base..base + rows_per_batch as i32,
+                ));
+                // Vary the payload per row so it does not collapse under compression.
+                let blobs: Vec<Vec<u8>> = (0..rows_per_batch)
+                    .map(|r| {
+                        let seed = (base as usize + r).wrapping_mul(2654435761);
+                        (0usize..8 * 1024)
+                            .map(|i| (i.wrapping_mul(31).wrapping_add(seed) & 0xff) as u8)
+                            .collect()
+                    })
+                    .collect();
+                let blob = Arc::new(arrow_array::LargeBinaryArray::from_iter_values(
+                    blobs.iter().map(|v| v.as_slice()),
+                ));
+                let s = StructArray::new(struct_fields.clone(), vec![a, blob as ArrayRef], None);
+                RecordBatch::try_new(schema.clone(), vec![Arc::new(s)]).unwrap()
+            })
+            .collect();
+
+        let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let dataset = Dataset::write(
+            reader,
+            "memory://test",
+            Some(WriteParams {
+                commit_handler: Some(Arc::new(RenameCommitHandler)),
+                data_storage_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset
+        dataset
+            .scan()
+            .project(&["s"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .try_into_batch()
+            .await
+            .unwrap();
+        let full_scan_bytes = dataset
+            .object_store
+            .as_ref()
+            .io_stats_incremental()
+            .read_bytes;
+
+        dataset
+            .scan()
+            .project(&["s"])
+            .unwrap()
+            .blob_handling(BlobHandling::AllBinary)
+            .filter("s.a = 100")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let io_stats = dataset.object_store.as_ref().io_stats_incremental();
+        assert_io_lt!(io_stats, read_bytes, full_scan_bytes);
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_project_nested(
@@ -9654,7 +10860,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       FilterExec: _distance@... IS NOT NULL
         SortExec: TopK(fetch=6), expr=...
           KNNVectorDistance: metric=l2
-            RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            CoalescePartitionsExec
               UnionExec
                 ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec]
                   FilterExec: _distance@... IS NOT NULL
@@ -9686,7 +10892,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             FilterExec: _distance@... IS NOT NULL
               SortExec: TopK(fetch=15), expr=...
                 KNNVectorDistance: metric=l2
-                  RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+                  CoalescePartitionsExec
                     UnionExec
                       ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec]
                         FilterExec: _distance@... IS NOT NULL
@@ -9714,7 +10920,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       FilterExec: _distance@... IS NOT NULL
         SortExec: TopK(fetch=5), expr=...
           KNNVectorDistance: metric=l2
-            RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            CoalescePartitionsExec
               UnionExec
                 ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
                   FilterExec: _distance@... IS NOT NULL
@@ -9736,7 +10942,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       FilterExec: _distance@... IS NOT NULL
         SortExec: TopK(fetch=5), expr=...
           KNNVectorDistance: metric=l2
-            RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            CoalescePartitionsExec
               UnionExec
                 ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
                   FilterExec: _distance@... IS NOT NULL
@@ -9835,7 +11041,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       FilterExec: _distance@... IS NOT NULL
         SortExec: TopK(fetch=8), expr=...
           KNNVectorDistance: metric=l2
-            RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            CoalescePartitionsExec
               UnionExec
                 ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
                   FilterExec: _distance@... IS NOT NULL
@@ -9871,7 +11077,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
       FilterExec: _distance@... IS NOT NULL
         SortExec: TopK(fetch=11), expr=...
           KNNVectorDistance: metric=l2
-            RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            CoalescePartitionsExec
               UnionExec
                 ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec]
                   FilterExec: _distance@... IS NOT NULL
@@ -9964,14 +11170,13 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         log::info!("Test case: Combined Scalar/non-scalar filtered read");
         let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[s@1 as s]
-  RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
-    UnionExec
-      Take: columns=\"_rowid, (s)\"
-        CoalesceBatchesExec: target_batch_size=8192
-          MaterializeIndex: query=[i > 10]@i_idx(BTree)
-      ProjectionExec: expr=[_rowid@2 as _rowid, s@1 as s]
-        FilterExec: i@0 > 10
-          LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"
+  UnionExec
+    Take: columns=\"_rowid, (s)\"
+      CoalesceBatchesExec: target_batch_size=8192
+        MaterializeIndex: query=[i > 10]@i_idx(BTree)
+    ProjectionExec: expr=[_rowid@2 as _rowid, s@1 as s]
+      FilterExec: i@0 > 10
+        LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "LanceRead: uri=..., projection=[s], num_fragments=5, range_before=None, \
             range_after=None, row_id=false, row_addr=false, full_filter=i > Int32(10), refine_filter=--
@@ -9987,13 +11192,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         log::info!("Test case: Combined Scalar/non-scalar filtered read with empty projection");
         let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[_rowaddr@0 as _rowaddr]
-  RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
-    UnionExec
-      AddRowAddrExec
-        MaterializeIndex: query=[i > 10]@i_idx(BTree)
-      ProjectionExec: expr=[_rowaddr@2 as _rowaddr, _rowid@1 as _rowid]
-        FilterExec: i@0 > 10
-          LanceScan: uri=..., projection=[i], row_id=true, row_addr=true, ordered=false, range=None"
+  UnionExec
+    AddRowAddrExec
+      MaterializeIndex: query=[i > 10]@i_idx(BTree)
+    ProjectionExec: expr=[_rowaddr@2 as _rowaddr, _rowid@1 as _rowid]
+      FilterExec: i@0 > 10
+        LanceScan: uri=..., projection=[i], row_id=true, row_addr=true, ordered=false, range=None"
         } else {
             "LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, \
             range_after=None, row_id=false, row_addr=true, full_filter=i > Int32(10), refine_filter=--
@@ -10016,14 +11220,13 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         log::info!("Test case: Dynamic projection");
         let expected = if data_storage_version == LanceFileVersion::Legacy {
             "ProjectionExec: expr=[regexp_match(s@1, .*) as matches]
-  RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
-    UnionExec
-      Take: columns=\"_rowid, (s)\"
-        CoalesceBatchesExec: target_batch_size=8192
-          MaterializeIndex: query=[i > 10]@i_idx(BTree)
-      ProjectionExec: expr=[_rowid@2 as _rowid, s@1 as s]
-        FilterExec: i@0 > 10
-          LanceScan: uri=..., row_id=true, row_addr=false, ordered=false, range=None"
+  UnionExec
+    Take: columns=\"_rowid, (s)\"
+      CoalesceBatchesExec: target_batch_size=8192
+        MaterializeIndex: query=[i > 10]@i_idx(BTree)
+    ProjectionExec: expr=[_rowid@2 as _rowid, s@1 as s]
+      FilterExec: i@0 > 10
+        LanceScan: uri=..., row_id=true, row_addr=false, ordered=false, range=None"
         } else {
             "ProjectionExec: expr=[regexp_match(s@0, .*) as matches]
   LanceRead: uri=..., projection=[s], num_fragments=5, range_before=None, \
@@ -10048,7 +11251,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello"#;
+      MatchQuery: column=s, query=[hello]"#;
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10082,8 +11285,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
       BoostQuery: negative_boost=1
-        MatchQuery: column=s, query=hello
-        MatchQuery: column=s, query=world"#;
+        MatchQuery: column=s, query=[hello]
+        MatchQuery: column=s, query=[world]"#;
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10105,8 +11308,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello
-        RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+      MatchQuery: column=s, query=[hello]
+        CoalescePartitionsExec
           UnionExec
             MaterializeIndex: query=[i > 10]@i_idx(BTree)
             ProjectionExec: expr=[_rowid@1 as _rowid]
@@ -10116,7 +11319,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello
+      MatchQuery: column=s, query=[hello]
         LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
           ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
         };
@@ -10134,15 +11337,31 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: Full text search with unindexed rows");
-        let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+        // The flat-FTS path now reads through `FilteredReadExec`, matching the
+        // brute-force KNN path. With no prefilter the scan still produces no
+        // pushdown, but the operator differs by storage version: legacy emits
+        // a `LanceScan`, v2 emits a `LanceRead` with empty filters.
+        let expected = if data_storage_version == LanceFileVersion::Legacy {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
       SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
-        RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+        CoalescePartitionsExec
           UnionExec
-            MatchQuery: column=s, query=hello
+            MatchQuery: column=s, query=[hello]
             FlatMatchQuery: column=s, query=hello
-              LanceScan: uri=..., projection=[s], row_id=true, row_addr=false, ordered=false, range=None"#;
+              LanceScan: uri=..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None"#
+        } else {
+            r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
+  Take: columns="_rowid, _score, (s)"
+    CoalesceBatchesExec: target_batch_size=8192
+      SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
+        CoalescePartitionsExec
+          UnionExec
+            MatchQuery: column=s, query=[hello]
+            FlatMatchQuery: column=s, query=hello
+              LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"#
+        };
         dataset.append_new_data().await?;
         assert_plan_equals(
             &dataset.dataset,
@@ -10159,7 +11378,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
-      MatchQuery: column=s, query=hello"#;
+      MatchQuery: column=s, query=[hello]"#;
         assert_plan_equals(
             &dataset.dataset,
             |scan| {
@@ -10175,36 +11394,46 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         .await?;
 
         log::info!("Test case: Full text search with unindexed rows and prefilter");
+        // After routing flat FTS through `FilteredReadExec`, the BTree on `i`
+        // pushes into the unindexed-fragment scan too — no more `FilterExec` on
+        // top of an unfiltered `LanceScan`. Legacy uses the `MaterializeIndex`
+        // shape, v2 uses `LanceRead` with `full_filter` set.
         let expected = if data_storage_version == LanceFileVersion::Legacy {
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
       SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
-        RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+        CoalescePartitionsExec
           UnionExec
-            MatchQuery: column=s, query=hello
-              RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+            MatchQuery: column=s, query=[hello]
+              CoalescePartitionsExec
                 UnionExec
                   MaterializeIndex: query=[i > 10]@i_idx(BTree)
                   ProjectionExec: expr=[_rowid@1 as _rowid]
                     FilterExec: i@0 > 10
                       LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None
             FlatMatchQuery: column=s, query=hello
-              FilterExec: i@1 > 10
-                LanceScan: uri=..., projection=[s, i], row_id=true, row_addr=false, ordered=false, range=None"#
+              CoalescePartitionsExec
+                UnionExec
+                  Take: columns="_rowid, (s)"
+                    CoalesceBatchesExec: target_batch_size=8192
+                      MaterializeIndex: query=[i > 10]@i_idx(BTree)
+                  ProjectionExec: expr=[_rowid@2 as _rowid, s@1 as s]
+                    FilterExec: i@0 > 10
+                      LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"#
         } else {
             r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid]
   Take: columns="_rowid, _score, (s)"
     CoalesceBatchesExec: target_batch_size=8192
       SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false]
-        RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+        CoalescePartitionsExec
           UnionExec
-            MatchQuery: column=s, query=hello
+            MatchQuery: column=s, query=[hello]
               LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
                 ScalarIndexQuery: query=[i > 10]@i_idx(BTree)
             FlatMatchQuery: column=s, query=hello
-              FilterExec: i@1 > 10
-                LanceScan: uri=..., projection=[s, i], row_id=true, row_addr=false, ordered=false, range=None"#
+              LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=--
+                ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"#
         };
         assert_plan_equals(
             &dataset.dataset,
@@ -10274,7 +11503,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
   FilterExec: _distance@2 IS NOT NULL
     SortExec: TopK(fetch=34), expr=[_distance@2 ASC NULLS LAST, _rowid@0 ASC NULLS LAST]...
       KNNVectorDistance: metric=l2
-        RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=2
+        CoalescePartitionsExec
           UnionExec
             ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec]
               FilterExec: _distance@2 IS NOT NULL
@@ -10327,6 +11556,41 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_query_index_field(&batch);
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_fts_multiple_unindexed_appends() {
+        // An FTS query over an indexed dataset plus more than one unindexed append
+        // must return matches from every unindexed fragment, not just the first. The
+        // flat search over unindexed data reads only a single input partition, so when
+        // the default parallelism splits the scan across partitions it used to drop
+        // every append but the first.
+        let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        test_ds.make_fts_index().await.unwrap();
+        // Two separate appends -> two unindexed fragments, each large enough that the
+        // physical optimizer parallelizes the flat scan across partitions.
+        test_ds.append_data_with_range(400, 5400).await.unwrap();
+        test_ds.append_data_with_range(5400, 10400).await.unwrap();
+
+        // Every row's `s` value contains the token "s", so FTS("s") matches all rows.
+        let total = test_ds.dataset.count_rows(None).await.unwrap();
+        let returned = test_ds
+            .dataset
+            .scan()
+            .full_text_search(FullTextSearchQuery::new("s".to_owned()))
+            .unwrap()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_fold(
+                0usize,
+                |acc, batch| async move { Ok(acc + batch.num_rows()) },
+            )
+            .await
+            .unwrap();
+        assert_eq!(returned, total);
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_fast_search_scalar_index_skips_unindexed_fragments(
@@ -10807,6 +12071,26 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_eq!(scanner.nearest_mut().unwrap().query_parallelism, -1);
     }
 
+    #[tokio::test]
+    async fn test_knn_approx_mode_defaults_and_setter() {
+        let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
+            .await
+            .unwrap();
+        let query_vector = Float32Array::from(vec![0.0; 32]);
+        let mut scanner = test_ds.dataset.scan();
+        scanner.nearest("vec", &query_vector, 5).unwrap();
+        assert_eq!(
+            scanner.nearest_mut().unwrap().approx_mode,
+            ApproxMode::Normal
+        );
+
+        scanner.approx_mode(ApproxMode::Accurate);
+        assert_eq!(
+            scanner.nearest_mut().unwrap().approx_mode,
+            ApproxMode::Accurate
+        );
+    }
+
     #[tokio::test]
     async fn test_ivf_pq_query_parallelism_returns_same_results() {
         let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false)
@@ -11410,7 +12694,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
     }
 
     fn find_filtered_read(plan: &dyn ExecutionPlan) -> Option<&FilteredReadExec> {
-        if let Some(f) = plan.as_any().downcast_ref::<FilteredReadExec>() {
+        if let Some(f) = plan.downcast_ref::<FilteredReadExec>() {
             return Some(f);
         }
         for child in plan.children() {
@@ -11447,6 +12731,47 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\")
         assert_eq!(filtered.options().io_buffer_size_bytes, Some(7777));
     }
 
+    #[tokio::test]
+    async fn test_batch_readahead_bounds_decode_concurrency() {
+        let data = lance_datagen::gen_batch()
+            .col("x", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(RowCount::from(8), BatchCount::from(1));
+        let dataset = Dataset::write(data, "memory://test_batch_readahead_concurrency", None)
+            .await
+            .unwrap();
+
+        // Default: threading mode falls back to get_num_compute_intensive_cpus().
+        let plan = dataset.scan().create_plan().await.unwrap();
+        let filtered = find_filtered_read(plan.as_ref())
+            .expect("expected a FilteredReadExec in the scan plan");
+        assert_eq!(
+            filtered.options().threading_mode,
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(get_num_compute_intensive_cpus()),
+        );
+
+        // Explicit batch_readahead(N) bounds the decode fan-out to N.
+        let mut scanner = dataset.scan();
+        scanner.batch_readahead(3);
+        let plan = scanner.create_plan().await.unwrap();
+        let filtered = find_filtered_read(plan.as_ref())
+            .expect("expected a FilteredReadExec in the scan plan");
+        assert_eq!(
+            filtered.options().threading_mode,
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(3),
+        );
+
+        let mut scanner = dataset.scan();
+        scanner.batch_readahead(0);
+        let Err(Error::InvalidInput { source, .. }) = scanner.create_plan().await else {
+            panic!("expected batch_readahead=0 to be rejected");
+        };
+        assert!(
+            source
+                .to_string()
+                .contains("batch_readahead must be greater than 0")
+        );
+    }
+
     // The env var key scopes serial_test's lock so this test only blocks others
     // that touch LANCE_DEFAULT_IO_BUFFER_SIZE — unrelated tests still run in
     // parallel.
diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs
index f5d792979df..40eac95e919 100644
--- a/rust/lance/src/dataset/schema_evolution.rs
+++ b/rust/lance/src/dataset/schema_evolution.rs
@@ -1,13 +1,18 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::{collections::HashSet, sync::Arc};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
 
 use super::fragment::FileFragment;
 use super::{
     Dataset,
     transaction::{Operation, Transaction},
+    write::cleanup_data_fragments,
 };
+use crate::index::DatasetIndexExt;
 use crate::{Error, Result, io::exec::Planner};
 use arrow::compute::CastOptions;
 use arrow::compute::can_cast_types;
@@ -145,24 +150,34 @@ impl ColumnAlteration {
 
 /// Limit casts to same type. This is mostly to filter out weird casts like
 /// casting a string to a boolean or float to string.
-fn is_upcast_downcast(from_type: &DataType, to_type: &DataType) -> bool {
+fn is_upcast_downcast(from_type: &DataType, to_type: &DataType, version: LanceFileVersion) -> bool {
     use DataType::*;
-    match from_type {
-        from_type if from_type.is_integer() => to_type.is_integer(),
-        from_type if from_type.is_floating() => to_type.is_floating(),
-        from_type if from_type.is_temporal() => to_type.is_temporal(),
-        Boolean => matches!(to_type, Boolean),
-        Utf8 | LargeUtf8 => matches!(to_type, Utf8 | LargeUtf8),
-        Binary | LargeBinary => matches!(to_type, Binary | LargeBinary),
-        Decimal128(_, _) | Decimal256(_, _) => {
-            matches!(to_type, Decimal128(_, _) | Decimal256(_, _))
+    match (from_type, to_type) {
+        // Legacy storage cannot materialize a fresh Dictionary column via
+        // alter because the writer expects `field.dictionary` metadata to be
+        // pre-populated, which the alter pipeline does not compute.
+        (_, Dictionary(_, _)) if matches!(version, LanceFileVersion::Legacy) => false,
+        // These need to be in front
+        (Dictionary(_, from_value_type), _) => {
+            is_upcast_downcast(from_value_type, to_type, version)
+        }
+        (_, Dictionary(_, to_value_type)) => is_upcast_downcast(from_type, to_value_type, version),
+        (from, to) if from.is_integer() => to.is_integer(),
+        (from, to) if from.is_floating() => to.is_floating(),
+        (from, to) if from.is_temporal() => to.is_temporal(),
+        (Boolean, to) => matches!(to, Boolean),
+        (Utf8 | LargeUtf8, to) => matches!(to, Utf8 | LargeUtf8),
+        (Binary | LargeBinary, to) => matches!(to, Binary | LargeBinary),
+        (Decimal128(_, _) | Decimal256(_, _), to) => {
+            matches!(to, Decimal128(_, _) | Decimal256(_, _))
         }
-        List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _) => match to_type {
+        (List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _), to) => match to {
             List(to_field) | LargeList(to_field) | FixedSizeList(to_field, _) => {
-                is_upcast_downcast(from_field.data_type(), to_field.data_type())
+                is_upcast_downcast(from_field.data_type(), to_field.data_type(), version)
             }
             _ => false,
         },
+
         _ => false,
     }
 }
@@ -239,7 +254,7 @@ pub(super) async fn add_columns_to_fragments(
     read_columns: Option<Vec<String>>,
     fragments: &[FileFragment],
     batch_size: Option<u32>,
-) -> Result<(Vec<Fragment>, Schema)> {
+) -> Result<(Vec<Fragment>, Schema, Vec<Fragment>)> {
     // Check names early (before calling add_columns_impl) to avoid extra work if
     // the names are wrong.
     let version = dataset.manifest.data_storage_format.lance_file_version()?;
@@ -261,10 +276,10 @@ pub(super) async fn add_columns_to_fragments(
     }
     let transforms = optimizer.optimize(dataset, transforms)?;
 
-    let (output_schema, fragments) = match transforms {
+    let (output_schema, new_fragments, fragments_to_cleanup) = match transforms {
         NewColumnTransform::BatchUDF(udf) => {
             check_names(udf.output_schema.as_ref())?;
-            let fragments = add_columns_impl(
+            let result = add_columns_impl(
                 fragments,
                 read_columns,
                 udf.mapper,
@@ -273,7 +288,11 @@ pub(super) async fn add_columns_to_fragments(
                 None,
             )
             .await?;
-            Result::Ok((udf.output_schema, fragments))
+            Result::Ok((
+                udf.output_schema,
+                result.fragments,
+                result.fragments_to_cleanup,
+            ))
         }
         NewColumnTransform::SqlExpressions(expressions) => {
             // We just transform the SQL expression into a UDF backed by DataFusion
@@ -336,31 +355,35 @@ pub(super) async fn add_columns_to_fragments(
             let mapper = Box::new(mapper);
 
             let read_columns = Some(read_schema.field_names().into_iter().cloned().collect());
-            let fragments =
+            let result =
                 add_columns_impl(fragments, read_columns, mapper, batch_size, None, None).await?;
-            Ok((output_schema, fragments))
+            Ok((output_schema, result.fragments, result.fragments_to_cleanup))
         }
         NewColumnTransform::Stream(stream) => {
             let output_schema = stream.schema();
             check_names(output_schema.as_ref())?;
             let fragments = add_columns_from_stream(fragments, stream, None, batch_size).await?;
-            Ok((output_schema, fragments))
+            Ok((output_schema, fragments.clone(), fragments))
         }
         NewColumnTransform::Reader(reader) => {
             let output_schema = reader.schema();
             check_names(output_schema.as_ref())?;
             let stream = reader.into_stream();
             let fragments = add_columns_from_stream(fragments, stream, None, batch_size).await?;
-            Ok((output_schema, fragments))
+            Ok((output_schema, fragments.clone(), fragments))
         }
         NewColumnTransform::AllNulls(output_schema) => {
             check_names(output_schema.as_ref())?;
 
-            // Check that the schema is compatible considering all the new columns must be nullable
-            let schema = Schema::try_from(output_schema.as_ref())?;
-            if !schema.all_fields_nullable() {
+            // AllNulls is metadata-only; missing columns are synthesized as nulls at
+            // read time, so only each new top-level column needs to be nullable.
+            if let Some(field) = output_schema.fields().iter().find(|f| !f.is_nullable()) {
                 return Err(Error::invalid_input_source(
-                    "All-null columns must be nullable.".into(),
+                    format!(
+                        "All-null columns must be nullable, but field '{}' is not.",
+                        field.name()
+                    )
+                    .into(),
                 ));
             }
 
@@ -379,14 +402,20 @@ pub(super) async fn add_columns_to_fragments(
                 ));
             }
 
-            Ok((output_schema, fragments))
+            Ok((output_schema, fragments, Vec::new()))
         }
     }?;
 
-    let mut schema = dataset.schema().merge(output_schema.as_ref())?;
+    let mut schema = match dataset.schema().merge(output_schema.as_ref()) {
+        Ok(schema) => schema,
+        Err(e) => {
+            cleanup_new_column_data_files(fragments, &fragments_to_cleanup).await;
+            return Err(e);
+        }
+    };
     schema.set_field_id(Some(dataset.manifest.max_field_id()));
 
-    Ok((fragments, schema))
+    Ok((new_fragments, schema, fragments_to_cleanup))
 }
 
 pub(super) async fn add_columns(
@@ -395,7 +424,7 @@ pub(super) async fn add_columns(
     read_columns: Option<Vec<String>>,
     batch_size: Option<u32>,
 ) -> Result<()> {
-    let (fragments, schema) = add_columns_to_fragments(
+    let (fragments, schema, fragments_to_cleanup) = add_columns_to_fragments(
         dataset,
         transforms,
         read_columns,
@@ -406,11 +435,76 @@ pub(super) async fn add_columns(
 
     let operation = Operation::Merge { fragments, schema };
     let transaction = Transaction::new(dataset.manifest.version, operation, None);
-    dataset
+    match dataset
         .apply_commit(transaction, &Default::default(), &Default::default())
-        .await?;
+        .await
+    {
+        Ok(()) => Ok(()),
+        Err(e) => {
+            cleanup_new_column_data_files(&dataset.get_fragments(), &fragments_to_cleanup).await;
+            Err(e)
+        }
+    }
+}
 
-    Ok(())
+async fn cleanup_new_column_data_files(fragments: &[FileFragment], new_fragments: &[Fragment]) {
+    let Some(first_fragment) = fragments.first() else {
+        return;
+    };
+
+    // add_columns rewrites fragment metadata in place, so cleanup must delete
+    // only files created by the current attempt and must not touch pre-existing
+    // files that still belong to the fragment.
+    let original_files_by_fragment = fragments
+        .iter()
+        .map(|fragment| {
+            let files = fragment
+                .metadata
+                .files
+                .iter()
+                .map(|file| (file.base_id, file.path.clone()))
+                .collect::<HashSet<_>>();
+            (fragment.id() as u64, files)
+        })
+        .collect::<HashMap<_, _>>();
+
+    let fragments_to_cleanup = new_fragments
+        .iter()
+        .filter_map(|fragment| {
+            let original_files = original_files_by_fragment.get(&fragment.id)?;
+            let files = fragment
+                .files
+                .iter()
+                .filter(|file| !original_files.contains(&(file.base_id, file.path.clone())))
+                .cloned()
+                .collect::<Vec<_>>();
+
+            if files.is_empty() {
+                None
+            } else {
+                let mut fragment = fragment.clone();
+                fragment.files = files;
+                Some(fragment)
+            }
+        })
+        .collect::<Vec<_>>();
+
+    cleanup_data_fragments(
+        &first_fragment.dataset().object_store,
+        &first_fragment.dataset().base,
+        None,
+        &fragments_to_cleanup,
+    )
+    .await;
+}
+
+struct AddColumnFragments {
+    /// Fragments produced by the add-columns operation and returned to the
+    /// caller for the final merge commit.
+    fragments: Vec<Fragment>,
+    /// Uncommitted fragments whose newly written data files must be removed if
+    /// the operation fails before the merge commit completes.
+    fragments_to_cleanup: Vec<Fragment>,
 }
 
 #[allow(clippy::type_complexity)]
@@ -421,63 +515,96 @@ async fn add_columns_impl(
     batch_size: Option<u32>,
     result_cache: Option<Arc<dyn UDFCheckpointStore>>,
     schemas: Option<(Schema, Schema)>,
-) -> Result<Vec<Fragment>> {
+) -> Result<AddColumnFragments> {
     let read_columns_ref = read_columns.as_deref();
     let mapper_ref = mapper.as_ref();
-    let fragments = futures::stream::iter(fragments)
-        .then(|fragment| {
-            let cache_ref = result_cache.clone();
-            let schemas_ref = &schemas;
-            async move {
-                if let Some(cache) = &cache_ref {
-                    let fragment_id = fragment.id() as u32;
-                    let fragment = cache.get_fragment(fragment_id)?;
-                    if let Some(fragment) = fragment {
-                        return Ok(fragment);
-                    }
+
+    let mut new_fragments = Vec::with_capacity(fragments.len());
+    let mut fragments_to_cleanup = Vec::with_capacity(fragments.len());
+
+    for fragment in fragments {
+        if let Some(cache) = &result_cache {
+            let fragment_id = fragment.id() as u32;
+            let fragment = match cache.get_fragment(fragment_id) {
+                Ok(fragment) => fragment,
+                Err(e) => {
+                    cleanup_new_column_data_files(fragments, &fragments_to_cleanup).await;
+                    return Err(e);
                 }
+            };
+            if let Some(fragment) = fragment {
+                new_fragments.push(fragment);
+                continue;
+            }
+        }
 
-                let mut updater = fragment
-                    .updater(read_columns_ref, schemas_ref.clone(), batch_size)
-                    .await?;
-
-                let mut batch_index = 0;
-                // TODO: the structure of the updater prevents batch-level parallelism here,
-                //       but there is no reason why we couldn't do this in parallel.
-                while let Some(batch) = updater.next().await? {
-                    let batch_info = BatchInfo {
-                        fragment_id: fragment.id() as u32,
-                        batch_index,
-                    };
+        let mut updater = match fragment
+            .updater(read_columns_ref, schemas.clone(), batch_size)
+            .await
+        {
+            Ok(updater) => updater,
+            Err(e) => {
+                cleanup_new_column_data_files(fragments, &fragments_to_cleanup).await;
+                return Err(e);
+            }
+        };
+        let fragment_result = async {
+            let mut batch_index = 0;
+            // TODO: the structure of the updater prevents batch-level parallelism here,
+            //       but there is no reason why we couldn't do this in parallel.
+            while let Some(batch) = updater.next().await? {
+                let batch_info = BatchInfo {
+                    fragment_id: fragment.id() as u32,
+                    batch_index,
+                };
 
-                    let new_batch = if let Some(cache) = &cache_ref {
-                        if let Some(batch) = cache.get_batch(&batch_info)? {
-                            batch
-                        } else {
-                            let new_batch = mapper_ref(batch)?;
-                            cache.insert_batch(batch_info, new_batch.clone())?;
-                            new_batch
-                        }
+                let new_batch = if let Some(cache) = &result_cache {
+                    if let Some(batch) = cache.get_batch(&batch_info)? {
+                        batch
                     } else {
-                        mapper_ref(batch)?
-                    };
+                        let new_batch = mapper_ref(batch)?;
+                        cache.insert_batch(batch_info, new_batch.clone())?;
+                        new_batch
+                    }
+                } else {
+                    mapper_ref(batch)?
+                };
 
-                    updater.update(new_batch).await?;
-                    batch_index += 1;
-                }
+                updater.update(new_batch).await?;
+                batch_index += 1;
+            }
 
-                let fragment = updater.finish().await?;
+            let new_fragment = updater.finish().await?;
+            fragments_to_cleanup.push(new_fragment.clone());
 
-                if let Some(cache) = &cache_ref {
-                    cache.insert_fragment(fragment.clone())?;
-                }
+            if let Some(cache) = &result_cache {
+                // Once the checkpoint store owns this fragment, retries may load
+                // it back instead of rewriting it. Removing it from the cleanup
+                // set avoids deleting data that has already been checkpointed.
+                cache.insert_fragment(new_fragment.clone())?;
+                fragments_to_cleanup.pop();
+            }
+
+            Ok::<_, Error>(new_fragment)
+        }
+        .await;
 
-                Ok::<_, Error>(fragment)
+        match fragment_result {
+            Ok(new_fragment) => {
+                new_fragments.push(new_fragment);
             }
-        })
-        .try_collect::<Vec<_>>()
-        .await?;
-    Ok(fragments)
+            Err(e) => {
+                updater.cleanup_unfinished_writer().await;
+                cleanup_new_column_data_files(fragments, &fragments_to_cleanup).await;
+                return Err(e);
+            }
+        }
+    }
+
+    Ok(AddColumnFragments {
+        fragments: new_fragments,
+        fragments_to_cleanup,
+    })
 }
 
 async fn add_columns_from_stream(
@@ -489,49 +616,80 @@ async fn add_columns_from_stream(
     let mut new_fragments = Vec::with_capacity(fragments.len());
     let mut last_seen_batch: Option<RecordBatch> = None;
     for fragment in fragments {
-        let mut updater = fragment
+        let mut updater = match fragment
             .updater::<String>(Some(&[]), schemas.clone(), batch_size)
-            .await?;
-        while let Some(batch) = updater.next().await? {
-            debug_assert_eq!(batch.num_columns(), 1);
-            let mut rows_remaining = batch.num_rows();
+            .await
+        {
+            Ok(updater) => updater,
+            Err(e) => {
+                cleanup_new_column_data_files(fragments, &new_fragments).await;
+                return Err(e);
+            }
+        };
+        let result: Result<Fragment> = async {
+            while let Some(batch) = updater.next().await? {
+                debug_assert_eq!(batch.num_columns(), 1);
+                let mut rows_remaining = batch.num_rows();
+
+                // The updater yields an empty batch when every row in a read batch
+                // has been deleted (e.g. a whole batch falls within the deletion
+                // vector). There is nothing to pull from the stream in that case, so
+                // feed an empty batch back to keep the updater in sync and continue.
+                if rows_remaining == 0 {
+                    updater
+                        .update(RecordBatch::new_empty(stream.schema()))
+                        .await?;
+                    continue;
+                }
 
-            let mut batches = Vec::new();
+                let mut batches = Vec::new();
 
-            while rows_remaining > 0 {
-                let next_batch = if let Some(last_seen_batch) = last_seen_batch {
-                    last_seen_batch
-                } else {
-                    stream.next().await.ok_or_else(|| {
-                        Error::invalid_input(
-                            "Stream ended before producing values for all rows in dataset",
-                        )
-                    })??
-                };
-                let num_rows = next_batch.num_rows();
-                if num_rows > rows_remaining {
-                    let new_batch = next_batch.slice(0, rows_remaining);
-                    batches.push(new_batch);
-                    last_seen_batch =
-                        Some(next_batch.slice(rows_remaining, num_rows - rows_remaining));
-                    rows_remaining = 0;
-                } else {
-                    batches.push(next_batch);
-                    rows_remaining -= num_rows;
-                    last_seen_batch = None;
+                while rows_remaining > 0 {
+                    let next_batch = if let Some(last_seen) = last_seen_batch.take() {
+                        last_seen
+                    } else {
+                        stream.next().await.ok_or_else(|| {
+                            Error::invalid_input(
+                                "Stream ended before producing values for all rows in dataset",
+                            )
+                        })??
+                    };
+                    let num_rows = next_batch.num_rows();
+                    if num_rows > rows_remaining {
+                        let new_batch = next_batch.slice(0, rows_remaining);
+                        batches.push(new_batch);
+                        last_seen_batch =
+                            Some(next_batch.slice(rows_remaining, num_rows - rows_remaining));
+                        rows_remaining = 0;
+                    } else {
+                        batches.push(next_batch);
+                        rows_remaining -= num_rows;
+                        last_seen_batch = None;
+                    }
                 }
-            }
 
-            let new_batch =
-                arrow_select::concat::concat_batches(&batches[0].schema(), batches.iter())?;
+                let new_batch =
+                    arrow_select::concat::concat_batches(&batches[0].schema(), batches.iter())?;
 
-            updater.update(new_batch).await?;
+                updater.update(new_batch).await?;
+            }
+            updater.finish().await
+        }
+        .await;
+
+        match result {
+            Ok(new_fragment) => new_fragments.push(new_fragment),
+            Err(e) => {
+                updater.cleanup_unfinished_writer().await;
+                cleanup_new_column_data_files(fragments, &new_fragments).await;
+                return Err(e);
+            }
         }
-        new_fragments.push(updater.finish().await?);
     }
 
     // Ensure the stream is fully consumed
     if last_seen_batch.is_some() || stream.next().await.is_some() {
+        cleanup_new_column_data_files(fragments, &new_fragments).await;
         return Err(Error::invalid_input_source(
             "Stream produced more values than expected for dataset".into(),
         ));
@@ -555,6 +713,7 @@ pub(super) async fn alter_columns(
     let mut cast_fields: Vec<(Field, Field)> = Vec::new();
 
     let mut next_field_id = dataset.manifest.max_field_id() + 1;
+    let version = dataset.manifest.data_storage_format.lance_file_version()?;
 
     for alteration in alterations {
         let field_src = dataset.schema().field(&alteration.path).ok_or_else(|| {
@@ -581,7 +740,7 @@ pub(super) async fn alter_columns(
 
         if let Some(data_type) = &alteration.data_type {
             if !(can_cast_types(&field_src.data_type(), data_type)
-                && is_upcast_downcast(&field_src.data_type(), data_type))
+                && is_upcast_downcast(&field_src.data_type(), data_type, version))
             {
                 return Err(Error::invalid_input(format!(
                     "Cannot cast column \"{}\" from {:?} to {:?}",
@@ -605,6 +764,41 @@ pub(super) async fn alter_columns(
 
     new_schema.validate()?;
 
+    // If any column being cast has an attached index, fail fast. Cast operations
+    // rewrite the underlying column data and silently invalidate any index on the
+    // affected column(s). The current behavior is to drop such indices without
+    // warning, which has caused production incidents where vector search silently
+    // regressed to brute-force scan. We require users to explicitly drop the
+    // index before altering the column type, so the action is never silent.
+    if !cast_fields.is_empty() {
+        let indices = dataset.load_indices().await?;
+        let affected: Vec<&lance_table::format::IndexMetadata> = indices
+            .iter()
+            .filter(|idx| {
+                cast_fields
+                    .iter()
+                    .any(|(old, _)| idx.fields.contains(&old.id))
+            })
+            .collect();
+        if !affected.is_empty() {
+            let affected_cols: Vec<String> = cast_fields
+                .iter()
+                .filter(|(old, _)| affected.iter().any(|i| i.fields.contains(&old.id)))
+                .map(|(old, _)| old.name.clone())
+                .collect();
+            let affected_idx_names: Vec<String> = affected.iter().map(|i| i.name.clone()).collect();
+            return Err(Error::invalid_input(format!(
+                "Cannot cast column(s) [{}] to a new type: they have {} index(es) \
+                 attached: [{}]. Cast rewrites column data and invalidates any index \
+                 on the affected column(s). Drop the index(es) with drop_index() \
+                 before altering, then recreate them after the cast completes.",
+                affected_cols.join(", "),
+                affected.len(),
+                affected_idx_names.join(", "),
+            )));
+        }
+    }
+
     // If we aren't casting a column, we don't need to touch the fragments.
     let transaction = if cast_fields.is_empty() {
         Transaction::new(
@@ -653,7 +847,7 @@ pub(super) async fn alter_columns(
         };
         let mapper = Box::new(mapper);
 
-        let fragments = add_columns_impl(
+        let result = add_columns_impl(
             &dataset.get_fragments(),
             Some(read_columns),
             mapper,
@@ -666,7 +860,8 @@ pub(super) async fn alter_columns(
         // Some data files may no longer contain any columns in the dataset (e.g. if every
         // remaining column has been altered into a different data file) and so we remove them
         let schema_field_ids = new_schema.field_ids().into_iter().collect::<Vec<_>>();
-        let fragments = fragments
+        let fragments = result
+            .fragments
             .into_iter()
             .map(|mut frag| {
                 frag.files.retain(|f| {
@@ -762,8 +957,7 @@ pub fn exclude(source: &Schema, other: &Schema, version: &LanceFileVersion) -> R
 
 #[cfg(test)]
 mod test {
-    use std::collections::HashMap;
-    use std::sync::Mutex;
+    use std::{collections::HashMap, fs, num::NonZero, path::Path as StdPath, sync::Mutex};
 
     use crate::dataset::WriteParams;
     use arrow_array::{
@@ -774,6 +968,7 @@ mod test {
     use arrow_schema::Fields as ArrowFields;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_file::version::LanceFileVersion;
+    use lance_table::format::{BasePath, DataFile};
     use rstest::rstest;
 
     // Used to validate that futures returned are Send.
@@ -781,6 +976,47 @@ mod test {
         t
     }
 
+    fn file_paths_in(dir: impl AsRef<StdPath>) -> Vec<String> {
+        fn collect_files(
+            base_dir: &StdPath,
+            dir: &StdPath,
+            files: &mut Vec<String>,
+        ) -> std::io::Result<()> {
+            if !dir.exists() {
+                return Ok(());
+            }
+            for entry in std::fs::read_dir(dir)? {
+                let path = entry?.path();
+                if path.is_dir() {
+                    collect_files(base_dir, &path, files)?;
+                } else if path.is_file()
+                    && path
+                        .file_name()
+                        .and_then(|name| name.to_str())
+                        .is_some_and(|file_name| !file_name.starts_with('.'))
+                {
+                    files.push(
+                        path.strip_prefix(base_dir)
+                            .unwrap()
+                            .to_string_lossy()
+                            .to_string(),
+                    );
+                }
+            }
+            Ok(())
+        }
+
+        let base_dir = dir.as_ref();
+        let mut files = Vec::new();
+        collect_files(base_dir, base_dir, &mut files).unwrap();
+        files.sort();
+        files
+    }
+
+    fn data_file_paths_in(base_dir: &str) -> Vec<String> {
+        file_paths_in(StdPath::new(base_dir).join("data"))
+    }
+
     #[tokio::test]
     async fn test_append_columns_exprs() -> Result<()> {
         let num_rows = 5;
@@ -836,30 +1072,710 @@ mod test {
             )
             .await?;
 
-        // Can derive a column from existing ones across multiple data files.
-        dataset
-            .add_columns(
-                NewColumnTransform::SqlExpressions(vec![(
-                    "triple_id".into(),
-                    "id + double_id".into(),
-                )]),
-                None,
-                None,
-            )
-            .await?;
+        // Can derive a column from existing ones across multiple data files.
+        dataset
+            .add_columns(
+                NewColumnTransform::SqlExpressions(vec![(
+                    "triple_id".into(),
+                    "id + double_id".into(),
+                )]),
+                None,
+                None,
+            )
+            .await?;
+
+        // These can be read back, the dataset is valid
+        dataset.validate().await?;
+
+        let data = dataset.scan().try_into_batch().await?;
+        let expected_schema = ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("value", DataType::Float64, true),
+            ArrowField::new("double_id", DataType::Int32, false),
+            ArrowField::new("triple_id", DataType::Int32, false),
+        ]);
+        assert_eq!(data.schema().as_ref(), &expected_schema);
+        assert_eq!(data.num_rows(), num_rows);
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_with_fully_deleted_batch() -> Result<()> {
+        // Regression test: when an entire read batch has been deleted, the
+        // updater yields a 0-row batch. The inner loop then never runs and
+        // `batches` stays empty, so `concat_batches(&batches[0]..)` used to
+        // panic with "index out of bounds: the len is 0 but the index is 0".
+        //
+        // A single fragment holds 105 rows; deleting the trailing 5 rows means
+        // that, when read with batch_size=50, the third batch [100..105) is
+        // fully filtered out and produces an empty batch.
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "i",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..105))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 200, // keep all rows in a single fragment
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        // Delete the entire trailing batch [100..105).
+        dataset.delete("i >= 100").await?;
+        assert_eq!(dataset.count_rows(None).await?, 100);
+
+        let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "j",
+            DataType::Int32,
+            false,
+        )]));
+        let new_batch = RecordBatch::try_new(
+            new_schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..100))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(new_batch)], new_schema.clone());
+
+        // Read with batch_size=50 so the deleted trailing rows form a full empty batch.
+        dataset
+            .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, Some(50))
+            .await?;
+
+        let data = dataset.scan().try_into_batch().await?;
+        assert_eq!(data.num_rows(), 100);
+        assert_eq!(
+            data.column_by_name("j").unwrap().as_ref(),
+            &Int32Array::from_iter_values(0..100)
+        );
+
+        Ok(())
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_add_columns_cleans_up_blob_v2_data_on_stream_error(
+        #[values(
+            ("inline", b"inline".to_vec()),
+            ("packed", vec![1u8; 128 * 1024]),
+            ("dedicated", vec![2u8; 5 * 1024 * 1024]),
+            ("external", b"external".to_vec())
+        )]
+        blob_case: (&str, Vec<u8>),
+    ) -> Result<()> {
+        let (blob_kind, payload) = blob_case;
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..1))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let external_dir = tempfile::tempdir()?;
+        let external_path = external_dir.path().join("blob.bin");
+        fs::write(&external_path, &payload)?;
+        let external_baseline_files = file_paths_in(external_dir.path());
+        let external_baseline_payload = fs::read(&external_path)?;
+
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                initial_bases: Some(vec![BasePath::new(
+                    1,
+                    external_dir.path().to_string_lossy().to_string(),
+                    Some("external".to_string()),
+                    false,
+                )]),
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let baseline_files = data_file_paths_in(test_uri);
+
+        let mut blob_builder = crate::BlobArrayBuilder::new(2);
+        if blob_kind == "external" {
+            blob_builder.push_uri(external_path.to_string_lossy())?;
+        } else {
+            blob_builder.push_bytes(payload)?;
+        }
+        blob_builder.push_bytes(b"extra")?;
+        let blob_array = blob_builder.finish()?;
+        let blob_schema = Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)]));
+        let blob_batch = RecordBatch::try_new(blob_schema.clone(), vec![blob_array])?;
+        let reader = RecordBatchIterator::new(vec![Ok(blob_batch)], blob_schema);
+
+        let err = dataset
+            .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, None)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("Stream produced more values than expected for dataset")
+        );
+
+        assert_eq!(
+            data_file_paths_in(test_uri),
+            baseline_files,
+            "add_columns should clean up new data files and blob v2 sidecars on failure"
+        );
+        assert_eq!(
+            file_paths_in(external_dir.path()),
+            external_baseline_files,
+            "cleanup must not delete external files"
+        );
+        assert_eq!(
+            fs::read(&external_path)?,
+            external_baseline_payload,
+            "cleanup must not modify external files"
+        );
+        dataset.validate().await?;
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_cleanup_preserves_checkpointed_fragment_files() -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..2))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 1,
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let original_fragments = dataset.get_fragments();
+        assert_eq!(original_fragments.len(), 2);
+
+        let data_dir = StdPath::new(test_uri).join("data");
+        let cached_file = data_dir.join("checkpointed.lance");
+        let cached_blob_dir = data_dir.join("checkpointed");
+        fs::write(&cached_file, b"checkpointed data")?;
+        fs::create_dir_all(&cached_blob_dir)?;
+        fs::write(
+            cached_blob_dir.join("00000000000000000000000000000001.blob"),
+            b"blob",
+        )?;
+
+        let mut checkpointed_fragment = original_fragments[0].metadata().clone();
+        checkpointed_fragment.files.push(DataFile::new(
+            "checkpointed.lance",
+            vec![dataset.manifest.max_field_id() + 1],
+            vec![0],
+            2,
+            2,
+            NonZero::new(17),
+            None,
+        ));
+
+        #[derive(Default)]
+        struct CheckpointedFragmentStore {
+            fragment: Mutex<Option<Fragment>>,
+        }
+
+        impl UDFCheckpointStore for CheckpointedFragmentStore {
+            fn get_batch(&self, _info: &BatchInfo) -> Result<Option<RecordBatch>> {
+                Ok(None)
+            }
+
+            fn insert_batch(&self, _info: BatchInfo, _batch: RecordBatch) -> Result<()> {
+                Ok(())
+            }
+
+            fn get_fragment(&self, fragment_id: u32) -> Result<Option<Fragment>> {
+                if fragment_id == 0 {
+                    Ok(self.fragment.lock().unwrap().clone())
+                } else {
+                    Ok(None)
+                }
+            }
+
+            fn insert_fragment(&self, _fragment: Fragment) -> Result<()> {
+                Ok(())
+            }
+        }
+
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(|_| Err(Error::invalid_input("injected UDF failure"))),
+            output_schema: Arc::new(ArrowSchema::new(vec![ArrowField::new(
+                "checkpointed",
+                DataType::Int32,
+                true,
+            )])),
+            result_checkpoint: Some(Arc::new(CheckpointedFragmentStore {
+                fragment: Mutex::new(Some(checkpointed_fragment)),
+            })),
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, None)
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("injected UDF failure"));
+
+        assert!(
+            cached_file.exists(),
+            "cleanup must not delete fragment files restored from a checkpoint"
+        );
+        assert!(
+            cached_blob_dir.exists(),
+            "cleanup must not delete blob sidecars restored from a checkpoint"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_cleans_current_blob_v2_writer_on_udf_error() -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..2))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let baseline_files = data_file_paths_in(test_uri);
+
+        let call_count = Arc::new(Mutex::new(0usize));
+        let mapper_call_count = call_count.clone();
+        let output_schema = Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)]));
+        let mapper = move |batch: &RecordBatch| {
+            let mut call_count = mapper_call_count.lock().unwrap();
+            *call_count += 1;
+            if *call_count == 2 {
+                return Err(Error::invalid_input("injected UDF failure"));
+            }
+
+            let mut blob_builder = crate::BlobArrayBuilder::new(batch.num_rows());
+            for _ in 0..batch.num_rows() {
+                blob_builder.push_bytes(vec![7u8; 5 * 1024 * 1024])?;
+            }
+            Ok(RecordBatch::try_new(
+                Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)])),
+                vec![blob_builder.finish()?],
+            )?)
+        };
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(mapper),
+            output_schema,
+            result_checkpoint: None,
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, Some(1))
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("injected UDF failure"));
+        assert_eq!(
+            data_file_paths_in(test_uri),
+            baseline_files,
+            "add_columns should clean files written by the current unfinished writer"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_preserves_checkpointed_blob_v2_fragment_on_checkpoint_lookup_error()
+    -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..2))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 1,
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        struct FailingLookupStore {
+            inserted: Arc<Mutex<Option<Fragment>>>,
+        }
+
+        impl UDFCheckpointStore for FailingLookupStore {
+            fn get_batch(&self, _info: &BatchInfo) -> Result<Option<RecordBatch>> {
+                Ok(None)
+            }
+
+            fn insert_batch(&self, _info: BatchInfo, _batch: RecordBatch) -> Result<()> {
+                Ok(())
+            }
+
+            fn get_fragment(&self, fragment_id: u32) -> Result<Option<Fragment>> {
+                if fragment_id == 1 {
+                    Err(Error::invalid_input("injected checkpoint lookup failure"))
+                } else {
+                    Ok(None)
+                }
+            }
+
+            fn insert_fragment(&self, fragment: Fragment) -> Result<()> {
+                *self.inserted.lock().unwrap() = Some(fragment);
+                Ok(())
+            }
+        }
+
+        let inserted = Arc::new(Mutex::new(None));
+        let output_schema = Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)]));
+        let mapper = move |batch: &RecordBatch| {
+            let mut blob_builder = crate::BlobArrayBuilder::new(batch.num_rows());
+            for _ in 0..batch.num_rows() {
+                blob_builder.push_bytes(vec![7u8; 5 * 1024 * 1024])?;
+            }
+            Ok(RecordBatch::try_new(
+                Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)])),
+                vec![blob_builder.finish()?],
+            )?)
+        };
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(mapper),
+            output_schema,
+            result_checkpoint: Some(Arc::new(FailingLookupStore {
+                inserted: inserted.clone(),
+            })),
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, None)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("injected checkpoint lookup failure")
+        );
+        let inserted = inserted.lock().unwrap().clone().unwrap();
+        let new_file = inserted
+            .files
+            .iter()
+            .find(|file| {
+                file.fields
+                    .iter()
+                    .any(|field| *field > dataset.manifest.max_field_id())
+            })
+            .expect("checkpoint should record the newly written data file");
+        let new_file_path = StdPath::new(test_uri).join("data").join(&new_file.path);
+        let new_blob_dir = StdPath::new(test_uri)
+            .join("data")
+            .join(StdPath::new(&new_file.path).file_stem().unwrap());
+        assert!(
+            new_file_path.exists(),
+            "cleanup must not delete data files after checkpoint takes ownership"
+        );
+        assert!(
+            new_blob_dir.exists(),
+            "cleanup must not delete blob sidecars after checkpoint takes ownership"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_cleans_finished_blob_v2_writer_on_checkpoint_insert_error()
+    -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..1))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let baseline_files = data_file_paths_in(test_uri);
+
+        struct FailingInsertStore;
+
+        impl UDFCheckpointStore for FailingInsertStore {
+            fn get_batch(&self, _info: &BatchInfo) -> Result<Option<RecordBatch>> {
+                Ok(None)
+            }
+
+            fn insert_batch(&self, _info: BatchInfo, _batch: RecordBatch) -> Result<()> {
+                Ok(())
+            }
+
+            fn get_fragment(&self, _fragment_id: u32) -> Result<Option<Fragment>> {
+                Ok(None)
+            }
+
+            fn insert_fragment(&self, _fragment: Fragment) -> Result<()> {
+                Err(Error::invalid_input("injected checkpoint insert failure"))
+            }
+        }
+
+        let output_schema = Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)]));
+        let mapper = move |batch: &RecordBatch| {
+            let mut blob_builder = crate::BlobArrayBuilder::new(batch.num_rows());
+            for _ in 0..batch.num_rows() {
+                blob_builder.push_bytes(vec![7u8; 5 * 1024 * 1024])?;
+            }
+            Ok(RecordBatch::try_new(
+                Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)])),
+                vec![blob_builder.finish()?],
+            )?)
+        };
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(mapper),
+            output_schema,
+            result_checkpoint: Some(Arc::new(FailingInsertStore)),
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, None)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("injected checkpoint insert failure")
+        );
+        assert_eq!(
+            data_file_paths_in(test_uri),
+            baseline_files,
+            "add_columns should clean finished writer files when checkpoint insert fails"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_cleans_blob_v2_files_on_declared_schema_merge_error() -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..1))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let baseline_files = data_file_paths_in(test_uri);
+
+        let mapper = move |batch: &RecordBatch| {
+            let mut blob_builder = crate::BlobArrayBuilder::new(batch.num_rows());
+            for _ in 0..batch.num_rows() {
+                blob_builder.push_bytes(vec![7u8; 5 * 1024 * 1024])?;
+            }
+            Ok(RecordBatch::try_new(
+                Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)])),
+                vec![blob_builder.finish()?],
+            )?)
+        };
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(mapper),
+            output_schema: Arc::new(ArrowSchema::new(vec![
+                ArrowField::new("declared", DataType::Int32, true),
+                ArrowField::new("declared", DataType::Int32, true),
+            ])),
+            result_checkpoint: None,
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, None)
+            .await
+            .unwrap_err();
+        assert!(matches!(err, Error::Schema { .. }));
+        assert_eq!(
+            data_file_paths_in(test_uri),
+            baseline_files,
+            "add_columns should clean files written before declared schema merge fails"
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_add_columns_preserves_checkpointed_blob_v2_fragment_after_later_failure()
+    -> Result<()> {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..2))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 1,
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        struct InsertThenFailStore {
+            inserted: Arc<Mutex<Option<Fragment>>>,
+        }
+
+        impl UDFCheckpointStore for InsertThenFailStore {
+            fn get_batch(&self, info: &BatchInfo) -> Result<Option<RecordBatch>> {
+                if info.fragment_id == 1 {
+                    Err(Error::invalid_input("injected later checkpoint failure"))
+                } else {
+                    Ok(None)
+                }
+            }
+
+            fn insert_batch(&self, _info: BatchInfo, _batch: RecordBatch) -> Result<()> {
+                Ok(())
+            }
+
+            fn get_fragment(&self, _fragment_id: u32) -> Result<Option<Fragment>> {
+                Ok(None)
+            }
 
-        // These can be read back, the dataset is valid
-        dataset.validate().await?;
+            fn insert_fragment(&self, fragment: Fragment) -> Result<()> {
+                *self.inserted.lock().unwrap() = Some(fragment);
+                Ok(())
+            }
+        }
 
-        let data = dataset.scan().try_into_batch().await?;
-        let expected_schema = ArrowSchema::new(vec![
-            ArrowField::new("id", DataType::Int32, false),
-            ArrowField::new("value", DataType::Float64, true),
-            ArrowField::new("double_id", DataType::Int32, false),
-            ArrowField::new("triple_id", DataType::Int32, false),
-        ]);
-        assert_eq!(data.schema().as_ref(), &expected_schema);
-        assert_eq!(data.num_rows(), num_rows);
+        let inserted = Arc::new(Mutex::new(None));
+        let output_schema = Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)]));
+        let mapper = move |batch: &RecordBatch| {
+            let mut blob_builder = crate::BlobArrayBuilder::new(batch.num_rows());
+            for _ in 0..batch.num_rows() {
+                blob_builder.push_bytes(vec![7u8; 5 * 1024 * 1024])?;
+            }
+            Ok(RecordBatch::try_new(
+                Arc::new(ArrowSchema::new(vec![crate::blob_field("blob", true)])),
+                vec![blob_builder.finish()?],
+            )?)
+        };
+        let transforms = NewColumnTransform::BatchUDF(BatchUDF {
+            mapper: Box::new(mapper),
+            output_schema,
+            result_checkpoint: Some(Arc::new(InsertThenFailStore {
+                inserted: inserted.clone(),
+            })),
+        });
+
+        let err = dataset
+            .add_columns(transforms, None, None)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("injected later checkpoint failure")
+        );
+
+        let inserted = inserted.lock().unwrap().clone().unwrap();
+        let new_file = inserted
+            .files
+            .iter()
+            .find(|file| {
+                file.fields
+                    .iter()
+                    .any(|field| *field > dataset.manifest.max_field_id())
+            })
+            .expect("checkpoint should record the newly written data file");
+        let new_file_path = StdPath::new(test_uri).join("data").join(&new_file.path);
+        let new_blob_dir = StdPath::new(test_uri)
+            .join("data")
+            .join(StdPath::new(&new_file.path).file_stem().unwrap());
+        assert!(
+            new_file_path.exists(),
+            "cleanup must not delete data files after checkpoint takes ownership"
+        );
+        assert!(
+            new_blob_dir.exists(),
+            "cleanup must not delete blob sidecars after checkpoint takes ownership"
+        );
 
         Ok(())
     }
@@ -1041,6 +1957,7 @@ mod test {
                     Ok(Some(Fragment {
                         files: vec![],
                         id: 0,
+                        overlays: vec![],
                         deletion_file: None,
                         row_id_meta: None,
                         physical_rows: Some(50),
@@ -1205,8 +2122,8 @@ mod test {
                 .await
                 .unwrap_err();
         assert!(
-            err.to_string()
-                .contains("All-null columns must be nullable.")
+            matches!(err, Error::InvalidInput { .. }),
+            "unexpected error: {err}"
         );
 
         let data = dataset.scan().try_into_batch().await?;
@@ -1220,6 +2137,107 @@ mod test {
         Ok(())
     }
 
+    /// `AllNulls` accepts any nullable top-level column whatever its inner-field
+    /// nullability (Map/List/Struct with non-null children); non-nullable ones are rejected.
+    #[tokio::test]
+    async fn test_add_column_all_nulls_nested() -> Result<()> {
+        let num_rows = 100;
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..num_rows))],
+        )?;
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+
+        let test_dir = TempStrDir::default();
+        let mut dataset = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                max_rows_per_file: 50,
+                max_rows_per_group: 25,
+                data_storage_version: Some(LanceFileVersion::Stable),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        let map_with_non_null_entries = DataType::Map(
+            Arc::new(ArrowField::new(
+                "entries",
+                DataType::Struct(ArrowFields::from(vec![
+                    ArrowField::new("key", DataType::Utf8, false),
+                    ArrowField::new("value", DataType::Float64, true),
+                ])),
+                false,
+            )),
+            false,
+        );
+        let list_with_non_null_items =
+            DataType::List(Arc::new(ArrowField::new("item", DataType::Utf8, false)));
+        let struct_with_non_null_child =
+            DataType::Struct(ArrowFields::from(vec![ArrowField::new(
+                "a",
+                DataType::Int32,
+                false,
+            )]));
+
+        dataset
+            .add_columns(
+                NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![
+                    ArrowField::new("cutoffs", map_with_non_null_entries.clone(), true),
+                    ArrowField::new("tags", list_with_non_null_items.clone(), true),
+                    ArrowField::new("info", struct_with_non_null_child.clone(), true),
+                ]))),
+                None,
+                None,
+            )
+            .await?;
+
+        let data = dataset.scan().try_into_batch().await?;
+        assert_eq!(data.num_rows(), num_rows as usize);
+        for (name, expected_type) in [
+            ("cutoffs", &map_with_non_null_entries),
+            ("tags", &list_with_non_null_items),
+            ("info", &struct_with_non_null_child),
+        ] {
+            let column = data.column_by_name(name).unwrap();
+            assert_eq!(
+                column.data_type(),
+                expected_type,
+                "type mismatch for {name}"
+            );
+            assert_eq!(
+                column.null_count(),
+                num_rows as usize,
+                "column {name} should be all-null"
+            );
+        }
+
+        // A non-nullable top-level field is still rejected, and the error names it.
+        let err =
+            dataset
+                .add_columns(
+                    NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![
+                        ArrowField::new("non_null_cutoffs", map_with_non_null_entries, false),
+                    ]))),
+                    None,
+                    None,
+                )
+                .await
+                .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidInput { .. }),
+            "unexpected error: {err}"
+        );
+
+        Ok(())
+    }
+
     #[tokio::test]
     async fn test_add_column_all_nulls_legacy() -> Result<()> {
         let num_rows = 100;
@@ -1784,7 +2802,6 @@ mod test {
     ) -> Result<()> {
         // Create a table with 2 scalar columns, 1 vector column
 
-        use crate::index::DatasetIndexExt;
         use arrow::datatypes::{Int32Type, Int64Type};
         use arrow_array::{Float16Array, Float32Array, Int64Array, ListArray};
         use half::f16;
@@ -1885,7 +2902,10 @@ mod test {
             assert_eq!(f.files.len(), 2);
         });
 
-        // Cast scalar column with index, should not keep index (TODO: keep it)
+        // Cast scalar column with index. The index must be dropped first; cast
+        // is now a fail-fast operation when an index is attached, see
+        // test_alter_columns_cast_fails_with_attached_index for that path.
+        dataset.drop_index("i_idx").await?;
         dataset
             .alter_columns(&[ColumnAlteration::new("i".into()).cast_to(DataType::Int64)])
             .await?;
@@ -1906,7 +2926,8 @@ mod test {
         ]);
         assert_eq!(&ArrowSchema::from(dataset.schema()), &expected_schema);
 
-        // We currently lose the index when casting a column
+        // The scalar index on `i` is gone (we dropped it); the vector index on
+        // `vec` is still present.
         let indices = dataset.load_indices().await?;
         assert_eq!(indices.len(), 1);
 
@@ -1915,7 +2936,8 @@ mod test {
             assert_eq!(f.files.len(), 3);
         });
 
-        // Cast vector column, should not keep index (TODO: keep it)
+        // Cast vector column. Drop its index first (same reason as above).
+        dataset.drop_index("vec_idx").await?;
         dataset
             .alter_columns(&[
                 ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList(
@@ -1983,6 +3005,293 @@ mod test {
         Ok(())
     }
 
+    /// Cast on a column with an attached index must fail fast rather than
+    /// silently dropping the index. This guards against the historical behavior
+    /// where cast would rewrite column data and the index would vanish without
+    /// any error or warning, causing vector search to silently regress to a
+    /// brute-force scan.
+    #[rstest]
+    #[tokio::test]
+    async fn test_alter_columns_cast_fails_with_attached_index(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) -> Result<()> {
+        use lance_arrow::FixedSizeListArrayExt;
+        use lance_index::IndexType;
+        use lance_linalg::distance::MetricType;
+        use lance_testing::datagen::generate_random_array;
+
+        use crate::index::vector::VectorIndexParams;
+
+        // Build a small dataset with one indexed vector column.
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "vec",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                64,
+            ),
+            false,
+        )]));
+        let nrows = 256;
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(
+                <arrow_array::FixedSizeListArray as FixedSizeListArrayExt>::try_new_from_values(
+                    generate_random_array(64 * nrows as usize),
+                    64,
+                )
+                .unwrap(),
+            )],
+        )?;
+
+        let test_dir = TempStrDir::default();
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        // Build an IVF_PQ index on the vector column.
+        let params = VectorIndexParams::ivf_pq(4, 8, 8, MetricType::L2, 50);
+        dataset
+            .create_index(&["vec"], IndexType::Vector, None, &params, false)
+            .await?;
+
+        let indices_before = dataset.load_indices().await?;
+        assert_eq!(indices_before.len(), 1, "precondition: index exists");
+        let index_name = indices_before[0].name.clone();
+
+        // Attempting to cast the indexed column must fail with a clear message
+        // that names the offending index(es).
+        let result = dataset
+            .alter_columns(&[
+                ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList(
+                    Arc::new(ArrowField::new("item", DataType::Float16, true)),
+                    64,
+                )),
+            ])
+            .await;
+        let err = result.expect_err("cast on indexed column should fail");
+        let msg = err.to_string();
+        assert!(
+            msg.contains("vec") && msg.contains(&index_name),
+            "error should mention column and index name, got: {msg}"
+        );
+        assert!(
+            msg.contains("drop_index"),
+            "error should suggest the remediation, got: {msg}"
+        );
+
+        // The dataset must be unchanged: schema is still float32, index still present.
+        assert_eq!(
+            dataset.schema().field("vec").unwrap().data_type(),
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                64,
+            ),
+        );
+        let indices_after = dataset.load_indices().await?;
+        assert_eq!(indices_after.len(), 1, "index should still exist");
+        assert_eq!(indices_after[0].name, index_name);
+
+        // Sanity check: after dropping the index, the same cast should succeed.
+        dataset.drop_index(&index_name).await?;
+        dataset
+            .alter_columns(&[
+                ColumnAlteration::new("vec".into()).cast_to(DataType::FixedSizeList(
+                    Arc::new(ArrowField::new("item", DataType::Float16, true)),
+                    64,
+                )),
+            ])
+            .await?;
+        assert_eq!(
+            dataset.schema().field("vec").unwrap().data_type(),
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float16, true)),
+                64,
+            ),
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_is_upcast_downcast_dictionary() {
+        use DataType::*;
+
+        let dict_i32_utf8 = Dictionary(Box::new(Int32), Box::new(Utf8));
+        let dict_i16_utf8 = Dictionary(Box::new(Int16), Box::new(Utf8));
+        let dict_i32_large_utf8 = Dictionary(Box::new(Int32), Box::new(LargeUtf8));
+        let dict_i32_int64 = Dictionary(Box::new(Int32), Box::new(Int64));
+        let stable = LanceFileVersion::Stable;
+        let legacy = LanceFileVersion::Legacy;
+
+        // Dict(_, Utf8) -> Utf8 / LargeUtf8 (decode direction): both versions.
+        assert!(is_upcast_downcast(&dict_i32_utf8, &Utf8, stable));
+        assert!(is_upcast_downcast(&dict_i32_utf8, &LargeUtf8, stable));
+        assert!(is_upcast_downcast(&dict_i32_utf8, &Utf8, legacy));
+        assert!(is_upcast_downcast(&dict_i32_utf8, &LargeUtf8, legacy));
+
+        // Utf8 / LargeUtf8 -> Dict(_, Utf8) (encode direction): stable only.
+        assert!(is_upcast_downcast(&Utf8, &dict_i32_utf8, stable));
+        assert!(is_upcast_downcast(&LargeUtf8, &dict_i32_utf8, stable));
+        assert!(!is_upcast_downcast(&Utf8, &dict_i32_utf8, legacy));
+        assert!(!is_upcast_downcast(&LargeUtf8, &dict_i32_utf8, legacy));
+
+        // Dict -> Dict with compatible value types, including different index
+        // types. Stable only; Legacy can't materialize a fresh dictionary.
+        assert!(is_upcast_downcast(&dict_i32_utf8, &dict_i16_utf8, stable));
+        assert!(is_upcast_downcast(
+            &dict_i32_utf8,
+            &dict_i32_large_utf8,
+            stable
+        ));
+        assert!(!is_upcast_downcast(&dict_i32_utf8, &dict_i16_utf8, legacy));
+
+        // Dict(_, Int64) <-> integer types (peel applies to non-string families).
+        assert!(is_upcast_downcast(&dict_i32_int64, &Int32, stable));
+        assert!(is_upcast_downcast(&Int32, &dict_i32_int64, stable));
+        assert!(is_upcast_downcast(&dict_i32_int64, &Int32, legacy));
+        assert!(!is_upcast_downcast(&Int32, &dict_i32_int64, legacy));
+
+        // Cross-family casts must still be rejected after peeling.
+        assert!(!is_upcast_downcast(&dict_i32_utf8, &Int32, stable));
+        assert!(!is_upcast_downcast(&Int32, &dict_i32_utf8, stable));
+        assert!(!is_upcast_downcast(&dict_i32_utf8, &Boolean, stable));
+        assert!(!is_upcast_downcast(&Boolean, &dict_i32_utf8, stable));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn test_cast_dictionary_to_string(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) -> Result<()> {
+        use arrow_array::DictionaryArray;
+        use arrow_array::types::Int32Type;
+
+        let dict_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "d",
+            dict_type.clone(),
+            false,
+        )]));
+
+        let values = ["alpha", "beta", "gamma", "alpha", "beta"];
+        let dict_array: DictionaryArray<Int32Type> = values.iter().copied().collect();
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(dict_array)])?;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone()),
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        dataset
+            .alter_columns(&[ColumnAlteration::new("d".into()).cast_to(DataType::Utf8)])
+            .await?;
+        dataset.validate().await?;
+        assert_eq!(
+            dataset.schema().field("d").unwrap().data_type(),
+            DataType::Utf8
+        );
+        let scanned = dataset.scan().try_into_batch().await?;
+        let decoded = scanned.column_by_name("d").unwrap();
+        let expected_decoded = StringArray::from(values.to_vec());
+        assert_eq!(
+            decoded.as_ref(),
+            &expected_decoded as &dyn arrow_array::Array
+        );
+
+        // Cross-family casts must still be rejected even through a dictionary.
+        let err = dataset
+            .alter_columns(&[ColumnAlteration::new("d".into()).cast_to(DataType::Int32)])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("Cannot cast column"));
+
+        Ok(())
+    }
+
+    // Stable can materialize a fresh Dictionary column via alter; Legacy
+    // cannot, because its writer requires `field.dictionary` metadata to be
+    // pre-populated, so the cast is rejected upfront with a clean error.
+    #[rstest]
+    #[tokio::test]
+    async fn test_cast_string_to_dictionary(
+        #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)]
+        data_storage_version: LanceFileVersion,
+    ) -> Result<()> {
+        use arrow_array::DictionaryArray;
+        use arrow_array::types::Int32Type;
+
+        let dict_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Utf8,
+            false,
+        )]));
+
+        let values = ["alpha", "beta", "gamma", "alpha", "beta"];
+        let string_array = StringArray::from(values.to_vec());
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(string_array)])?;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = &test_dir;
+
+        let mut dataset = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone()),
+            test_uri,
+            Some(WriteParams {
+                data_storage_version: Some(data_storage_version),
+                ..Default::default()
+            }),
+        )
+        .await?;
+
+        let result = dataset
+            .alter_columns(&[ColumnAlteration::new("s".into()).cast_to(dict_type.clone())])
+            .await;
+
+        match data_storage_version {
+            LanceFileVersion::Legacy => {
+                let err = result.unwrap_err();
+                assert!(
+                    err.to_string().contains("Cannot cast column"),
+                    "expected upfront rejection on Legacy, got: {err}"
+                );
+            }
+            _ => {
+                result?;
+                dataset.validate().await?;
+                assert_eq!(
+                    dataset.schema().field("s").unwrap().data_type(),
+                    dict_type.clone()
+                );
+                let scanned = dataset.scan().try_into_batch().await?;
+                let encoded = scanned.column_by_name("s").unwrap();
+                let expected_encoded: DictionaryArray<Int32Type> = values.iter().copied().collect();
+                assert_eq!(
+                    encoded.as_ref(),
+                    &expected_encoded as &dyn arrow_array::Array
+                );
+            }
+        }
+
+        Ok(())
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_drop_columns(
diff --git a/rust/lance/src/dataset/schema_evolution/optimize.rs b/rust/lance/src/dataset/schema_evolution/optimize.rs
index cdfdb82b87b..19bb6a6d46c 100644
--- a/rust/lance/src/dataset/schema_evolution/optimize.rs
+++ b/rust/lance/src/dataset/schema_evolution/optimize.rs
@@ -70,7 +70,7 @@ impl SqlToAllNullsOptimizer {
         match expr {
             Expr::Cast(cast) => {
                 if matches!(cast.expr.as_ref(), Expr::Literal(ScalarValue::Null, _)) {
-                    let data_type = cast.data_type.clone();
+                    let data_type = cast.field.data_type().clone();
                     AllNullsResult::AllNulls(data_type)
                 } else {
                     AllNullsResult::NotAllNulls
diff --git a/rust/lance/src/dataset/statistics.rs b/rust/lance/src/dataset/statistics.rs
index 9e52c9653e1..627ccfc1081 100644
--- a/rust/lance/src/dataset/statistics.rs
+++ b/rust/lance/src/dataset/statistics.rs
@@ -5,11 +5,16 @@
 
 use std::{collections::HashMap, future::Future, sync::Arc};
 
+use datafusion::scalar::ScalarValue;
 use futures::{StreamExt, TryStreamExt};
-use lance_core::Result;
+use lance_core::{Error, Result};
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::zonemap::ZoneMapIndex;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
+use roaring::RoaringBitmap;
 
 use super::{Dataset, fragment::FileFragment};
+use crate::index::{DatasetIndexExt, DatasetIndexInternalExt};
 
 /// Statistics about a single field in the dataset
 pub struct FieldStatistics {
@@ -82,3 +87,161 @@ impl DatasetStatisticsExt for Dataset {
         })
     }
 }
+
+/// A read-only handle for cheap, index-derived statistics about a [`Dataset`].
+///
+/// Obtained via [`Dataset::statistics`]. Groups statistics accessors behind one
+/// handle instead of accreting one-off methods on [`Dataset`]. Every accessor is
+/// served from index metadata and never scans data.
+#[derive(Debug, Clone, Copy)]
+pub struct DatasetStatistics<'a> {
+    dataset: &'a Dataset,
+}
+
+impl<'a> DatasetStatistics<'a> {
+    pub(crate) fn new(dataset: &'a Dataset) -> Self {
+        Self { dataset }
+    }
+
+    /// Global `[min, max]` for `column` from its min/max-capable scalar index
+    /// (currently ZoneMap), without a scan.
+    ///
+    /// `None` unless the column's index segments *jointly* cover every live
+    /// fragment and the column can be soundly bounded — fragments appended after
+    /// the index was built, or a NaN-bearing column, yield `None`. The disjoint
+    /// segments of a multi-segment index are folded together.
+    ///
+    /// When `Some`, the range is a superset of live values, conservative under
+    /// deletion vectors: safe to prune with. See [`ScalarIndex::value_range`].
+    ///
+    /// [`ScalarIndex::value_range`]: lance_index::scalar::ScalarIndex::value_range
+    pub async fn column_value_range(
+        &self,
+        column: &str,
+    ) -> Result<Option<(ScalarValue, ScalarValue)>> {
+        let dataset = self.dataset;
+        let Some(field) = dataset.schema().field(column) else {
+            return Err(Error::invalid_input(format!(
+                "column_value_range: column '{column}' not found in dataset schema"
+            )));
+        };
+        let field_id = field.id;
+        let field_path = dataset.schema().field_path(field_id)?;
+
+        // A multi-segment ZoneMap is several index entries over the same column,
+        // each covering a disjoint fragment subset. Match the field, then the
+        // details type (the column may also carry e.g. a BTree).
+        let indices = dataset.load_indices().await?;
+        let segments: Vec<_> = indices
+            .iter()
+            .filter(|idx| matches!(idx.fields.as_slice(), [only] if *only == field_id))
+            .filter(|idx| {
+                idx.index_details
+                    .as_ref()
+                    .is_some_and(|d| d.type_url.ends_with("ZoneMapIndexDetails"))
+            })
+            .collect();
+        if segments.is_empty() {
+            return Ok(None);
+        }
+
+        // Soundness: the segments must *jointly* cover every live fragment, else
+        // the fold sees only a subset and could prune live rows (e.g. fragments
+        // appended after the index was built). Extra dead fragments are harmless.
+        let mut covered = RoaringBitmap::new();
+        for idx in &segments {
+            let Some(bitmap) = idx.fragment_bitmap.as_ref() else {
+                return Ok(None);
+            };
+            covered |= bitmap.clone();
+        }
+        if !dataset.fragment_bitmap.as_ref().is_subset(&covered) {
+            return Ok(None);
+        }
+
+        // Keep the opened indices alive so the `ZoneMapIndex` refs we fold over
+        // stay borrowed.
+        let mut opened = Vec::with_capacity(segments.len());
+        for idx in &segments {
+            opened.push(
+                dataset
+                    .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector)
+                    .await?,
+            );
+        }
+        let Some(zonemaps) = opened
+            .iter()
+            .map(|index| index.as_any().downcast_ref::<ZoneMapIndex>())
+            .collect::<Option<Vec<_>>>()
+        else {
+            return Ok(None);
+        };
+        Ok(ZoneMapIndex::value_range_over(zonemaps))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator};
+    use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+    use lance_core::utils::tempfile::TempStrDir;
+    use lance_file::version::LanceFileVersion;
+
+    use crate::dataset::WriteParams;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_calculate_data_stats_after_dropping_wide_dataset_columns() {
+        let num_columns = 64;
+        let num_rows = 128;
+        let schema = Arc::new(ArrowSchema::new(
+            (0..num_columns)
+                .map(|idx| ArrowField::new(format!("col_{idx}"), DataType::Int32, true))
+                .collect::<Vec<_>>(),
+        ));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            (0..num_columns)
+                .map(|column_idx| {
+                    Arc::new(Int32Array::from_iter_values(
+                        (0..num_rows).map(|row_idx| row_idx + column_idx),
+                    )) as ArrayRef
+                })
+                .collect::<Vec<_>>(),
+        )
+        .unwrap();
+
+        let test_dir = TempStrDir::default();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let columns_to_drop = (1..num_columns)
+            .map(|idx| format!("col_{idx}"))
+            .collect::<Vec<_>>();
+        let column_refs = columns_to_drop
+            .iter()
+            .map(String::as_str)
+            .collect::<Vec<_>>();
+        dataset.drop_columns(&column_refs).await.unwrap();
+
+        let stats = Arc::new(dataset).calculate_data_stats().await.unwrap();
+        assert_eq!(stats.fields.len(), 1);
+        assert_eq!(stats.fields[0].id, 0);
+        assert!(
+            stats.fields[0].bytes_on_disk > 0,
+            "bytes_on_disk should include the remaining column after drop_columns"
+        );
+    }
+}
diff --git a/rust/lance/src/dataset/tests/dataset_aggregate.rs b/rust/lance/src/dataset/tests/dataset_aggregate.rs
index ef2a90e6315..81aa945527d 100644
--- a/rust/lance/src/dataset/tests/dataset_aggregate.rs
+++ b/rust/lance/src/dataset/tests/dataset_aggregate.rs
@@ -22,7 +22,7 @@ use datafusion_substrait::substrait::proto::{
         reference_segment::{self, StructField},
     },
     extensions::{
-        SimpleExtensionDeclaration, SimpleExtensionUri,
+        SimpleExtensionDeclaration, SimpleExtensionUrn,
         simple_extension_declaration::{ExtensionFunction, MappingType},
     },
     function_argument::ArgType,
@@ -44,6 +44,7 @@ use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, assert_pla
 use lance_arrow::FixedSizeListArrayExt;
 use lance_index::IndexType;
 use lance_index::scalar::FullTextSearchQuery;
+use lance_index::scalar::ScalarIndexParams;
 use lance_index::scalar::inverted::InvertedIndexParams;
 use lance_linalg::distance::MetricType;
 
@@ -94,17 +95,6 @@ fn create_aggregate_rel(
             git_hash: String::new(),
             producer: "lance-test".to_string(),
         }),
-        #[allow(deprecated)]
-        extension_uris: vec![
-            SimpleExtensionUri {
-                extension_uri_anchor: 1,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
-            },
-            SimpleExtensionUri {
-                extension_uri_anchor: 2,
-                uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(),
-            },
-        ],
         extensions,
         relations: vec![PlanRel {
             rel_type: Some(datafusion_substrait::substrait::proto::plan_rel::RelType::Root(
@@ -116,7 +106,16 @@ fn create_aggregate_rel(
         }],
         advanced_extensions: None,
         expected_type_urls: vec![],
-        extension_urns: vec![],
+        extension_urns: vec![
+            SimpleExtensionUrn {
+                extension_urn_anchor: 1,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(),
+            },
+            SimpleExtensionUrn {
+                extension_urn_anchor: 2,
+                urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(),
+            },
+        ],
         parameter_bindings: vec![],
         type_aliases: vec![],
     };
@@ -128,9 +127,7 @@ fn create_aggregate_rel(
 fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration {
     SimpleExtensionDeclaration {
         mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction {
-            #[allow(deprecated)]
-            extension_uri_reference: 1,
-            extension_urn_reference: 0,
+            extension_urn_reference: 1,
             function_anchor: anchor,
             name: name.to_string(),
         })),
@@ -268,7 +265,9 @@ async fn test_count_star_single_fragment() {
         vec![],
     );
 
-    // Verify COUNT(*) has empty projection optimization
+    // COUNT(*) is rewritten by CountPushdown into a Final aggregate
+    // over CountFromMaskExec, which answers from manifest metadata + the
+    // deletion mask instead of scanning column data.
     let mut scanner = ds.scan();
     scanner
         .aggregate(AggregateExpr::substrait(agg_bytes.clone()))
@@ -276,8 +275,8 @@ async fn test_count_star_single_fragment() {
     let plan = scanner.create_plan().await.unwrap();
     assert_plan_node_equals(
         plan,
-        "AggregateExec: mode=Single, gby=[], aggr=[count(...)]
-  LanceRead: uri=..., projection=[], num_fragments=1, range_before=None, range_after=None, row_id=false, row_addr=true, full_filter=--, refine_filter=--",
+        "AggregateExec: mode=Final, gby=[], aggr=[count(...)]
+  CountFromMask",
     )
     .await
     .unwrap();
@@ -1204,11 +1203,12 @@ async fn test_scanner_count_rows() {
         .unwrap();
     let plan = scanner.create_plan().await.unwrap();
 
-    // COUNT(*) should have empty projection (optimized to not read any columns)
+    // COUNT(*) is rewritten by CountPushdown into a Final aggregate
+    // over CountFromMaskExec.
     assert_plan_node_equals(
         plan.clone(),
-        "AggregateExec: mode=Single, gby=[], aggr=[count(Int32(1))]
-  LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, row_id=false, row_addr=true, full_filter=--, refine_filter=--",
+        "AggregateExec: mode=Final, gby=[], aggr=[count(Int32(1))]
+  CountFromMask",
     )
     .await
     .unwrap();
@@ -1255,6 +1255,264 @@ async fn test_scanner_count_rows_with_filter() {
     );
 }
 
+#[tokio::test]
+async fn test_scanner_count_rows_with_indexed_filter() {
+    // When the filter is fully evaluable by a scalar index that covers
+    // every dataset fragment, the rule rewrites COUNT(*) into a Final
+    // aggregate over CountFromMaskExec, with the ScalarIndexExec
+    // wired in as the prefilter — no LanceRead, no column scan.
+    let mut ds = create_numeric_dataset("memory://test_count_indexed", 2, 50).await;
+    ds.create_index(
+        &["x"],
+        IndexType::BTree,
+        None,
+        &ScalarIndexParams::default(),
+        true,
+    )
+    .await
+    .unwrap();
+
+    let mut scanner = ds.scan();
+    scanner.filter("x < 50").unwrap();
+    scanner
+        .aggregate(AggregateExpr::builder().count_star().build())
+        .unwrap();
+    let plan = scanner.create_plan().await.unwrap();
+
+    assert_plan_node_equals(
+        plan.clone(),
+        "AggregateExec: mode=Final, gby=[], aggr=[count(Int32(1))]
+  CountFromMask
+    ScalarIndexQuery: query=[x < 50]@x_idx(BTree)",
+    )
+    .await
+    .unwrap();
+
+    let stream = execute_plan(plan, LanceExecutionOptions::default()).unwrap();
+    let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+    assert_eq!(batches.len(), 1);
+    assert_eq!(
+        batches[0].column(0).as_primitive::<Int64Type>().value(0),
+        50,
+    );
+}
+
+#[tokio::test]
+async fn test_scanner_count_rows_with_indexed_filter_stable_row_ids() {
+    // Indexed-filter count under stable row ids, with deletions in both
+    // fragments. The rule fires and the cross-fragment count stays correct.
+    let tmp = tempdir().unwrap();
+    let uri = tmp.path().to_str().unwrap();
+    let mut ds = gen_batch()
+        .col("x", array::step::<Int64Type>())
+        .col("y", array::step_custom::<Int64Type>(0, 2))
+        .col("category", array::cycle::<Int64Type>(vec![1, 2, 3]))
+        .into_dataset_with_params(
+            uri,
+            FragmentCount::from(2),
+            FragmentRowCount::from(50),
+            Some(crate::dataset::WriteParams {
+                max_rows_per_file: 50,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+    ds.create_index(
+        &["x"],
+        IndexType::BTree,
+        None,
+        &ScalarIndexParams::default(),
+        true,
+    )
+    .await
+    .unwrap();
+    // Delete one row from each fragment (x=10 in frag 0, x=70 in frag 1).
+    ds.delete("x = 10 OR x = 70").await.unwrap();
+
+    let mut scanner = ds.scan();
+    scanner.filter("x < 100").unwrap();
+    scanner
+        .aggregate(AggregateExpr::builder().count_star().build())
+        .unwrap();
+    let plan = scanner.create_plan().await.unwrap();
+
+    assert_plan_node_equals(
+        plan.clone(),
+        "AggregateExec: mode=Final, gby=[], aggr=[count(Int32(1))]
+  CountFromMask
+    ScalarIndexQuery: query=[x < 100]@x_idx(BTree)",
+    )
+    .await
+    .unwrap();
+
+    let stream = execute_plan(plan, LanceExecutionOptions::default()).unwrap();
+    let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+    assert_eq!(batches.len(), 1);
+    // 100 rows match `x < 100`, minus the two deletions.
+    assert_eq!(
+        batches[0].column(0).as_primitive::<Int64Type>().value(0),
+        98,
+    );
+}
+
+#[tokio::test]
+async fn test_scanner_count_rows_indexed_filter_stable_row_ids_after_compaction() {
+    // Update rewrites a scattered subset of rows under stable row ids; the
+    // rewritten copies keep their stable ids, so compaction folds the surviving
+    // and rewritten halves of a fragment into row-id segments whose ranges
+    // overlap. The indexed-filter count must still see every live row.
+    let tmp = tempdir().unwrap();
+    let uri = tmp.path().to_str().unwrap();
+    let ds = gen_batch()
+        .col("x", array::step::<Int64Type>())
+        .col("category", array::cycle::<Int64Type>(vec![1, 2, 3]))
+        .into_dataset_with_params(
+            uri,
+            FragmentCount::from(2),
+            FragmentRowCount::from(50),
+            Some(crate::dataset::WriteParams {
+                max_rows_per_file: 50,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+    // Update every third row (category == 1), scattered across stable ids.
+    let res = crate::dataset::UpdateBuilder::new(Arc::new(ds))
+        .update_where("category = 1")
+        .unwrap()
+        .set("category", "0")
+        .unwrap()
+        .build()
+        .unwrap()
+        .execute()
+        .await
+        .unwrap();
+    let mut ds = res.new_dataset.as_ref().clone();
+    // Compaction merges the surviving and rewritten fragments, producing a
+    // fragment whose row-id sequence has overlapping segments.
+    crate::dataset::optimize::compact_files(&mut ds, Default::default(), None)
+        .await
+        .unwrap();
+    // Index after compaction: it covers every fragment and no deletions remain,
+    // so the count is answered from the stable-id universe (the path the
+    // overlapping segments corrupt).
+    ds.create_index(
+        &["x"],
+        IndexType::BTree,
+        None,
+        &ScalarIndexParams::default(),
+        true,
+    )
+    .await
+    .unwrap();
+
+    let mut scanner = ds.scan();
+    scanner.filter("x < 100").unwrap();
+    scanner
+        .aggregate(AggregateExpr::builder().count_star().build())
+        .unwrap();
+    let plan = scanner.create_plan().await.unwrap();
+
+    assert_plan_node_equals(
+        plan.clone(),
+        "AggregateExec: mode=Final, gby=[], aggr=[count(Int32(1))]
+  CountFromMask
+    ScalarIndexQuery: query=[x < 100]@x_idx(BTree)",
+    )
+    .await
+    .unwrap();
+
+    let stream = execute_plan(plan, LanceExecutionOptions::default()).unwrap();
+    let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+    assert_eq!(batches.len(), 1);
+    // No deletions remain after compaction; all 100 rows match `x < 100`.
+    assert_eq!(
+        batches[0].column(0).as_primitive::<Int64Type>().value(0),
+        100,
+    );
+}
+
+#[tokio::test]
+async fn test_scanner_count_rows_with_partial_index_coverage() {
+    // Index covers the first two fragments, then a third fragment is
+    // appended. The rule cannot answer the count from the index alone for
+    // the appended fragment, so it emits a split plan: CountFromMaskExec
+    // over the indexed fragments + AggregateExec(Partial)/FilteredReadExec
+    // over the rest, both unioned and summed by AggregateExec(Final).
+    let tmp = tempdir().unwrap();
+    let uri = tmp.path().to_str().unwrap();
+    let mut ds = create_numeric_dataset(uri, 2, 50).await;
+    ds.create_index(
+        &["x"],
+        IndexType::BTree,
+        None,
+        &ScalarIndexParams::default(),
+        true,
+    )
+    .await
+    .unwrap();
+
+    // Append a third fragment that the index does not cover.
+    let reader = gen_batch()
+        .col("x", array::step_custom::<Int64Type>(100, 1))
+        .col("y", array::step_custom::<Int64Type>(0, 2))
+        .col("category", array::cycle::<Int64Type>(vec![1, 2, 3]))
+        .into_reader_rows(
+            lance_datagen::RowCount::from(50),
+            lance_datagen::BatchCount::from(1),
+        );
+    let ds = Dataset::write(
+        reader,
+        uri,
+        Some(crate::dataset::WriteParams {
+            mode: crate::dataset::WriteMode::Append,
+            max_rows_per_file: 50,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(ds.get_fragments().len(), 3);
+
+    let mut scanner = ds.scan();
+    // `x < 1000` matches every row (values are 0..100 + 100..150). The
+    // pushdown branch contributes the first 100 from the indexed fragments;
+    // the scan branch contributes the 50 rows from the appended fragment.
+    scanner.filter("x < 1000").unwrap();
+    scanner
+        .aggregate(AggregateExpr::builder().count_star().build())
+        .unwrap();
+    // Pin target_parallelism=1 so EnforceDistribution produces a deterministic
+    // plan snapshot regardless of the machine's CPU count.
+    scanner.target_parallelism(1);
+    let plan = scanner.create_plan().await.unwrap();
+
+    assert_plan_node_equals(
+        plan.clone(),
+        "AggregateExec: mode=Final, gby=[], aggr=[count(Int32(1))]
+  CoalescePartitionsExec
+    UnionExec
+      CountFromMask
+        ScalarIndexQuery: query=[x < 1000]@x_idx(BTree)
+      AggregateExec: mode=Partial, gby=[], aggr=[count(Int32(1))]
+        LanceRead: uri=..., projection=[], num_fragments=1, range_before=None, range_after=None, row_id=false, row_addr=true, full_filter=x < Int64(1000), refine_filter=--",
+    )
+    .await
+    .unwrap();
+
+    let stream = execute_plan(plan, LanceExecutionOptions::default()).unwrap();
+    let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+    assert_eq!(batches.len(), 1);
+    assert_eq!(
+        batches[0].column(0).as_primitive::<Int64Type>().value(0),
+        150,
+    );
+}
+
 #[tokio::test]
 async fn test_scanner_count_rows_empty_result() {
     let ds = create_numeric_dataset("memory://test_count_rows_empty", 1, 100).await;
@@ -1340,7 +1598,7 @@ async fn test_scanner_count_rows_with_fts() {
     assert_plan_node_equals(
         plan.clone(),
         "AggregateExec: mode=Single, gby=[], aggr=[count(Int32(1))]
-  MatchQuery: column=text, query=document",
+  MatchQuery: column=text, query=[document]",
     )
     .await
     .unwrap();
diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs
index e785de7bee4..b12a7198e5a 100644
--- a/rust/lance/src/dataset/tests/dataset_index.rs
+++ b/rust/lance/src/dataset/tests/dataset_index.rs
@@ -34,8 +34,10 @@ use lance_core::utils::tempfile::TempStrDir;
 use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch};
 use lance_file::reader::{FileReader, FileReaderOptions};
 use lance_file::version::LanceFileVersion;
+use lance_index::optimize::OptimizeOptions;
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::{
+    InvertedListFormatVersion,
     query::{BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery},
     tokenizer::InvertedIndexParams,
 };
@@ -861,6 +863,148 @@ async fn test_fts_on_multiple_columns() {
     assert_eq!(results.num_rows(), 1);
 }
 
+fn nested_fts_batch(
+    ids: Vec<u64>,
+    a_values: Vec<Option<&str>>,
+    b_values: Vec<Option<&str>>,
+) -> RecordBatch {
+    let a_values = Arc::new(StringArray::from(a_values)) as ArrayRef;
+    let b_values = Arc::new(StringArray::from(b_values)) as ArrayRef;
+    let struct_array = StructArray::from(vec![
+        (
+            Arc::new(Field::new("a", DataType::Utf8, true)),
+            a_values.clone(),
+        ),
+        (
+            Arc::new(Field::new("b", DataType::Utf8, true)),
+            b_values.clone(),
+        ),
+    ]);
+    let struct_type = struct_array.data_type().clone();
+    RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            Field::new("id", DataType::UInt64, false),
+            Field::new("s", struct_type, true),
+        ])),
+        vec![
+            Arc::new(UInt64Array::from(ids)) as ArrayRef,
+            Arc::new(struct_array) as ArrayRef,
+        ],
+    )
+    .unwrap()
+}
+
+async fn nested_fts_result_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec<u64> {
+    let batch = dataset
+        .scan()
+        .full_text_search(query)
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let mut ids = batch["id"].as_primitive::<UInt64Type>().values().to_vec();
+    ids.sort_unstable();
+    ids
+}
+
+#[tokio::test]
+async fn test_fts_on_nested_fields() {
+    let batch = nested_fts_batch(
+        vec![0, 1, 2, 3],
+        vec![
+            Some("lance nested alpha"),
+            Some("plain text"),
+            None,
+            Some("phrase target here"),
+        ],
+        vec![
+            Some("metadata only"),
+            Some("database nested beta"),
+            Some("lance beta"),
+            Some("other"),
+        ],
+    );
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let test_uri = TempStrDir::default();
+    let mut dataset = Dataset::write(batches, &test_uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["s.a"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default().with_position(true),
+            true,
+        )
+        .await
+        .unwrap();
+    dataset
+        .create_index(
+            &["s.b"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let indices = dataset.load_indices().await.unwrap();
+    let indexed_fields = indices
+        .iter()
+        .map(|index| dataset.schema().field_path(index.fields[0]).unwrap())
+        .collect::<HashSet<_>>();
+    assert_eq!(
+        indexed_fields,
+        HashSet::from(["s.a".to_string(), "s.b".to_string()])
+    );
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+        MatchQuery::new("alpha".to_owned()).with_column(Some("s.a".to_owned())),
+    ));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0]);
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::Match(
+        MatchQuery::new("beta".to_owned()).with_column(Some("s.b".to_owned())),
+    ));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![1, 2]);
+
+    assert_eq!(
+        nested_fts_result_ids(&dataset, FullTextSearchQuery::new("lance".to_owned())).await,
+        vec![0, 2]
+    );
+
+    let query = FullTextSearchQuery::new_query(FtsQuery::MultiMatch(MultiMatchQuery {
+        match_queries: vec![
+            MatchQuery::new("nested".to_owned()).with_column(Some("s.a".to_owned())),
+            MatchQuery::new("nested".to_owned()).with_column(Some("s.b".to_owned())),
+        ],
+    }));
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0, 1]);
+
+    let query = FullTextSearchQuery::new_query(
+        PhraseQuery::new("phrase target".to_owned())
+            .with_column(Some("s.a".to_owned()))
+            .into(),
+    );
+    assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![3]);
+
+    let append_batch = nested_fts_batch(
+        vec![4, 5],
+        vec![Some("fresh lance append"), Some("plain append")],
+        vec![Some("other"), Some("fresh beta append")],
+    );
+    let schema = append_batch.schema();
+    let batches = RecordBatchIterator::new(vec![append_batch].into_iter().map(Ok), schema);
+    dataset.append(batches, None).await.unwrap();
+
+    assert_eq!(
+        nested_fts_result_ids(&dataset, FullTextSearchQuery::new("fresh".to_owned())).await,
+        vec![4, 5]
+    );
+}
+
 #[tokio::test]
 async fn test_fts_unindexed_data() {
     let params = InvertedIndexParams::default();
@@ -936,6 +1080,68 @@ async fn test_fts_unindexed_data() {
     assert_eq!(results.num_rows(), 1);
 }
 
+#[tokio::test]
+async fn test_fts_v1_remains_queryable_after_append_optimize() {
+    let params = InvertedIndexParams::default().format_version(InvertedListFormatVersion::V1);
+    let text_col = StringArray::from(vec!["alpha original", "beta original"]);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![Field::new(
+            "text",
+            text_col.data_type().to_owned(),
+            false,
+        )])
+        .into(),
+        vec![Arc::new(text_col) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(batches, "memory://test.lance", None)
+        .await
+        .unwrap();
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+    assert_eq!(dataset.load_indices().await.unwrap()[0].index_version, 1);
+
+    let appended = StringArray::from(vec!["alpha appended"]);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![Field::new(
+            "text",
+            appended.data_type().to_owned(),
+            false,
+        )])
+        .into(),
+        vec![Arc::new(appended) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    dataset.append(batches, None).await.unwrap();
+    dataset
+        .optimize_indices(&OptimizeOptions::append())
+        .await
+        .unwrap();
+
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("alpha".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 2);
+    assert!(
+        dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .all(|index| index.index_version == 1)
+    );
+}
+
 #[tokio::test]
 async fn test_fts_unindexed_data_with_stop_words() {
     // When indexed data has avg_doc_length < 1.0 (e.g. single-word stop words
@@ -1137,6 +1343,78 @@ async fn test_fts_without_index() {
     assert_eq!(results.num_rows(), 1);
 }
 
+#[tokio::test]
+async fn test_fts_without_index_uses_scalar_index_for_prefilter() {
+    // Verify that flat FTS (no inverted index on text) routes its prefilter
+    // through `FilteredReadExec` so a scalar index on the filter column is
+    // actually used. Six rows with two distinct ids: a prefilter of `id = 1`
+    // must match exactly the three text rows tagged with id=1.
+    let text = StringArray::from(vec![
+        "alpha bravo",
+        "charlie delta",
+        "alpha echo",
+        "foxtrot",
+        "alpha golf",
+        "hotel india",
+    ]);
+    let ids = Int32Array::from(vec![1, 1, 1, 2, 2, 2]);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![
+            Field::new("text", text.data_type().to_owned(), false),
+            Field::new("id", ids.data_type().to_owned(), false),
+        ])
+        .into(),
+        vec![Arc::new(text) as ArrayRef, Arc::new(ids) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let test_uri = TempStrDir::default();
+    let mut dataset = Dataset::write(batches, &test_uri, None).await.unwrap();
+
+    // Scalar index on `id` only — no FTS index on `text`.
+    dataset
+        .create_index(
+            &["id"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let mut scan = dataset.scan();
+    scan.prefilter(true)
+        .full_text_search(
+            FullTextSearchQuery::new("alpha".to_owned())
+                .with_columns(&["text".to_string()])
+                .unwrap(),
+        )
+        .unwrap()
+        .filter("id = 1")
+        .unwrap();
+
+    let plan = scan.analyze_plan().await.unwrap();
+    // The flat-FTS path now reads via `FilteredReadExec` (prints as `LanceRead`)
+    // with the prefilter plumbed into it, so the scalar index on `id` is used.
+    assert_contains!(&plan, "FlatMatchQuery");
+    assert_contains!(&plan, "LanceRead");
+    assert_contains!(&plan, "full_filter=id = Int32(1)");
+    // The legacy plan ran a `LanceScan` wrapped in a manual `LanceFilterExec`;
+    // make sure we did not regress to that shape.
+    assert_not_contains!(&plan, "LanceScan:");
+
+    let results = scan.try_into_batch().await.unwrap();
+    // Only rows with id=1 AND text matching "alpha": rows 0 ("alpha bravo")
+    // and 2 ("alpha echo"). Row 4 ("alpha golf") has id=2 and must be excluded.
+    assert_eq!(
+        results.num_rows(),
+        2,
+        "expected the two id=1 rows that match `alpha`, got plan:\n{plan}"
+    );
+}
+
 #[tokio::test]
 async fn test_fts_rank() {
     let params = InvertedIndexParams::default();
@@ -1204,6 +1482,83 @@ async fn test_fts_rank() {
     assert_eq!(row_ids, &[0]);
 }
 
+#[tokio::test]
+async fn test_fts_unfiltered_after_filtered_returns_real_row_ids() {
+    // After a filtered FTS scan populates the per-partition cache,
+    // the next unfiltered scan must still return real row_ids, not
+    // partition-local doc_ids. Needs >1 fragment so the two differ
+    // (fragment N's row_ids start at N << 32).
+    let text_col = GenericStringArray::<i32>::from(vec![
+        "alpha first",
+        "alpha second",
+        "alpha third",
+        "alpha fourth",
+    ]);
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            text_col.data_type().to_owned(),
+            false,
+        )])
+        .into(),
+        vec![Arc::new(text_col) as ArrayRef],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let test_uri = TempStrDir::default();
+    let mut dataset = Dataset::write(
+        RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema),
+        &test_uri,
+        Some(WriteParams {
+            max_rows_per_file: 1,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    dataset
+        .create_index(
+            &["text"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let fts = |ds: &Dataset, filter: Option<&str>| {
+        let mut s = ds.scan();
+        s.with_row_id()
+            .full_text_search(FullTextSearchQuery::new("alpha".to_owned()))
+            .unwrap();
+        if let Some(f) = filter {
+            s.prefilter(true).filter(f).unwrap();
+        }
+        s
+    };
+    let sorted_row_ids = |b: &RecordBatch| {
+        let mut v: Vec<u64> = b[ROW_ID].as_primitive::<UInt64Type>().values().to_vec();
+        v.sort();
+        v
+    };
+
+    let fresh = sorted_row_ids(&fts(&dataset, None).try_into_batch().await.unwrap());
+    assert_eq!(fresh.len(), 4);
+
+    // Reopen so the baseline scan's cached LazyDocSet doesn't mask
+    // the regression -- the filtered scan needs to be the first
+    // thing that touches the DocSet.
+    let dataset = Dataset::open(test_uri.as_str()).await.unwrap();
+    fts(&dataset, Some("text LIKE 'alpha first%'"))
+        .try_into_batch()
+        .await
+        .unwrap();
+
+    let after = sorted_row_ids(&fts(&dataset, None).try_into_batch().await.unwrap());
+    assert_eq!(after, fresh);
+}
+
 async fn create_fts_dataset<
     Offset: arrow::array::OffsetSizeTrait,
     ListOffset: arrow::array::OffsetSizeTrait,
@@ -1590,6 +1945,169 @@ async fn test_fts_index_with_large_string() {
     test_fts_index::<i64, i64>(true).await;
 }
 
+#[tokio::test]
+async fn test_fts_list_index_uses_row_level_documents() {
+    let tempdir = TempStrDir::default();
+    let uri = tempdir.to_owned();
+    drop(tempdir);
+
+    let mut list_col = GenericListBuilder::<i32, _>::new(GenericStringBuilder::<i32>::new());
+    list_col.values().append_value("lance");
+    list_col.values().append_value("lance database");
+    list_col.append(true);
+    list_col.values().append_value("database");
+    list_col.append(true);
+    list_col.append(true);
+    list_col.values().append_null();
+    list_col.append(true);
+    list_col.append(false);
+
+    let docs = Arc::new(list_col.finish()) as ArrayRef;
+    let ids = Arc::new(UInt64Array::from_iter_values(0..docs.len() as u64)) as ArrayRef;
+    let batch = RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("doc", docs.data_type().clone(), true),
+            ArrowField::new("id", DataType::UInt64, false),
+        ])),
+        vec![docs, ids],
+    )
+    .unwrap();
+    let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    dataset
+        .create_index(
+            &["doc"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new("lance".to_owned()).limit(Some(10)))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[0]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(FullTextSearchQuery::new("database".to_owned()).limit(Some(10)))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let mut ids = result["id"]
+        .as_primitive::<UInt64Type>()
+        .values()
+        .iter()
+        .copied()
+        .collect::<Vec<_>>();
+    ids.sort_unstable();
+    assert_eq!(ids, vec![0, 1], "{:?}", result);
+}
+
+#[tokio::test]
+async fn test_fts_list_phrase_query_can_cross_elements() {
+    assert_fts_list_phrase_query_can_cross_elements::<i32>().await;
+}
+
+#[tokio::test]
+async fn test_fts_large_list_phrase_query_can_cross_elements() {
+    assert_fts_list_phrase_query_can_cross_elements::<i64>().await;
+}
+
+async fn assert_fts_list_phrase_query_can_cross_elements<Offset: arrow::array::OffsetSizeTrait>() {
+    let tempdir = TempStrDir::default();
+    let uri = tempdir.to_owned();
+    drop(tempdir);
+
+    let mut list_col = GenericListBuilder::<Offset, _>::new(GenericStringBuilder::<Offset>::new());
+    let rows: &[&[&str]] = &[
+        &["alpha", "beta"],
+        &["want the", "apple"],
+        &["want", "apple"],
+    ];
+    for values in rows.iter().copied() {
+        for value in values {
+            list_col.values().append_value(value);
+        }
+        list_col.append(true);
+    }
+
+    let docs = Arc::new(list_col.finish()) as ArrayRef;
+    let ids = Arc::new(UInt64Array::from(vec![0u64, 1, 2])) as ArrayRef;
+    let batch = RecordBatch::try_new(
+        Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("doc", docs.data_type().clone(), true),
+            ArrowField::new("id", DataType::UInt64, false),
+        ])),
+        vec![docs, ids],
+    )
+    .unwrap();
+    let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+    let mut dataset = Dataset::write(batches, &uri, None).await.unwrap();
+
+    let params = InvertedIndexParams::default()
+        .with_position(true)
+        .remove_stop_words(true);
+    dataset
+        .create_index(&["doc"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("alpha beta".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[0]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("want the apple".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[1]);
+
+    let result = dataset
+        .scan()
+        .project(&["id"])
+        .unwrap()
+        .full_text_search(
+            FullTextSearchQuery::new_query(PhraseQuery::new("want apple".to_owned()).into())
+                .limit(Some(10)),
+        )
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(result["id"].as_primitive::<UInt64Type>().values(), &[2]);
+}
+
 #[tokio::test]
 async fn test_fts_accented_chars() {
     let ds = create_fts_dataset::<i32, i32>(false, false, InvertedIndexParams::default()).await;
@@ -2001,11 +2519,7 @@ mod fts_serializing_backend {
         ) -> Option<CacheEntry> {
             let guard = self.serialized.lock().await;
             if let Some((bytes, stored_codec, _)) = guard.get(key) {
-                return Some(
-                    stored_codec
-                        .deserialize(&bytes.clone())
-                        .expect("deserialization should succeed"),
-                );
+                return stored_codec.deserialize(&bytes.clone()).hit();
             }
             drop(guard);
             self.passthrough.get(key, codec).await
@@ -3534,3 +4048,115 @@ async fn test_legacy_dataset_uses_v2_0_for_indexes() {
         "Index files should never use legacy format, even for legacy datasets"
     );
 }
+
+#[tokio::test]
+async fn test_manifest_read_recovers_from_stale_size() {
+    // A cached `ManifestLocation.size` can lag the real object: a reader may pick
+    // up a size from a stale listing/hint while another writer is committing
+    // concurrently. Reading the manifest (or its index section) with that stale
+    // size must not fail with a spurious "file size is too small" error. The
+    // reader should drop the cached size, fetch the true size, and succeed.
+    use crate::session::Session;
+    use lance_table::io::commit::ManifestLocation;
+    use lance_table::io::manifest::read_manifest_indexes;
+
+    let test_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "id",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from((0..100).collect::<Vec<i32>>()))],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+
+    let mut dataset = Dataset::write(reader, &test_uri, None).await.unwrap();
+    dataset
+        .create_index(
+            &["id"],
+            IndexType::BTree,
+            Some("id_idx".to_string()),
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+
+    let real_location = dataset.manifest_location().clone();
+    assert!(real_location.size.is_some());
+
+    // A deliberately-too-small size stands in for a stale cached size. Without the
+    // retry, both reads below decode a bogus footer offset and fail with
+    // "file size is too small".
+    let stale_location = ManifestLocation {
+        size: Some(1),
+        ..real_location.clone()
+    };
+
+    let session = Session::default();
+    let manifest = Dataset::load_manifest(
+        dataset.object_store.as_ref(),
+        &stale_location,
+        test_uri.as_ref(),
+        &session,
+    )
+    .await
+    .expect("load_manifest should recover from a stale manifest size");
+    assert_eq!(manifest.version, real_location.version);
+
+    let indices = read_manifest_indexes(dataset.object_store.as_ref(), &stale_location, &manifest)
+        .await
+        .expect("read_manifest_indexes should recover from a stale manifest size");
+    assert_eq!(indices.len(), 1);
+    assert_eq!(indices[0].name, "id_idx");
+}
+
+/// `load_segment_params` must match the fully opened segment's params,
+/// including `custom_stop_words` — the field `InvertedIndexDetails` loses.
+#[tokio::test]
+async fn test_load_segment_params_full_fidelity() {
+    use crate::index::DatasetIndexInternalExt;
+    use lance_index::metrics::NoOpMetricsCollector;
+    use lance_index::scalar::inverted::InvertedIndex;
+
+    let batch = RecordBatch::try_new(
+        arrow_schema::Schema::new(vec![Field::new("text", DataType::Utf8, false)]).into(),
+        vec![Arc::new(StringArray::from(vec![
+            "the quick brown fox",
+            "lazy dogs sleep",
+        ]))],
+    )
+    .unwrap();
+    let schema = batch.schema();
+    let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema);
+    let mut dataset = Dataset::write(stream, "memory://test/segment_params", None)
+        .await
+        .unwrap();
+
+    let params = InvertedIndexParams::default().custom_stop_words(Some(vec!["quick".to_string()]));
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    let segments = crate::index::scalar::load_segments(&dataset, "text")
+        .await
+        .unwrap()
+        .expect("FTS index segments");
+    let read = crate::index::scalar::load_segment_params(&dataset, &segments[0])
+        .await
+        .unwrap();
+
+    let generic = dataset
+        .open_generic_index("text", &segments[0].uuid, &NoOpMetricsCollector)
+        .await
+        .unwrap();
+    let opened = generic
+        .as_any()
+        .downcast_ref::<InvertedIndex>()
+        .expect("inverted index");
+    assert_eq!(&read, opened.params());
+}
diff --git a/rust/lance/src/dataset/tests/dataset_io.rs b/rust/lance/src/dataset/tests/dataset_io.rs
index c0c90fc71c9..f9618914037 100644
--- a/rust/lance/src/dataset/tests/dataset_io.rs
+++ b/rust/lance/src/dataset/tests/dataset_io.rs
@@ -12,10 +12,11 @@ use crate::dataset::WriteMode::Overwrite;
 use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::{ManifestWriteConfig, write_manifest_file};
 use crate::session::Session;
+use crate::session::caches::ManifestKey;
 use crate::{Dataset, Error, Result};
 use lance_table::format::DataStorageFormat;
 
-use crate::dataset::write::{WriteMode, WriteParams};
+use crate::dataset::write::{CommitBuilder, InsertBuilder, WriteMode, WriteParams};
 use arrow::array::as_struct_array;
 use arrow::compute::concat_batches;
 use arrow_array::RecordBatch;
@@ -33,7 +34,10 @@ use lance_arrow::bfloat16::{self, BFLOAT16_EXT_NAME};
 use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
 use lance_core::utils::tempfile::{TempStdDir, TempStrDir};
 use lance_datagen::{BatchCount, RowCount, array, gen_batch};
-use lance_file::version::LanceFileVersion;
+use lance_file::{
+    version::LanceFileVersion,
+    writer::{FileWriter, FileWriterOptions},
+};
 use lance_io::assert_io_eq;
 use lance_table::feature_flags;
 use lance_table::format::BasePath;
@@ -241,6 +245,62 @@ async fn test_with_object_store_wrappers_wraps_base_store_params() {
     assert!(request_tracker.incremental_stats().read_iops > 0);
 }
 
+#[tokio::test]
+async fn test_store_params_for_base_resolves_base_scoped_options() {
+    let test_dir = TempStdDir::default();
+    create_file(&test_dir, WriteMode::Create, LanceFileVersion::Stable).await;
+    let uri = test_dir.to_str().unwrap();
+    let dataset = Arc::new(Dataset::open(uri).await.unwrap());
+
+    let base_dir = tempfile::tempdir().unwrap();
+    let base_uri = file_object_store_uri(base_dir.path());
+    let base = BasePath::new(1, base_uri, Some("base".to_string()), true);
+    dataset.add_bases(vec![base.clone()], None).await.unwrap();
+
+    // Reopen with a single flat storage options map carrying base-scoped
+    // entries (`base_<id>.<key>`) next to shared defaults.
+    let dataset = DatasetBuilder::from_uri(uri)
+        .with_storage_options(HashMap::from([
+            ("shared_option".to_string(), "shared".to_string()),
+            (
+                "base_1.scoped_option".to_string(),
+                "base1-value".to_string(),
+            ),
+        ]))
+        .load()
+        .await
+        .unwrap();
+
+    // The registered base resolves the scoped entry on top of shared defaults.
+    let base_path = dataset.manifest.base_paths.get(&1).unwrap().clone();
+    let params = dataset.store_params_for_base(Some(&base_path));
+    assert_eq!(
+        params.storage_options().unwrap(),
+        &HashMap::from([
+            ("shared_option".to_string(), "shared".to_string()),
+            ("scoped_option".to_string(), "base1-value".to_string()),
+        ])
+    );
+
+    // The default scope keeps only the shared defaults.
+    let params = dataset.store_params_for_base(None);
+    assert_eq!(
+        params.storage_options().unwrap(),
+        &HashMap::from([("shared_option".to_string(), "shared".to_string())])
+    );
+
+    // The base store resolved from scoped options is usable end to end.
+    let base_store = dataset.object_store(Some(1)).await.unwrap();
+    let probe = base
+        .extract_path(dataset.session().store_registry())
+        .unwrap()
+        .join("data")
+        .join("probe.lance");
+    base_store.put(&probe, b"hello").await.unwrap();
+    let read = base_store.inner.get_range(&probe, 0..5).await.unwrap();
+    assert_eq!(read.as_ref(), b"hello");
+}
+
 #[tokio::test]
 async fn test_with_object_store_wrappers_wraps_refs_store() {
     let test_dir = tempfile::tempdir().unwrap();
@@ -327,6 +387,118 @@ async fn test_create_data_file_uses_base_object_store() {
     assert!(tracker.incremental_stats().read_iops > 0);
 }
 
+#[tokio::test]
+async fn test_create_data_file_rejects_nested_schema_mismatch() {
+    let dataset_uri = format!(
+        "memory://test_create_data_file_rejects_nested_schema_mismatch/{}",
+        uuid::Uuid::new_v4()
+    );
+
+    let dataset_struct_fields = vec![
+        ArrowField::new("a", DataType::Int32, true),
+        ArrowField::new("b", DataType::Int32, true),
+    ];
+    let dataset_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "s",
+        DataType::Struct(dataset_struct_fields.clone().into()),
+        true,
+    )]));
+    let dataset_struct = StructArray::try_new(
+        dataset_struct_fields.clone().into(),
+        vec![
+            Arc::new(Int32Array::from(vec![1])) as ArrayRef,
+            Arc::new(Int32Array::from(vec![2])) as ArrayRef,
+        ],
+        None,
+    )
+    .unwrap();
+    let dataset_batch =
+        RecordBatch::try_new(dataset_schema.clone(), vec![Arc::new(dataset_struct)]).unwrap();
+    let dataset = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(dataset_batch)], dataset_schema),
+        &dataset_uri,
+        Some(WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_2),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    async fn write_replacement_file(
+        dataset: &Dataset,
+        file_name: &str,
+        struct_fields: Vec<ArrowField>,
+    ) {
+        let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "s",
+            DataType::Struct(struct_fields.clone().into()),
+            true,
+        )]));
+        let values = struct_fields
+            .iter()
+            .enumerate()
+            .map(|(idx, _)| Arc::new(Int32Array::from(vec![idx as i32])) as ArrayRef)
+            .collect::<Vec<_>>();
+        let struct_array = StructArray::try_new(struct_fields.into(), values, None).unwrap();
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array) as ArrayRef]).unwrap();
+
+        let object_writer = dataset
+            .object_store
+            .create(&dataset.data_dir().join(file_name))
+            .await
+            .unwrap();
+        let mut writer = FileWriter::try_new(
+            object_writer,
+            crate::datatypes::Schema::try_from(schema.as_ref()).unwrap(),
+            FileWriterOptions {
+                format_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            },
+        )
+        .unwrap();
+        writer.write_batch(&batch).await.unwrap();
+        writer.finish().await.unwrap();
+    }
+
+    write_replacement_file(
+        &dataset,
+        "nested_reordered.lance",
+        vec![
+            ArrowField::new("b", DataType::Int32, true),
+            ArrowField::new("a", DataType::Int32, true),
+        ],
+    )
+    .await;
+    let err = dataset
+        .create_data_file("nested_reordered.lance", None)
+        .await
+        .unwrap_err();
+    assert!(
+        err.to_string().contains("Schema mismatch"),
+        "unexpected error: {err}"
+    );
+
+    write_replacement_file(
+        &dataset,
+        "nested_unknown.lance",
+        vec![
+            ArrowField::new("x", DataType::Int32, true),
+            ArrowField::new("y", DataType::Int32, true),
+        ],
+    )
+    .await;
+    let err = dataset
+        .create_data_file("nested_unknown.lance", None)
+        .await
+        .unwrap_err();
+    assert!(
+        err.to_string().contains("Schema mismatch"),
+        "unexpected error: {err}"
+    );
+}
+
 #[tokio::test]
 async fn test_shallow_clone_base_artifacts_use_base_object_store() {
     let source_dir = tempfile::tempdir().unwrap();
@@ -692,12 +864,82 @@ async fn test_load_manifest_iops() {
         .await
         .unwrap();
 
-    // There should be only two IOPS:
-    // 1. List _versions directory to get the latest manifest location
-    // 2. Read the manifest file. (The manifest is small enough to be read in one go.
-    //    Larger manifests would result in more IOPS.)
+    // The write above committed on this same Session, so the manifest is already
+    // in the metadata cache. Opening therefore issues a single IOP:
+    // 1. List _versions directory to resolve the latest manifest location.
+    // The manifest body is served from the cache instead of being read from storage.
     let io_stats = _dataset.object_store.as_ref().io_stats_incremental();
-    assert_io_eq!(io_stats, read_iops, 2);
+    assert_io_eq!(io_stats, read_iops, 1);
+}
+
+#[tokio::test]
+async fn test_checkout_removed_version_not_served_from_cache() {
+    let test_uri = TempStrDir::default();
+    let session = Arc::new(Session::default());
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from_iter_values(0..10_i32))],
+    )
+    .unwrap();
+    let dataset = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema.clone()),
+        &test_uri,
+        Some(WriteParams {
+            session: Some(session.clone()),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let version = dataset.manifest().version;
+    let location = dataset.manifest_location().clone();
+    let cache = session.metadata_cache.for_dataset(&dataset.uri);
+
+    assert!(
+        cache
+            .get_with_key(&ManifestKey {
+                version,
+                e_tag: location.e_tag.as_deref(),
+            })
+            .await
+            .is_some(),
+        "manifest should be cached after the write"
+    );
+    dataset.checkout_version(version).await.unwrap();
+
+    // Remove the version from storage, as cleanup (or a manual delete) would.
+    dataset.object_store.delete(&location.path).await.unwrap();
+
+    let resolved = dataset
+        .commit_handler
+        .resolve_version_location(&dataset.base, version, &dataset.object_store.inner)
+        .await
+        .unwrap();
+    assert!(
+        resolved.size.is_none(),
+        "resolving a removed version must fall back to a size-less location, got {:?}",
+        resolved.size
+    );
+
+    cache
+        .insert_with_key(
+            &ManifestKey {
+                version,
+                e_tag: None,
+            },
+            Arc::new(dataset.manifest().clone()),
+        )
+        .await;
+    assert!(
+        dataset.checkout_version(version).await.is_err(),
+        "checkout of a version removed from storage must not be served from cache"
+    );
 }
 
 #[rstest]
@@ -890,6 +1132,164 @@ async fn test_write_manifest(
     assert!(matches!(write_result, Err(Error::NotSupported { .. })));
 }
 
+#[tokio::test]
+async fn test_rle_v2_v23_write_and_append() {
+    let test_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![7; 1000]))],
+    )
+    .unwrap();
+
+    let batches = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone());
+    let mut dataset = Dataset::write(
+        batches,
+        &test_uri,
+        Some(WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_3),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let manifest = read_manifest(
+        dataset.object_store.as_ref(),
+        &dataset
+            .commit_handler
+            .resolve_latest_location(&dataset.base, dataset.object_store.as_ref())
+            .await
+            .unwrap()
+            .path,
+        None,
+    )
+    .await
+    .unwrap();
+    assert_eq!(
+        manifest.data_storage_format.lance_file_version().unwrap(),
+        LanceFileVersion::V2_3
+    );
+
+    let append_batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![9; 1000]))],
+    )
+    .unwrap();
+    let append_batches =
+        RecordBatchIterator::new(vec![Ok(append_batch)].into_iter(), schema.clone());
+    dataset = Dataset::write(
+        append_batches,
+        &test_uri,
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    assert_eq!(
+        dataset
+            .manifest
+            .data_storage_format
+            .lance_file_version()
+            .unwrap(),
+        LanceFileVersion::V2_3
+    );
+
+    let actual = dataset.scan().try_into_batch().await.unwrap();
+    let expected = RecordBatch::try_new(
+        schema,
+        vec![Arc::new(Int32Array::from(
+            [vec![7; 1000], vec![9; 1000]].concat(),
+        ))],
+    )
+    .unwrap();
+    assert_eq!(actual, expected);
+}
+
+#[tokio::test]
+async fn test_rle_v2_uncommitted_create_commits_v23_storage() {
+    let test_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![7; 1000]))],
+    )
+    .unwrap();
+
+    let transaction = InsertBuilder::new(test_uri.as_str())
+        .with_params(&WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_3),
+            ..Default::default()
+        })
+        .execute_uncommitted(vec![batch])
+        .await
+        .unwrap();
+
+    let dataset = CommitBuilder::new(test_uri.as_str())
+        .execute(transaction)
+        .await
+        .unwrap();
+    assert_eq!(
+        dataset
+            .manifest
+            .data_storage_format
+            .lance_file_version()
+            .unwrap(),
+        LanceFileVersion::V2_3
+    );
+}
+
+#[tokio::test]
+async fn test_rle_v2_shallow_clone_preserves_v23_storage() {
+    let test_uri = TempStrDir::default();
+    let clone_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::Int32,
+        false,
+    )]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![7; 1000]))],
+    )
+    .unwrap();
+
+    let mut dataset = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema),
+        &test_uri,
+        Some(WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_3),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let clone = dataset
+        .shallow_clone(clone_uri.as_str(), dataset.version().version, None)
+        .await
+        .unwrap();
+    assert_eq!(
+        clone
+            .manifest
+            .data_storage_format
+            .lance_file_version()
+            .unwrap(),
+        LanceFileVersion::V2_3
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn append_dataset(
diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs
index 6374a0d2867..4b81f62dc6c 100644
--- a/rust/lance/src/dataset/tests/dataset_merge_update.rs
+++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs
@@ -1,20 +1,24 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use std::collections::HashSet;
 use std::sync::Arc;
+use std::time::Duration;
 use std::vec;
 
 use crate::dataset::ROW_ID;
 use crate::dataset::WriteDestination;
 use crate::dataset::optimize::{CompactionOptions, compact_files};
 use crate::dataset::transaction::{DataReplacementGroup, Operation};
-use crate::dataset::{AutoCleanupParams, MergeInsertBuilder, ProjectionRequest};
+use crate::dataset::{AutoCleanupParams, MergeInsertBuilder, ProjectionRequest, UpdateBuilder};
 use crate::index::DatasetIndexExt;
 use crate::{Dataset, Error};
 use lance_core::ROW_ADDR;
 use lance_index::IndexType;
 use lance_index::optimize::OptimizeOptions;
+use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::ScalarIndexParams;
+use lance_index::scalar::inverted::tokenizer::InvertedIndexParams;
 use mock_instant::thread_local::MockClock;
 
 use crate::dataset::write::{InsertBuilder, WriteMode, WriteParams};
@@ -26,7 +30,7 @@ use arrow_array::{
     ArrayRef, Float32Array, Int32Array, ListArray, RecordBatchIterator, StringArray,
     types::Int32Type,
 };
-use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
+use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema};
 use lance_arrow::BLOB_META_KEY;
 use lance_core::utils::tempfile::{TempDir, TempStrDir};
 use lance_datafusion::utils::reader_to_stream;
@@ -34,7 +38,7 @@ use lance_datagen::{BatchCount, RowCount, array, gen_batch};
 use lance_file::version::LanceFileVersion;
 use lance_file::writer::FileWriter;
 use lance_io::utils::CachedFileSize;
-use lance_table::format::DataFile;
+use lance_table::format::{BasePath, DataFile, Fragment};
 
 use crate::dataset::write::merge_insert::{WhenMatched, WhenNotMatched};
 use futures::TryStreamExt;
@@ -1045,7 +1049,8 @@ async fn test_datafile_replacement_error() {
         Operation::DataReplacement {
             replacements: vec![DataReplacementGroup(0, new_data_file)],
         },
-        Some(2),
+        // read at the current version (after the Merge above)
+        Some(dataset.manifest.version),
         None,
         None,
         Arc::new(Default::default()),
@@ -1624,32 +1629,47 @@ async fn test_merge_insert_with_reordered_columns_and_index() {
     final_dataset.validate().await.unwrap();
 }
 
-/// DataReplacement should invalidate index fragment bitmaps for replaced fields.
+/// With stable row ids, updating a top-level struct column keeps a scalar index on a
+/// nested child field correct. The update API rejects nested column references, so a
+/// nested field can only be changed by setting its whole struct column; that update must
+/// not wrongly extend the child-field index over the rewritten fragment (which would
+/// leave the updated value unscanned and silently dropped).
 #[tokio::test]
-async fn test_data_replacement_invalidates_index_bitmap() {
+async fn test_update_struct_column_keeps_nested_index() {
+    let struct_fields = Fields::from(vec![ArrowField::new("x", DataType::Int32, true)]);
     let schema = Arc::new(ArrowSchema::new(vec![
-        ArrowField::new("a", DataType::Int32, true),
-        ArrowField::new("b", DataType::Int32, true),
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("s", DataType::Struct(struct_fields.clone()), true),
     ]));
-
-    // Create dataset with 2 columns
+    let s_arr = StructArray::new(
+        struct_fields.clone(),
+        vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef],
+        None,
+    );
     let batch = RecordBatch::try_new(
         schema.clone(),
         vec![
-            Arc::new(Int32Array::from(vec![1, 2, 3])),
-            Arc::new(Int32Array::from(vec![10, 20, 30])),
+            Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef,
+            Arc::new(s_arr) as ArrayRef,
         ],
     )
     .unwrap();
     let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
-    let mut dataset = Dataset::write(reader, "memory://test_replacement_idx", None)
-        .await
-        .unwrap();
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_update_nested_index",
+        Some(WriteParams {
+            enable_stable_row_ids: true,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
 
-    // Create scalar index on column 'a'
+    // BTree index on the NESTED field `s.x`.
     dataset
         .create_index(
-            &["a"],
+            &["s.x"],
             IndexType::BTree,
             None,
             &ScalarIndexParams::default(),
@@ -1658,85 +1678,98 @@ async fn test_data_replacement_invalidates_index_bitmap() {
         .await
         .unwrap();
 
-    // Verify fragment 0 is in the index bitmap
-    let indices = dataset.load_indices().await.unwrap();
-    let a_index = indices.iter().find(|idx| idx.name == "a_idx").unwrap();
-    assert!(a_index.fragment_bitmap.as_ref().unwrap().contains(0));
-
-    // Write a replacement data file for column 'a'
-    let single_col_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
-        "a",
-        DataType::Int32,
-        true,
-    )]));
-    let replacement_batch = RecordBatch::try_new(
-        single_col_schema.clone(),
-        vec![Arc::new(Int32Array::from(vec![4, 5, 6]))],
-    )
-    .unwrap();
-
-    let object_writer = dataset
-        .object_store
-        .create(&Path::from("data/replacement.lance"))
+    let pre = dataset
+        .scan()
+        .filter("s.x = 20")
+        .unwrap()
+        .try_into_batch()
         .await
         .unwrap();
-    let mut writer = FileWriter::try_new(
-        object_writer,
-        single_col_schema.as_ref().try_into().unwrap(),
-        Default::default(),
-    )
-    .unwrap();
-    writer.write_batch(&replacement_batch).await.unwrap();
-    writer.finish().await.unwrap();
+    assert_eq!(pre.num_rows(), 1, "precondition: s.x=20 should match id=2");
 
-    // Build replacement DataFile matching the existing data file for column 'a'
-    let frag = dataset.get_fragment(0).unwrap();
-    let data_file = frag.data_file_for_field(0).unwrap();
-    let mut new_data_file = data_file.clone();
-    new_data_file.path = "replacement.lance".to_string();
-
-    // Commit DataReplacement
-    let read_version = dataset.version().version;
-    let dataset = Dataset::commit(
-        WriteDestination::Dataset(Arc::new(dataset)),
-        Operation::DataReplacement {
-            replacements: vec![DataReplacementGroup(0, new_data_file)],
-        },
-        Some(read_version),
-        None,
-        None,
-        Arc::new(Default::default()),
-        false,
-    )
-    .await
-    .unwrap();
+    // Nested column references are rejected by `set`, so update the whole struct column
+    // `s` for id=2, changing s.x 20 -> 999.
+    let update_result = UpdateBuilder::new(Arc::new(dataset.clone()))
+        .update_where("id = 2")
+        .unwrap()
+        .set("s", "named_struct('x', cast(999 as int))")
+        .unwrap()
+        .build()
+        .unwrap()
+        .execute()
+        .await
+        .unwrap();
+    let dataset = update_result.new_dataset;
 
-    // The index bitmap for 'a' should no longer contain fragment 0
-    let indices = dataset.load_indices().await.unwrap();
-    let a_index = indices.iter().find(|idx| idx.name == "a_idx").unwrap();
-    let effective = a_index
+    // The nested `s.x` index must NOT be extended to the rewritten fragment: its
+    // effective coverage stays {0}, so the rewritten fragment is left unindexed and
+    // fully scanned.
+    let sx_idx = dataset
+        .load_indices()
+        .await
+        .unwrap()
+        .iter()
+        .find(|i| i.fields.len() == 1)
+        .expect("nested s.x index")
+        .clone();
+    let effective = sx_idx
         .effective_fragment_bitmap(&dataset.fragment_bitmap)
+        .expect("index has a fragment bitmap");
+    assert_eq!(
+        effective.iter().collect::<Vec<_>>(),
+        vec![0],
+        "nested-field index must not be extended to the rewritten fragment"
+    );
+
+    // The updated value must be found, and the stale value gone.
+    let new = dataset
+        .scan()
+        .filter("s.x = 999")
+        .unwrap()
+        .try_into_batch()
+        .await
         .unwrap();
-    assert!(
-        !effective.contains(0),
-        "Fragment 0 should be removed from index bitmap after DataReplacement on indexed column"
+    assert_eq!(
+        new.num_rows(),
+        1,
+        "updated value s.x=999 must be found after the struct-column update"
     );
+    let old = dataset
+        .scan()
+        .filter("s.x = 20")
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(old.num_rows(), 0, "s.x=20 should no longer match any row");
 }
 
-/// DataReplacement on an indexed column should remove the fragment from
-/// fragment_bitmap AND add it to invalidated_fragment_bitmap so that
-/// stale index entries are blocked at query time.
-#[tokio::test]
-async fn test_data_replacement_populates_invalidated_bitmap() {
-    use lance_file::writer::FileWriter;
-    use object_store::path::Path;
+/// Sum a named execution metric (e.g. `fragments_scanned`) across every node of a
+/// physical plan. Used to observe FilteredReadExec data-scan behavior.
+fn sum_scan_metric(plan: &Arc<dyn datafusion::physical_plan::ExecutionPlan>, name: &str) -> usize {
+    let mut total = plan
+        .metrics()
+        .and_then(|m| m.sum_by_name(name))
+        .map(|v| v.as_usize())
+        .unwrap_or(0);
+    for child in plan.children() {
+        total += sum_scan_metric(child, name);
+    }
+    total
+}
 
+/// Control: with stable row ids, a merge_insert full-row update of a *flat*
+/// indexed column is handled correctly. The flat field's id is in
+/// `fields_for_preserving_frag_bitmap`, so the RewriteRows index-maintenance path
+/// does NOT extend the index bitmap to the rewritten fragment; that fragment stays
+/// unindexed and is fully scanned, so the new value is found and the old is not.
+/// Contrast with `test_merge_insert_nested_index_stable_row_id`.
+#[tokio::test]
+async fn test_merge_insert_flat_index_stable_row_id() {
     let schema = Arc::new(ArrowSchema::new(vec![
         ArrowField::new("id", DataType::Int32, false),
-        ArrowField::new("value", DataType::Int32, true),
+        ArrowField::new("val", DataType::Int32, false),
     ]));
-
-    // Create dataset with one fragment
     let batch = RecordBatch::try_new(
         schema.clone(),
         vec![
@@ -1746,14 +1779,21 @@ async fn test_data_replacement_populates_invalidated_bitmap() {
     )
     .unwrap();
     let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
-    let mut dataset = Dataset::write(reader, "memory://test_replacement_invalidated", None)
-        .await
-        .unwrap();
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_mi_flat_index",
+        Some(WriteParams {
+            enable_stable_row_ids: true,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
 
-    // Create BTree index on 'value'
+    // BTree index on the flat indexed column `val`.
     dataset
         .create_index(
-            &["value"],
+            &["val"],
             IndexType::BTree,
             None,
             &ScalarIndexParams::default(),
@@ -1762,170 +1802,424 @@ async fn test_data_replacement_populates_invalidated_bitmap() {
         .await
         .unwrap();
 
-    // Verify initial state: fragment 0 in bitmap, no invalidated fragments
-    let indices = dataset.load_indices().await.unwrap();
-    let idx = indices.iter().find(|i| i.name == "value_idx").unwrap();
-    assert!(idx.fragment_bitmap.as_ref().unwrap().contains(0));
+    // Sanity: index returns id=2 for val=20 before the update.
+    let pre = dataset
+        .scan()
+        .filter("val = 20")
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(pre.num_rows(), 1, "precondition: val=20 should match id=2");
 
-    // Write a replacement data file for column 'value'
-    let value_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
-        "value",
-        DataType::Int32,
-        true,
-    )]));
-    let replacement_batch = RecordBatch::try_new(
-        value_schema.clone(),
-        vec![Arc::new(Int32Array::from(vec![40, 50, 60]))],
+    // Full-row update of id=2 changing val 20 -> 999. Only id=2 is in the source
+    // (all matched, no inserts) so the new fragment is a pure rewrite-rows fragment.
+    let source = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![2])),
+            Arc::new(Int32Array::from(vec![999])),
+        ],
     )
     .unwrap();
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone()));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
 
-    let object_writer = dataset
-        .object_store
-        .create(&Path::from("data/replacement_inv.lance"))
+    // Index bitmap coverage: the guard did NOT extend the bitmap, so the index
+    // still covers only the original fragment 0, not the rewritten fragment.
+    let val_idx = dataset
+        .load_indices()
         .await
-        .unwrap();
-    let mut writer = FileWriter::try_new(
-        object_writer,
-        value_schema.as_ref().try_into().unwrap(),
-        Default::default(),
-    )
-    .unwrap();
-    writer.write_batch(&replacement_batch).await.unwrap();
-    writer.finish().await.unwrap();
+        .unwrap()
+        .iter()
+        .find(|i| i.name == "val_idx")
+        .unwrap()
+        .clone();
+    assert_eq!(
+        val_idx
+            .fragment_bitmap
+            .as_ref()
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>(),
+        vec![0],
+        "flat index must not be extended to the rewritten fragment"
+    );
 
-    // Build replacement DataFile
-    let frag = dataset.get_fragment(0).unwrap();
-    let lance_schema: lance_core::datatypes::Schema = schema.as_ref().try_into().unwrap();
-    let value_field_id = lance_schema.field("value").unwrap().id;
-    let data_file = frag.data_file_for_field(value_field_id as u32).unwrap();
-    let mut new_data_file = data_file.clone();
-    new_data_file.path = "replacement_inv.lance".to_string();
+    // The scan is PARTIALLY indexed, not fully indexed: the index's effective
+    // coverage ({0}) is a strict subset of the live fragments ({0, 1}), so the
+    // rewritten fragment is left to a full scan.
+    assert_eq!(
+        dataset.fragment_bitmap.iter().collect::<Vec<_>>(),
+        vec![0, 1],
+        "the update should have produced a second fragment"
+    );
+    let effective = val_idx
+        .effective_fragment_bitmap(&dataset.fragment_bitmap)
+        .expect("index has a fragment bitmap");
+    assert_eq!(
+        effective.iter().collect::<Vec<_>>(),
+        vec![0],
+        "index effectively covers only fragment 0"
+    );
+    assert!(
+        effective.len() < dataset.fragment_bitmap.len(),
+        "scan must be partially indexed: the rewritten fragment is not covered by the index"
+    );
 
-    // Commit DataReplacement
-    let read_version = dataset.version().version;
-    let dataset = Dataset::commit(
-        WriteDestination::Dataset(Arc::new(dataset)),
-        Operation::DataReplacement {
-            replacements: vec![DataReplacementGroup(0, new_data_file)],
-        },
-        Some(read_version),
-        None,
-        None,
-        Arc::new(Default::default()),
-        false,
+    // Project only _rowid. The scan is PARTIALLY indexed: because the rewritten
+    // fragment is not covered by the index, FilteredReadExec must DATA-SCAN it to
+    // surface the moved value, which shows up in its `fragments_scanned` metric.
+    let mut scanner = dataset.scan();
+    scanner
+        .empty_project()
+        .unwrap()
+        .with_row_id()
+        .filter("val = 999")
+        .unwrap();
+    let plan = scanner.create_plan().await.unwrap();
+    let batches = datafusion::physical_plan::collect(
+        plan.clone(),
+        Arc::new(datafusion::execution::TaskContext::default()),
     )
     .await
     .unwrap();
-
-    // Check: fragment 0 removed from fragment_bitmap
-    let indices = dataset.load_indices().await.unwrap();
-    let idx = indices.iter().find(|i| i.name == "value_idx").unwrap();
-    assert!(
-        !idx.fragment_bitmap.as_ref().unwrap().contains(0),
-        "Fragment 0 should be removed from fragment_bitmap"
+    // Exactly one fragment — the uncovered (rewritten) one — is data-scanned.
+    let fragments_scanned = sum_scan_metric(&plan, "fragments_scanned");
+    assert_eq!(
+        fragments_scanned, 1,
+        "partially-indexed scan must data-scan exactly the one uncovered fragment, but fragments_scanned={fragments_scanned}"
+    );
+    let val999_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+    assert_eq!(
+        val999_rows, 1,
+        "val=999 lives only in the unindexed fragment and must be found by the full scan"
     );
+
+    // The stale old value is gone (the index-covered fragment's row was moved out).
+    let val20_rows = dataset
+        .scan()
+        .empty_project()
+        .unwrap()
+        .with_row_id()
+        .filter("val = 20")
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap()
+        .num_rows();
+    assert_eq!(val20_rows, 0, "val=20 should no longer match any row");
 }
 
-/// Regression test (lance-format/lance#6283): after in-place update via
-/// DataReplacement, stale FTS index entries for the replaced fragment must
-/// be blocked at query time so searches reflect the new data.
+/// With stable row ids, a full-row merge_insert update of a column covered by a
+/// NESTED-field scalar index keeps that index correct.
+///
+/// The updated row keeps its row id and moves to a new fragment.
+/// `register_pure_rewrite_rows_update_frags_in_indices` decides whether to extend each
+/// index's bitmap to that new fragment by testing `index.fields` against
+/// `fields_for_preserving_frag_bitmap`, which merge_insert builds via `fields_pre_order()`
+/// so that nested leaf ids are included. The nested `s.x` index must therefore NOT be
+/// extended over the rewritten fragment: that fragment stays unindexed and is fully
+/// scanned, so the updated value is found.
+///
+/// Regression guard: before the fix the set was built from top-level `schema().fields`,
+/// omitting the nested leaf id, so the index was wrongly extended over the rewritten
+/// fragment and the updated value was silently dropped.
 #[tokio::test]
-async fn test_fts_stale_entries_after_data_replacement() {
-    use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
-
+async fn test_merge_insert_nested_index_stable_row_id() {
+    let struct_fields = Fields::from(vec![ArrowField::new("x", DataType::Int32, false)]);
     let schema = Arc::new(ArrowSchema::new(vec![
         ArrowField::new("id", DataType::Int32, false),
-        ArrowField::new("text", DataType::Utf8, true),
+        ArrowField::new("s", DataType::Struct(struct_fields.clone()), false),
     ]));
+    let make_batch = |ids: Vec<i32>, xs: Vec<i32>| {
+        let s = StructArray::new(
+            struct_fields.clone(),
+            vec![Arc::new(Int32Array::from(xs)) as ArrayRef],
+            None,
+        );
+        RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(ids)) as ArrayRef,
+                Arc::new(s) as ArrayRef,
+            ],
+        )
+        .unwrap()
+    };
 
-    // Step 1: Create dataset with 2 rows in separate fragments
-    let batch = RecordBatch::try_new(
+    let reader = RecordBatchIterator::new(
+        vec![Ok(make_batch(vec![1, 2, 3], vec![10, 20, 30]))],
         schema.clone(),
-        vec![
-            Arc::new(Int32Array::from(vec![0, 1])),
-            Arc::new(StringArray::from(vec![
-                "the quick brown fox",
-                "the lazy dog",
-            ])),
-        ],
-    )
-    .unwrap();
-    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    );
     let mut dataset = Dataset::write(
         reader,
-        "memory://test_fts_incremental_reindex",
+        "memory://test_mi_nested_index",
         Some(WriteParams {
-            max_rows_per_file: 1, // Force 2 fragments
+            enable_stable_row_ids: true,
             ..Default::default()
         }),
     )
     .await
     .unwrap();
 
-    // Step 2: Create FTS inverted index on 'text'
-    let params = InvertedIndexParams::default();
+    // BTree index on the NESTED field `s.x`.
     dataset
-        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .create_index(
+            &["s.x"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
         .await
         .unwrap();
 
-    // Sanity check: "quick" and "lazy" should each return 1 result
-    let results = dataset
+    // Sanity: index finds id=2 for s.x = 20.
+    let pre = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .filter("s.x = 20")
         .unwrap()
         .try_into_batch()
         .await
         .unwrap();
-    assert_eq!(results.num_rows(), 1);
-    let results = dataset
-        .scan()
-        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+    assert_eq!(pre.num_rows(), 1, "precondition: s.x=20 should match id=2");
+
+    // Full-row merge_insert update of id=2 changing s.x 20 -> 999 (pure rewrite-rows fragment).
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
         .unwrap()
-        .try_into_batch()
-        .await
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .try_build()
         .unwrap();
-    assert_eq!(results.num_rows(), 1);
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(make_batch(vec![2], vec![999]))],
+        schema.clone(),
+    ));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
 
-    // Step 3: Replace fragment 1's data file via DataReplacement.
-    // The fragment ID stays the same, but the text changes from
-    // "the lazy dog" to "a speedy cat". This prunes fragment 1
-    // from the FTS index's fragment_bitmap.
-    let frag1 = dataset.get_fragment(1).unwrap();
-    let old_data_file = frag1.metadata().files[0].clone();
+    // The rewritten fragment must NOT be covered by the nested `s.x` index, so
+    // FilteredReadExec data-scans exactly that one fragment and finds the updated value.
+    // (Before the fix the index was wrongly extended over the rewritten fragment, so it
+    // was never data-scanned — fragments_scanned == 0 — and the value was dropped.)
+    let mut scanner = dataset.scan();
+    scanner
+        .empty_project()
+        .unwrap()
+        .with_row_id()
+        .filter("s.x = 999")
+        .unwrap();
+    let plan = scanner.create_plan().await.unwrap();
+    let batches = datafusion::physical_plan::collect(
+        plan.clone(),
+        Arc::new(datafusion::execution::TaskContext::default()),
+    )
+    .await
+    .unwrap();
+    let fragments_scanned = sum_scan_metric(&plan, "fragments_scanned");
+    assert_eq!(
+        fragments_scanned, 1,
+        "the rewritten fragment must be data-scanned exactly once, but fragments_scanned={fragments_scanned}"
+    );
+    let sx999_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+    assert_eq!(
+        sx999_rows, 1,
+        "updated value s.x=999 must be found after the nested-field merge_insert update"
+    );
+}
 
-    // Write replacement data file with updated text
-    let replacement_batch = RecordBatch::try_new(
+/// With stable row ids, a merge_insert full-row update invalidates EVERY column index
+/// for the rewritten rows — not only indices on columns that actually changed — because
+/// merge_insert treats the whole schema as modified (it does not detect which columns
+/// changed). Here `col1` is updated and `col2` is left unchanged, yet both `col1_idx`
+/// and `col2_idx` drop the rewritten fragment from their coverage.
+#[tokio::test]
+async fn test_merge_insert_flat_index_stable_row_id_multiple_indexes() {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("col1", DataType::Int32, false),
+        ArrowField::new("col2", DataType::Int32, false),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2])),
+            Arc::new(Int32Array::from(vec![10, 20])),
+            Arc::new(Int32Array::from(vec![100, 200])),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_mi_flat_multi_index",
+        Some(WriteParams {
+            enable_stable_row_ids: true,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    for col in ["col1", "col2"] {
+        dataset
+            .create_index(
+                &[col],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+    }
+
+    // Full-row update of id=1: col1 10 -> 999, col2 left unchanged (100).
+    let source = RecordBatch::try_new(
         schema.clone(),
         vec![
             Arc::new(Int32Array::from(vec![1])),
-            Arc::new(StringArray::from(vec!["a speedy cat"])),
+            Arc::new(Int32Array::from(vec![999])),
+            Arc::new(Int32Array::from(vec![100])),
         ],
     )
     .unwrap();
-    let replacement_path = dataset.data_dir().join("replacement.lance");
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone()));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
+
+    // The update produced a second fragment holding the rewritten row id=1.
+    assert_eq!(
+        dataset.fragment_bitmap.iter().collect::<Vec<_>>(),
+        vec![0, 1],
+        "update should have produced a second fragment"
+    );
+
+    let indices = dataset.load_indices().await.unwrap();
+    let covered = |name: &str| {
+        indices
+            .iter()
+            .find(|i| i.name == name)
+            .unwrap()
+            .fragment_bitmap
+            .as_ref()
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>()
+    };
+
+    // The index on the CHANGED column is invalidated (the rewritten fragment is not
+    // covered) -- expected.
+    assert_eq!(
+        covered("col1_idx"),
+        vec![0],
+        "col1 index must drop the rewritten fragment (col1 changed)"
+    );
+
+    // ALL column indexes are invalidated, not only the changed ones: the index on the
+    // UNCHANGED column `col2` also drops the rewritten fragment.
+    //
+    // TODO(stable-row-id optimization): merge_insert treats the whole schema as modified
+    // because it does not detect which columns actually changed, so every index is
+    // invalidated for the rewritten rows. With stable row ids the moved rows keep their
+    // row ids and col2's values are unchanged, so `col2_idx` could instead be EXTENDED to
+    // the rewritten fragment (preserving its coverage) rather than invalidated, avoiding
+    // an unnecessary reindex. See `register_pure_rewrite_rows_update_frags_in_indices`.
+    assert_eq!(
+        covered("col2_idx"),
+        vec![0],
+        "col2 index is also invalidated even though col2 was not changed (see TODO)"
+    );
+}
+
+/// DataReplacement should invalidate index fragment bitmaps for replaced fields.
+#[tokio::test]
+async fn test_data_replacement_invalidates_index_bitmap() {
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("a", DataType::Int32, true),
+        ArrowField::new("b", DataType::Int32, true),
+    ]));
+
+    // Create dataset with 2 columns
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2, 3])),
+            Arc::new(Int32Array::from(vec![10, 20, 30])),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(reader, "memory://test_replacement_idx", None)
+        .await
+        .unwrap();
+
+    // Create scalar index on column 'a'
+    dataset
+        .create_index(
+            &["a"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+    // Verify fragment 0 is in the index bitmap
+    let indices = dataset.load_indices().await.unwrap();
+    let a_index = indices.iter().find(|idx| idx.name == "a_idx").unwrap();
+    assert!(a_index.fragment_bitmap.as_ref().unwrap().contains(0));
+
+    // Write a replacement data file for column 'a'
+    let single_col_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "a",
+        DataType::Int32,
+        true,
+    )]));
+    let replacement_batch = RecordBatch::try_new(
+        single_col_schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![4, 5, 6]))],
+    )
+    .unwrap();
+
     let object_writer = dataset
         .object_store
-        .create(&replacement_path)
+        .create(&Path::from("data/replacement.lance"))
         .await
         .unwrap();
     let mut writer = FileWriter::try_new(
         object_writer,
-        schema.as_ref().try_into().unwrap(),
+        single_col_schema.as_ref().try_into().unwrap(),
         Default::default(),
     )
     .unwrap();
     writer.write_batch(&replacement_batch).await.unwrap();
     writer.finish().await.unwrap();
 
-    let mut new_data_file = old_data_file.clone();
+    // Build replacement DataFile matching the existing data file for column 'a'
+    let frag = dataset.get_fragment(0).unwrap();
+    let data_file = frag.data_file_for_field(0).unwrap();
+    let mut new_data_file = data_file.clone();
     new_data_file.path = "replacement.lance".to_string();
 
-    let read_version = dataset.manifest.version;
+    // Commit DataReplacement
+    let read_version = dataset.version().version;
     let dataset = Dataset::commit(
         WriteDestination::Dataset(Arc::new(dataset)),
         Operation::DataReplacement {
-            replacements: vec![DataReplacementGroup(1, new_data_file)],
+            replacements: vec![DataReplacementGroup(0, new_data_file)],
         },
         Some(read_version),
         None,
@@ -1936,173 +2230,280 @@ async fn test_fts_stale_entries_after_data_replacement() {
     .await
     .unwrap();
 
-    // Verify the replacement worked — fragment 1 now has the new text
-    let batch = dataset.scan().try_into_batch().await.unwrap();
-    assert_eq!(batch.num_rows(), 2);
+    // The index bitmap for 'a' should no longer contain fragment 0
+    let indices = dataset.load_indices().await.unwrap();
+    let a_index = indices.iter().find(|idx| idx.name == "a_idx").unwrap();
+    let effective = a_index
+        .effective_fragment_bitmap(&dataset.fragment_bitmap)
+        .unwrap();
+    assert!(
+        !effective.contains(0),
+        "Fragment 0 should be removed from index bitmap after DataReplacement on indexed column"
+    );
+}
 
-    // Step 4: FTS search should reflect the new data, not the old.
-    // Fragment 1 is unindexed (pruned from fragment_bitmap), so the
-    // scanner does a flat FTS scan on it and uses the index for fragment 0.
+/// Run a predicate over `col` twice -- once index-served, once via a forced flat scan
+/// (`use_scalar_index(false)`) -- assert the two agree, and return the matching `col`
+/// values sorted. Equality is the index-consistency invariant: a divergence means the
+/// index served rows that disagree with the underlying data.
+async fn index_consistent_values(dataset: &Dataset, col: &str, predicate: &str) -> Vec<i32> {
+    let sorted = |batch: &RecordBatch| -> Vec<i32> {
+        let mut v: Vec<i32> = batch
+            .column_by_name(col)
+            .unwrap()
+            .as_primitive::<Int32Type>()
+            .iter()
+            .flatten()
+            .collect();
+        v.sort();
+        v
+    };
 
-    // "speedy" is in the new text for fragment 1 — found via flat scan.
-    let results = dataset
+    let indexed = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("speedy".to_owned()))
+        .filter(predicate)
         .unwrap()
         .try_into_batch()
         .await
         .unwrap();
-    assert_eq!(results.num_rows(), 1);
-
-    // "lazy" was in the OLD text for fragment 1. The index has stale
-    // posting entries for it, but fragment 1 was pruned from the
-    // fragment_bitmap. Flat scan of fragment 1 sees "a speedy cat"
-    // which doesn't match. So 0 results.
-    let results = dataset
+    let flat = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .use_scalar_index(false)
+        .filter(predicate)
         .unwrap()
         .try_into_batch()
         .await
         .unwrap();
+
+    let indexed_vals = sorted(&indexed);
+    let flat_vals = sorted(&flat);
     assert_eq!(
-        results.num_rows(),
-        0,
-        "Expected 0 results for 'lazy' (stale data, fragment 1 pruned from index)"
+        indexed_vals, flat_vals,
+        "index-served `{predicate}` disagrees with a flat scan"
     );
+    indexed_vals
+}
 
-    // "quick" is in fragment 0 which is still indexed — should still work.
-    let results = dataset
-        .scan()
-        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
-        .unwrap()
-        .try_into_batch()
-        .await
-        .unwrap();
-    assert_eq!(results.num_rows(), 1);
+/// Build a Merge overlay fragment that rewrites a single `field_id` in place: tombstone
+/// (-2) the field in `prev`'s existing data files and back it with `new_file` (a new
+/// single-column file) instead. A file left with no live field is dropped. This is the
+/// manifest shape an in-place column rewrite produces when it falls back from a
+/// DataReplacement to a Merge.
+fn build_overlay_frag(prev: &Fragment, field_id: i32, new_file: &str) -> Fragment {
+    let mut overlay = prev.clone();
+    overlay.files = prev
+        .files
+        .iter()
+        .filter_map(|df| {
+            let masked: Vec<i32> = df
+                .fields
+                .iter()
+                .map(|&f| if f == field_id { -2 } else { f })
+                .collect();
+            if masked.iter().all(|&f| f == -2) {
+                return None; // file holds only the tombstoned field
+            }
+            let mut m = df.clone();
+            m.fields = masked.into();
+            Some(m)
+        })
+        .collect();
+    overlay.add_file(
+        new_file,
+        vec![field_id],
+        vec![0],
+        &LanceFileVersion::default(),
+        None,
+    );
+    overlay
 }
 
-/// Same scenario as test_fts_index_incremental_reindex_after_in_place_update
-/// but with a vector (IVF_PQ) index instead of FTS.
+/// A `Merge` that rewrites an indexed column's data in place must keep that column's
+/// index consistent: a query the index serves must return the same rows as a flat scan
+/// of the rewritten data. The overlay fragment tombstones (-2) the column's field id in
+/// the existing data file and appends a new file for it, so the field stays in the
+/// schema and its index is retained -- the rewritten fragment must be pruned from that
+/// index. This is the shape produced when an in-place column rewrite cannot be expressed
+/// as a DataReplacement (e.g. an `update` has merged the fragment's column files) and
+/// falls back to a Merge overlay.
 #[tokio::test]
-async fn test_vector_index_after_data_replacement() {
-    use arrow_array::FixedSizeListArray;
-    use lance_arrow::FixedSizeListArrayExt;
-    use lance_index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
-    use lance_testing::datagen::generate_random_array;
-
-    const DIM: usize = 32;
-    const ROWS_PER_FRAG: usize = 256;
-    const TOTAL: usize = ROWS_PER_FRAG * 2;
-
-    let fsl_field = ArrowField::new(
-        "vector",
-        DataType::FixedSizeList(
-            Arc::new(ArrowField::new("item", DataType::Float32, true)),
-            DIM as i32,
-        ),
-        true,
-    );
+async fn test_merge_rewriting_indexed_column_keeps_index_consistent() {
     let schema = Arc::new(ArrowSchema::new(vec![
-        ArrowField::new("id", DataType::Int32, false),
-        fsl_field,
+        ArrowField::new("a", DataType::Int32, true),
+        ArrowField::new("b", DataType::Int32, true),
     ]));
-
-    // Step 1: Create dataset with TOTAL rows in 2 fragments.
-    let vectors = generate_random_array(TOTAL * DIM);
-    let vector_array =
-        Arc::new(FixedSizeListArray::try_new_from_values(vectors, DIM as i32).unwrap());
-    let ids: Vec<i32> = (0..TOTAL as i32).collect();
     let batch = RecordBatch::try_new(
         schema.clone(),
-        vec![Arc::new(Int32Array::from(ids)), vector_array.clone()],
+        vec![
+            Arc::new(Int32Array::from(vec![1, 2, 3, 4])),
+            Arc::new(Int32Array::from(vec![10, 20, 30, 40])),
+        ],
     )
     .unwrap();
     let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
-    let mut dataset = Dataset::write(
-        reader,
-        "memory://test_vec_data_replacement",
-        Some(WriteParams {
-            max_rows_per_file: ROWS_PER_FRAG,
-            ..Default::default()
-        }),
-    )
-    .await
-    .unwrap();
-    assert_eq!(dataset.get_fragments().len(), 2);
+    let mut dataset = Dataset::write(reader, "memory://merge_index_rewrite", None)
+        .await
+        .unwrap();
 
-    // Step 2: Create IVF_PQ vector index
-    let ivf_params = IvfBuildParams::new(2);
-    let pq_params = PQBuildParams {
-        num_sub_vectors: 1,
-        ..Default::default()
-    };
-    let params = crate::index::vector::VectorIndexParams::with_ivf_pq_params(
-        lance_linalg::distance::MetricType::L2,
-        ivf_params,
-        pq_params,
-    );
     dataset
-        .create_index(&["vector"], IndexType::Vector, None, &params, true)
+        .create_index(
+            &["a"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
         .await
         .unwrap();
 
-    // Sanity: nearest to all-zeros query with refine should return
-    // results from both fragments.
-    let query_zeros = Float32Array::from(vec![0.0_f32; DIM]);
-    let results = dataset
-        .scan()
-        .nearest("vector", &query_zeros, 10)
-        .unwrap()
-        .refine(10)
-        .try_into_batch()
-        .await
-        .unwrap();
-    assert_eq!(results.num_rows(), 10);
+    // Baseline: the index serves correct results before the rewrite.
+    assert_eq!(
+        index_consistent_values(&dataset, "a", "a >= 3").await,
+        vec![3, 4]
+    );
 
-    // Step 3: DataReplacement — replace fragment 1's data.
-    // Write all-999.0 vectors so they are very far from origin.
-    let frag1 = dataset.get_fragment(1).unwrap();
-    let frag1_id = frag1.id() as u64;
-    let old_data_file = frag1.metadata().files[0].clone();
+    let a_field_id = 0i32;
 
-    let far_values: Vec<f32> = vec![999.0_f32; ROWS_PER_FRAG * DIM];
-    let far_vectors = Float32Array::from(far_values);
-    let far_vector_array =
-        FixedSizeListArray::try_new_from_values(far_vectors, DIM as i32).unwrap();
-    let replacement_ids: Vec<i32> = (ROWS_PER_FRAG as i32..(TOTAL as i32)).collect();
-    let replacement_batch = RecordBatch::try_new(
+    // Write a new single-column file holding `a`'s replacement values.
+    let a_only = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "a",
+        DataType::Int32,
+        true,
+    )]));
+    let new_a = RecordBatch::try_new(
+        a_only.clone(),
+        vec![Arc::new(Int32Array::from(vec![91, 92, 93, 94]))],
+    )
+    .unwrap();
+    let new_a_path = dataset.data_dir().join("merge_new_a.lance");
+    let object_writer = dataset.object_store.create(&new_a_path).await.unwrap();
+    let mut writer = FileWriter::try_new(
+        object_writer,
+        a_only.as_ref().try_into().unwrap(),
+        Default::default(),
+    )
+    .unwrap();
+    writer.write_batch(&new_a).await.unwrap();
+    writer.finish().await.unwrap();
+
+    // Overlay that file onto fragment 0, rewriting `a` in place.
+    let prev = dataset.get_fragment(0).unwrap().metadata().clone();
+    let overlay = build_overlay_frag(&prev, a_field_id, "merge_new_a.lance");
+
+    let read_version = dataset.version().version;
+    let dataset = Dataset::commit(
+        WriteDestination::Dataset(Arc::new(dataset)),
+        Operation::Merge {
+            fragments: vec![overlay],
+            schema: schema.as_ref().try_into().unwrap(),
+        },
+        Some(read_version),
+        None,
+        None,
+        Arc::new(Default::default()),
+        false,
+    )
+    .await
+    .unwrap();
+
+    // `a` now holds [91, 92, 93, 94]; an index-served query must reflect that.
+    assert_eq!(
+        index_consistent_values(&dataset, "a", "a >= 90").await,
+        vec![91, 92, 93, 94],
+        "index-served `a >= 90` must return the rewritten values"
+    );
+    assert!(
+        index_consistent_values(&dataset, "a", "a < 90")
+            .await
+            .is_empty(),
+        "no row satisfies `a < 90` after the rewrite"
+    );
+}
+
+/// DataReplacement on an indexed column should remove the fragment from
+/// fragment_bitmap AND add it to invalidated_fragment_bitmap so that
+/// stale index entries are blocked at query time.
+#[tokio::test]
+async fn test_data_replacement_populates_invalidated_bitmap() {
+    use lance_file::writer::FileWriter;
+    use object_store::path::Path;
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("value", DataType::Int32, true),
+    ]));
+
+    // Create dataset with one fragment
+    let batch = RecordBatch::try_new(
         schema.clone(),
         vec![
-            Arc::new(Int32Array::from(replacement_ids)),
-            Arc::new(far_vector_array),
+            Arc::new(Int32Array::from(vec![1, 2, 3])),
+            Arc::new(Int32Array::from(vec![10, 20, 30])),
         ],
     )
     .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(reader, "memory://test_replacement_invalidated", None)
+        .await
+        .unwrap();
+
+    // Create BTree index on 'value'
+    dataset
+        .create_index(
+            &["value"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+    // Verify initial state: fragment 0 in bitmap, no invalidated fragments
+    let indices = dataset.load_indices().await.unwrap();
+    let idx = indices.iter().find(|i| i.name == "value_idx").unwrap();
+    assert!(idx.fragment_bitmap.as_ref().unwrap().contains(0));
+
+    // Write a replacement data file for column 'value'
+    let value_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "value",
+        DataType::Int32,
+        true,
+    )]));
+    let replacement_batch = RecordBatch::try_new(
+        value_schema.clone(),
+        vec![Arc::new(Int32Array::from(vec![40, 50, 60]))],
+    )
+    .unwrap();
 
-    let replacement_path = dataset.data_dir().join("replacement.lance");
     let object_writer = dataset
         .object_store
-        .create(&replacement_path)
+        .create(&Path::from("data/replacement_inv.lance"))
         .await
         .unwrap();
     let mut writer = FileWriter::try_new(
         object_writer,
-        schema.as_ref().try_into().unwrap(),
+        value_schema.as_ref().try_into().unwrap(),
         Default::default(),
     )
     .unwrap();
     writer.write_batch(&replacement_batch).await.unwrap();
     writer.finish().await.unwrap();
 
-    let mut new_data_file = old_data_file.clone();
-    new_data_file.path = "replacement.lance".to_string();
+    // Build replacement DataFile
+    let frag = dataset.get_fragment(0).unwrap();
+    let lance_schema: lance_core::datatypes::Schema = schema.as_ref().try_into().unwrap();
+    let value_field_id = lance_schema.field("value").unwrap().id;
+    let data_file = frag.data_file_for_field(value_field_id as u32).unwrap();
+    let mut new_data_file = data_file.clone();
+    new_data_file.path = "replacement_inv.lance".to_string();
 
-    let read_version = dataset.manifest.version;
+    // Commit DataReplacement
+    let read_version = dataset.version().version;
     let dataset = Dataset::commit(
         WriteDestination::Dataset(Arc::new(dataset)),
         Operation::DataReplacement {
-            replacements: vec![DataReplacementGroup(frag1_id, new_data_file)],
+            replacements: vec![DataReplacementGroup(0, new_data_file)],
         },
         Some(read_version),
         None,
@@ -2113,38 +2514,20 @@ async fn test_vector_index_after_data_replacement() {
     .await
     .unwrap();
 
-    // Step 4: Search — nearest to all-zeros WITHOUT refine.
-    // Fragment 1's vectors are now all-999.0 (very far from origin).
-    // The top-10 nearest should all come from fragment 0.
-    // If stale index entries leak through, results from fragment 1
-    // would appear with their old (closer) PQ-approximated distances.
-    let results = dataset
-        .scan()
-        .nearest("vector", &query_zeros, 10)
-        .unwrap()
-        .try_into_batch()
-        .await
-        .unwrap();
-    let ids = results["id"].as_any().downcast_ref::<Int32Array>().unwrap();
-    for i in 0..results.num_rows() {
-        assert!(
-            (ids.value(i) as usize) < ROWS_PER_FRAG,
-            "Result {} has id={} which is from fragment 1 (stale index entry leaked through)",
-            i,
-            ids.value(i)
-        );
-    }
+    // Check: fragment 0 removed from fragment_bitmap
+    let indices = dataset.load_indices().await.unwrap();
+    let idx = indices.iter().find(|i| i.name == "value_idx").unwrap();
+    assert!(
+        !idx.fragment_bitmap.as_ref().unwrap().contains(0),
+        "Fragment 0 should be removed from fragment_bitmap"
+    );
 }
 
-/// Regression test: inverted (FTS) index should not carry stale data after
-/// merge_insert + compact + optimize_indices.
-///
-/// This is the FTS equivalent of test_merge_insert_with_reordered_columns_and_index.
-/// The inverted index's update() ignores the valid_old_fragments filter, so stale
-/// posting list entries from pruned fragments survive the merge and cause errors
-/// when queries try to resolve the old row addresses.
+/// Regression test (lance-format/lance#6283): after in-place update via
+/// DataReplacement, stale FTS index entries for the replaced fragment must
+/// be blocked at query time so searches reflect the new data.
 #[tokio::test]
-async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
+async fn test_fts_stale_entries_after_data_replacement() {
     use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
 
     let schema = Arc::new(ArrowSchema::new(vec![
@@ -2167,7 +2550,7 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
     let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
     let mut dataset = Dataset::write(
         reader,
-        "memory://test_fts_stale",
+        "memory://test_fts_incremental_reindex",
         Some(WriteParams {
             max_rows_per_file: 1, // Force 2 fragments
             ..Default::default()
@@ -2183,7 +2566,7 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
         .await
         .unwrap();
 
-    // Sanity check: searching "quick" should return 1 result
+    // Sanity check: "quick" and "lazy" should each return 1 result
     let results = dataset
         .scan()
         .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
@@ -2192,73 +2575,86 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
         .await
         .unwrap();
     assert_eq!(results.num_rows(), 1);
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
 
-    // Step 3: merge_insert with reversed column order (text, id)
-    // This triggers the RewriteColumns/DataReplacement path, which prunes the
-    // index fragment bitmap for the 'text' column.
-    let reversed_schema = Arc::new(ArrowSchema::new(vec![
-        ArrowField::new("text", DataType::Utf8, true),
-        ArrowField::new("id", DataType::Int32, false),
-    ]));
-    let source_batch = RecordBatch::try_new(
-        reversed_schema.clone(),
+    // Step 3: Replace fragment 1's data file via DataReplacement.
+    // The fragment ID stays the same, but the text changes from
+    // "the lazy dog" to "a speedy cat". This prunes fragment 1
+    // from the FTS index's fragment_bitmap.
+    let frag1 = dataset.get_fragment(1).unwrap();
+    let old_data_file = frag1.metadata().files[0].clone();
+
+    // Write replacement data file with updated text
+    let replacement_batch = RecordBatch::try_new(
+        schema.clone(),
         vec![
-            Arc::new(StringArray::from(vec![
-                "updated fox text",
-                "new entry here",
-            ])),
-            Arc::new(Int32Array::from(vec![1, 2])),
+            Arc::new(Int32Array::from(vec![1])),
+            Arc::new(StringArray::from(vec!["a speedy cat"])),
         ],
     )
     .unwrap();
-
-    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
-        .unwrap()
-        .when_matched(WhenMatched::UpdateAll)
-        .when_not_matched(WhenNotMatched::InsertAll)
-        .try_build()
-        .unwrap();
-
-    let reader = Box::new(RecordBatchIterator::new(
-        vec![Ok(source_batch)],
-        reversed_schema.clone(),
-    ));
-    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
-    let mut dataset = dataset.as_ref().clone();
-
-    // Step 4: compact_files — moves rows to new fragment(s)
-    compact_files(&mut dataset, CompactionOptions::default(), None)
+    let replacement_path = dataset.data_dir().join("replacement.lance");
+    let object_writer = dataset
+        .object_store
+        .create(&replacement_path)
         .await
         .unwrap();
+    let mut writer = FileWriter::try_new(
+        object_writer,
+        schema.as_ref().try_into().unwrap(),
+        Default::default(),
+    )
+    .unwrap();
+    writer.write_batch(&replacement_batch).await.unwrap();
+    writer.finish().await.unwrap();
 
-    // Step 5: optimize_indices — should rebuild the FTS index without stale data.
-    // With the current bug, the inverted index ignores valid_old_fragments and
-    // merges stale posting list entries pointing at now-deleted fragments.
-    dataset
-        .optimize_indices(&OptimizeOptions::default())
-        .await
-        .unwrap();
+    let mut new_data_file = old_data_file.clone();
+    new_data_file.path = "replacement.lance".to_string();
 
-    // Step 6: FTS search should not error and should return correct results.
-    // "quick" appeared in the original data for id=0 (never updated), so it
-    // should still be found.
+    let read_version = dataset.manifest.version;
+    let dataset = Dataset::commit(
+        WriteDestination::Dataset(Arc::new(dataset)),
+        Operation::DataReplacement {
+            replacements: vec![DataReplacementGroup(1, new_data_file)],
+        },
+        Some(read_version),
+        None,
+        None,
+        Arc::new(Default::default()),
+        false,
+    )
+    .await
+    .unwrap();
+
+    // Verify the replacement worked — fragment 1 now has the new text
+    let batch = dataset.scan().try_into_batch().await.unwrap();
+    assert_eq!(batch.num_rows(), 2);
+
+    // Step 4: FTS search should reflect the new data, not the old.
+    // Fragment 1 is unindexed (pruned from fragment_bitmap), so the
+    // scanner does a flat FTS scan on it and uses the index for fragment 0.
+
+    // "speedy" is in the new text for fragment 1 — found via flat scan.
     let results = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .full_text_search(FullTextSearchQuery::new("speedy".to_owned()))
         .unwrap()
         .try_into_batch()
         .await
         .unwrap();
-    assert_eq!(
-        results.num_rows(),
-        1,
-        "Expected 1 result for 'quick' after optimize, got {}",
-        results.num_rows()
-    );
+    assert_eq!(results.num_rows(), 1);
 
-    // "lazy" was in the original text for id=1, but id=1 was updated to
-    // "updated fox text". The old posting for "lazy" should have been filtered
-    // out during the index update.
+    // "lazy" was in the OLD text for fragment 1. The index has stale
+    // posting entries for it, but fragment 1 was pruned from the
+    // fragment_bitmap. Flat scan of fragment 1 sees "a speedy cat"
+    // which doesn't match. So 0 results.
     let results = dataset
         .scan()
         .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
@@ -2269,70 +2665,183 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
     assert_eq!(
         results.num_rows(),
         0,
-        "Expected 0 results for 'lazy' (stale data should be filtered), got {}",
-        results.num_rows()
+        "Expected 0 results for 'lazy' (stale data, fragment 1 pruned from index)"
     );
 
-    // "updated" should be found (new text for id=1)
+    // "quick" is in fragment 0 which is still indexed — should still work.
     let results = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("updated".to_owned()))
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
         .unwrap()
         .try_into_batch()
         .await
         .unwrap();
     assert_eq!(results.num_rows(), 1);
+}
 
-    // "entry" should be found (new row id=2)
+/// Same scenario as test_fts_index_incremental_reindex_after_in_place_update
+/// but with a vector (IVF_PQ) index instead of FTS.
+#[tokio::test]
+async fn test_vector_index_after_data_replacement() {
+    use arrow_array::FixedSizeListArray;
+    use lance_arrow::FixedSizeListArrayExt;
+    use lance_index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+    use lance_testing::datagen::generate_random_array;
+
+    const DIM: usize = 32;
+    const ROWS_PER_FRAG: usize = 256;
+    const TOTAL: usize = ROWS_PER_FRAG * 2;
+
+    let fsl_field = ArrowField::new(
+        "vector",
+        DataType::FixedSizeList(
+            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+            DIM as i32,
+        ),
+        true,
+    );
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        fsl_field,
+    ]));
+
+    // Step 1: Create dataset with TOTAL rows in 2 fragments.
+    let vectors = generate_random_array(TOTAL * DIM);
+    let vector_array =
+        Arc::new(FixedSizeListArray::try_new_from_values(vectors, DIM as i32).unwrap());
+    let ids: Vec<i32> = (0..TOTAL as i32).collect();
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(Int32Array::from(ids)), vector_array.clone()],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_vec_data_replacement",
+        Some(WriteParams {
+            max_rows_per_file: ROWS_PER_FRAG,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(dataset.get_fragments().len(), 2);
+
+    // Step 2: Create IVF_PQ vector index
+    let ivf_params = IvfBuildParams::new(2);
+    let pq_params = PQBuildParams {
+        num_sub_vectors: 1,
+        ..Default::default()
+    };
+    let params = crate::index::vector::VectorIndexParams::with_ivf_pq_params(
+        lance_linalg::distance::MetricType::L2,
+        ivf_params,
+        pq_params,
+    );
+    dataset
+        .create_index(&["vector"], IndexType::Vector, None, &params, true)
+        .await
+        .unwrap();
+
+    // Sanity: nearest to all-zeros query with refine should return
+    // results from both fragments.
+    let query_zeros = Float32Array::from(vec![0.0_f32; DIM]);
     let results = dataset
         .scan()
-        .full_text_search(FullTextSearchQuery::new("entry".to_owned()))
+        .nearest("vector", &query_zeros, 10)
         .unwrap()
+        .refine(10)
         .try_into_batch()
         .await
         .unwrap();
-    assert_eq!(results.num_rows(), 1);
+    assert_eq!(results.num_rows(), 10);
 
-    // Step 7: Another merge_insert should NOT error
-    let source_batch2 = RecordBatch::try_new(
+    // Step 3: DataReplacement — replace fragment 1's data.
+    // Write all-999.0 vectors so they are very far from origin.
+    let frag1 = dataset.get_fragment(1).unwrap();
+    let frag1_id = frag1.id() as u64;
+    let old_data_file = frag1.metadata().files[0].clone();
+
+    let far_values: Vec<f32> = vec![999.0_f32; ROWS_PER_FRAG * DIM];
+    let far_vectors = Float32Array::from(far_values);
+    let far_vector_array =
+        FixedSizeListArray::try_new_from_values(far_vectors, DIM as i32).unwrap();
+    let replacement_ids: Vec<i32> = (ROWS_PER_FRAG as i32..(TOTAL as i32)).collect();
+    let replacement_batch = RecordBatch::try_new(
         schema.clone(),
         vec![
-            Arc::new(Int32Array::from(vec![1])),
-            Arc::new(StringArray::from(vec!["final text"])),
+            Arc::new(Int32Array::from(replacement_ids)),
+            Arc::new(far_vector_array),
         ],
     )
     .unwrap();
 
-    let merge_job2 = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
-        .unwrap()
-        .when_matched(WhenMatched::UpdateAll)
-        .when_not_matched(WhenNotMatched::InsertAll)
-        .try_build()
+    let replacement_path = dataset.data_dir().join("replacement.lance");
+    let object_writer = dataset
+        .object_store
+        .create(&replacement_path)
+        .await
         .unwrap();
+    let mut writer = FileWriter::try_new(
+        object_writer,
+        schema.as_ref().try_into().unwrap(),
+        Default::default(),
+    )
+    .unwrap();
+    writer.write_batch(&replacement_batch).await.unwrap();
+    writer.finish().await.unwrap();
 
-    let reader2 = Box::new(RecordBatchIterator::new(
-        vec![Ok(source_batch2)],
-        schema.clone(),
-    ));
-    let (final_dataset, _) = merge_job2.execute(reader_to_stream(reader2)).await.unwrap();
-    final_dataset.validate().await.unwrap();
+    let mut new_data_file = old_data_file.clone();
+    new_data_file.path = "replacement.lance".to_string();
+
+    let read_version = dataset.manifest.version;
+    let dataset = Dataset::commit(
+        WriteDestination::Dataset(Arc::new(dataset)),
+        Operation::DataReplacement {
+            replacements: vec![DataReplacementGroup(frag1_id, new_data_file)],
+        },
+        Some(read_version),
+        None,
+        None,
+        Arc::new(Default::default()),
+        false,
+    )
+    .await
+    .unwrap();
+
+    // Step 4: Search — nearest to all-zeros WITHOUT refine.
+    // Fragment 1's vectors are now all-999.0 (very far from origin).
+    // The top-10 nearest should all come from fragment 0.
+    // If stale index entries leak through, results from fragment 1
+    // would appear with their old (closer) PQ-approximated distances.
+    let results = dataset
+        .scan()
+        .nearest("vector", &query_zeros, 10)
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    let ids = results["id"].as_any().downcast_ref::<Int32Array>().unwrap();
+    for i in 0..results.num_rows() {
+        assert!(
+            (ids.value(i) as usize) < ROWS_PER_FRAG,
+            "Result {} has id={} which is from fragment 1 (stale index entry leaked through)",
+            i,
+            ids.value(i)
+        );
+    }
 }
 
-/// Regression test: when rows are updated in-place, the FTS index must
-/// invalidate old entries and allow re-indexing incrementally.
+/// Regression test: inverted (FTS) index should not carry stale data after
+/// merge_insert + compact + optimize_indices.
 ///
-/// Sequence:
-/// 1. Write fragments 1 and 2.
-/// 2. Create FTS index covering fragments 1 and 2.
-/// 3. Update fragment 1 in-place via merge_insert (DataReplacement path).
-///    This removes fragment 1 from the index's fragment_bitmap.
-/// 4. Call optimize_indices (append) to create a new index segment covering
-///    the updated fragment 1.
-/// 5. Call optimize_indices (merge) to merge both segments. The first segment
-///    contains the old, invalidated values for fragment 1; the second segment
-///    contains the new, valid values. We must keep only the new values.
+/// This is the FTS equivalent of test_merge_insert_with_reordered_columns_and_index.
+/// The inverted index's update() ignores the valid_old_fragments filter, so stale
+/// posting list entries from pruned fragments survive the merge and cause errors
+/// when queries try to resolve the old row addresses.
 #[tokio::test]
-async fn test_fts_index_incremental_reindex_after_in_place_update() {
+async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() {
     use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
 
     let schema = Arc::new(ArrowSchema::new(vec![
@@ -2355,7 +2864,7 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() {
     let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
     let mut dataset = Dataset::write(
         reader,
-        "memory://test_fts_incremental_reindex",
+        "memory://test_fts_stale",
         Some(WriteParams {
             max_rows_per_file: 1, // Force 2 fragments
             ..Default::default()
@@ -2371,7 +2880,7 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() {
         .await
         .unwrap();
 
-    // Sanity check: "quick" and "lazy" should each return 1 result
+    // Sanity check: searching "quick" should return 1 result
     let results = dataset
         .scan()
         .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
@@ -2380,19 +2889,10 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() {
         .await
         .unwrap();
     assert_eq!(results.num_rows(), 1);
-    let results = dataset
-        .scan()
-        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
-        .unwrap()
-        .try_into_batch()
-        .await
-        .unwrap();
-    assert_eq!(results.num_rows(), 1);
 
-    // Step 3: merge_insert with reversed column order to trigger
-    // RewriteColumns/DataReplacement path, which prunes the index
-    // fragment bitmap for the updated fragment.
-    // Update id=1 ("the lazy dog" -> "a speedy cat")
+    // Step 3: merge_insert with reversed column order (text, id)
+    // This triggers the RewriteColumns/DataReplacement path, which prunes the
+    // index fragment bitmap for the 'text' column.
     let reversed_schema = Arc::new(ArrowSchema::new(vec![
         ArrowField::new("text", DataType::Utf8, true),
         ArrowField::new("id", DataType::Int32, false),
@@ -2400,12 +2900,209 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() {
     let source_batch = RecordBatch::try_new(
         reversed_schema.clone(),
         vec![
-            Arc::new(StringArray::from(vec!["a speedy cat"])),
-            Arc::new(Int32Array::from(vec![1])),
-        ],
-    )
-    .unwrap();
-
+            Arc::new(StringArray::from(vec![
+                "updated fox text",
+                "new entry here",
+            ])),
+            Arc::new(Int32Array::from(vec![1, 2])),
+        ],
+    )
+    .unwrap();
+
+    let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap();
+
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source_batch)],
+        reversed_schema.clone(),
+    ));
+    let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap();
+    let mut dataset = dataset.as_ref().clone();
+
+    // Step 4: compact_files — moves rows to new fragment(s)
+    compact_files(&mut dataset, CompactionOptions::default(), None)
+        .await
+        .unwrap();
+
+    // Step 5: optimize_indices — should rebuild the FTS index without stale data.
+    // With the current bug, the inverted index ignores valid_old_fragments and
+    // merges stale posting list entries pointing at now-deleted fragments.
+    dataset
+        .optimize_indices(&OptimizeOptions::default())
+        .await
+        .unwrap();
+
+    // Step 6: FTS search should not error and should return correct results.
+    // "quick" appeared in the original data for id=0 (never updated), so it
+    // should still be found.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        1,
+        "Expected 1 result for 'quick' after optimize, got {}",
+        results.num_rows()
+    );
+
+    // "lazy" was in the original text for id=1, but id=1 was updated to
+    // "updated fox text". The old posting for "lazy" should have been filtered
+    // out during the index update.
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(
+        results.num_rows(),
+        0,
+        "Expected 0 results for 'lazy' (stale data should be filtered), got {}",
+        results.num_rows()
+    );
+
+    // "updated" should be found (new text for id=1)
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("updated".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // "entry" should be found (new row id=2)
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("entry".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // Step 7: Another merge_insert should NOT error
+    let source_batch2 = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![1])),
+            Arc::new(StringArray::from(vec!["final text"])),
+        ],
+    )
+    .unwrap();
+
+    let merge_job2 = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap();
+
+    let reader2 = Box::new(RecordBatchIterator::new(
+        vec![Ok(source_batch2)],
+        schema.clone(),
+    ));
+    let (final_dataset, _) = merge_job2.execute(reader_to_stream(reader2)).await.unwrap();
+    final_dataset.validate().await.unwrap();
+}
+
+/// Regression test: when rows are updated in-place, the FTS index must
+/// invalidate old entries and allow re-indexing incrementally.
+///
+/// Sequence:
+/// 1. Write fragments 1 and 2.
+/// 2. Create FTS index covering fragments 1 and 2.
+/// 3. Update fragment 1 in-place via merge_insert (DataReplacement path).
+///    This removes fragment 1 from the index's fragment_bitmap.
+/// 4. Call optimize_indices (append) to create a new index segment covering
+///    the updated fragment 1.
+/// 5. Call optimize_indices (merge) to merge both segments. The first segment
+///    contains the old, invalidated values for fragment 1; the second segment
+///    contains the new, valid values. We must keep only the new values.
+#[tokio::test]
+async fn test_fts_index_incremental_reindex_after_in_place_update() {
+    use lance_index::scalar::{FullTextSearchQuery, inverted::InvertedIndexParams};
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("text", DataType::Utf8, true),
+    ]));
+
+    // Step 1: Create dataset with 2 rows in separate fragments
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![0, 1])),
+            Arc::new(StringArray::from(vec![
+                "the quick brown fox",
+                "the lazy dog",
+            ])),
+        ],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+    let mut dataset = Dataset::write(
+        reader,
+        "memory://test_fts_incremental_reindex",
+        Some(WriteParams {
+            max_rows_per_file: 1, // Force 2 fragments
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // Step 2: Create FTS inverted index on 'text'
+    let params = InvertedIndexParams::default();
+    dataset
+        .create_index(&["text"], IndexType::Inverted, None, &params, true)
+        .await
+        .unwrap();
+
+    // Sanity check: "quick" and "lazy" should each return 1 result
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("quick".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+    let results = dataset
+        .scan()
+        .full_text_search(FullTextSearchQuery::new("lazy".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(results.num_rows(), 1);
+
+    // Step 3: merge_insert with reversed column order to trigger
+    // RewriteColumns/DataReplacement path, which prunes the index
+    // fragment bitmap for the updated fragment.
+    // Update id=1 ("the lazy dog" -> "a speedy cat")
+    let reversed_schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("text", DataType::Utf8, true),
+        ArrowField::new("id", DataType::Int32, false),
+    ]));
+    let source_batch = RecordBatch::try_new(
+        reversed_schema.clone(),
+        vec![
+            Arc::new(StringArray::from(vec!["a speedy cat"])),
+            Arc::new(Int32Array::from(vec![1])),
+        ],
+    )
+    .unwrap();
+
     let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
         .unwrap()
         .when_matched(WhenMatched::UpdateAll)
@@ -2631,3 +3328,969 @@ async fn test_sub_schema_merge_insert_binary_v2_2() {
     assert_eq!(binary_arr.value(0), data_a.as_slice());
     assert_eq!(binary_arr.value(1), data_b.as_slice());
 }
+
+#[tokio::test]
+async fn test_fts_unfiltered_after_compaction_returns_remapped_row_ids() {
+    // After `compact_files` with `defer_index_remap = true`, queries
+    // read the old FTS index but must apply the dataset's
+    // FragReuseIndex remap. Otherwise the deferred-row_id path
+    // returns pre-compaction row_ids that no longer exist.
+    use arrow::datatypes::UInt64Type;
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("text", DataType::Utf8, false),
+    ]));
+    let batch = RecordBatch::try_new(
+        schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(vec![0, 1, 2, 3])),
+            Arc::new(StringArray::from(vec![
+                "alpha first",
+                "alpha second",
+                "alpha third",
+                "alpha fourth",
+            ])),
+        ],
+    )
+    .unwrap();
+    let mut dataset = Dataset::write(
+        RecordBatchIterator::new(vec![Ok(batch)], schema),
+        "memory://test_fts_frag_reuse",
+        Some(WriteParams {
+            max_rows_per_file: 1, // 4 fragments -> 4 partitions
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    dataset
+        .create_index(
+            &["text"],
+            IndexType::Inverted,
+            None,
+            &InvertedIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+    compact_files(
+        &mut dataset,
+        CompactionOptions {
+            target_rows_per_fragment: 1000,
+            defer_index_remap: true,
+            ..Default::default()
+        },
+        None,
+    )
+    .await
+    .unwrap();
+
+    let after = dataset
+        .scan()
+        .with_row_id()
+        .full_text_search(FullTextSearchQuery::new("alpha".to_owned()))
+        .unwrap()
+        .try_into_batch()
+        .await
+        .unwrap();
+    assert_eq!(after.num_rows(), 4);
+    let returned: Vec<u64> = after[ROW_ID].as_primitive::<UInt64Type>().values().to_vec();
+    let live: std::collections::HashSet<u64> =
+        dataset.scan().with_row_id().try_into_batch().await.unwrap()[ROW_ID]
+            .as_primitive::<UInt64Type>()
+            .values()
+            .iter()
+            .copied()
+            .collect();
+    for id in &returned {
+        assert!(live.contains(id), "stale row_id {id}");
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Multi-base tests: merge insert on datasets whose data lives across multiple
+// registered base paths, with and without routing new fragments to bases.
+// ---------------------------------------------------------------------------
+
+/// Fixture: primary storage plus two external bases. base1 is a dataset-root
+/// style base (files under `{base1}/data/`), base2 is a plain data directory
+/// (files directly under `{base2}/`). Initial data: ids 0..6 in two fragments
+/// in base1, ids 6..9 in one fragment in primary storage.
+struct MultiBaseFixture {
+    _tmp: TempDir,
+    dataset: Dataset,
+    base1_dir: std::path::PathBuf,
+    base2_dir: std::path::PathBuf,
+}
+
+fn multi_base_schema() -> Arc<ArrowSchema> {
+    Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("a", DataType::Int32, false),
+        ArrowField::new("b", DataType::Utf8, true),
+    ]))
+}
+
+fn multi_base_batch(ids: &[i32], a_offset: i32, b_prefix: &str) -> RecordBatch {
+    RecordBatch::try_new(
+        multi_base_schema(),
+        vec![
+            Arc::new(Int32Array::from(ids.to_vec())),
+            Arc::new(Int32Array::from(
+                ids.iter().map(|id| id + a_offset).collect::<Vec<_>>(),
+            )),
+            Arc::new(StringArray::from(
+                ids.iter()
+                    .map(|id| format!("{}{}", b_prefix, id))
+                    .collect::<Vec<_>>(),
+            )),
+        ],
+    )
+    .unwrap()
+}
+
+async fn multi_base_fixture(indexed: bool) -> MultiBaseFixture {
+    let tmp = TempDir::default();
+    let primary_dir = tmp.std_path().join("primary");
+    let base1_dir = tmp.std_path().join("base1");
+    let base2_dir = tmp.std_path().join("base2");
+    std::fs::create_dir_all(&base1_dir).unwrap();
+    std::fs::create_dir_all(&base2_dir).unwrap();
+    let primary_uri = format!("file://{}", primary_dir.display());
+
+    let reader = RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[0, 1, 2, 3, 4, 5], 100, "orig"))],
+        multi_base_schema(),
+    );
+    let dataset = Dataset::write(
+        reader,
+        &primary_uri,
+        Some(WriteParams {
+            mode: WriteMode::Create,
+            max_rows_per_file: 3,
+            initial_bases: Some(vec![
+                BasePath {
+                    id: 1,
+                    name: Some("base1".to_string()),
+                    is_dataset_root: true,
+                    path: format!("file://{}", base1_dir.display()),
+                },
+                BasePath {
+                    id: 2,
+                    name: Some("base2".to_string()),
+                    is_dataset_root: false,
+                    path: format!("file://{}", base2_dir.display()),
+                },
+            ]),
+            target_bases: Some(vec![1]),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    let reader = RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[6, 7, 8], 100, "orig"))],
+        multi_base_schema(),
+    );
+    let mut dataset = Dataset::write(
+        reader,
+        Arc::new(dataset),
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    if indexed {
+        dataset
+            .create_index(
+                &["id"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                false,
+            )
+            .await
+            .unwrap();
+    }
+
+    let fragments = dataset.get_fragments();
+    assert_eq!(fragments.len(), 3);
+    for fragment in &fragments[..2] {
+        for file in &fragment.metadata.files {
+            assert_eq!(file.base_id, Some(1));
+        }
+    }
+    for file in &fragments[2].metadata.files {
+        assert_eq!(file.base_id, None);
+    }
+
+    MultiBaseFixture {
+        _tmp: tmp,
+        dataset,
+        base1_dir,
+        base2_dir,
+    }
+}
+
+/// Collect (id, a, b) rows sorted by id.
+async fn collect_multi_base_rows(dataset: &Dataset) -> Vec<(i32, i32, Option<String>)> {
+    let mut scan = dataset.scan();
+    scan.project(&["id", "a", "b"]).unwrap();
+    let batches = scan
+        .try_into_stream()
+        .await
+        .unwrap()
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+    let mut rows = vec![];
+    for batch in batches {
+        let ids = batch.column(0).as_primitive::<Int32Type>();
+        let a = batch.column(1).as_primitive::<Int32Type>();
+        let b = batch.column(2).as_string::<i32>();
+        for i in 0..batch.num_rows() {
+            let b_val = if b.is_null(i) {
+                None
+            } else {
+                Some(b.value(i).to_string())
+            };
+            rows.push((ids.value(i), a.value(i), b_val));
+        }
+    }
+    rows.sort_unstable();
+    rows
+}
+
+fn expected_row(id: i32, a_offset: i32, b_prefix: &str) -> (i32, i32, Option<String>) {
+    (id, id + a_offset, Some(format!("{}{}", b_prefix, id)))
+}
+
+/// Merge insert against a multi-base table without routing: every path must
+/// read fragments from external bases correctly and write all new files to
+/// primary storage with no base id. `indexed` toggles the v2 plan path
+/// (false) vs the legacy indexed-scan path (true).
+#[rstest]
+#[tokio::test]
+async fn test_merge_insert_on_multi_base_table(#[values(false, true)] indexed: bool) {
+    let fixture = multi_base_fixture(indexed).await;
+    let dataset = Arc::new(fixture.dataset);
+
+    // Update one row in each existing fragment (two in base1, one in
+    // primary), insert two new rows.
+    let source = multi_base_batch(&[1, 4, 7, 10, 11], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+
+    assert_eq!(stats.num_updated_rows, 3);
+    assert_eq!(stats.num_inserted_rows, 2);
+    assert_eq!(dataset.count_rows(None).await.unwrap(), 11);
+
+    for fragment in dataset.get_fragments() {
+        let metadata = &fragment.metadata;
+        if metadata.id >= 3 {
+            // Fragments written by the merge live in primary storage.
+            for file in &metadata.files {
+                assert_eq!(file.base_id, None);
+            }
+        } else {
+            // Pre-existing fragments keep their base and get local deletion
+            // files for the rewritten rows.
+            let expected_base = if metadata.id < 2 { Some(1) } else { None };
+            for file in &metadata.files {
+                assert_eq!(file.base_id, expected_base);
+            }
+            let deletion = metadata.deletion_file.as_ref().unwrap();
+            assert_eq!(deletion.base_id, None);
+        }
+    }
+
+    let mut expected = vec![];
+    for id in [0, 2, 3, 5, 6, 8] {
+        expected.push(expected_row(id, 100, "orig"));
+    }
+    for id in [1, 4, 7, 10, 11] {
+        expected.push(expected_row(id, 1000, "new"));
+    }
+    expected.sort_unstable();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+
+    // Re-open from scratch to make sure the result is readable without any
+    // cached state.
+    let dataset = Dataset::open(dataset.uri()).await.unwrap();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+}
+
+/// Merge insert routing new fragments to target bases, by id and by name,
+/// covering both base layouts (dataset-root and plain data directory).
+#[rstest]
+#[tokio::test]
+async fn test_merge_insert_route_to_target_bases(#[values(false, true)] indexed: bool) {
+    let fixture = multi_base_fixture(indexed).await;
+    let dataset = Arc::new(fixture.dataset);
+
+    let source = multi_base_batch(&[1, 4, 10, 11], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![2])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+    assert_eq!(stats.num_updated_rows, 2);
+    assert_eq!(stats.num_inserted_rows, 2);
+
+    // New fragments land in base2, which is a plain data directory, so the
+    // files sit directly under it.
+    let mut merge_files = 0;
+    for fragment in dataset.get_fragments() {
+        let metadata = &fragment.metadata;
+        if metadata.id >= 3 {
+            for file in &metadata.files {
+                assert_eq!(file.base_id, Some(2));
+                let on_disk = fixture.base2_dir.join(file.path.as_str());
+                assert!(on_disk.exists(), "missing data file {:?}", on_disk);
+                merge_files += 1;
+            }
+        }
+    }
+    assert!(merge_files > 0);
+
+    let max_fragment_id = dataset.manifest.max_fragment_id().unwrap();
+
+    // A second merge referencing a base by name: base1 is a dataset-root
+    // base, so files go under `{base1}/data/`.
+    let source = multi_base_batch(&[12, 13], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_base_names_or_paths(vec!["base1".to_string()])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+    assert_eq!(stats.num_inserted_rows, 2);
+
+    let mut merge_files = 0;
+    for fragment in dataset.get_fragments() {
+        let metadata = &fragment.metadata;
+        if metadata.id > max_fragment_id {
+            for file in &metadata.files {
+                assert_eq!(file.base_id, Some(1));
+                let on_disk = fixture.base1_dir.join("data").join(file.path.as_str());
+                assert!(on_disk.exists(), "missing data file {:?}", on_disk);
+                merge_files += 1;
+            }
+        }
+    }
+    assert!(merge_files > 0);
+
+    let mut expected = vec![];
+    for id in [0, 2, 3, 5, 6, 7, 8] {
+        expected.push(expected_row(id, 100, "orig"));
+    }
+    for id in [1, 4, 10, 11, 12, 13] {
+        expected.push(expected_row(id, 1000, "new"));
+    }
+    expected.sort_unstable();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+
+    let dataset = Dataset::open(dataset.uri()).await.unwrap();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+}
+
+/// Partial-schema merge insert on a multi-base table: column patches for
+/// existing fragments stay in primary storage (mixing with data files in
+/// external bases within the same fragment) while inserted rows route to the
+/// requested base. Requires an index on the join key to reach the in-place
+/// update path.
+#[tokio::test]
+async fn test_merge_insert_partial_schema_multi_base() {
+    let fixture = multi_base_fixture(true).await;
+    let dataset = Arc::new(fixture.dataset);
+
+    // Update column `a` for all rows of fragment 0 (full column rewrite) and
+    // one row of fragment 1 (incremental update), insert ids 10 and 11.
+    let partial_schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new("id", DataType::Int32, false),
+        ArrowField::new("a", DataType::Int32, false),
+    ]));
+    let ids = vec![0, 1, 2, 3, 10, 11];
+    let source = RecordBatch::try_new(
+        partial_schema.clone(),
+        vec![
+            Arc::new(Int32Array::from(ids.clone())),
+            Arc::new(Int32Array::from(
+                ids.iter().map(|id| id + 1000).collect::<Vec<_>>(),
+            )),
+        ],
+    )
+    .unwrap();
+
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![2])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(vec![Ok(source)], partial_schema));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+    assert_eq!(stats.num_updated_rows, 4);
+    assert_eq!(stats.num_inserted_rows, 2);
+
+    for fragment in dataset.get_fragments() {
+        let metadata = &fragment.metadata;
+        match metadata.id {
+            0 | 1 => {
+                // Patched fragments: the original file in base1 plus a column
+                // patch written to primary storage.
+                assert_eq!(metadata.files.len(), 2);
+                assert_eq!(metadata.files[0].base_id, Some(1));
+                assert_eq!(metadata.files[1].base_id, None);
+            }
+            2 => {
+                assert_eq!(metadata.files.len(), 1);
+                assert_eq!(metadata.files[0].base_id, None);
+            }
+            _ => {
+                // Inserted rows route to base2.
+                for file in &metadata.files {
+                    assert_eq!(file.base_id, Some(2));
+                    let on_disk = fixture.base2_dir.join(file.path.as_str());
+                    assert!(on_disk.exists(), "missing data file {:?}", on_disk);
+                }
+            }
+        }
+    }
+
+    // Updated rows keep their `b` values, inserted rows have no `b`.
+    let mut expected = vec![];
+    for id in [4, 5, 6, 7, 8] {
+        expected.push(expected_row(id, 100, "orig"));
+    }
+    for id in [0, 1, 2, 3] {
+        expected.push((id, id + 1000, Some(format!("orig{}", id))));
+    }
+    for id in [10, 11] {
+        expected.push((id, id + 1000, None));
+    }
+    expected.sort_unstable();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+
+    let dataset = Dataset::open(dataset.uri()).await.unwrap();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+}
+
+/// Round-robin distribution across multiple target bases within a single
+/// merge insert. New data files are cut at `max_rows_per_file` (the write
+/// default of 1Mi rows), so inserting more rows than that produces multiple
+/// files, which must alternate between the target bases.
+#[tokio::test]
+async fn test_merge_insert_round_robin_target_bases() {
+    let tmp = TempDir::default();
+    let primary_dir = tmp.std_path().join("primary");
+    let base1_dir = tmp.std_path().join("base1");
+    let base2_dir = tmp.std_path().join("base2");
+    std::fs::create_dir_all(&base1_dir).unwrap();
+    std::fs::create_dir_all(&base2_dir).unwrap();
+    let primary_uri = format!("file://{}", primary_dir.display());
+
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "id",
+        DataType::Int32,
+        false,
+    )]));
+    let reader = RecordBatchIterator::new(
+        vec![Ok(RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(Int32Array::from_iter_values(0..10))],
+        )
+        .unwrap())],
+        schema.clone(),
+    );
+    let dataset = Dataset::write(
+        reader,
+        &primary_uri,
+        Some(WriteParams {
+            mode: WriteMode::Create,
+            initial_bases: Some(vec![
+                BasePath {
+                    id: 1,
+                    name: Some("base1".to_string()),
+                    is_dataset_root: true,
+                    path: format!("file://{}", base1_dir.display()),
+                },
+                BasePath {
+                    id: 2,
+                    name: Some("base2".to_string()),
+                    is_dataset_root: false,
+                    path: format!("file://{}", base2_dir.display()),
+                },
+            ]),
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    // 1.2Mi new rows -> two data files -> one per base.
+    const BATCH_ROWS: i32 = 100_000;
+    let batches: Vec<_> = (0..12)
+        .map(|i| {
+            let start = 1000 + i * BATCH_ROWS;
+            Ok(RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(
+                    start..start + BATCH_ROWS,
+                ))],
+            )
+            .unwrap())
+        })
+        .collect();
+    let job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()])
+        .unwrap()
+        .target_bases(vec![1, 2])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(batches, schema.clone()));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+    assert_eq!(stats.num_inserted_rows, 1_200_000);
+    assert_eq!(dataset.count_rows(None).await.unwrap(), 1_200_010);
+
+    let merge_file_bases: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|fragment| fragment.metadata.id >= 1)
+        .flat_map(|fragment| fragment.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(merge_file_bases, vec![Some(1), Some(2)]);
+}
+
+/// Target base validation across build and execution paths.
+#[tokio::test]
+async fn test_merge_insert_target_bases_validation() {
+    let fixture = multi_base_fixture(false).await;
+    let dataset = Arc::new(fixture.dataset);
+
+    // Both selectors set fails at build time.
+    let err = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![1])
+        .target_base_names_or_paths(vec!["base2".to_string()])
+        .try_build()
+        .err()
+        .unwrap();
+    assert!(
+        err.to_string()
+            .contains("Cannot specify both target_base_names_or_paths and target_bases"),
+        "unexpected error: {}",
+        err
+    );
+
+    // Unknown base id fails at execution.
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![99])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[1], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let err = job.execute(reader_to_stream(reader)).await.unwrap_err();
+    assert!(
+        err.to_string()
+            .contains("Target base ID 99 not found in available bases"),
+        "unexpected error: {}",
+        err
+    );
+
+    // An empty target base list is rejected rather than silently ignored.
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[1], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let err = job.execute(reader_to_stream(reader)).await.unwrap_err();
+    assert!(
+        err.to_string().contains("target_bases cannot be empty"),
+        "unexpected error: {}",
+        err
+    );
+
+    // Unknown base name fails at execution.
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_base_names_or_paths(vec!["nonexistent".to_string()])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[1], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let err = job.execute(reader_to_stream(reader)).await.unwrap_err();
+    assert!(
+        err.to_string()
+            .contains("Base reference 'nonexistent' not found in available bases"),
+        "unexpected error: {}",
+        err
+    );
+
+    // Delete-only merges write no data files but still validate target bases.
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::Delete)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .target_bases(vec![99])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[1], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let err = job.execute(reader_to_stream(reader)).await.unwrap_err();
+    assert!(
+        err.to_string()
+            .contains("Target base ID 99 not found in available bases"),
+        "unexpected error: {}",
+        err
+    );
+
+    // Valid target bases on a delete-only merge are a no-op.
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::Delete)
+        .when_not_matched(WhenNotMatched::DoNothing)
+        .target_bases(vec![2])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[5], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let (dataset, stats) = job.execute(reader_to_stream(reader)).await.unwrap();
+    assert_eq!(stats.num_deleted_rows, 1);
+    assert_eq!(dataset.count_rows(None).await.unwrap(), 8);
+
+    // Datasets with no registered bases reject target bases.
+    let plain_dir = TempStrDir::default();
+    let reader = RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[0, 1], 100, "orig"))],
+        multi_base_schema(),
+    );
+    let plain_dataset = Dataset::write(reader, plain_dir.as_str(), None)
+        .await
+        .unwrap();
+    let job = MergeInsertBuilder::try_new(Arc::new(plain_dataset), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![1])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(multi_base_batch(&[1], 1000, "new"))],
+        multi_base_schema(),
+    ));
+    let err = job.execute(reader_to_stream(reader)).await.unwrap_err();
+    assert!(
+        err.to_string()
+            .contains("Target base ID 1 not found in available bases"),
+        "unexpected error: {}",
+        err
+    );
+}
+
+/// Base id 0 and the dataset URI include primary storage in the merge insert
+/// target rotation.
+#[tokio::test]
+async fn test_merge_insert_target_bases_include_primary() {
+    let fixture = multi_base_fixture(false).await;
+    let dataset = Arc::new(fixture.dataset);
+    let primary_uri = dataset.uri().to_string();
+
+    // Single new file: the first slot (primary) receives it.
+    let source = multi_base_batch(&[1, 10], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![0, 2])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, _) = job.execute(reader_to_stream(reader)).await.unwrap();
+    let new_files: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata.id >= 3)
+        .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(new_files, vec![None]);
+
+    // Flipped order: the first slot is base 2.
+    let max_id = dataset.manifest.max_fragment_id().unwrap();
+    let source = multi_base_batch(&[11], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![2, 0])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, _) = job.execute(reader_to_stream(reader)).await.unwrap();
+    let new_files: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata.id > max_id)
+        .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(new_files, vec![Some(2)]);
+
+    // Names variant: the dataset's URI selects primary storage.
+    let max_id = dataset.manifest.max_fragment_id().unwrap();
+    let source = multi_base_batch(&[12], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_base_names_or_paths(vec![primary_uri])
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, _) = job.execute(reader_to_stream(reader)).await.unwrap();
+    let new_files: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata.id > max_id)
+        .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(new_files, vec![None]);
+
+    let mut expected = vec![];
+    for id in [0, 2, 3, 4, 5, 6, 7, 8] {
+        expected.push(expected_row(id, 100, "orig"));
+    }
+    for id in [1, 10, 11, 12] {
+        expected.push(expected_row(id, 1000, "new"));
+    }
+    expected.sort_unstable();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+
+    let dataset = Dataset::open(dataset.uri()).await.unwrap();
+    assert_eq!(collect_multi_base_rows(&dataset).await, expected);
+}
+
+/// Merge insert attempts discarded by a retryable commit conflict must clean
+/// up the data files they routed to target bases; after concurrent merges the
+/// bases must contain only files referenced by the final manifest.
+#[tokio::test]
+async fn test_merge_insert_conflict_retry_cleans_routed_files() {
+    let fixture = multi_base_fixture(false).await;
+    let dataset = Arc::new(fixture.dataset);
+    let concurrency: u32 = 5;
+
+    let barrier = Arc::new(tokio::sync::Barrier::new(concurrency as usize));
+    let mut handles = Vec::new();
+    for i in 0..concurrency {
+        // Every task starts from the same dataset version and updates the same
+        // row, so all but one attempt per round hit a retryable conflict.
+        let dataset = dataset.clone();
+        let barrier = barrier.clone();
+        handles.push(tokio::spawn(async move {
+            barrier.wait().await;
+            let source = multi_base_batch(&[1, 100 + i as i32], 1000 + i as i32, "new");
+            let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .conflict_retries(20)
+                .retry_timeout(Duration::from_secs(60))
+                .target_bases(vec![1, 2])
+                .try_build()
+                .unwrap();
+            let reader = Box::new(RecordBatchIterator::new(
+                vec![Ok(source)],
+                multi_base_schema(),
+            ));
+            job.execute(reader_to_stream(reader)).await.unwrap()
+        }));
+    }
+    let mut total_attempts = 0;
+    for handle in handles {
+        let (_dataset, stats) = handle.await.unwrap();
+        total_attempts += stats.num_attempts;
+    }
+    assert!(
+        total_attempts > concurrency,
+        "expected at least one conflicted attempt, got {} attempts",
+        total_attempts
+    );
+
+    let dataset = Dataset::open(dataset.uri()).await.unwrap();
+    assert_eq!(
+        dataset.count_rows(None).await.unwrap(),
+        9 + concurrency as usize
+    );
+
+    let mut referenced: HashSet<(Option<u32>, String)> = HashSet::new();
+    for fragment in dataset.get_fragments() {
+        for file in &fragment.metadata.files {
+            referenced.insert((file.base_id, file.path.to_string()));
+        }
+    }
+    let list_files = |dir: &std::path::Path| -> Vec<String> {
+        if !dir.exists() {
+            return vec![];
+        }
+        std::fs::read_dir(dir)
+            .unwrap()
+            .filter_map(|entry| {
+                let path = entry.unwrap().path();
+                if path.extension().is_some_and(|ext| ext == "lance") {
+                    Some(path.file_name().unwrap().to_string_lossy().to_string())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    };
+    for name in list_files(&fixture.base1_dir.join("data")) {
+        assert!(
+            referenced.contains(&(Some(1), name.clone())),
+            "orphaned file in base1: {}",
+            name
+        );
+    }
+    for name in list_files(&fixture.base2_dir) {
+        assert!(
+            referenced.contains(&(Some(2), name.clone())),
+            "orphaned file in base2: {}",
+            name
+        );
+    }
+}
+
+/// `target_all_bases` on merge insert resolves to every registered base at
+/// execution time, with primary storage first when included.
+#[tokio::test]
+async fn test_merge_insert_target_all_bases() {
+    let fixture = multi_base_fixture(false).await;
+    let dataset = Arc::new(fixture.dataset);
+
+    // Single new file: with primary included it takes the first slot.
+    let source = multi_base_batch(&[20], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_all_bases(true)
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, _) = job.execute(reader_to_stream(reader)).await.unwrap();
+    let new_files: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata.id >= 3)
+        .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(new_files, vec![None]);
+
+    // Without primary the first slot is the lowest registered base id.
+    let max_id = dataset.manifest.max_fragment_id().unwrap();
+    let source = multi_base_batch(&[21], 1000, "new");
+    let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_all_bases(false)
+        .try_build()
+        .unwrap();
+    let reader = Box::new(RecordBatchIterator::new(
+        vec![Ok(source)],
+        multi_base_schema(),
+    ));
+    let (dataset, _) = job.execute(reader_to_stream(reader)).await.unwrap();
+    let new_files: Vec<_> = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata.id > max_id)
+        .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+        .collect();
+    assert_eq!(new_files, vec![Some(1)]);
+
+    // Cannot be combined with explicit target bases.
+    let err = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()])
+        .unwrap()
+        .when_matched(WhenMatched::UpdateAll)
+        .when_not_matched(WhenNotMatched::InsertAll)
+        .target_bases(vec![1])
+        .target_all_bases(true)
+        .try_build()
+        .err()
+        .unwrap();
+    assert!(
+        err.to_string()
+            .contains("Cannot specify target_all_bases together with"),
+        "unexpected error: {}",
+        err
+    );
+
+    let expected_new: Vec<_> = [20, 21]
+        .iter()
+        .map(|id| expected_row(*id, 1000, "new"))
+        .collect();
+    let all_rows = collect_multi_base_rows(&dataset).await;
+    for row in expected_new {
+        assert!(all_rows.contains(&row), "missing row {:?}", row);
+    }
+}
diff --git a/rust/lance/src/dataset/tests/dataset_scanner.rs b/rust/lance/src/dataset/tests/dataset_scanner.rs
index dcc64aa0632..363698d9966 100644
--- a/rust/lance/src/dataset/tests/dataset_scanner.rs
+++ b/rust/lance/src/dataset/tests/dataset_scanner.rs
@@ -42,6 +42,82 @@ use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::{DEFAULT_QUERY_PARALLELISM, Query};
 use pretty_assertions::assert_eq;
 
+#[tokio::test]
+async fn test_scan_wide_fixed_size_list_at_batch_boundary() {
+    const DIM_A: usize = 140_000;
+    const DIM_B: usize = 4_096;
+    const SHORT_ROWS: usize = 68;
+    const LONG_ROWS: usize = 128;
+
+    fn make_batch(schema: SchemaRef, rows: usize, base: usize) -> RecordBatch {
+        let values_a = Float32Array::from_iter_values(
+            (0..rows * DIM_A).map(|idx| ((idx + base) % 1009) as f32 / 1009.0),
+        );
+        let values_b = Float32Array::from_iter_values(
+            (0..rows * DIM_B).map(|idx| ((idx + base) % 251) as f32 / 251.0),
+        );
+        let arr_a = FixedSizeListArray::try_new_from_values(values_a, DIM_A as i32).unwrap();
+        let arr_b = FixedSizeListArray::try_new_from_values(values_b, DIM_B as i32).unwrap();
+        RecordBatch::try_new(schema, vec![Arc::new(arr_a), Arc::new(arr_b)]).unwrap()
+    }
+
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new(
+            "a",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                DIM_A as i32,
+            ),
+            true,
+        ),
+        ArrowField::new(
+            "b",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                DIM_B as i32,
+            ),
+            true,
+        ),
+    ]));
+
+    let batches = vec![
+        make_batch(schema.clone(), SHORT_ROWS, 0),
+        make_batch(schema.clone(), LONG_ROWS, 17),
+    ];
+    let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+    let write_params = WriteParams {
+        data_storage_version: Some(LanceFileVersion::V2_1),
+        ..WriteParams::default()
+    };
+    let dir = tempfile::tempdir().unwrap();
+    let dataset = Dataset::write(reader, dir.path().to_str().unwrap(), Some(write_params))
+        .await
+        .unwrap();
+
+    // The first column splits into 9 read chunks. The second column is a
+    // higher-priority request that can reserve the remaining buffer while the
+    // first column is still awaited.
+    let mut scanner = dataset.scan();
+    scanner.io_buffer_size(70 * 1024 * 1024);
+    scanner
+        .limit(Some(LONG_ROWS as i64), Some(SHORT_ROWS as i64))
+        .unwrap();
+    let mut stream = tokio::time::timeout(
+        std::time::Duration::from_secs(20),
+        scanner.try_into_stream(),
+    )
+    .await
+    .expect("stream creation timed out")
+    .unwrap();
+    let batch = tokio::time::timeout(std::time::Duration::from_secs(20), stream.try_next())
+        .await
+        .expect("first batch timed out")
+        .unwrap()
+        .unwrap();
+
+    assert_eq!(batch.num_rows(), LONG_ROWS);
+}
+
 #[tokio::test]
 async fn test_vector_filter_fts_search() {
     let dataset = prepare_query_filter_dataset().await;
@@ -62,6 +138,7 @@ async fn test_vector_filter_fts_search() {
         use_index: true,
         query_parallelism: DEFAULT_QUERY_PARALLELISM,
         dist_q_c: 0.0,
+        approx_mode: Default::default(),
     };
 
     // Case 1: search with prefilter=true, query_filter=vector([300,300,300,300])
diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs
index 3e2a4caa3b3..74790d301c1 100644
--- a/rust/lance/src/dataset/tests/dataset_transactions.rs
+++ b/rust/lance/src/dataset/tests/dataset_transactions.rs
@@ -272,6 +272,39 @@ pub(super) fn assert_results<T: Array + PartialEq + 'static>(
     )
 }
 
+fn gen_rows() -> impl arrow_array::RecordBatchReader + Send + 'static {
+    lance_datagen::gen_batch()
+        .col("key", array::step::<Int32Type>())
+        .into_reader_rows(RowCount::from(10), BatchCount::from(1))
+}
+
+/// Write a dataset with `versions` versions of 10 rows each.
+async fn write_versions(uri: &str, versions: usize, enable_v2_manifest_paths: bool) -> Dataset {
+    let mut ds = Dataset::write(
+        gen_rows(),
+        uri,
+        Some(WriteParams {
+            enable_v2_manifest_paths,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    for _ in 1..versions {
+        ds.append(
+            gen_rows(),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                enable_v2_manifest_paths,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+    }
+    ds
+}
+
 #[tokio::test]
 async fn test_inline_transaction() {
     use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
@@ -382,6 +415,197 @@ async fn test_inline_transaction() {
     assert!(ds_new.manifest.transaction_file.is_some());
     let read_tx = ds_new.read_transaction().await.unwrap().unwrap();
     assert_eq!(read_tx, tx);
+
+    // The direct read takes the same external-file fallback.
+    let version_transaction = ds_new
+        .read_version_transaction(location.version)
+        .await
+        .unwrap();
+    assert_eq!(version_transaction.transaction, Some(tx));
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_does_not_populate_caches() {
+    use lance_index::IndexType;
+    use lance_index::scalar::ScalarIndexParams;
+
+    let test_uri = TempStrDir::default();
+    let mut dataset = write_versions(&test_uri, 1, true).await;
+    // Index the table so historical manifests carry an IndexSection that a
+    // caching read path would decode.
+    dataset
+        .create_index(
+            &["key"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap(); // version 2
+    for _ in 0..18 {
+        dataset
+            .append(
+                gen_rows(),
+                Some(WriteParams {
+                    mode: WriteMode::Append,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap();
+    }
+    let latest_version = dataset.version().version;
+    assert_eq!(latest_version, 20);
+
+    // Fresh session so any cache insertion by the API under test shows as growth.
+    let session = Arc::new(Session::default());
+    let dataset = DatasetBuilder::from_uri(&test_uri)
+        .with_session(session.clone())
+        .load()
+        .await
+        .unwrap();
+
+    let metadata_stats_before = session.metadata_cache_stats().await;
+    let index_stats_before = session.index_cache_stats().await;
+
+    let mut actual = Vec::with_capacity(latest_version as usize);
+    for version in 1..=latest_version {
+        let version_transaction = dataset.read_version_transaction(version).await.unwrap();
+        assert_eq!(version_transaction.version, version);
+        actual.push(version_transaction);
+    }
+
+    let metadata_stats_after = session.metadata_cache_stats().await;
+    let index_stats_after = session.index_cache_stats().await;
+    assert_eq!(
+        metadata_stats_after.num_entries,
+        metadata_stats_before.num_entries
+    );
+    assert_eq!(
+        metadata_stats_after.size_bytes,
+        metadata_stats_before.size_bytes
+    );
+    assert_eq!(
+        index_stats_after.num_entries,
+        index_stats_before.num_entries
+    );
+    assert_eq!(index_stats_after.size_bytes, index_stats_before.size_bytes);
+
+    // Results match a full checkout.
+    for version_transaction in &actual {
+        let checked_out = dataset
+            .checkout_version(version_transaction.version)
+            .await
+            .unwrap();
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+        assert_eq!(
+            version_transaction.timestamp,
+            checked_out.version().timestamp
+        );
+        assert!(version_transaction.transaction.is_some());
+    }
+
+    // A missing (e.g. cleaned up) version errors as DatasetNotFound, matching
+    // the historical checkout_version-based contract of the public API.
+    let err = dataset.read_version_transaction(9999).await.unwrap_err();
+    assert!(
+        matches!(err, crate::Error::DatasetNotFound { .. }),
+        "expected DatasetNotFound for a missing version, got {err:?}"
+    );
+}
+
+#[tokio::test]
+async fn test_read_transaction_recovers_from_stale_manifest_size() {
+    let test_uri = TempStrDir::default();
+    let ds = write_versions(&test_uri, 1, true).await;
+    let manifest = ds.manifest().clone();
+    // Only meaningful for the inline path; a plain write inlines the transaction.
+    assert!(manifest.transaction_section.is_some());
+
+    // A size at/under the transaction offset makes the first read_message fail
+    // "file size is too small"; only the retry at the true size can recover.
+    let mut stale = ds.manifest_location().clone();
+    stale.size = Some(1);
+    let recovered = ds
+        .read_transaction_from_storage(&manifest, &stale)
+        .await
+        .unwrap();
+    assert_eq!(recovered, ds.read_transaction().await.unwrap());
+    assert!(recovered.is_some());
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_v1_manifest_naming() {
+    let test_uri = TempStrDir::default();
+    let ds = write_versions(&test_uri, 3, false).await;
+    assert_eq!(
+        ds.manifest_location().naming_scheme,
+        ManifestNamingScheme::V1
+    );
+
+    for version in 1..=3 {
+        let version_transaction = ds.read_version_transaction(version).await.unwrap();
+        let checked_out = ds.checkout_version(version).await.unwrap();
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+        assert_eq!(
+            version_transaction.timestamp,
+            checked_out.version().timestamp
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_read_version_transaction_on_branch() {
+    let test_uri = TempStrDir::default();
+    let mut main_ds = write_versions(&test_uri, 1, true).await;
+    let branch_ds = main_ds.create_branch("dev", 1, None).await.unwrap();
+
+    // Commit on the branch.
+    let branch_ds = Dataset::write(
+        gen_rows(),
+        branch_ds.uri(),
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(branch_ds.manifest().branch.as_deref(), Some("dev"));
+
+    // Versions resolve against the branch chain and match a full checkout.
+    for version in branch_ds.versions().await.unwrap() {
+        let version_transaction = branch_ds
+            .read_version_transaction(version.version)
+            .await
+            .unwrap();
+        assert_eq!(version_transaction.version, version.version);
+        assert_eq!(version_transaction.timestamp, version.timestamp);
+        let checked_out = branch_ds.checkout_version(version.version).await.unwrap();
+        assert_eq!(checked_out.manifest().branch.as_deref(), Some("dev"));
+        assert_eq!(
+            version_transaction.transaction,
+            checked_out.read_transaction().await.unwrap()
+        );
+    }
+
+    // The append on the branch is the branch's own transaction.
+    let latest = branch_ds.version().version;
+    let version_transaction = branch_ds.read_version_transaction(latest).await.unwrap();
+    assert!(matches!(
+        version_transaction.transaction,
+        Some(Transaction {
+            operation: Operation::Append { .. },
+            ..
+        })
+    ));
 }
 
 #[tokio::test]
diff --git a/rust/lance/src/dataset/tests/dataset_versioning.rs b/rust/lance/src/dataset/tests/dataset_versioning.rs
index 5ac01c498b2..c04dd0f3183 100644
--- a/rust/lance/src/dataset/tests/dataset_versioning.rs
+++ b/rust/lance/src/dataset/tests/dataset_versioning.rs
@@ -211,6 +211,77 @@ async fn test_version_id_fast_path() {
     assert_eq!(historical.latest_version_id().await.unwrap(), 2);
 }
 
+#[rstest]
+#[tokio::test]
+async fn test_stale_checks_cover_fast_successor_and_latest_version(
+    #[values(false, true)] enable_v2_manifest_paths: bool,
+) {
+    let expected_scheme = if enable_v2_manifest_paths {
+        ManifestNamingScheme::V2
+    } else {
+        ManifestNamingScheme::V1
+    };
+    let test_uri = TempStrDir::default();
+    let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+        "i",
+        DataType::UInt32,
+        false,
+    )]));
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(UInt32Array::from_iter_values(0..5))],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema.clone());
+
+    let original = Dataset::write(
+        reader,
+        &test_uri,
+        Some(WriteParams {
+            enable_v2_manifest_paths,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    assert_eq!(original.manifest_location().naming_scheme, expected_scheme);
+    assert!(!original.is_stale().await.unwrap());
+    assert!(!original.has_successor_version().await.unwrap());
+
+    let data = RecordBatch::try_new(
+        schema.clone(),
+        vec![Arc::new(UInt32Array::from_iter_values(5..10))],
+    )
+    .unwrap();
+    let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema);
+    let updated = Dataset::write(
+        reader,
+        &test_uri,
+        Some(WriteParams {
+            mode: WriteMode::Append,
+            enable_v2_manifest_paths,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+
+    assert!(original.is_stale().await.unwrap());
+    assert!(original.has_successor_version().await.unwrap());
+    assert_eq!(updated.manifest_location().naming_scheme, expected_scheme);
+    assert!(!updated.is_stale().await.unwrap());
+    assert!(!updated.has_successor_version().await.unwrap());
+
+    let historical = updated.checkout_version(1).await.unwrap();
+    assert_eq!(
+        historical.manifest_location().naming_scheme,
+        expected_scheme
+    );
+    assert!(historical.is_stale().await.unwrap());
+    assert!(historical.has_successor_version().await.unwrap());
+}
+
 #[rstest]
 #[tokio::test]
 async fn test_restore(
@@ -565,6 +636,65 @@ async fn test_fragment_id_never_reset() {
     assert_eq!(dataset.manifest.max_fragment_id(), Some(4));
 }
 
+/// create_branch and shallow_clone must read the SOURCE ref's chain, not the
+/// receiver's. Both chains get a version 2 with diverged row counts so a clone
+/// that wrongly resolves the version under the receiver succeeds silently with
+/// the wrong data.
+#[tokio::test]
+async fn test_create_branch_and_shallow_clone_from_other_branch() {
+    let tempdir = TempDir::default();
+    let test_uri = tempdir.path_str();
+
+    let gen_rows = |start: i32, rows: u64| {
+        gen_batch()
+            .col("id", array::step_custom::<Int32Type>(start, 1))
+            .into_reader_rows(RowCount::from(rows), BatchCount::from(1))
+    };
+    let write = |uri: String, start: i32, rows: u64, mode: WriteMode| async move {
+        Dataset::write(
+            gen_rows(start, rows),
+            uri.as_str(),
+            Some(WriteParams {
+                mode,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap()
+    };
+
+    // main v1: 50 rows.
+    let mut main_ds = write(test_uri.clone(), 0, 50, WriteMode::Create).await;
+    // dev: forked at v1, appended 30 rows -> dev v2 has 80 rows.
+    let dev_ds = main_ds.create_branch("dev", 1, None).await.unwrap();
+    write(dev_ds.uri().to_string(), 1000, 30, WriteMode::Append).await;
+    // Diverge main to the same version number with a different row count.
+    let mut main_ds = write(test_uri.clone(), 5000, 10, WriteMode::Append).await; // main v2: 60 rows
+
+    // Cross-source create_branch: receiver is main, source is dev.
+    let child_ds = main_ds
+        .create_branch("child", ("dev", 2), None)
+        .await
+        .unwrap();
+    assert_eq!(
+        child_ds.count_rows(None).await.unwrap(),
+        80,
+        "child must clone dev@2, not main@2"
+    );
+
+    // Cross-source shallow_clone: same rule.
+    let clone_uri = format!("{}_clone", test_uri);
+    let cloned_ds = main_ds
+        .shallow_clone(&clone_uri, ("dev", 2), None)
+        .await
+        .unwrap();
+    assert_eq!(
+        cloned_ds.count_rows(None).await.unwrap(),
+        80,
+        "shallow clone must read dev@2, not main@2"
+    );
+}
+
 #[tokio::test]
 async fn test_branch() {
     let tempdir = TempDir::default();
@@ -797,6 +927,86 @@ async fn test_branch() {
         "branch1"
     );
 
+    // Opening at a branch-pointing tag through the builder must check out the
+    // tag's branch chain, not main's chain at the tag's version number.
+    let tag_open = DatasetBuilder::from_uri(&test_uri)
+        .with_tag("tag1")
+        .load()
+        .await
+        .unwrap();
+    assert_eq!(tag_open.manifest.branch.as_deref(), Some("dev/branch2"));
+    assert_eq!(tag_open.version().version, 3);
+    assert_eq!(tag_open.count_rows(None).await.unwrap(), 100);
+
+    // Malformed branch names are rejected at the boundary
+    for bad_name in ["", "branch1/"] {
+        let err = main_dataset
+            .checkout_version((Some(bad_name), None::<u64>))
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidRef { .. }),
+            "checkout of {:?} must be rejected as InvalidRef, got: {}",
+            bad_name,
+            err
+        );
+        let err = DatasetBuilder::from_uri(&test_uri)
+            .with_branch(bad_name, None)
+            .load()
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(err, Error::InvalidRef { .. }),
+            "open of {:?} must be rejected as InvalidRef, got: {}",
+            bad_name,
+            err
+        );
+    }
+
+    // "main" stays a valid spelling of the main branch on checkout; the JNI
+    // bindings construct Ref::Version(Some("main"), _) directly.
+    let main_by_name = checkout_branch1.checkout_branch("main").await.unwrap();
+    assert_eq!(main_by_name.manifest.branch, None);
+    assert_eq!(main_by_name.version().version, 1);
+    let main_by_ref = checkout_branch1
+        .checkout_version(crate::dataset::refs::Ref::Version(
+            Some("main".to_string()),
+            None,
+        ))
+        .await
+        .unwrap();
+    assert_eq!(main_by_ref.manifest.branch, None);
+
+    // A checkout whose resolved manifest is not on the requested branch must
+    // error loudly instead of handing back another branch's data: stage main's
+    // manifest under a branch path that was never created, so resolution finds
+    // a manifest belonging to main.
+    use object_store::ObjectStoreExt as _;
+    let staged_manifest = main_dataset.manifest_location().path.clone();
+    let staged_copy = Path::parse(format!(
+        "{}/tree/ghost/_versions/{}",
+        test_uri,
+        staged_manifest.filename().unwrap()
+    ))
+    .unwrap();
+    main_dataset
+        .object_store
+        .inner
+        .copy(&staged_manifest, &staged_copy)
+        .await
+        .unwrap();
+    let err = main_dataset.checkout_branch("ghost").await.unwrap_err();
+    assert!(
+        err.to_string().contains("resolved a manifest belonging to"),
+        "expected the branch-mismatch guardrail, got: {}",
+        err
+    );
+    main_dataset
+        .object_store
+        .remove_dir_all(Path::parse(format!("{}/tree/ghost", test_uri)).unwrap())
+        .await
+        .unwrap();
+
     let mut dataset = main_dataset;
     // Finally delete all branches
     assert!(matches!(
diff --git a/rust/lance/src/dataset/transaction.rs b/rust/lance/src/dataset/transaction.rs
index 3f96b9964d5..79f67f74690 100644
--- a/rust/lance/src/dataset/transaction.rs
+++ b/rust/lance/src/dataset/transaction.rs
@@ -17,11 +17,11 @@ use super::write::merge_insert::inserted_rows::KeyExistenceFilter;
 use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows};
 use crate::index::mem_wal::update_mem_wal_index_merged_generations;
 use crate::utils::temporal::timestamp_to_nanos;
-use deepsize::DeepSizeOf;
 use lance_core::datatypes::{
     LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY,
     LANCE_UNENFORCED_PRIMARY_KEY_POSITION,
 };
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result, datatypes::Schema};
 use lance_file::{datatypes::Fields, version::LanceFileVersion};
 use lance_index::mem_wal::MergedGeneration;
@@ -32,7 +32,8 @@ use lance_table::rowids::read_row_ids;
 use lance_table::{
     format::{
         BasePath, DataFile, DataStorageFormat, Fragment, IndexFile, IndexMetadata, Manifest,
-        RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, pb,
+        RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta,
+        overlay::DataOverlayFile, pb,
     },
     io::{
         commit::CommitHandler,
@@ -258,6 +259,17 @@ pub struct Transaction {
 #[derive(Debug, Clone, DeepSizeOf, PartialEq)]
 pub struct DataReplacementGroup(pub u64, pub DataFile);
 
+/// Overlay files to append to a single fragment, in order (the last entry is
+/// newest). The overlays are appended to the fragment's existing `overlays`
+/// list rather than replacing it, so overlays written by concurrent commits are
+/// preserved. Each overlay's `committed_version` is stamped to the new dataset
+/// version at commit time (re-stamped on retry).
+#[derive(Debug, Clone, DeepSizeOf, PartialEq)]
+pub struct DataOverlayGroup {
+    pub fragment_id: u64,
+    pub overlays: Vec<DataOverlayFile>,
+}
+
 /// An entry for a map update. If value is None, the key will be removed from the map.
 #[derive(Debug, Clone, DeepSizeOf, PartialEq)]
 pub struct UpdateMapEntry {
@@ -367,6 +379,11 @@ pub enum Operation {
     DataReplacement {
         replacements: Vec<DataReplacementGroup>,
     },
+    /// Attach overlay files to fragments, supplying new values for a subset of
+    /// `(physical offset, field)` cells without rewriting the fragments' base
+    /// data files. See [`DataOverlayFile`] and the Data Overlay Files
+    /// specification for resolution, coverage, and versioning rules.
+    DataOverlay { groups: Vec<DataOverlayGroup> },
     /// Merge a new column in
     /// 'fragments' is the final fragments include all data files, the new fragments must align with old ones at rows.
     /// 'schema' is not forced to include existed columns, which means we could use Merge to drop column data
@@ -476,7 +493,7 @@ pub enum UpdateMode {
 pub struct UpdatedFragmentOffsets(pub HashMap<u64, RoaringBitmap>);
 
 impl DeepSizeOf for UpdatedFragmentOffsets {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.0.iter().fold(0_usize, |acc, (frag_id, bitmap)| {
             acc + frag_id.deep_size_of_children(context)
                 + (bitmap.len() as usize).saturating_mul(std::mem::size_of::<u32>())
@@ -499,6 +516,7 @@ impl std::fmt::Display for Operation {
             Self::Project { .. } => write!(f, "Project"),
             Self::UpdateConfig { .. } => write!(f, "UpdateConfig"),
             Self::DataReplacement { .. } => write!(f, "DataReplacement"),
+            Self::DataOverlay { .. } => write!(f, "DataOverlay"),
             Self::Clone { .. } => write!(f, "Clone"),
             Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"),
             Self::UpdateBases { .. } => write!(f, "UpdateBases"),
@@ -1345,6 +1363,8 @@ impl PartialEq for Operation {
             (Self::Clone { .. }, Self::UpdateBases { .. }) => {
                 std::mem::discriminant(self) == std::mem::discriminant(other)
             }
+            (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b),
+            (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false,
         }
     }
 }
@@ -1361,7 +1381,7 @@ pub struct RewrittenIndex {
 }
 
 impl DeepSizeOf for RewrittenIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.new_index_details
             .type_url
             .deep_size_of_children(context)
@@ -1521,6 +1541,7 @@ impl Operation {
             Self::Project { .. } => "Project",
             Self::UpdateConfig { .. } => "UpdateConfig",
             Self::DataReplacement { .. } => "DataReplacement",
+            Self::DataOverlay { .. } => "DataOverlay",
             Self::UpdateMemWalState { .. } => "UpdateMemWalState",
             Self::Clone { .. } => "Clone",
             Self::UpdateBases { .. } => "UpdateBases",
@@ -1861,6 +1882,8 @@ impl Transaction {
                     ))
                 });
 
+        let new_version = current_manifest.map_or(1, |m| m.version + 1);
+
         match &self.operation {
             Operation::Clone { .. } => {
                 return Err(Error::internal(
@@ -1875,7 +1898,6 @@ impl Transaction {
                 if let Some(next_row_id) = &mut next_row_id {
                     Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?;
                     // Add version metadata for all new fragments
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     for fragment in new_fragments.iter_mut() {
                         let version_meta = build_version_meta(fragment, new_version);
                         fragment.last_updated_at_version_meta = version_meta.clone();
@@ -1923,7 +1945,20 @@ impl Transaction {
                             return None;
                         }
                         if let Some(updated) = updated_fragments.iter().find(|uf| uf.id == f.id) {
-                            Some(updated.clone())
+                            let mut updated = updated.clone();
+                            // Carry forward the fragment's current overlays (which
+                            // may include ones added by a concurrent commit). An
+                            // in-place column rewrite then tombstones the overlaid
+                            // fields it rewrote, since the fresh base values
+                            // supersede them.
+                            updated.overlays = f.overlays.clone();
+                            if matches!(update_mode, Some(RewriteColumns)) {
+                                lance_table::format::overlay::tombstone_overlay_fields(
+                                    &mut updated.overlays,
+                                    fields_modified,
+                                );
+                            }
+                            Some(updated)
                         } else {
                             Some(f.clone())
                         }
@@ -1945,7 +1980,6 @@ impl Transaction {
                     && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets
                     && !off_map.is_empty()
                 {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     let prev_version = current_manifest.map(|m| m.version).unwrap_or(0);
                     for fragment in final_fragments.iter_mut() {
                         let Some(bitmap) = off_map.get(&fragment.id) else {
@@ -1988,7 +2022,6 @@ impl Transaction {
                 }
 
                 if next_row_id.is_some() {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     resolve_update_version_metadata(
                         existing_fragments,
                         new_fragments.as_mut_slice(),
@@ -2033,7 +2066,7 @@ impl Transaction {
                 if !merged_generations.is_empty() {
                     update_mem_wal_index_merged_generations(
                         &mut final_indices,
-                        current_manifest.map_or(1, |m| m.version + 1),
+                        new_version,
                         merged_generations.clone(),
                     )?;
                 }
@@ -2045,7 +2078,6 @@ impl Transaction {
                 if let Some(next_row_id) = &mut next_row_id {
                     Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?;
                     // Add version metadata for all new fragments
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
                     for fragment in new_fragments.iter_mut() {
                         let version_meta = build_version_meta(fragment, new_version);
                         fragment.last_updated_at_version_meta = version_meta.clone();
@@ -2083,6 +2115,12 @@ impl Transaction {
                     Self::handle_rewrite_indices(&mut final_indices, rewritten_indices, groups)?;
                 }
 
+                // A full compaction materializes a fragment's overlays into fresh
+                // base data. Any index older than one of those overlays was built on
+                // the pre-overlay values, so drop the rewritten fragment from its
+                // coverage to keep it from serving stale values.
+                Self::prune_overlay_stale_fields_from_indices(&mut final_indices, groups);
+
                 if let Some(frag_reuse_index) = frag_reuse_index {
                     final_indices.retain(|idx| idx.name != frag_reuse_index.name);
                     final_indices.push(frag_reuse_index.clone());
@@ -2093,6 +2131,53 @@ impl Transaction {
                 removed_indices,
             } => {
                 final_fragments.extend(maybe_existing_fragments?.clone());
+
+                // The transaction may have been staged against an older version,
+                // and the pairwise conflict checks cannot enumerate every
+                // hazardous concurrent operation. Enforce the invariants that
+                // make it safe to apply here, where the final manifest is known:
+                // every indexed field must still exist in the schema (a
+                // concurrent projection or column cast drops or re-ids fields),
+                // and every non-system segment staged for removal must still be
+                // present (a concurrent drop or replacement would otherwise be
+                // silently undone). System-index removals are reconciled during
+                // conflict rebasing and are exempt.
+                let field_ids = schema
+                    .fields_pre_order()
+                    .map(|f| f.id)
+                    .collect::<HashSet<_>>();
+                for new_index in new_indices {
+                    if let Some(field_id) = new_index
+                        .fields
+                        .iter()
+                        .find(|field_id| !field_ids.contains(*field_id))
+                    {
+                        return Err(Error::incompatible_transaction_source(
+                            format!(
+                                "CreateIndex: field {} covered by index '{}' no longer \
+                                 exists in the schema",
+                                field_id, new_index.name
+                            )
+                            .into(),
+                        ));
+                    }
+                }
+                if let Some(missing) = removed_indices.iter().find(|removed| {
+                    !is_system_index(removed)
+                        && !final_indices.iter().any(|idx| idx.uuid == removed.uuid)
+                }) {
+                    return Err(Error::retryable_commit_conflict_source(
+                        current_manifest.map(|m| m.version).unwrap_or_default(),
+                        format!(
+                            "CreateIndex: segment {} of index '{}' staged for removal was \
+                             removed by a concurrent transaction; please re-stage against \
+                             the latest version",
+                            missing.uuid, missing.name
+                        )
+                        .into(),
+                    ));
+                }
+
                 let removed_uuids = removed_indices
                     .iter()
                     .map(|old_index| old_index.uuid)
@@ -2111,13 +2196,11 @@ impl Transaction {
                 final_fragments.extend(maybe_existing_fragments?.clone());
             }
             Operation::Merge { fragments, .. } => {
+                let existing_fragments = maybe_existing_fragments?;
                 let mut merged_fragments = fragments.clone();
                 if next_row_id.is_some() {
-                    let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1);
-                    let prev_by_id: HashMap<u64, &Fragment> = maybe_existing_fragments?
-                        .iter()
-                        .map(|f| (f.id, f))
-                        .collect();
+                    let prev_by_id: HashMap<u64, &Fragment> =
+                        existing_fragments.iter().map(|f| (f.id, f)).collect();
                     for fragment in merged_fragments.iter_mut() {
                         match prev_by_id.get(&fragment.id) {
                             Some(prev) => {
@@ -2144,6 +2227,15 @@ impl Transaction {
                 }
                 final_fragments.extend(merged_fragments);
 
+                // A Merge can rewrite a column's data file in place; the field stays
+                // in the schema, so the index is retained -- prune its now-stale
+                // entries for the rewritten fragments.
+                Self::prune_merge_rewritten_fields_from_indices(
+                    &mut final_indices,
+                    existing_fragments,
+                    fragments,
+                );
+
                 // Some fields that have indices may have been removed, so we should
                 // remove those indices as well.
                 Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments)
@@ -2240,27 +2332,24 @@ impl Transaction {
                             && file.file_major_version == new_file.file_major_version
                             && file.file_minor_version == new_file.file_minor_version
                         {
-                            // assign the new file path / size to the fragment
+                            // assign the new file path / size / base to the fragment
                             file.path = new_file.path.clone();
                             file.file_size_bytes = new_file.file_size_bytes.clone();
+                            file.base_id = new_file.base_id;
                         }
                         columns_covered.extend(file.fields.iter());
                     }
                     // SPECIAL CASE: if the column(s) being replaced are not covered by the fragment
                     // Then it means it's a all-NULL column that is being replaced with real data
-                    // just add it to the final fragments
+                    // just add it to the final fragments. Push the DataFile as
+                    // given so every field (including base_id) is preserved.
                     if columns_covered.is_disjoint(&new_file.fields.iter().collect()) {
-                        new_frag.add_file(
-                            new_file.path.clone(),
-                            new_file.fields.to_vec(),
-                            new_file.column_indices.to_vec(),
-                            &LanceFileVersion::try_from_major_minor(
-                                new_file.file_major_version,
-                                new_file.file_minor_version,
-                            )
-                            .expect("Expected valid file version"),
-                            new_file.file_size_bytes.get(),
-                        );
+                        LanceFileVersion::try_from_major_minor(
+                            new_file.file_major_version,
+                            new_file.file_minor_version,
+                        )
+                        .expect("Expected valid file version");
+                        new_frag.files.push(new_file.clone());
                     }
 
                     // Nothing changed in the current fragment, which is not expected -- error out
@@ -2269,6 +2358,15 @@ impl Transaction {
                             "Expected to modify the fragment but no changes were made. This means the new data files does not align with any exiting datafiles. Please check if the schema of the new data files matches the schema of the old data files including the file major and minor versions",
                         ));
                     }
+
+                    // New base values for these fields supersede any overlay
+                    // still shadowing them; tombstone the overlaid fields so the
+                    // replacement is not silently masked.
+                    lance_table::format::overlay::tombstone_overlay_fields(
+                        &mut new_frag.overlays,
+                        &replaced_fields,
+                    );
+
                     final_fragments.push(new_frag);
                 }
 
@@ -2300,10 +2398,57 @@ impl Transaction {
                     &replaced_fields,
                 );
             }
+            Operation::DataOverlay { groups } => {
+                // Stamp each overlay with the version this commit is producing.
+                // build_manifest re-runs on every retry with an updated
+                // current_manifest, so this is naturally re-stamped on retry.
+                let new_version = current_manifest.map_or(1, |m| m.version + 1);
+
+                let existing_fragments = maybe_existing_fragments?;
+                // Multiple groups may target the same fragment; merge them in
+                // order rather than letting a HashMap collapse drop all but the
+                // last group's overlays.
+                let mut overlays_by_fragment: HashMap<u64, Vec<&DataOverlayFile>> = HashMap::new();
+                for group in groups {
+                    overlays_by_fragment
+                        .entry(group.fragment_id)
+                        .or_default()
+                        .extend(group.overlays.iter());
+                }
+
+                // Every group must target an existing fragment. Build a set of
+                // existing ids once so this is O(groups + fragments) rather than
+                // O(groups * fragments).
+                let existing_fragment_ids: HashSet<u64> =
+                    existing_fragments.iter().map(|f| f.id).collect();
+                for fragment_id in overlays_by_fragment.keys() {
+                    if !existing_fragment_ids.contains(fragment_id) {
+                        return Err(Error::invalid_input(format!(
+                            "DataOverlay targets fragment {fragment_id}, which does not exist"
+                        )));
+                    }
+                }
+
+                for fragment in existing_fragments {
+                    let mut fragment = fragment.clone();
+                    if let Some(new_overlays) = overlays_by_fragment.get(&fragment.id) {
+                        // Appended (not replaced) so concurrently-written overlays
+                        // survive; later entries are newer.
+                        fragment
+                            .overlays
+                            .extend(new_overlays.iter().map(|&overlay| {
+                                let mut overlay = overlay.clone();
+                                overlay.committed_version = new_version;
+                                overlay
+                            }));
+                    }
+                    final_fragments.push(fragment);
+                }
+            }
             Operation::UpdateMemWalState { merged_generations } => {
                 update_mem_wal_index_merged_generations(
                     &mut final_indices,
-                    current_manifest.map_or(1, |m| m.version + 1),
+                    new_version,
                     merged_generations.clone(),
                 )?;
             }
@@ -2320,6 +2465,15 @@ impl Transaction {
         // Clean up data files that only contain tombstoned fields
         Self::remove_tombstoned_data_files(&mut final_fragments);
 
+        // Enforce the newest-last overlay ordering invariant at the write
+        // boundary. Load normalizes with a sort; this rejects any commit path
+        // that assembled a fragment's overlays out of order.
+        for fragment in &final_fragments {
+            if !fragment.overlays.is_empty() {
+                lance_table::format::overlay::verify_overlays_newest_last(&fragment.overlays)?;
+            }
+        }
+
         let user_requested_version = match (&config.storage_format, config.use_legacy_format) {
             (Some(storage_format), _) => Some(storage_format.lance_file_version()?),
             (None, Some(true)) => Some(LanceFileVersion::Legacy),
@@ -2623,6 +2777,110 @@ impl Transaction {
         }
     }
 
+    /// Map each (non-tombstoned) field id in a fragment to the path of the data
+    /// file that backs it.
+    fn fragment_field_paths(frag: &Fragment) -> HashMap<i32, &str> {
+        let mut map = HashMap::new();
+        for file in &frag.files {
+            for &field_id in file.fields.iter() {
+                if field_id >= 0 {
+                    map.insert(field_id, file.path.as_str());
+                }
+            }
+        }
+        map
+    }
+
+    /// A `Merge` can rewrite a column's data *in place* -- the field stays in the
+    /// schema but its backing data file changes (the overlay fragment carries a new
+    /// file for the field and tombstones its old field id). `retain_relevant_indices`
+    /// only drops indices for *removed* fields, so without this the index keeps
+    /// covering the rewritten fragments with stale entries. Remove each such fragment
+    /// from any index covering a field whose backing data file changed.
+    fn prune_merge_rewritten_fields_from_indices(
+        indices: &mut [IndexMetadata],
+        prev_fragments: &[Fragment],
+        new_fragments: &[Fragment],
+    ) {
+        let prev_by_id: HashMap<u64, &Fragment> =
+            prev_fragments.iter().map(|f| (f.id, f)).collect();
+        for new_frag in new_fragments {
+            let Some(prev) = prev_by_id.get(&new_frag.id) else {
+                continue; // brand-new fragment: nothing stale to prune
+            };
+            let prev_paths = Self::fragment_field_paths(prev);
+            let new_paths = Self::fragment_field_paths(new_frag);
+            // Fields still present whose backing file path changed == rewritten data.
+            let changed: Vec<u32> = prev_paths
+                .iter()
+                .filter(|(field_id, prev_path)| {
+                    new_paths
+                        .get(*field_id)
+                        .is_some_and(|new_path| new_path != *prev_path)
+                })
+                .map(|(field_id, _)| *field_id as u32)
+                .collect();
+            if changed.is_empty() {
+                continue;
+            }
+            Self::prune_updated_fields_from_indices(
+                indices,
+                std::slice::from_ref(new_frag),
+                &changed,
+            );
+        }
+    }
+
+    /// After a `Rewrite` fully compacts a fragment, its data overlays are baked
+    /// into the new fragment's base data. An index built *before* one of those
+    /// overlays (`overlay.committed_version > index.dataset_version`) indexed the
+    /// stale pre-overlay values -- and unlike a live overlay, the compacted
+    /// fragment no longer signals that staleness to the query path. Drop each
+    /// rewritten (new) fragment from the coverage of any index covering a field
+    /// such an overlay supplied, so those rows fall back to a flat scan.
+    fn prune_overlay_stale_fields_from_indices(
+        indices: &mut [IndexMetadata],
+        groups: &[RewriteGroup],
+    ) {
+        for group in groups {
+            // field id -> newest overlay committed_version supplying that field
+            let mut overlaid_field_versions: HashMap<i32, u64> = HashMap::new();
+            for old_frag in &group.old_fragments {
+                for overlay in &old_frag.overlays {
+                    for &field_id in overlay.data_file.fields.iter() {
+                        if field_id < 0 {
+                            // Tombstoned (obsolete) overlay field: supplies nothing.
+                            continue;
+                        }
+                        let entry = overlaid_field_versions.entry(field_id).or_insert(0);
+                        *entry = (*entry).max(overlay.committed_version);
+                    }
+                }
+            }
+            if overlaid_field_versions.is_empty() {
+                continue;
+            }
+
+            let new_fragment_ids = group
+                .new_fragments
+                .iter()
+                .map(|f| f.id as u32)
+                .collect::<Vec<_>>();
+            for index in indices.iter_mut() {
+                let is_stale = index.fields.iter().any(|field_id| {
+                    overlaid_field_versions
+                        .get(field_id)
+                        .is_some_and(|&overlay_version| overlay_version > index.dataset_version)
+                });
+                if is_stale && let Some(fragment_bitmap) = &mut index.fragment_bitmap {
+                    for new_id in &new_fragment_ids {
+                        fragment_bitmap.remove(*new_id);
+                    }
+                }
+            }
+        }
+    }
+
     fn is_vector_index(index: &IndexMetadata) -> bool {
         if let Some(details) = &index.index_details {
             details.type_url.ends_with("VectorIndexDetails")
@@ -3007,6 +3265,34 @@ impl TryFrom<pb::transaction::DataReplacementGroup> for DataReplacementGroup {
     }
 }
 
+impl From<&DataOverlayGroup> for pb::transaction::DataOverlayGroup {
+    fn from(group: &DataOverlayGroup) -> Self {
+        Self {
+            fragment_id: group.fragment_id,
+            overlays: group
+                .overlays
+                .iter()
+                .map(pb::DataOverlayFile::from)
+                .collect(),
+        }
+    }
+}
+
+impl TryFrom<pb::transaction::DataOverlayGroup> for DataOverlayGroup {
+    type Error = Error;
+
+    fn try_from(message: pb::transaction::DataOverlayGroup) -> Result<Self> {
+        Ok(Self {
+            fragment_id: message.fragment_id,
+            overlays: message
+                .overlays
+                .into_iter()
+                .map(DataOverlayFile::try_from)
+                .collect::<Result<Vec<_>>>()?,
+        })
+    }
+}
+
 impl TryFrom<pb::Transaction> for Transaction {
     type Error = Error;
 
@@ -3063,7 +3349,7 @@ impl TryFrom<pb::Transaction> for Transaction {
                         .into_iter()
                         .map(Fragment::try_from)
                         .collect::<Result<Vec<_>>>()?,
-                    schema: Schema::from(&Fields(schema)),
+                    schema: Schema::try_from(&Fields(schema))?,
                     config_upsert_values: config_upsert_option,
                     initial_bases: if initial_bases.is_empty() {
                         None
@@ -3131,7 +3417,7 @@ impl TryFrom<pb::Transaction> for Transaction {
                     .into_iter()
                     .map(Fragment::try_from)
                     .collect::<Result<Vec<_>>>()?,
-                schema: Schema::from(&Fields(schema)),
+                schema: Schema::try_from(&Fields(schema))?,
             },
             Some(pb::transaction::Operation::Restore(pb::transaction::Restore { version })) => {
                 Operation::Restore { version }
@@ -3185,7 +3471,7 @@ impl TryFrom<pb::Transaction> for Transaction {
             },
             Some(pb::transaction::Operation::Project(pb::transaction::Project { schema })) => {
                 Operation::Project {
-                    schema: Schema::from(&Fields(schema)),
+                    schema: Schema::try_from(&Fields(schema))?,
                 }
             }
             Some(pb::transaction::Operation::UpdateConfig(update_config)) => {
@@ -3289,6 +3575,14 @@ impl TryFrom<pb::Transaction> for Transaction {
             })) => Operation::UpdateBases {
                 new_bases: new_bases.into_iter().map(BasePath::from).collect(),
             },
+            Some(pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay {
+                groups,
+            })) => Operation::DataOverlay {
+                groups: groups
+                    .into_iter()
+                    .map(DataOverlayGroup::try_from)
+                    .collect::<Result<Vec<_>>>()?,
+            },
             None => {
                 return Err(Error::internal(
                     "Transaction message did not contain an operation".to_string(),
@@ -3559,6 +3853,14 @@ impl From<&Transaction> for pb::Transaction {
                         .collect(),
                 })
             }
+            Operation::DataOverlay { groups } => {
+                pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay {
+                    groups: groups
+                        .iter()
+                        .map(pb::transaction::DataOverlayGroup::from)
+                        .collect(),
+                })
+            }
             Operation::UpdateMemWalState { merged_generations } => {
                 pb::transaction::Operation::UpdateMemWalState(pb::transaction::UpdateMemWalState {
                     merged_generations: merged_generations
@@ -3837,6 +4139,7 @@ mod tests {
     use lance_core::{ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_LAST_UPDATED_AT_VERSION};
     use lance_file::version::LanceFileVersion;
     use lance_io::utils::CachedFileSize;
+    use lance_table::format::overlay::OverlayCoverage;
     use lance_table::format::{
         RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta,
     };
@@ -4148,6 +4451,7 @@ mod tests {
             physical_rows: Some(100),
             row_id_meta: None,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4180,6 +4484,7 @@ mod tests {
             physical_rows: Some(50),
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4212,6 +4517,7 @@ mod tests {
             physical_rows: Some(50), // More physical rows than existing row IDs
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4247,6 +4553,7 @@ mod tests {
             physical_rows: Some(50), // Less physical rows than existing row IDs
             row_id_meta: Some(RowIdMeta::Inline(serialized)),
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4275,6 +4582,7 @@ mod tests {
                 physical_rows: Some(30), // No existing row IDs
                 row_id_meta: None,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 last_updated_at_version_meta: None,
                 created_at_version_meta: None,
@@ -4284,6 +4592,7 @@ mod tests {
                 physical_rows: Some(25), // Partial existing row IDs
                 row_id_meta: Some(RowIdMeta::Inline(serialized)),
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 last_updated_at_version_meta: None,
                 created_at_version_meta: None,
@@ -4328,6 +4637,7 @@ mod tests {
             physical_rows: None,
             row_id_meta: None,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             last_updated_at_version_meta: None,
             created_at_version_meta: None,
@@ -4835,6 +5145,7 @@ mod tests {
         let fragment = Fragment {
             id: 1,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -5104,6 +5415,7 @@ mod tests {
                 None,
             )],
             physical_rows: Some(10),
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             last_updated_at_version_meta: None,
@@ -5195,6 +5507,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![mk_file("before.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -5267,6 +5580,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![data_file.clone()],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: row_id_meta.clone(),
             physical_rows: Some(5),
@@ -5286,6 +5600,7 @@ mod tests {
         let merged_fragment = Fragment {
             id: 0,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta,
             physical_rows: Some(5),
@@ -5335,6 +5650,7 @@ mod tests {
         let prev_fragment = Fragment {
             id: 0,
             files: vec![mk_file("before.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(5),
@@ -5399,6 +5715,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 0,
             files: vec![mk_file("existing.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0))),
             physical_rows: Some(3),
@@ -5421,6 +5738,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 1,
             files: vec![mk_file("new.lance")],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1))),
             physical_rows: Some(4),
@@ -5483,6 +5801,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(3),
@@ -5496,6 +5815,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5538,6 +5858,7 @@ mod tests {
             Fragment {
                 id: 1,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq))),
                 physical_rows: Some(2),
@@ -5549,6 +5870,7 @@ mod tests {
             Fragment {
                 id: 2,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq))),
                 physical_rows: Some(3),
@@ -5564,6 +5886,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5605,6 +5928,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5619,6 +5943,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5663,6 +5988,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5676,6 +6002,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 20,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(4),
@@ -5709,6 +6036,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5720,6 +6048,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(1),
@@ -5748,6 +6077,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5758,6 +6088,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(3),
@@ -5788,6 +6119,7 @@ mod tests {
         let existing_fragment = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))),
             physical_rows: Some(2),
@@ -5801,6 +6133,7 @@ mod tests {
         let new_fragment = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(1),
@@ -5842,6 +6175,7 @@ mod tests {
         let in_range_frag = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq))),
             physical_rows: Some(2),
@@ -5862,6 +6196,7 @@ mod tests {
         let out_of_range_frag = Fragment {
             id: 2,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq))),
             physical_rows: Some(2),
@@ -5876,6 +6211,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5914,6 +6250,7 @@ mod tests {
         let existing = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq))),
             physical_rows: Some(3),
@@ -5926,6 +6263,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -5974,6 +6312,7 @@ mod tests {
         let src_frag = Fragment {
             id: 1,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq))),
             physical_rows: Some(100),
@@ -5988,6 +6327,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(100),
@@ -6035,6 +6375,7 @@ mod tests {
             Fragment {
                 id: 1,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a))),
                 physical_rows: Some(3),
@@ -6046,6 +6387,7 @@ mod tests {
             Fragment {
                 id: 2,
                 files: vec![],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b))),
                 physical_rows: Some(3),
@@ -6061,6 +6403,7 @@ mod tests {
         let new_frag = Fragment {
             id: 10,
             files: vec![],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))),
             physical_rows: Some(2),
@@ -6127,4 +6470,311 @@ mod tests {
         assert!(!left.modifies_same_metadata(&different_key));
         assert!(left.modifies_same_metadata(&replace));
     }
+
+    #[test]
+    fn test_data_overlay_operation_roundtrips() {
+        // A DataOverlay operation survives the protobuf round-trip, preserving
+        // the target fragment, the overlay's coverage, and its committed_version.
+        let mut bitmap = roaring::RoaringBitmap::new();
+        bitmap.insert(1);
+        bitmap.insert(4);
+        let overlay = DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None),
+            coverage: OverlayCoverage::dense(bitmap.clone()),
+            committed_version: 6,
+        };
+        let pb_overlay = pb::DataOverlayFile::from(&overlay);
+
+        let message = pb::Transaction {
+            read_version: 1,
+            uuid: Uuid::new_v4().to_string(),
+            operation: Some(pb::transaction::Operation::DataOverlay(
+                pb::transaction::DataOverlay {
+                    groups: vec![pb::transaction::DataOverlayGroup {
+                        fragment_id: 7,
+                        overlays: vec![pb_overlay],
+                    }],
+                },
+            )),
+            ..Default::default()
+        };
+
+        let txn = Transaction::try_from(message).unwrap();
+        match txn.operation {
+            Operation::DataOverlay { groups } => {
+                assert_eq!(groups.len(), 1);
+                assert_eq!(groups[0].fragment_id, 7);
+                assert_eq!(groups[0].overlays.len(), 1);
+                assert_eq!(groups[0].overlays[0].committed_version, 6);
+                assert_eq!(
+                    *groups[0].overlays[0].coverage_for_field(0).unwrap(),
+                    bitmap
+                );
+            }
+            other => panic!("expected DataOverlay, got {other:?}"),
+        }
+    }
+
+    fn overlay_with_field(field: i32, committed_version: u64) -> DataOverlayFile {
+        DataOverlayFile {
+            data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None),
+            coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])),
+            committed_version,
+        }
+    }
+
+    #[test]
+    fn test_prune_overlay_stale_fields_from_indices() {
+        // Fragment 0 carried an overlay on field 1 committed at v5, and was
+        // fully compacted into new fragment 7.
+        let mut old_frag = Fragment::new(0);
+        old_frag.overlays = vec![overlay_with_field(1, 5)];
+        let groups = vec![RewriteGroup {
+            old_fragments: vec![old_frag],
+            new_fragments: vec![Fragment::new(7)],
+        }];
+
+        // Post-remap state: every index already covers the new fragment (7).
+        let covering = || Some(RoaringBitmap::from_iter([7u32]));
+        let mut indices = vec![
+            // Stale: covers the overlaid field 1, built (v2) before the overlay.
+            create_test_index("stale", 1, 2, covering(), false),
+            // Not stale: covers field 1 but built at the overlay's version (v5);
+            // `committed_version > dataset_version` is false at equality.
+            create_test_index("fresh", 1, 5, covering(), false),
+            // Unrelated: covers field 2, which the overlay never touched.
+            create_test_index("unrelated", 2, 2, covering(), false),
+        ];
+
+        Transaction::prune_overlay_stale_fields_from_indices(&mut indices, &groups);
+
+        assert!(
+            !indices[0].fragment_bitmap.as_ref().unwrap().contains(7),
+            "stale index must drop the rewritten fragment from its coverage"
+        );
+        assert!(
+            indices[1].fragment_bitmap.as_ref().unwrap().contains(7),
+            "an index built at/after the overlay is not stale"
+        );
+        assert!(
+            indices[2].fragment_bitmap.as_ref().unwrap().contains(7),
+            "an index on an un-overlaid field is unaffected"
+        );
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_multi_fragment() {
+        // Overlays targeting two distinct fragments are each applied and stamped.
+        // A targeted fragment already carrying an overlay (committed at v3) gets
+        // the new overlay appended and stamped while its existing overlay is
+        // preserved, and a fragment the operation does not target is passed
+        // through with its existing overlays untouched.
+        let mut frag0 = Fragment::new(0);
+        frag0.overlays = vec![overlay_with_field(5, 3)]; // targeted, pre-existing at v3
+        let frag1 = Fragment::new(1);
+        let mut frag2 = Fragment::new(2);
+        frag2.overlays = vec![overlay_with_field(9, 3)]; // untargeted, committed at v3
+        let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]);
+        let mut manifest = Manifest::new(
+            LanceSchema::try_from(&schema).unwrap(),
+            Arc::new(vec![frag0, frag1, frag2]),
+            lance_table::format::DataStorageFormat::new(LanceFileVersion::V2_0),
+            HashMap::new(),
+        );
+        // The pre-existing overlays were committed at v3, so the current
+        // manifest must be at least that version; the new commit then stamps
+        // its overlay at v4, keeping the fragment's overlays newest-last.
+        manifest.version = 3;
+
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(1, 0)],
+                    },
+                    DataOverlayGroup {
+                        fragment_id: 1,
+                        overlays: vec![overlay_with_field(2, 0)],
+                    },
+                ],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let frag = |id: u64| {
+            result
+                .fragments
+                .iter()
+                .find(|f| f.id == id)
+                .unwrap_or_else(|| panic!("fragment {id} missing from result"))
+        };
+        // The already-overlaid target keeps its v3 overlay and appends the new
+        // one, stamped to the new version.
+        assert_eq!(frag(0).overlays.len(), 2);
+        assert_eq!(frag(0).overlays[0].committed_version, 3);
+        assert_eq!(frag(0).overlays[1].committed_version, result.version);
+        // The fresh target gets its overlay, stamped to the new version.
+        assert_eq!(frag(1).overlays.len(), 1);
+        assert_eq!(frag(1).overlays[0].committed_version, result.version);
+        // The untargeted fragment is unchanged: same overlay, original version.
+        assert_eq!(frag(2).overlays.len(), 1);
+        assert_eq!(frag(2).overlays[0].committed_version, 3);
+        assert!(result.version > manifest.version);
+    }
+
+    #[test]
+    fn test_data_replacement_tombstones_overlaid_fields() {
+        // A DataReplacement writing new base values for field 5 must stop any
+        // overlay from shadowing those cells: field 5 is tombstoned in place
+        // (preserving the overlay's field 3), and an overlay covering only field
+        // 5 is dropped entirely.
+        let mut fragment = Fragment::new(0);
+        fragment.files = vec![
+            DataFile::new_legacy_from_fields("f3.lance", vec![3], None),
+            DataFile::new_legacy_from_fields("f5.lance", vec![5], None),
+        ];
+        fragment.overlays = vec![
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("o35.lance", vec![3, 5], None),
+                coverage: OverlayCoverage::sparse(vec![
+                    roaring::RoaringBitmap::from_iter([0u32]),
+                    roaring::RoaringBitmap::from_iter([0u32]),
+                ]),
+                committed_version: 3,
+            },
+            DataOverlayFile {
+                data_file: DataFile::new_legacy_from_fields("o5.lance", vec![5], None),
+                coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])),
+                committed_version: 3,
+            },
+        ];
+
+        let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]);
+        let manifest = Manifest::new(
+            LanceSchema::try_from(&schema).unwrap(),
+            Arc::new(vec![fragment]),
+            lance_table::format::DataStorageFormat::new(LanceFileVersion::V2_0),
+            HashMap::new(),
+        );
+
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataReplacement {
+                replacements: vec![DataReplacementGroup(
+                    0,
+                    DataFile::new_legacy_from_fields("f5-new.lance", vec![5], None),
+                )],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let frag = &result.fragments[0];
+        // The base data file for field 5 was swapped in.
+        assert!(frag.files.iter().any(|f| f.path == "f5-new.lance"));
+        // The [3, 5] overlay keeps field 3 and tombstones field 5; the [5]-only
+        // overlay is dropped.
+        assert_eq!(frag.overlays.len(), 1);
+        assert_eq!(frag.overlays[0].data_file.fields.as_ref(), &[3, -2]);
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_merges_duplicate_groups() {
+        // Two groups targeting the same fragment must both survive (a HashMap
+        // collapse would have dropped the first).
+        let manifest = sample_manifest();
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(1, 0)],
+                    },
+                    DataOverlayGroup {
+                        fragment_id: 0,
+                        overlays: vec![overlay_with_field(2, 0)],
+                    },
+                ],
+            },
+            None,
+        );
+
+        let (result, _) = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap();
+
+        let overlays = &result.fragments[0].overlays;
+        assert_eq!(overlays.len(), 2);
+        assert_eq!(overlays[0].data_file.fields.as_ref(), [1i32].as_slice());
+        assert_eq!(overlays[1].data_file.fields.as_ref(), [2i32].as_slice());
+    }
+
+    #[test]
+    fn test_data_overlay_build_manifest_rejects_unknown_fragment() {
+        let manifest = sample_manifest();
+        let txn = Transaction::new(
+            manifest.version,
+            Operation::DataOverlay {
+                groups: vec![DataOverlayGroup {
+                    fragment_id: 99,
+                    overlays: vec![overlay_with_field(1, 0)],
+                }],
+            },
+            None,
+        );
+        let err = txn
+            .build_manifest(
+                Some(&manifest),
+                vec![],
+                "txn",
+                &ManifestWriteConfig::default(),
+            )
+            .unwrap_err();
+        assert!(err.to_string().contains("does not exist"), "{err}");
+    }
+
+    #[test]
+    fn test_data_overlay_operation_eq() {
+        let overlay = |field: i32| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id: 0,
+                overlays: vec![overlay_with_field(field, 1)],
+            }],
+        };
+        // Reflexive and value-based (the arm previously returned false for self).
+        assert_eq!(overlay(1), overlay(1));
+        assert_ne!(overlay(1), overlay(2));
+        // Not equal to a different operation kind (previously returned true vs Rewrite).
+        let rewrite = Operation::Rewrite {
+            groups: vec![],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
+        };
+        assert_ne!(overlay(1), rewrite);
+    }
 }
diff --git a/rust/lance/src/dataset/udtf.rs b/rust/lance/src/dataset/udtf.rs
index 75c0388bc24..2144b108859 100644
--- a/rust/lance/src/dataset/udtf.rs
+++ b/rust/lance/src/dataset/udtf.rs
@@ -13,7 +13,6 @@ use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD};
 use lance_index::scalar::FullTextSearchQuery;
 use lance_index::scalar::inverted::parser::from_json;
 use serde_json::Value;
-use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::sync::Arc;
@@ -61,10 +60,6 @@ impl FtsTableProvider {
 
 #[async_trait]
 impl TableProvider for FtsTableProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.full_schema.clone()
     }
diff --git a/rust/lance/src/dataset/updater.rs b/rust/lance/src/dataset/updater.rs
index b9bc34f8706..752a479e5b0 100644
--- a/rust/lance/src/dataset/updater.rs
+++ b/rust/lance/src/dataset/updater.rs
@@ -6,13 +6,13 @@ use futures::StreamExt;
 use lance_core::datatypes::{OnMissing, OnTypeMismatch};
 use lance_core::utils::deletion::DeletionVector;
 use lance_core::{Error, Result, datatypes::Schema};
-use lance_table::format::Fragment;
+use lance_table::format::{DataFile, Fragment};
 use lance_table::utils::stream::ReadBatchFutStream;
 
 use super::Dataset;
 use super::fragment::FragmentReader;
 use super::scanner::get_default_batch_size;
-use super::write::{GenericWriter, open_writer};
+use super::write::{GenericWriter, cleanup_data_fragments, open_update_writer};
 use crate::dataset::FileFragment;
 use crate::dataset::utils::SchemaAdapter;
 
@@ -146,13 +146,7 @@ impl Updater {
             .data_storage_format
             .lance_file_version()?;
 
-        open_writer(
-            &self.fragment.dataset().object_store,
-            &schema,
-            &self.fragment.dataset().base,
-            data_storage_version,
-        )
-        .await
+        open_update_writer(self.dataset(), &schema, data_storage_version).await
     }
 
     /// Update one batch.
@@ -221,6 +215,35 @@ impl Updater {
         Ok(self.fragment.metadata().clone())
     }
 
+    /// Clean up any data file and blob sidecars created by the current unfinished writer.
+    pub(super) async fn cleanup_unfinished_writer(&mut self) {
+        let Some(writer) = self.writer.take() else {
+            return;
+        };
+        let (path, base_id) = writer.data_file_path();
+        let path = path.to_string();
+        drop(writer);
+
+        if path.is_empty() {
+            return;
+        }
+
+        let mut fragment = Fragment::new(self.fragment.id() as u64);
+        // cleanup_data_fragments only needs path/base_id to remove the unfinished
+        // data file and any blob sidecars. Build a minimal synthetic fragment so
+        // we can reuse the shared cleanup path without fabricating full metadata.
+        fragment
+            .files
+            .push(DataFile::new(path, vec![], vec![], 0, 0, None, base_id));
+        cleanup_data_fragments(
+            &self.dataset().object_store,
+            &self.dataset().base,
+            None,
+            &[fragment],
+        )
+        .await;
+    }
+
     /// Get the final schema of the fragment after the update.
     ///
     /// This may be None if the schema is not known. This can happen if it was
diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs
index 8be2753cb96..b35cfeb3eed 100644
--- a/rust/lance/src/dataset/write.rs
+++ b/rust/lance/src/dataset/write.rs
@@ -6,7 +6,11 @@ use chrono::TimeDelta;
 use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use futures::{Stream, StreamExt, TryStreamExt};
-use lance_arrow::BLOB_META_KEY;
+use lance_arrow::{
+    ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+    BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+    BLOB_V2_EXT_NAME,
+};
 use lance_core::datatypes::{
     NullabilityComparison, OnMissing, OnTypeMismatch, SchemaCompareOptions,
 };
@@ -22,20 +26,26 @@ use lance_file::previous::writer::{
 };
 use lance_file::version::LanceFileVersion;
 use lance_file::writer::{self as current_writer, FileWriterOptions};
-use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry};
+use lance_io::object_store::{
+    ObjectStore, ObjectStoreParams, ObjectStoreRegistry, parse_base_scoped_key,
+};
 use lance_table::format::{BasePath, DataFile, Fragment};
 use lance_table::io::commit::{CommitHandler, commit_handler_from_url};
 use lance_table::io::manifest::ManifestDescribing;
 use object_store::path::Path;
-use std::collections::{HashMap, HashSet};
+use std::borrow::Cow;
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::num::NonZero;
 use std::sync::Arc;
 use std::sync::atomic::AtomicUsize;
 use tracing::{info, instrument};
 
 use crate::Dataset;
+use crate::blob::normalize_prepared_blob_schema;
 use crate::dataset::blob::{
-    BlobPreprocessor, ExternalBaseCandidate, ExternalBaseResolver, preprocess_blob_batches,
+    BlobPreprocessor, ExternalBaseCandidate, ExternalBaseResolver,
+    blob_dedicated_threshold_from_metadata, blob_inline_threshold_from_metadata,
+    blob_pack_file_threshold_from_metadata, preprocess_blob_batches,
 };
 use crate::session::Session;
 
@@ -170,6 +180,74 @@ fn validate_external_blob_write_params(params: &WriteParams) -> Result<()> {
     Ok(())
 }
 
+fn validate_blob_threshold_metadata_for_append(
+    input_schema: &Schema,
+    dataset_schema: &Schema,
+) -> Result<()> {
+    for input_field in &input_schema.fields {
+        let Some(dataset_field) = dataset_schema.field(&input_field.name) else {
+            continue;
+        };
+        validate_blob_threshold_metadata_for_field_recursive(input_field, dataset_field)?;
+    }
+
+    Ok(())
+}
+
+fn validate_blob_threshold_metadata_for_field_recursive(
+    input_field: &lance_core::datatypes::Field,
+    dataset_field: &lance_core::datatypes::Field,
+) -> Result<()> {
+    let input_is_blob_v2 = input_field
+        .metadata
+        .get(ARROW_EXT_NAME_KEY)
+        .is_some_and(|extension_name| extension_name == BLOB_V2_EXT_NAME);
+    let dataset_is_blob_v2 = dataset_field
+        .metadata
+        .get(ARROW_EXT_NAME_KEY)
+        .is_some_and(|extension_name| extension_name == BLOB_V2_EXT_NAME);
+    if input_is_blob_v2 || dataset_is_blob_v2 {
+        for (key, read_threshold) in [
+            (
+                BLOB_INLINE_SIZE_THRESHOLD_META_KEY,
+                blob_inline_threshold_from_metadata
+                    as fn(&HashMap<String, String>, &str) -> Result<usize>,
+            ),
+            (
+                BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY,
+                blob_dedicated_threshold_from_metadata,
+            ),
+            (
+                BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY,
+                blob_pack_file_threshold_from_metadata,
+            ),
+        ] {
+            if !input_field.metadata.contains_key(key) {
+                continue;
+            }
+            let input_value = read_threshold(&input_field.metadata, &input_field.name)?;
+            let dataset_value = read_threshold(&dataset_field.metadata, &dataset_field.name)?;
+            if input_value != dataset_value {
+                return Err(Error::invalid_input(format!(
+                    "Cannot append data with blob threshold metadata {key}={input_value} for \
+                     field '{}'; the dataset schema has effective value {dataset_value}. Blob \
+                     thresholds for existing columns are stored in the dataset schema.",
+                    input_field.name,
+                )));
+            }
+        }
+    }
+
+    for input_child in &input_field.children {
+        let Some(dataset_child) = dataset_field.child(&input_child.name) else {
+            continue;
+        };
+        validate_blob_threshold_metadata_for_field_recursive(input_child, dataset_child)?;
+    }
+
+    Ok(())
+}
+
 /// Auto cleanup parameters
 #[derive(Debug, Clone)]
 pub struct AutoCleanupParams {
@@ -211,8 +289,16 @@ pub struct WriteParams {
     /// Write mode
     pub mode: WriteMode,
 
+    /// Default object store params for the write.
+    ///
+    /// Storage options may carry base-scoped entries (`base_<id>.<key>`) that
+    /// apply only to the registered base path with that id, overriding the
+    /// unscoped options that every base inherits.
     pub store_params: Option<ObjectStoreParams>,
 
+    /// Exact object store params per base path URI, taking precedence over
+    /// `base_<id>.<key>` storage options in [`Self::store_params`]. See
+    /// [`Self::with_base_store_params`].
     pub base_store_params: Option<HashMap<String, ObjectStoreParams>>,
 
     pub progress: Arc<dyn WriteFragmentProgress>,
@@ -237,6 +323,7 @@ pub struct WriteParams {
     ///
     /// Newer versions are more efficient but the data can only be read by more recent versions
     /// of lance.
+    /// Lance file version 2.3 enables RLE v2 run length widths by default.
     ///
     /// If not specified then the latest stable version will be used.
     pub data_storage_version: Option<LanceFileVersion>,
@@ -295,13 +382,23 @@ pub struct WriteParams {
     /// The IDs must correspond to either:
     /// - IDs in initial_bases (for CREATE/OVERWRITE modes)
     /// - IDs already registered in the existing dataset manifest (for APPEND mode)
+    /// - [`PRIMARY_BASE_ID`] (0), which targets the dataset's primary storage
+    ///   and participates in the round-robin like any other entry
     pub target_bases: Option<Vec<u32>>,
 
     /// Target base names or paths as strings (unresolved).
     /// These will be resolved to IDs when the write operation executes.
     /// Resolution happens at builder execution time when dataset context is available.
+    /// An entry equal to the dataset's URI targets the dataset's primary storage.
     pub target_base_names_or_paths: Option<Vec<String>>,
 
+    /// Target every base registered in the dataset manifest, resolved when the
+    /// write executes. `Some(include_primary)`: when `include_primary` is true
+    /// the dataset's primary storage participates in the rotation as the first
+    /// slot. Cannot be combined with `target_bases` or
+    /// `target_base_names_or_paths`.
+    pub target_all_bases: Option<bool>,
+
     /// Allow writing external blob URIs that cannot be mapped to any registered
     /// non-dataset-root base path. When disabled, such rows are rejected.
     pub allow_external_blob_outside_bases: bool,
@@ -339,6 +436,7 @@ impl Default for WriteParams {
             initial_bases: None,
             target_bases: None,
             target_base_names_or_paths: None,
+            target_all_bases: None,
             allow_external_blob_outside_bases: false,
             external_blob_mode: ExternalBlobMode::Reference,
             blob_pack_file_size_threshold: None,
@@ -369,8 +467,10 @@ impl WriteParams {
 
     /// Set exact runtime object store params for a registered base path.
     ///
-    /// These params are used as-is for that base. The write-level default
-    /// `store_params` remain the fallback for bases without an explicit binding.
+    /// These params are used as-is for that base, taking precedence over
+    /// `base_<id>.<key>` storage options in `store_params`. The write-level
+    /// default `store_params` remain the fallback for bases without an
+    /// explicit binding.
     pub fn with_base_store_params(
         mut self,
         base_path: impl AsRef<str>,
@@ -436,6 +536,19 @@ impl WriteParams {
         }
     }
 
+    /// Target every base registered in the dataset manifest, resolved when the
+    /// write executes. When `include_primary` is true the dataset's primary
+    /// storage participates in the rotation as the first slot.
+    ///
+    /// Cannot be combined with [`Self::with_target_bases`] or
+    /// [`Self::with_target_base_names_or_paths`].
+    pub fn with_target_all_bases(self, include_primary: bool) -> Self {
+        Self {
+            target_all_bases: Some(include_primary),
+            ..self
+        }
+    }
+
     /// Configure whether external blobs outside registered bases are allowed.
     pub fn with_allow_external_blob_outside_bases(self, allow: bool) -> Self {
         Self {
@@ -507,7 +620,7 @@ pub async fn do_write_fragments(
     };
 
     let external_base_resolver = if storage_version >= LanceFileVersion::V2_2
-        && schema.fields.iter().any(|field| field.is_blob_v2())
+        && schema.fields_pre_order().any(|field| field.is_blob_v2())
     {
         Some(Arc::new(
             build_external_base_resolver(dataset, &params).await?,
@@ -520,6 +633,8 @@ pub async fn do_write_fragments(
         .unwrap_or_else(|| params.store_registry());
     let source_store_params = params.store_params.clone().unwrap_or_default();
 
+    // Keep a copy so failure paths can clean up files written to target bases.
+    let cleanup_bases = target_bases_info.clone();
     let writer_generator = WriterGenerator::new(
         object_store.clone(),
         base_dir,
@@ -600,7 +715,13 @@ pub async fn do_write_fragments(
         // Drop the writer so its in-progress file is cleaned up (LocalWriter
         // removes its temp file; ObjectWriter aborts the multipart upload).
         drop(writer.take());
-        cleanup_data_fragments(&object_store, base_dir, &fragments).await;
+        cleanup_data_fragments(
+            &object_store,
+            base_dir,
+            cleanup_bases.as_deref(),
+            &fragments,
+        )
+        .await;
         return Err(e);
     }
 
@@ -625,7 +746,13 @@ pub async fn do_write_fragments(
             }
             Err(e) => {
                 drop(writer);
-                cleanup_data_fragments(&object_store, base_dir, &fragments).await;
+                cleanup_data_fragments(
+                    &object_store,
+                    base_dir,
+                    cleanup_bases.as_deref(),
+                    &fragments,
+                )
+                .await;
                 return Err(e);
             }
         }
@@ -639,53 +766,67 @@ pub async fn do_write_fragments(
 /// Contract:
 /// - Errors from individual `delete` calls are logged and swallowed, never returned —
 ///   callers should propagate the original write error.
-/// - Only files in the dataset's default storage (`base_id == None`) are deleted;
-///   files in external bases are skipped because we don't have their object stores here.
+/// - Files in the dataset's default storage (`base_id == None`) are deleted via
+///   `object_store`; files whose `base_id` matches an entry in `target_bases` are
+///   deleted via that base's object store. Files in bases not listed in
+///   `target_bases` are skipped because we don't have their object stores here.
 /// - Safe to call with an empty slice.
 /// - Must be called before the fragments are committed, otherwise live data may be deleted.
 pub(crate) async fn cleanup_data_fragments(
     object_store: &ObjectStore,
     base_dir: &Path,
+    target_bases: Option<&[TargetBaseInfo]>,
     fragments: &[Fragment],
 ) {
     let data_dir = base_dir.clone().join(DATA_DIR);
     let mut skipped_external = 0usize;
     for fragment in fragments {
         for file in &fragment.files {
-            if file.base_id.is_none() {
-                let path = data_dir.clone().join(file.path.as_str());
-                if let Err(e) = object_store.delete(&path).await {
-                    log::warn!("Failed to clean up orphaned data file '{}': {}", path, e);
+            let (store, file_dir) = if let Some(base_id) = file.base_id {
+                match target_bases.and_then(|bases| bases.iter().find(|b| b.base_id == base_id)) {
+                    Some(base_info) => {
+                        let dir = if base_info.is_dataset_root {
+                            base_info.base_dir.clone().join(DATA_DIR)
+                        } else {
+                            base_info.base_dir.clone()
+                        };
+                        (base_info.object_store.as_ref(), dir)
+                    }
+                    None => {
+                        skipped_external += 1;
+                        continue;
+                    }
                 }
+            } else {
+                (object_store, data_dir.clone())
+            };
 
-                // Clean up any blob v2 sidecars that might exist for this data file.
-                // Blob v2 sidecars are written to `data/{data_file_key}/{blob_id}.blob`.
-                // The `data_file_key` is the file stem of the .lance file.
-                if let Some(stem) = std::path::Path::new(file.path.as_str())
-                    .file_stem()
-                    .and_then(|s| s.to_str())
-                {
-                    let blob_dir = data_dir.clone().join(stem);
-                    match object_store.remove_dir_all(blob_dir.clone()).await {
-                        Err(e) if !matches!(e, Error::NotFound { .. }) => {
-                            log::warn!(
-                                "Failed to clean up orphaned blob dir '{}': {}",
-                                blob_dir,
-                                e
-                            );
-                        }
-                        _ => {}
+            let path = file_dir.clone().join(file.path.as_str());
+            if let Err(e) = store.delete(&path).await {
+                log::warn!("Failed to clean up orphaned data file '{}': {}", path, e);
+            }
+
+            // Clean up any blob v2 sidecars that might exist for this data file.
+            // Blob v2 sidecars are written to `data/{data_file_key}/{blob_id}.blob`.
+            // The `data_file_key` is the file stem of the .lance file.
+            if let Some(stem) = std::path::Path::new(file.path.as_str())
+                .file_stem()
+                .and_then(|s| s.to_str())
+            {
+                let blob_dir = file_dir.clone().join(stem);
+                match store.remove_dir_all(blob_dir.clone()).await {
+                    Err(e) if !matches!(e, Error::NotFound { .. }) => {
+                        log::warn!("Failed to clean up orphaned blob dir '{}': {}", blob_dir, e);
                     }
+                    _ => {}
                 }
-            } else {
-                skipped_external += 1;
             }
         }
     }
     if skipped_external > 0 {
         log::warn!(
             "Skipped cleanup of {} orphaned data file(s) in external bases: \
-             cleanup not supported for external bases",
+             their object stores are not available here",
             skipped_external
         );
     }
@@ -709,6 +850,12 @@ pub async fn validate_and_resolve_target_bases(
         ));
     }
 
+    if params.target_all_bases.is_some() {
+        return Err(Error::invalid_input(
+            "target_all_bases requires dataset context to resolve; use the write or merge insert APIs to apply it.",
+        ));
+    }
+
     // Step 2: Assign IDs to initial_bases and add them to all_bases
     let mut all_bases: HashMap<u32, BasePath> = existing_base_paths.cloned().unwrap_or_default();
     if let Some(initial_bases) = &mut params.initial_bases {
@@ -722,6 +869,11 @@ pub async fn validate_and_resolve_target_bases(
             all_bases.insert(base_path.id, base_path.clone());
         }
     }
+    log_unregistered_base_scoped_options(
+        params.store_params.as_ref(),
+        &all_bases,
+        log::Level::Warn,
+    );
 
     // Step 3: Resolve target_base_names_or_paths to IDs
     let target_base_ids = if let Some(ref names_or_paths) = params.target_base_names_or_paths {
@@ -757,6 +909,13 @@ pub async fn validate_and_resolve_target_bases(
         .unwrap_or_default();
 
     if let Some(target_bases) = &target_base_ids {
+        // An empty list would panic in round-robin selection; reject it
+        // instead of silently writing to primary storage.
+        if target_bases.is_empty() {
+            return Err(Error::invalid_input(
+                "target_bases cannot be empty. Omit the option to write to primary storage.",
+            ));
+        }
         let mut bases_info = Vec::new();
 
         for &target_base_id in target_bases {
@@ -767,7 +926,7 @@ pub async fn validate_and_resolve_target_bases(
                 ))
             })?;
 
-            let store_params = write_store_params_for_base(params, &base_path.path);
+            let store_params = write_store_params_for_base(params, base_path);
             let (target_object_store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
@@ -789,6 +948,136 @@ pub async fn validate_and_resolve_target_bases(
     }
 }
 
+/// Like [`validate_and_resolve_target_bases`], but also resolves references to
+/// the dataset's primary storage: base id [`PRIMARY_BASE_ID`] in
+/// `target_bases`, or an entry equal to `primary_uri` in
+/// `target_base_names_or_paths`. Primary slots participate in the round-robin
+/// like any other target base; files written through them carry no base id.
+pub(crate) async fn validate_and_resolve_target_bases_with_primary(
+    params: &mut WriteParams,
+    existing_base_paths: Option<&HashMap<u32, BasePath>>,
+    primary_object_store: &Arc<ObjectStore>,
+    primary_base_dir: &Path,
+    primary_uri: &str,
+) -> Result<Option<Vec<TargetBaseInfo>>> {
+    // Expand an all-bases request into an explicit id list (primary first,
+    // then registered bases in ascending id order) and continue below.
+    if let Some(include_primary) = params.target_all_bases {
+        if params.target_bases.is_some() || params.target_base_names_or_paths.is_some() {
+            return Err(Error::invalid_input(
+                "Cannot specify target_all_bases together with target_bases or target_base_names_or_paths.",
+            ));
+        }
+        let mut ids: Vec<u32> = existing_base_paths
+            .map(|bases| bases.keys().copied().collect())
+            .unwrap_or_default();
+        // CREATE mode registers initial_bases in the same write; assign their
+        // ids here (the delegate keeps non-zero ids as-is) so they join the
+        // rotation.
+        if let Some(initial_bases) = &mut params.initial_bases {
+            let mut next_id = ids.iter().max().map(|id| id + 1).unwrap_or(1);
+            for base_path in initial_bases.iter_mut() {
+                if base_path.id == 0 {
+                    base_path.id = next_id;
+                    next_id += 1;
+                }
+                ids.push(base_path.id);
+            }
+        }
+        ids.sort_unstable();
+        ids.dedup();
+        if include_primary {
+            ids.insert(0, PRIMARY_BASE_ID);
+        }
+        if ids.is_empty() {
+            return Err(Error::invalid_input(
+                "target_all_bases found no registered bases and include_primary is false. \
+                 Register bases or include primary storage.",
+            ));
+        }
+        params.target_bases = Some(ids);
+        params.target_all_bases = None;
+    }
+
+    let has_primary_ids = params
+        .target_bases
+        .as_ref()
+        .is_some_and(|ids| ids.contains(&PRIMARY_BASE_ID));
+    let has_primary_refs = params
+        .target_base_names_or_paths
+        .as_ref()
+        .is_some_and(|refs| refs.iter().any(|r| r == primary_uri));
+    if !has_primary_ids && !has_primary_refs {
+        return validate_and_resolve_target_bases(params, existing_base_paths).await;
+    }
+
+    // The delegate below may be skipped when only primary slots remain, so
+    // validate mutual exclusion here as well.
+    if params.target_base_names_or_paths.is_some() && params.target_bases.is_some() {
+        return Err(Error::invalid_input(
+            "Cannot specify both target_base_names_or_paths and target_bases. Use one or the other.",
+        ));
+    }
+
+    // Strip the primary slots, resolve the remaining references through the
+    // normal path, then splice the primary slots back into their original
+    // positions so the round-robin order matches what the caller asked for.
+    let is_primary_slot: Vec<bool> = if let Some(ids) = &params.target_bases {
+        ids.iter().map(|id| *id == PRIMARY_BASE_ID).collect()
+    } else {
+        params
+            .target_base_names_or_paths
+            .as_ref()
+            .unwrap()
+            .iter()
+            .map(|r| r == primary_uri)
+            .collect()
+    };
+
+    let mut shim = params.clone();
+    if let Some(ids) = &params.target_bases {
+        let rest: Vec<u32> = ids
+            .iter()
+            .copied()
+            .filter(|id| *id != PRIMARY_BASE_ID)
+            .collect();
+        shim.target_bases = if rest.is_empty() { None } else { Some(rest) };
+    } else {
+        let rest: Vec<String> = params
+            .target_base_names_or_paths
+            .as_ref()
+            .unwrap()
+            .iter()
+            .filter(|r| *r != primary_uri)
+            .cloned()
+            .collect();
+        shim.target_base_names_or_paths = if rest.is_empty() { None } else { Some(rest) };
+    }
+
+    let resolved_rest = validate_and_resolve_target_bases(&mut shim, existing_base_paths).await?;
+    // The delegate assigns ids to initial_bases in place; propagate that side
+    // effect back so CREATE-mode transactions register properly assigned ids.
+    params.initial_bases = shim.initial_bases;
+
+    let mut rest_iter = resolved_rest.unwrap_or_default().into_iter();
+    let mut bases_info = Vec::with_capacity(is_primary_slot.len());
+    for is_primary in is_primary_slot {
+        if is_primary {
+            bases_info.push(TargetBaseInfo {
+                base_id: PRIMARY_BASE_ID,
+                object_store: primary_object_store.clone(),
+                base_dir: primary_base_dir.clone(),
+                is_dataset_root: true,
+            });
+        } else {
+            bases_info.push(rest_iter.next().ok_or_else(|| {
+                Error::internal("target base resolution returned fewer bases than requested")
+            })?);
+        }
+    }
+    Ok(Some(bases_info))
+}
+
 fn append_external_base_candidate(
     base_path: &BasePath,
     store_prefix: String,
@@ -810,22 +1099,53 @@ fn append_external_base_candidate(
     }
 }
 
-fn write_store_params_for_base(params: &WriteParams, base_path: &str) -> ObjectStoreParams {
-    params
-        .base_store_params
-        .as_ref()
-        .and_then(|base_store_params| base_store_params.get(base_path))
-        .cloned()
-        .unwrap_or_else(|| params.store_params.clone().unwrap_or_default())
+/// Log base-scoped storage options (`base_<id>.<key>`) whose id does not
+/// match any registered base path. Unregistered entries are ignored during
+/// resolution. The open path logs at debug, since options may legitimately be
+/// vended for bases the loaded version does not register; the write path logs
+/// at warn, since ids are already assigned there and an unmatched id is much
+/// more likely a mistake.
+pub(crate) fn log_unregistered_base_scoped_options(
+    store_params: Option<&ObjectStoreParams>,
+    base_paths: &HashMap<u32, BasePath>,
+    level: log::Level,
+) {
+    if !log::log_enabled!(level) {
+        return;
+    }
+    let Some(options) = store_params.and_then(|params| params.storage_options()) else {
+        return;
+    };
+    let unregistered = options
+        .keys()
+        .filter_map(|key| parse_base_scoped_key(key).map(|(id, _)| id))
+        .filter(|id| !base_paths.contains_key(id))
+        .collect::<BTreeSet<_>>();
+    if !unregistered.is_empty() {
+        log::log!(
+            level,
+            "Ignoring base-scoped storage options for unregistered base path ids: {:?}",
+            unregistered
+        );
+    }
 }
 
-fn dataset_store_params_for_base(dataset: &Dataset, base_path: &str) -> ObjectStoreParams {
-    dataset
+fn write_store_params_for_base(params: &WriteParams, base_path: &BasePath) -> ObjectStoreParams {
+    // Exact per-URI bindings are used as-is. Otherwise the write-level default
+    // params are resolved for the base scope: `base_<id>.<key>` storage
+    // options overlay the shared defaults for that base.
+    if let Some(store_params) = params
         .base_store_params
         .as_ref()
-        .and_then(|base_store_params| base_store_params.get(base_path))
-        .cloned()
-        .unwrap_or_else(|| dataset.store_params.as_deref().cloned().unwrap_or_default())
+        .and_then(|base_store_params| base_store_params.get(&base_path.path))
+    {
+        return store_params.clone();
+    }
+    let default_params = params.store_params.clone().unwrap_or_default();
+    match default_params.scoped_to_base(Some(base_path.id)) {
+        Cow::Owned(scoped_params) => scoped_params,
+        Cow::Borrowed(_) => default_params,
+    }
 }
 
 async fn append_external_initial_bases(
@@ -837,7 +1157,7 @@ async fn append_external_initial_bases(
 ) -> Result<()> {
     if let Some(initial_bases) = initial_bases {
         for base_path in initial_bases {
-            let store_params = write_store_params_for_base(params, &base_path.path);
+            let store_params = write_store_params_for_base(params, base_path);
             let (store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
@@ -870,7 +1190,7 @@ async fn build_external_base_resolver(
 
     if let Some(dataset) = dataset {
         for base_path in dataset.manifest.base_paths.values() {
-            let store_params = dataset_store_params_for_base(dataset, &base_path.path);
+            let store_params = dataset.store_params_for_base(Some(base_path));
             let (store, extracted_path) = ObjectStore::from_uri_and_params(
                 store_registry.clone(),
                 &base_path.path,
@@ -936,12 +1256,13 @@ pub async fn write_fragments_internal(
     // Make sure the max rows per group is not larger than the max rows per file
     params.max_rows_per_group = std::cmp::min(params.max_rows_per_group, params.max_rows_per_file);
     validate_external_blob_write_params(&params)?;
+    let normalized_converted_schema = normalize_prepared_blob_schema(&converted_schema)?;
 
     let (schema, storage_version) = if let Some(dataset) = dataset {
         match params.mode {
             WriteMode::Append | WriteMode::Create => {
                 // Append mode, so we need to check compatibility
-                converted_schema.check_compatible(
+                normalized_converted_schema.check_compatible(
                     dataset.schema(),
                     &SchemaCompareOptions {
                         // We don't care if the user claims their data is nullable / non-nullable.  We will
@@ -953,8 +1274,12 @@ pub async fn write_fragments_internal(
                         ..Default::default()
                     },
                 )?;
+                validate_blob_threshold_metadata_for_append(
+                    &normalized_converted_schema,
+                    dataset.schema(),
+                )?;
                 let write_schema = dataset.schema().project_by_schema(
-                    &converted_schema,
+                    &normalized_converted_schema,
                     OnMissing::Error,
                     OnTypeMismatch::Error,
                 )?;
@@ -975,16 +1300,20 @@ pub async fn write_fragments_internal(
                         .data_storage_format
                         .lance_file_version()?,
                 );
-                (converted_schema, data_storage_version)
+                (normalized_converted_schema, data_storage_version)
             }
         }
     } else {
         // Brand new dataset, use the schema from the data and the storage version
         // from the user or the default.
-        (converted_schema, params.storage_version_or_default())
+        (
+            normalized_converted_schema,
+            params.storage_version_or_default(),
+        )
     };
 
-    if storage_version < LanceFileVersion::V2_2 && schema.fields.iter().any(|f| f.is_blob_v2()) {
+    if storage_version < LanceFileVersion::V2_2 && schema.fields_pre_order().any(|f| f.is_blob_v2())
+    {
         return Err(Error::invalid_input(format!(
             "Blob v2 requires file version >= 2.2 (got {:?})",
             storage_version
@@ -992,13 +1321,10 @@ pub async fn write_fragments_internal(
     }
 
     if storage_version >= LanceFileVersion::V2_2
-        && schema
-            .fields
-            .iter()
-            .any(|f| f.metadata.contains_key(BLOB_META_KEY))
+        && let Some(blob_field_path) = legacy_blob_field_path(&schema)
     {
         return Err(Error::invalid_input(format!(
-            "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)."
+            "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Found legacy blob field: {blob_field_path}. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)."
         )));
     }
 
@@ -1017,10 +1343,23 @@ pub async fn write_fragments_internal(
     Ok((fragments, schema))
 }
 
+fn legacy_blob_field_path(schema: &Schema) -> Option<String> {
+    schema
+        .fields_pre_order()
+        .find(|field| field.metadata.contains_key(BLOB_META_KEY))
+        .map(|field| {
+            schema
+                .field_path(field.id)
+                .unwrap_or_else(|_| field.name.clone())
+        })
+}
+
 #[async_trait::async_trait]
 pub trait GenericWriter: Send {
     /// Write the given batches to the file
     async fn write(&mut self, batches: &[RecordBatch]) -> Result<()>;
+    /// Get the file path and base ID for the data file being written.
+    fn data_file_path(&self) -> (&str, Option<u32>);
     /// Get the current position in the file
     ///
     /// We use this to know when the file is too large and we need to start
@@ -1047,17 +1386,20 @@ where
     async fn write(&mut self, batches: &[RecordBatch]) -> Result<()> {
         self.writer.write(batches).await
     }
+    fn data_file_path(&self) -> (&str, Option<u32>) {
+        (&self.path, self.base_id)
+    }
     async fn tell(&mut self) -> Result<u64> {
         Ok(self.writer.tell().await? as u64)
     }
     async fn finish(&mut self) -> Result<(u32, DataFile)> {
-        let size_bytes = self.writer.tell().await?;
+        let summary = self.writer.finish().await?;
         Ok((
-            self.writer.finish().await? as u32,
+            summary.num_rows as u32,
             DataFile::new_legacy(
                 self.path.clone(),
                 self.writer.schema(),
-                NonZero::new(size_bytes as u64),
+                NonZero::new(summary.size_bytes),
                 self.base_id,
             ),
         ))
@@ -1086,6 +1428,9 @@ impl GenericWriter for V2WriterAdapter {
         }
         Ok(())
     }
+    fn data_file_path(&self) -> (&str, Option<u32>) {
+        (&self.path, self.base_id)
+    }
     async fn tell(&mut self) -> Result<u64> {
         Ok(self.writer.tell().await?)
     }
@@ -1106,17 +1451,17 @@ impl GenericWriter for V2WriterAdapter {
             .map(|(_, column_index)| *column_index as i32)
             .collect::<Vec<_>>();
         let (major, minor) = self.writer.version().to_numbers();
-        let num_rows = self.writer.finish().await? as u32;
+        let write_summary = self.writer.finish().await?;
         let data_file = DataFile::new(
             std::mem::take(&mut self.path),
             field_ids,
             column_indices,
             major,
             minor,
-            NonZero::new(self.writer.tell().await?),
+            NonZero::new(write_summary.size_bytes),
             self.base_id,
         );
-        Ok((num_rows, data_file))
+        Ok((write_summary.num_rows as u32, data_file))
     }
 }
 
@@ -1139,6 +1484,39 @@ pub async fn open_writer(
     .await
 }
 
+pub(super) async fn open_update_writer(
+    dataset: &Dataset,
+    schema: &Schema,
+    storage_version: LanceFileVersion,
+) -> Result<Box<dyn GenericWriter>> {
+    // add_columns / alter_columns reuse the normal writer stack, but they do not
+    // flow through WriteParams. Rebuild the external base resolver here so blob
+    // v2 reference columns can resolve dataset-registered external URIs.
+    let external_base_resolver = if storage_version >= LanceFileVersion::V2_2
+        && schema.fields_pre_order().any(|f| f.is_blob_v2())
+    {
+        Some(Arc::new(
+            build_external_base_resolver(Some(dataset), &WriteParams::default()).await?,
+        ))
+    } else {
+        None
+    };
+
+    open_writer_with_options(
+        &dataset.object_store,
+        schema,
+        &dataset.base,
+        storage_version,
+        WriterOptions {
+            add_data_dir: true,
+            external_base_resolver,
+            source_store_registry: dataset.session.store_registry(),
+            ..Default::default()
+        },
+    )
+    .await
+}
+
 #[derive(Default)]
 struct WriterOptions {
     add_data_dir: bool,
@@ -1215,7 +1593,7 @@ async fn open_writer_with_options(
                 source_store_registry,
                 source_store_params,
                 blob_pack_file_size_threshold,
-            ))
+            )?)
         } else {
             None
         };
@@ -1230,9 +1608,18 @@ async fn open_writer_with_options(
     Ok(writer)
 }
 
+/// Reserved base id that refers to the dataset's primary storage in
+/// [`WriteParams::target_bases`]. Real base ids are assigned starting from 1,
+/// so 0 is never a registered base. Files written through a primary slot
+/// carry no base id, exactly like a write without target bases.
+pub const PRIMARY_BASE_ID: u32 = 0;
+
 /// Information about a target base for writing.
 /// Contains the base ID, object store, directory path, and whether it's a dataset root.
+#[derive(Clone)]
 pub struct TargetBaseInfo {
+    /// The registered base id, or [`PRIMARY_BASE_ID`] for the dataset's
+    /// primary storage.
     pub base_id: u32,
     pub object_store: Arc<ObjectStore>,
     /// The base directory path (without /data subdirectory)
@@ -1316,7 +1703,9 @@ impl WriterGenerator {
                 self.storage_version,
                 WriterOptions {
                     add_data_dir: base_info.is_dataset_root,
-                    base_id: Some(base_info.base_id),
+                    // Primary-storage slots stamp no base id, like a write
+                    // without target bases.
+                    base_id: (base_info.base_id != PRIMARY_BASE_ID).then_some(base_info.base_id),
                     external_base_resolver: self.external_base_resolver.clone(),
                     allow_external_blob_outside_bases: self.allow_external_blob_outside_bases,
                     external_blob_mode: self.external_blob_mode,
@@ -1976,7 +2365,28 @@ mod tests {
             .with_base_store_params("az://container/path-a", azure_store_params("account-a"))
             .with_base_store_params("az://container/path-b", azure_store_params("account-b"));
 
-        let existing_base_paths = HashMap::from([
+        let existing_base_paths = azure_base_paths_a_b();
+
+        let target_bases =
+            validate_and_resolve_target_bases(&mut params, Some(&existing_base_paths))
+                .await
+                .unwrap()
+                .unwrap();
+
+        assert_eq!(target_bases.len(), 2);
+        assert_eq!(
+            target_bases[0].object_store.store_prefix,
+            "az$container@account-a"
+        );
+        assert_eq!(
+            target_bases[1].object_store.store_prefix,
+            "az$container@account-b"
+        );
+    }
+
+    #[cfg(feature = "azure")]
+    fn azure_base_paths_a_b() -> HashMap<u32, BasePath> {
+        HashMap::from([
             (
                 1,
                 BasePath::new(
@@ -1995,7 +2405,32 @@ mod tests {
                     false,
                 ),
             ),
-        ]);
+        ])
+    }
+
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_validate_and_resolve_target_bases_uses_base_scoped_storage_options() {
+        // A single flat storage options map carries per-base credentials via
+        // the `base_<id>.<key>` convention; unscoped keys are shared defaults.
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), "account-shared".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                    ("base_1.account_name".to_string(), "account-a".to_string()),
+                    ("base_2.account_name".to_string(), "account-b".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+        let mut params = WriteParams {
+            store_params: Some(store_params),
+            ..Default::default()
+        }
+        .with_target_bases(vec![1, 2]);
+
+        let existing_base_paths = azure_base_paths_a_b();
 
         let target_bases =
             validate_and_resolve_target_bases(&mut params, Some(&existing_base_paths))
@@ -2014,6 +2449,43 @@ mod tests {
         );
     }
 
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_base_store_params_take_precedence_over_base_scoped_options() {
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                    (
+                        "base_1.account_name".to_string(),
+                        "account-scoped".to_string(),
+                    ),
+                ]),
+            ))),
+            ..Default::default()
+        };
+        let mut params = WriteParams {
+            store_params: Some(store_params),
+            ..Default::default()
+        }
+        .with_target_bases(vec![1])
+        .with_base_store_params("az://container/path-a", azure_store_params("account-exact"));
+
+        let existing_base_paths = azure_base_paths_a_b();
+
+        let target_bases =
+            validate_and_resolve_target_bases(&mut params, Some(&existing_base_paths))
+                .await
+                .unwrap()
+                .unwrap();
+
+        assert_eq!(target_bases.len(), 1);
+        assert_eq!(
+            target_bases[0].object_store.store_prefix,
+            "az$container@account-exact"
+        );
+    }
+
     #[tokio::test]
     async fn test_explicit_data_file_bases_writer_generator() {
         use arrow::datatypes::{DataType, Field as ArrowField, Schema as ArrowSchema};
@@ -2412,52 +2884,126 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_multi_base_overwrite() {
+    async fn test_multi_base_write_read_with_base_scoped_storage_options() {
+        use crate::dataset::builder::DatasetBuilder;
+        use lance_core::utils::tempfile::TempStrDir;
         use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
 
-        // Create initial dataset
-        let test_uri = "memory://overwrite_test";
-        let primary_uri = format!("{}/primary", test_uri);
-        let base1_uri = format!("{}/base1", test_uri);
-        let base2_uri = format!("{}/base2", test_uri);
-        let _base3_uri = format!("{}/base3", test_uri);
+        let primary_dir = TempStrDir::default();
+        let base1_dir = TempStrDir::default();
+
+        // Local stores ignore these options; this verifies base-scoped entries
+        // flow through the full write/read path without breaking anything.
+        let scoped_options = HashMap::from([
+            ("shared_option".to_string(), "shared".to_string()),
+            (
+                "base_1.scoped_option".to_string(),
+                "base1-value".to_string(),
+            ),
+        ]);
+        let store_params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                scoped_options.clone(),
+            ))),
+            ..Default::default()
+        };
 
         let mut data_gen =
             BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
-
         let dataset = Dataset::write(
-            data_gen.batch(3),
-            &primary_uri,
+            data_gen.batch(5),
+            primary_dir.as_str(),
             Some(WriteParams {
                 mode: WriteMode::Create,
-                initial_bases: Some(vec![
-                    BasePath {
-                        id: 1,
-                        name: Some("base1".to_string()),
-                        path: base1_uri.clone(),
-                        is_dataset_root: true,
-                    },
-                    BasePath {
-                        id: 2,
-                        name: Some("base2".to_string()),
-                        path: base2_uri.clone(),
-                        is_dataset_root: true,
-                    },
-                ]),
+                store_params: Some(store_params),
+                initial_bases: Some(vec![BasePath {
+                    id: 1,
+                    name: Some("base1".to_string()),
+                    path: base1_dir.as_str().to_string(),
+                    is_dataset_root: true,
+                }]),
                 target_bases: Some(vec![1]),
                 ..Default::default()
             }),
         )
         .await
         .unwrap();
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 5);
+        for fragment in dataset.get_fragments() {
+            assert!(
+                fragment
+                    .metadata
+                    .files
+                    .iter()
+                    .all(|file| file.base_id == Some(1))
+            );
+        }
+
+        // Reopen with the same flat options and scan through the base store.
+        let dataset = DatasetBuilder::from_uri(primary_dir.as_str())
+            .with_storage_options(scoped_options)
+            .load()
+            .await
+            .unwrap();
+        let batches = dataset
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let num_rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+        assert_eq!(num_rows, 5);
+    }
+
+    #[tokio::test]
+    async fn test_multi_base_overwrite() {
+        use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
+
+        // Create initial dataset
+        let test_uri = "memory://overwrite_test";
+        let primary_uri = format!("{}/primary", test_uri);
+        let base1_uri = format!("{}/base1", test_uri);
+        let base2_uri = format!("{}/base2", test_uri);
+        let _base3_uri = format!("{}/base3", test_uri);
+
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+
+        let dataset = Dataset::write(
+            data_gen.batch(3),
+            &primary_uri,
+            Some(WriteParams {
+                mode: WriteMode::Create,
+                initial_bases: Some(vec![
+                    BasePath {
+                        id: 1,
+                        name: Some("base1".to_string()),
+                        path: base1_uri.clone(),
+                        is_dataset_root: true,
+                    },
+                    BasePath {
+                        id: 2,
+                        name: Some("base2".to_string()),
+                        path: base2_uri.clone(),
+                        is_dataset_root: true,
+                    },
+                ]),
+                target_bases: Some(vec![1]),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 3);
+
+        // Overwrite - should inherit existing base configuration (base1, base2)
+        // Write to base2
+        let mut data_gen2 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
 
-        assert_eq!(dataset.count_rows(None).await.unwrap(), 3);
-
-        // Overwrite - should inherit existing base configuration (base1, base2)
-        // Write to base2
-        let mut data_gen2 =
-            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
-
         let dataset = Dataset::write(
             data_gen2.batch(2),
             std::sync::Arc::new(dataset),
@@ -3434,6 +3980,7 @@ mod tests {
         let fragments = vec![Fragment {
             id: 0,
             files: vec![external_file, local_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(0),
@@ -3441,7 +3988,7 @@ mod tests {
             last_updated_at_version_meta: None,
         }];
 
-        cleanup_data_fragments(&object_store, &base_dir, &fragments).await;
+        cleanup_data_fragments(&object_store, &base_dir, None, &fragments).await;
 
         // The local file should be removed; the external file is skipped without
         // erroring (its base store isn't known here).
@@ -3451,4 +3998,487 @@ mod tests {
             "Local data file should be deleted by cleanup"
         );
     }
+
+    /// Verifies the target-base branch in `cleanup_data_fragments`: files whose
+    /// `base_id` matches a provided [`TargetBaseInfo`] are deleted via that base's
+    /// object store (respecting `is_dataset_root` layout), while files in bases
+    /// without a provided store are still skipped.
+    #[tokio::test]
+    async fn test_cleanup_data_fragments_deletes_target_base_files() {
+        use lance_core::utils::tempfile::TempStrDir;
+
+        let primary_dir = TempStrDir::default();
+        let base1_dir = TempStrDir::default();
+        let base2_dir = TempStrDir::default();
+
+        let (object_store, base_dir) = ObjectStore::from_uri_and_params(
+            Default::default(),
+            primary_dir.as_str(),
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+        let (base1_store, base1_path) = ObjectStore::from_uri_and_params(
+            Default::default(),
+            base1_dir.as_str(),
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+        let (base2_store, base2_path) = ObjectStore::from_uri_and_params(
+            Default::default(),
+            base2_dir.as_str(),
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+
+        // base2 is a plain data directory: files sit at its root, not under data/.
+        let count_plain_files = |dir: &str| {
+            std::fs::read_dir(dir)
+                .map(|entries| {
+                    entries
+                        .filter(|e| e.as_ref().unwrap().path().is_file())
+                        .count()
+                })
+                .unwrap_or(0)
+        };
+
+        // base1 is a dataset root (files under data/), base2 is a plain data dir.
+        let base1_file_path = base1_path.clone().join(DATA_DIR).join("one.lance");
+        base1_store.put(&base1_file_path, b"x").await.unwrap();
+        let base2_file_path = base2_path.clone().join("two.lance");
+        base2_store.put(&base2_file_path, b"x").await.unwrap();
+        assert_eq!(count_data_files(base1_dir.as_str()), 1);
+        assert_eq!(count_plain_files(base2_dir.as_str()), 1);
+
+        let mut base1_file = DataFile::new_unstarted("one.lance", 2, 1);
+        base1_file.base_id = Some(1);
+        let mut base2_file = DataFile::new_unstarted("two.lance", 2, 1);
+        base2_file.base_id = Some(2);
+        let mut unknown_file = DataFile::new_unstarted("unknown.lance", 2, 1);
+        unknown_file.base_id = Some(42);
+        let fragments = vec![Fragment {
+            id: 0,
+            files: vec![base1_file, base2_file, unknown_file],
+            overlays: vec![],
+            deletion_file: None,
+            row_id_meta: None,
+            physical_rows: Some(0),
+            created_at_version_meta: None,
+            last_updated_at_version_meta: None,
+        }];
+
+        let target_bases = vec![
+            TargetBaseInfo {
+                base_id: 1,
+                object_store: base1_store,
+                base_dir: base1_path,
+                is_dataset_root: true,
+            },
+            TargetBaseInfo {
+                base_id: 2,
+                object_store: base2_store,
+                base_dir: base2_path,
+                is_dataset_root: false,
+            },
+        ];
+
+        cleanup_data_fragments(&object_store, &base_dir, Some(&target_bases), &fragments).await;
+
+        assert_eq!(
+            count_data_files(base1_dir.as_str()),
+            0,
+            "File in dataset-root target base should be deleted"
+        );
+        assert_eq!(
+            count_plain_files(base2_dir.as_str()),
+            0,
+            "File in plain-directory target base should be deleted"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cleanup_routed_data_files_on_failed_write() {
+        // Files already completed in target bases must be removed when the
+        // write later fails.
+        use lance_core::utils::tempfile::TempStrDir;
+
+        let primary_dir = TempStrDir::default();
+        let base1_dir = TempStrDir::default();
+
+        let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new(
+            "id",
+            DataType::Int32,
+            false,
+        )]));
+        let schema = Schema::try_from(arrow_schema.as_ref()).unwrap();
+
+        let (object_store, base_dir) = ObjectStore::from_uri_and_params(
+            Default::default(),
+            primary_dir.as_str(),
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+        let (base1_store, base1_path) = ObjectStore::from_uri_and_params(
+            Default::default(),
+            base1_dir.as_str(),
+            &Default::default(),
+        )
+        .await
+        .unwrap();
+
+        let good_batch = RecordBatch::try_new(
+            arrow_schema.clone(),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+
+        // 3 rows per file: the first batch fills and completes a file in the
+        // target base, then the stream fails.
+        let items: Vec<std::result::Result<RecordBatch, DataFusionError>> = vec![
+            Ok(good_batch.clone()),
+            Ok(good_batch.clone()),
+            Err(DataFusionError::External("injected failure".into())),
+        ];
+        let stream = Box::pin(RecordBatchStreamAdapter::new(
+            arrow_schema.clone(),
+            futures::stream::iter(items),
+        ));
+
+        let target_bases = vec![TargetBaseInfo {
+            base_id: 1,
+            object_store: base1_store,
+            base_dir: base1_path,
+            is_dataset_root: true,
+        }];
+
+        let result = do_write_fragments(
+            None,
+            object_store.clone(),
+            &base_dir,
+            &schema,
+            stream,
+            WriteParams {
+                max_rows_per_file: 3,
+                ..Default::default()
+            },
+            LanceFileVersion::V2_1,
+            Some(target_bases),
+        )
+        .await;
+
+        assert!(result.is_err(), "Expected write to fail");
+        assert_eq!(
+            count_data_files(base1_dir.as_str()),
+            0,
+            "Data files routed to the target base should be cleaned up on failure"
+        );
+        assert_eq!(count_data_files(primary_dir.as_str()), 0);
+    }
+
+    /// PRIMARY_BASE_ID (0) and the dataset URI include primary storage in the
+    /// target rotation, alongside registered bases.
+    #[tokio::test]
+    async fn test_multi_base_target_primary_and_bases() {
+        use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
+
+        let test_uri = "memory://primary_slot_test";
+        let primary_uri = format!("{}/primary", test_uri);
+        let base1_uri = format!("{}/base1", test_uri);
+        let base2_uri = format!("{}/base2", test_uri);
+
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+
+        // CREATE mode targeting primary + a new base: also verifies the id
+        // assignment on initial_bases reaches the committed manifest.
+        let dataset = Dataset::write(
+            data_gen.batch(6),
+            &primary_uri,
+            Some(WriteParams {
+                mode: WriteMode::Create,
+                max_rows_per_file: 3,
+                initial_bases: Some(vec![
+                    BasePath {
+                        id: 1,
+                        name: Some("base1".to_string()),
+                        is_dataset_root: true,
+                        path: base1_uri.clone(),
+                    },
+                    BasePath {
+                        id: 2,
+                        name: Some("base2".to_string()),
+                        is_dataset_root: false,
+                        path: base2_uri.clone(),
+                    },
+                ]),
+                target_bases: Some(vec![PRIMARY_BASE_ID, 1]),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(dataset.manifest.base_paths.len(), 2);
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![None, Some(1)]);
+
+        // APPEND across primary + both bases, one file per slot in order.
+        let mut data_gen2 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen2.batch(9),
+            Arc::new(dataset),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                max_rows_per_file: 3,
+                target_bases: Some(vec![PRIMARY_BASE_ID, 1, 2]),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .skip(2)
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![None, Some(1), Some(2)]);
+
+        // Names variant: the dataset's own URI selects primary storage.
+        let mut data_gen3 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen3.batch(6),
+            Arc::new(dataset),
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                max_rows_per_file: 3,
+                target_base_names_or_paths: Some(vec![primary_uri.clone(), "base2".to_string()]),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .skip(5)
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![None, Some(2)]);
+
+        assert_eq!(dataset.count_rows(None).await.unwrap(), 21);
+    }
+
+    /// `target_all_bases` resolves to every registered base at execution
+    /// time, with primary storage as the first slot when included.
+    #[tokio::test]
+    async fn test_multi_base_target_all_bases() {
+        use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
+
+        let test_uri = "memory://all_bases_test";
+        let primary_uri = format!("{}/primary", test_uri);
+        let base1_uri = format!("{}/base1", test_uri);
+        let base2_uri = format!("{}/base2", test_uri);
+
+        let mut data_gen =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen.batch(3),
+            &primary_uri,
+            Some(WriteParams {
+                mode: WriteMode::Create,
+                initial_bases: Some(vec![
+                    BasePath {
+                        id: 1,
+                        name: Some("base1".to_string()),
+                        is_dataset_root: true,
+                        path: base1_uri.clone(),
+                    },
+                    BasePath {
+                        id: 2,
+                        name: Some("base2".to_string()),
+                        is_dataset_root: false,
+                        path: base2_uri.clone(),
+                    },
+                ]),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // All bases including primary: slots are [primary, base1, base2].
+        let mut data_gen2 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen2.batch(9),
+            Arc::new(dataset),
+            Some(
+                WriteParams {
+                    mode: WriteMode::Append,
+                    max_rows_per_file: 3,
+                    ..Default::default()
+                }
+                .with_target_all_bases(true),
+            ),
+        )
+        .await
+        .unwrap();
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .skip(1)
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![None, Some(1), Some(2)]);
+
+        // Without primary: slots are [base1, base2].
+        let mut data_gen3 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen3.batch(6),
+            Arc::new(dataset),
+            Some(
+                WriteParams {
+                    mode: WriteMode::Append,
+                    max_rows_per_file: 3,
+                    ..Default::default()
+                }
+                .with_target_all_bases(false),
+            ),
+        )
+        .await
+        .unwrap();
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .skip(4)
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![Some(1), Some(2)]);
+
+        // Cannot be combined with explicit target bases.
+        let mut data_gen4 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let result = Dataset::write(
+            data_gen4.batch(3),
+            Arc::new(dataset),
+            Some(
+                WriteParams {
+                    mode: WriteMode::Append,
+                    target_bases: Some(vec![1]),
+                    ..Default::default()
+                }
+                .with_target_all_bases(true),
+            ),
+        )
+        .await;
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("Cannot specify target_all_bases together with")
+        );
+
+        // On a dataset with no registered bases: include_primary=true is a
+        // no-op rotation over primary, false is rejected.
+        let plain_uri = "memory://all_bases_plain";
+        let mut data_gen5 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let plain = Dataset::write(data_gen5.batch(3), plain_uri, None)
+            .await
+            .unwrap();
+        let mut data_gen6 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let plain = Dataset::write(
+            data_gen6.batch(3),
+            Arc::new(plain),
+            Some(
+                WriteParams {
+                    mode: WriteMode::Append,
+                    ..Default::default()
+                }
+                .with_target_all_bases(true),
+            ),
+        )
+        .await
+        .unwrap();
+        assert!(
+            plain.get_fragments().iter().all(|f| f
+                .metadata
+                .files
+                .iter()
+                .all(|file| file.base_id.is_none()))
+        );
+        let mut data_gen7 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let result = Dataset::write(
+            data_gen7.batch(3),
+            Arc::new(plain),
+            Some(
+                WriteParams {
+                    mode: WriteMode::Append,
+                    ..Default::default()
+                }
+                .with_target_all_bases(false),
+            ),
+        )
+        .await;
+        assert!(
+            result
+                .unwrap_err()
+                .to_string()
+                .contains("target_all_bases found no registered bases")
+        );
+
+        // CREATE mode: initial_bases join the rotation before their ids are
+        // committed to a manifest.
+        let create_uri = "memory://all_bases_create";
+        let mut data_gen8 =
+            BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned())));
+        let dataset = Dataset::write(
+            data_gen8.batch(9),
+            create_uri,
+            Some(
+                WriteParams {
+                    mode: WriteMode::Create,
+                    max_rows_per_file: 3,
+                    initial_bases: Some(vec![
+                        BasePath {
+                            id: 0,
+                            name: Some("base1".to_string()),
+                            is_dataset_root: true,
+                            path: format!("{}/base1", create_uri),
+                        },
+                        BasePath {
+                            id: 0,
+                            name: Some("base2".to_string()),
+                            is_dataset_root: false,
+                            path: format!("{}/base2", create_uri),
+                        },
+                    ]),
+                    ..Default::default()
+                }
+                .with_target_all_bases(true),
+            ),
+        )
+        .await
+        .unwrap();
+        assert_eq!(dataset.manifest.base_paths.len(), 2);
+        let file_bases: Vec<_> = dataset
+            .get_fragments()
+            .iter()
+            .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id))
+            .collect();
+        assert_eq!(file_bases, vec![None, Some(1), Some(2)]);
+    }
 }
diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs
index 2ab34441997..e0e9b32e998 100644
--- a/rust/lance/src/dataset/write/commit.rs
+++ b/rust/lance/src/dataset/write/commit.rs
@@ -104,6 +104,10 @@ impl<'a> CommitBuilder<'a> {
     }
 
     /// Pass a commit handler to use for the dataset.
+    ///
+    /// Takes precedence over the destination dataset's own handler. If not
+    /// set, a `Dataset` destination commits through its own handler and a
+    /// `Uri` destination resolves one from the uri.
     pub fn with_commit_handler(mut self, commit_handler: Arc<dyn CommitHandler>) -> Self {
         self.commit_handler = Some(commit_handler);
         self
@@ -241,7 +245,9 @@ impl<'a> CommitBuilder<'a> {
             WriteDestination::Dataset(dataset) => (
                 dataset.object_store.clone(),
                 dataset.base.clone(),
-                dataset.commit_handler.clone(),
+                self.commit_handler
+                    .clone()
+                    .unwrap_or_else(|| dataset.commit_handler.clone()),
             ),
             WriteDestination::Uri(uri) => {
                 let commit_handler = if let (Some(_), Some(commit_handler)) =
@@ -339,7 +345,6 @@ impl<'a> CommitBuilder<'a> {
         } else {
             self.use_stable_row_ids.unwrap_or(false)
         };
-
         // Validate storage format matches existing dataset
         if let Some(ds) = dest.dataset()
             && let Some(storage_format) = self.storage_format
@@ -497,7 +502,6 @@ impl<'a> CommitBuilder<'a> {
             },
             read_version,
             tag: None,
-            //TODO: handle batch transaction merges in the future
             transaction_properties: None,
         };
         let dataset = self.execute(merged.clone()).await?;
@@ -545,6 +549,7 @@ mod tests {
                 file_size_bytes: CachedFileSize::new(100),
                 base_id: None,
             }],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(10),
@@ -624,10 +629,12 @@ mod tests {
             .unwrap();
         assert_eq!(new_ds.manifest().version, 7);
         // Session should still be re-used
-        // However, the dataset needs to be loaded and the read version checked out,
-        // so an additional 4 IOPs are needed.
+        // However, the dataset needs to be loaded and the read version checked out.
+        // The read version's manifest body is served from the session cache (it
+        // was cached when v1 was first created), so the checkout only pays the
+        // version-resolution head, not a manifest read.
         let io_stats = dataset.object_store.as_ref().io_stats_incremental();
-        assert_io_eq!(io_stats, read_iops, 5, "load dataset + check version");
+        assert_io_eq!(io_stats, read_iops, 3, "load dataset + check version");
         assert_io_eq!(io_stats, write_iops, 2, "write txn + manifest");
 
         // Commit transaction with URI and new session. Re-use the store
diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs
index 20209ed7f30..144cc2bac6e 100644
--- a/rust/lance/src/dataset/write/insert.rs
+++ b/rust/lance/src/dataset/write/insert.rs
@@ -8,8 +8,8 @@ use arrow_array::{RecordBatch, RecordBatchIterator};
 use datafusion::execution::SendableRecordBatchStream;
 use humantime::format_duration;
 use lance_core::datatypes::{NullabilityComparison, Schema, SchemaCompareOptions};
+use lance_core::is_system_column;
 use lance_core::utils::tracing::{DATASET_WRITING_EVENT, TRACE_DATASET_EVENTS};
-use lance_core::{ROW_ADDR, ROW_ID, ROW_OFFSET};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_file::version::LanceFileVersion;
 use lance_io::object_store::ObjectStore;
@@ -19,10 +19,13 @@ use lance_table::io::commit::CommitHandler;
 use object_store::path::Path;
 
 use crate::Dataset;
+use crate::blob::normalize_prepared_blob_schema;
 use crate::dataset::ReadParams;
 use crate::dataset::builder::DatasetBuilder;
 use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder};
-use crate::dataset::write::{validate_and_resolve_target_bases, write_fragments_internal};
+use crate::dataset::write::{
+    validate_and_resolve_target_bases_with_primary, write_fragments_internal,
+};
 use crate::{Error, Result};
 use tracing::info;
 
@@ -201,8 +204,14 @@ impl<'a> InsertBuilder<'a> {
         self.validate_write(&mut context, &schema)?;
 
         let existing_base_paths = context.dest.dataset().map(|ds| &ds.manifest.base_paths);
-        let target_base_info =
-            validate_and_resolve_target_bases(&mut context.params, existing_base_paths).await?;
+        let target_base_info = validate_and_resolve_target_bases_with_primary(
+            &mut context.params,
+            existing_base_paths,
+            &context.object_store,
+            &context.base_path,
+            &context.dest.uri(),
+        )
+        .await?;
 
         let (written_fragments, written_schema) = write_fragments_internal(
             context.dest.dataset(),
@@ -315,12 +324,16 @@ impl<'a> InsertBuilder<'a> {
                 ..Default::default()
             };
 
-            data_schema.check_compatible(dataset.schema(), &schema_cmp_opts)?;
+            let normalized_data_schema = normalize_prepared_blob_schema(data_schema)?;
+            normalized_data_schema.check_compatible(dataset.schema(), &schema_cmp_opts)?;
         }
 
-        // Make sure we aren't using any reserved column names
+        // The system columns (`_rowid`, `_rowaddr`, `_rowoffset`, and the row-version
+        // columns) are virtual: they're injected into scan results at read time and
+        // never stored. A stored column sharing one of these names would collide with
+        // the system column on read, so reject it at write time.
         for field in data_schema.fields.iter() {
-            if field.name == ROW_ID || field.name == ROW_ADDR || field.name == ROW_OFFSET {
+            if is_system_column(&field.name) {
                 return Err(Error::invalid_input_source(
                     format!(
                         "The column {} is a reserved name and cannot be used in a Lance dataset",
@@ -442,7 +455,7 @@ struct WriteContext<'a> {
 mod test {
     use std::collections::HashMap;
 
-    use arrow_array::{BinaryArray, Int32Array, RecordBatchReader, StructArray};
+    use arrow_array::{ArrayRef, BinaryArray, Int32Array, RecordBatchReader, StructArray};
     use arrow_schema::{ArrowError, DataType, Field, Schema};
     use lance_arrow::BLOB_META_KEY;
 
@@ -496,6 +509,38 @@ mod test {
         );
     }
 
+    #[rstest::rstest]
+    #[case::row_id("_rowid")]
+    #[case::row_addr("_rowaddr")]
+    #[case::row_offset("_rowoffset")]
+    #[case::row_created_at_version("_row_created_at_version")]
+    #[case::row_last_updated_at_version("_row_last_updated_at_version")]
+    #[tokio::test]
+    async fn rejects_reserved_system_column_names(#[case] reserved_name: &str) {
+        // Every system column name must be rejected on write. The row-version
+        // columns (`_row_created_at_version`, `_row_last_updated_at_version`) are
+        // computed at read time and appended by `Projection::to_schema`; a user
+        // data column sharing one of those names would otherwise pass ingest and
+        // later collide with the appended field.
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            reserved_name,
+            DataType::Int32,
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))])
+            .unwrap();
+
+        let result = InsertBuilder::new("memory://")
+            .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()))
+            .await;
+
+        let err = result.expect_err("writing a reserved system column name should fail");
+        assert!(
+            err.to_string().contains("reserved name"),
+            "unexpected error for {reserved_name}: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn allow_overwrite_to_v2_2_without_blob_upgrade() {
         let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
@@ -559,6 +604,41 @@ mod test {
         }
     }
 
+    #[tokio::test]
+    async fn create_v2_2_dataset_rejects_nested_legacy_blob_schema() {
+        let image_field = Field::new("image_bytes", DataType::Binary, true).with_metadata(
+            HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]),
+        );
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "summary_image_nested",
+            DataType::Struct(vec![image_field.clone()].into()),
+            true,
+        )]));
+        let image_values: ArrayRef = Arc::new(BinaryArray::from(vec![Some(b"abc".as_slice())]));
+        let nested_values = StructArray::from(vec![(Arc::new(image_field), image_values)]);
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_values)]).unwrap();
+
+        let dataset = InsertBuilder::new("memory://forced-nested-blob-v2")
+            .with_params(&WriteParams {
+                mode: WriteMode::Create,
+                data_storage_version: Some(LanceFileVersion::V2_2),
+                ..Default::default()
+            })
+            .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()))
+            .await;
+
+        let err = dataset.unwrap_err();
+        match err {
+            Error::InvalidInput { source, .. } => {
+                let message = source.to_string();
+                assert!(message.contains("Legacy blob columns"));
+                assert!(message.contains("summary_image_nested.image_bytes"));
+                assert!(message.contains("lance.blob.v2"));
+            }
+            other => panic!("unexpected error: {other:?}"),
+        }
+    }
+
     mod external_error {
         use super::*;
         use std::fmt;
diff --git a/rust/lance/src/dataset/write/merge_insert.rs b/rust/lance/src/dataset/write/merge_insert.rs
index f6776852db3..12b5cb1651d 100644
--- a/rust/lance/src/dataset/write/merge_insert.rs
+++ b/rust/lance/src/dataset/write/merge_insert.rs
@@ -43,7 +43,10 @@ use inserted_rows::KeyExistenceFilter;
 
 use super::cleanup_data_fragments;
 use super::retry::{RetryConfig, RetryExecutor, execute_with_retry};
-use super::{CommitBuilder, WriteParams, write_fragments_internal};
+use super::{
+    CommitBuilder, TargetBaseInfo, WriteMode, WriteParams,
+    validate_and_resolve_target_bases_with_primary, write_fragments_internal,
+};
 use crate::dataset::rowids::get_row_id_index;
 use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows};
 use crate::dataset::utils::CapturedRowIds;
@@ -58,7 +61,9 @@ use crate::{
     },
     index::DatasetIndexInternalExt,
     io::exec::{
-        AddRowAddrExec, Planner, TakeExec, project, scalar_index::MapIndexExec, utils::ReplayExec,
+        AddRowAddrExec, Planner, TakeExec, project,
+        scalar_index::{IndexLookup, MapIndexExec},
+        utils::ReplayExec,
     },
 };
 use arrow_array::{
@@ -95,6 +100,7 @@ use futures::{
     Stream, StreamExt, TryStreamExt,
     stream::{self},
 };
+use lance_arrow::json::{convert_json_columns, has_json_fields, is_arrow_json_field};
 use lance_arrow::{RecordBatchExt, SchemaExt, interleave_batches};
 use lance_core::datatypes::NullabilityComparison;
 use lance_core::utils::address::RowAddress;
@@ -121,6 +127,7 @@ use lance_table::format::{Fragment, IndexMetadata, RowIdMeta};
 use log::info;
 use roaring::RoaringTreemap;
 use snafu::ResultExt;
+use std::collections::HashMap;
 use std::{
     collections::{BTreeMap, HashSet},
     sync::{
@@ -348,6 +355,14 @@ struct MergeInsertParams {
     source_dedupe_behavior: SourceDedupeBehavior,
     // Number of inner commit retries for manifest version conflicts. Default is 20.
     commit_retries: Option<u32>,
+    // Target base IDs for routing new fragments, mirroring WriteParams::target_bases.
+    target_bases: Option<Vec<u32>>,
+    // Target base names or path URIs (unresolved), mirroring
+    // WriteParams::target_base_names_or_paths. Resolved at execution time.
+    target_base_names_or_paths: Option<Vec<String>>,
+    // Target all registered bases, mirroring WriteParams::target_all_bases.
+    // Some(include_primary); resolved at execution time.
+    target_all_bases: Option<bool>,
 }
 
 /// A MergeInsertJob inserts new rows, deletes old rows, and updates existing rows all as
@@ -468,6 +483,9 @@ impl MergeInsertBuilder {
                 use_index: true,
                 source_dedupe_behavior: SourceDedupeBehavior::Fail,
                 commit_retries: None,
+                target_bases: None,
+                target_base_names_or_paths: None,
+                target_all_bases: None,
             },
         })
     }
@@ -566,6 +584,46 @@ impl MergeInsertBuilder {
         self
     }
 
+    /// Write new fragments produced by this merge insert to these base IDs.
+    ///
+    /// New data files are distributed across the target bases round-robin,
+    /// the same way a normal write with [`WriteParams::target_bases`] routes
+    /// them. The IDs must be registered in the dataset manifest, or
+    /// [`super::PRIMARY_BASE_ID`] (0) to include the dataset's primary
+    /// storage in the rotation (e.g. `vec![0, 1, 2]` spreads across primary
+    /// plus bases 1 and 2). Data files that patch existing fragments and
+    /// deletion files are always written to the dataset's primary storage.
+    ///
+    /// Cannot be combined with [`Self::target_base_names_or_paths`].
+    pub fn target_bases(&mut self, base_ids: Vec<u32>) -> &mut Self {
+        self.params.target_bases = Some(base_ids);
+        self
+    }
+
+    /// Like [`Self::target_bases`], but referencing bases by name or path URI.
+    ///
+    /// References are resolved against the base paths registered in the
+    /// dataset manifest when the merge insert executes. An entry equal to the
+    /// dataset's URI includes the dataset's primary storage in the rotation.
+    ///
+    /// Cannot be combined with [`Self::target_bases`].
+    pub fn target_base_names_or_paths(&mut self, refs: Vec<String>) -> &mut Self {
+        self.params.target_base_names_or_paths = Some(refs);
+        self
+    }
+
+    /// Write new fragments produced by this merge insert to every base
+    /// registered in the dataset manifest, resolved when the merge executes.
+    /// When `include_primary` is true the dataset's primary storage
+    /// participates in the rotation as the first slot.
+    ///
+    /// Cannot be combined with [`Self::target_bases`] or
+    /// [`Self::target_base_names_or_paths`].
+    pub fn target_all_bases(&mut self, include_primary: bool) -> &mut Self {
+        self.params.target_all_bases = Some(include_primary);
+        self
+    }
+
     /// Crate a merge insert job
     pub fn try_build(&mut self) -> Result<MergeInsertJob> {
         if !self.params.insert_not_matched
@@ -576,6 +634,19 @@ impl MergeInsertBuilder {
                 "The merge insert job is not configured to change the data in any way",
             ));
         }
+        if self.params.target_bases.is_some() && self.params.target_base_names_or_paths.is_some() {
+            return Err(Error::invalid_input(
+                "Cannot specify both target_base_names_or_paths and target_bases. Use one or the other.",
+            ));
+        }
+        if self.params.target_all_bases.is_some()
+            && (self.params.target_bases.is_some()
+                || self.params.target_base_names_or_paths.is_some())
+        {
+            return Err(Error::invalid_input(
+                "Cannot specify target_all_bases together with target_bases or target_base_names_or_paths.",
+            ));
+        }
         Ok(MergeInsertJob {
             dataset: self.dataset.clone(),
             params: self.params.clone(),
@@ -583,6 +654,45 @@ impl MergeInsertBuilder {
     }
 }
 
+/// Resolve the merge insert target bases against the base paths registered in
+/// the dataset manifest. Returns `None` when no target bases were requested.
+/// Base id [`super::PRIMARY_BASE_ID`] and the dataset's URI refer to the
+/// dataset's primary storage.
+///
+/// Resolution runs once per execution attempt so retries validate against the
+/// manifest version they are writing to.
+async fn resolve_target_bases(
+    dataset: &Dataset,
+    params: &MergeInsertParams,
+) -> Result<Option<Vec<TargetBaseInfo>>> {
+    if params.target_bases.is_none()
+        && params.target_base_names_or_paths.is_none()
+        && params.target_all_bases.is_none()
+    {
+        return Ok(None);
+    }
+    // Reuse the normal write path resolution (validation, name/path lookup,
+    // and per-base credential handling) through a parameter shim.
+    let mut write_params = WriteParams {
+        mode: WriteMode::Append,
+        target_bases: params.target_bases.clone(),
+        target_base_names_or_paths: params.target_base_names_or_paths.clone(),
+        target_all_bases: params.target_all_bases,
+        session: Some(dataset.session.clone()),
+        store_params: dataset.store_params.as_deref().cloned(),
+        base_store_params: dataset.base_store_params.as_deref().cloned(),
+        ..Default::default()
+    };
+    validate_and_resolve_target_bases_with_primary(
+        &mut write_params,
+        Some(&dataset.manifest.base_paths),
+        &dataset.object_store,
+        &dataset.base,
+        dataset.uri(),
+    )
+    .await
+}
+
 enum SchemaComparison {
     FullCompatible,
     Subschema,
@@ -624,30 +734,67 @@ impl MergeInsertJob {
             .map(|_| SchemaComparison::Subschema)
     }
 
-    async fn join_key_as_scalar_index(&self) -> Result<Option<IndexMetadata>> {
-        if self.params.on.len() != 1 {
-            // joining on more than one column
-            Ok(None)
-        } else {
-            let col = &self.params.on[0];
-            self.dataset
+    /// Collect every join column that has a scalar index supporting exact
+    /// equality.
+    ///
+    /// For a single-column join this matches the previous behavior. For a
+    /// multi-column (composite key) join, every indexed column contributes
+    /// an additional `IsIn` probe inside one [`MapIndexExec`]: their AND
+    /// yields the row addresses where every indexed column matches some
+    /// source value. The downstream hash join still filters by the full
+    /// composite key, so unindexed columns simply do not prune the
+    /// candidate set — they are checked by the post-filter.
+    ///
+    /// Returns an empty vec when no join column has a usable scalar index;
+    /// callers should then fall through to the full-scan path.
+    async fn indexed_join_keys(&self) -> Result<Vec<(String, IndexMetadata)>> {
+        let mut indexed = Vec::with_capacity(self.params.on.len());
+        for col in &self.params.on {
+            if let Some(idx) = self
+                .dataset
                 .load_scalar_index(
                     IndexCriteria::default()
                         .for_column(col)
                         // Unclear if this would work if the index does not support exact equality
                         .supports_exact_equality(),
                 )
-                .await
+                .await?
+            {
+                indexed.push((col.clone(), idx));
+            }
+        }
+        Ok(indexed)
+    }
+
+    /// Fragments that cannot be reached by every index in `indexed_keys`
+    /// and therefore must be scanned separately and unioned in alongside
+    /// the indexed take.  A fragment is "reachable" by the composite probe
+    /// only if it is in the intersection of all the indices' fragment
+    /// bitmaps; everything else falls into the unindexed set.
+    async fn unindexed_fragments_for_keys(
+        &self,
+        indexed_keys: &[(String, IndexMetadata)],
+    ) -> Result<Vec<Fragment>> {
+        let mut unindexed: HashMap<u64, Fragment> = HashMap::new();
+        for (_, index) in indexed_keys {
+            for frag in self.dataset.unindexed_fragments(&index.name).await? {
+                unindexed.entry(frag.id).or_insert(frag);
+            }
         }
+        Ok(unindexed.into_values().collect())
     }
 
     async fn create_indexed_scan_joined_stream(
         &self,
         source: SendableRecordBatchStream,
-        index: IndexMetadata,
+        indexed_keys: Vec<(String, IndexMetadata)>,
     ) -> Result<SendableRecordBatchStream> {
         // This relies on a few non-standard physical operators and so we cannot use the
         // datafusion dataframe API and need to construct the plan manually :'(
+        debug_assert!(
+            !indexed_keys.is_empty(),
+            "create_indexed_scan_joined_stream requires at least one indexed key"
+        );
         let schema = source.schema();
         let add_row_addr = match self.check_compatible_schema(&schema)? {
             SchemaComparison::FullCompatible => false,
@@ -662,22 +809,25 @@ impl MergeInsertJob {
         // the new data into memory.  In the future, we can do better
         let shared_input = Arc::new(ReplayExec::new(Capacity::Unbounded, input));
 
-        // 3 - Use the index to map input to row addresses
-        // First, we need to project to the key column
-        let field = schema.field_with_name(&self.params.on[0])?;
-        let index_mapper_input = Arc::new(project(
-            shared_input.clone(),
-            // schema for only the key join column
-            &Schema::new(vec![field.clone()]),
-        )?);
-
-        // Then we pass the key column into the index mapper
-        let index_column = self.params.on[0].clone();
-        let mut index_mapper: Arc<dyn ExecutionPlan> = Arc::new(MapIndexExec::new(
-            // create index from original data and key column
+        // 3 - Probe every indexed join column.  For composite keys this is
+        //     the AND of one `IsIn` query per indexed column, which yields
+        //     a tighter candidate set than probing a single column.  The
+        //     downstream hash join still filters by the full composite key,
+        //     so unindexed `on` columns simply do not prune the candidates.
+        let lookup_fields = indexed_keys
+            .iter()
+            .map(|(col, _)| Ok(schema.field_with_name(col)?.clone()))
+            .collect::<Result<Vec<_>>>()?;
+        let index_mapper_input =
+            Arc::new(project(shared_input.clone(), &Schema::new(lookup_fields))?);
+
+        let lookups = indexed_keys
+            .iter()
+            .map(|(col, idx)| IndexLookup::new(col.clone(), idx.name.clone()))
+            .collect::<Vec<_>>();
+        let mut index_mapper: Arc<dyn ExecutionPlan> = Arc::new(MapIndexExec::new_multi(
             self.dataset.clone(),
-            index_column.clone(),
-            index.name.clone(),
+            lookups,
             index_mapper_input,
         ));
 
@@ -722,8 +872,12 @@ impl MergeInsertJob {
             .filter(|name| name.as_str() != ROW_ID && name.as_str() != ROW_ADDR)
             .collect::<Vec<_>>();
 
-        // 5a - We also need to scan any new unindexed data and union it in
-        let unindexed_fragments = self.dataset.unindexed_fragments(&index.name).await?;
+        // 5a - We also need to scan any new unindexed data and union it in.
+        //      A row can be reached by the composite index probe only if it
+        //      lives in a fragment covered by *every* chosen index, so the
+        //      "unindexed" set is the union of fragments missing from any
+        //      one of them.
+        let unindexed_fragments = self.unindexed_fragments_for_keys(&indexed_keys).await?;
         if !unindexed_fragments.is_empty() {
             let mut builder = self.dataset.scan();
             if add_row_addr {
@@ -748,22 +902,47 @@ impl MergeInsertJob {
         // field names (DF doesn't support this as of version 44)
         target = Self::prefix_columns_phys(target, "target_");
 
-        // 6 - Finally, join the input (source table) with the taken data (target table)
-        let source_key = Column::new_with_schema(&index_column, shared_input.schema().as_ref())?;
-        let target_key = Column::new_with_schema(
-            &format!("target_{}", index_column),
-            target.schema().as_ref(),
-        )?;
+        // 6 - Join the source against the taken target rows on the full
+        //     composite key.  Probing the index produces a super-set of the
+        //     actual matches (when not every key column has an index, or
+        //     even when they do — the per-column `IsIn` lists do not
+        //     correlate values across the tuple), so this join is what
+        //     trims candidates down to the exact composite-key matches.
+        let on_keys = self
+            .params
+            .on
+            .iter()
+            .map(|col| {
+                let source_key = Column::new_with_schema(col, shared_input.schema().as_ref())?;
+                let target_key =
+                    Column::new_with_schema(&format!("target_{}", col), target.schema().as_ref())?;
+                Ok::<_, Error>((
+                    Arc::new(source_key) as Arc<dyn PhysicalExpr>,
+                    Arc::new(target_key) as Arc<dyn PhysicalExpr>,
+                ))
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        // Use standard SQL NULL semantics for composite keys so this path
+        // produces the same result as the full-scan path.  The
+        // single-column case keeps its historical `NullEqualsNull` behavior
+        // to avoid changing semantics for existing callers.
+        let null_equality = if self.params.on.len() == 1 {
+            NullEquality::NullEqualsNull
+        } else {
+            NullEquality::NullEqualsNothing
+        };
+
         let joined = Arc::new(
             HashJoinExec::try_new(
                 shared_input,
                 target,
-                vec![(Arc::new(source_key), Arc::new(target_key))],
+                on_keys,
                 None,
                 &JoinType::Full,
                 None,
                 PartitionMode::CollectLeft,
-                NullEquality::NullEqualsNull,
+                null_equality,
                 false,
             )
             .unwrap(),
@@ -881,9 +1060,15 @@ impl MergeInsertJob {
                 WhenNotMatchedBySource::Keep
             )
         {
-            // keeping unmatched rows, no deletion
-            if let Some(index) = self.join_key_as_scalar_index().await? {
-                return self.create_indexed_scan_joined_stream(source, index).await;
+            // keeping unmatched rows, no deletion. Use the indexed-scan path
+            // only when EVERY join column is indexed; a partially-indexed
+            // composite key under-matches there (the probe can't resolve the
+            // full tuple), so fall through to the correct full-table join.
+            let indexed_keys = self.indexed_join_keys().await?;
+            if indexed_keys.len() == self.params.on.len() {
+                return self
+                    .create_indexed_scan_joined_stream(source, indexed_keys)
+                    .await;
             }
         }
 
@@ -903,7 +1088,11 @@ impl MergeInsertJob {
         dataset: Arc<Dataset>,
         source: SendableRecordBatchStream,
         current_version: u64,
+        target_bases_info: Option<Vec<TargetBaseInfo>>,
     ) -> Result<(Vec<Fragment>, Vec<Fragment>, Vec<u32>)> {
+        // Shared across the per-group tasks spawned below; only new fragments
+        // are routed to target bases, column patches stay in primary storage.
+        let target_bases_info = Arc::new(target_bases_info);
         // Expected source schema: _rowaddr, updated_cols*
         use datafusion::logical_expr::{col, lit};
         let session_ctx = get_session_context(&LanceExecutionOptions {
@@ -924,7 +1113,7 @@ impl MergeInsertJob {
         // sort node so each input batch fits in the memory pool.
         let capped_plan = sorted_plan
             .transform_down(|node| {
-                if node.as_any().downcast_ref::<SortExec>().is_some() {
+                if node.downcast_ref::<SortExec>().is_some() {
                     let children = node.children();
                     let new_children: Vec<Arc<dyn ExecutionPlan>> = children
                         .into_iter()
@@ -951,7 +1140,38 @@ impl MergeInsertJob {
         let reservation =
             MemoryConsumer::new("MergeInsert").register(session_ctx.task_ctx().memory_pool());
 
-        while let Some((frag_id, batches)) = group_stream.next().await.transpose()? {
+        // Best-effort removal of uncommitted files after a mid-update failure.
+        // Aborts in-flight tasks first, then deletes the new fragments written
+        // so far (including ones routed to target bases). Column-patch files
+        // from completed tasks stay in primary storage where regular dataset
+        // cleanup can reclaim them.
+        async fn cleanup_on_failure(
+            dataset: &Dataset,
+            target_bases_info: &Option<Vec<TargetBaseInfo>>,
+            new_fragments: &Mutex<Vec<Fragment>>,
+            tasks: &mut JoinSet<Result<usize>>,
+        ) {
+            tasks.shutdown().await;
+            let written = new_fragments.lock().unwrap().clone();
+            cleanup_data_fragments(
+                &dataset.object_store,
+                &dataset.base,
+                target_bases_info.as_deref(),
+                &written,
+            )
+            .await;
+        }
+
+        loop {
+            let (frag_id, batches) = match group_stream.next().await.transpose() {
+                Ok(Some(group)) => group,
+                Ok(None) => break,
+                Err(e) => {
+                    cleanup_on_failure(&dataset, &target_bases_info, &new_fragments, &mut tasks)
+                        .await;
+                    return Err(e.into());
+                }
+            };
             async fn handle_fragment(
                 dataset: Arc<Dataset>,
                 fragment: FileFragment,
@@ -1003,6 +1223,20 @@ impl MergeInsertJob {
                             Err(e) => Err(e),
                         })?;
 
+                    // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary/JSONB)
+                    // before writing. Without this, Utf8 data is written raw while the
+                    // schema says LargeBinary, causing decoder panics on subsequent reads.
+                    let needs_json_conversion = batches[0]
+                        .schema()
+                        .fields()
+                        .iter()
+                        .any(|f| is_arrow_json_field(f) || has_json_fields(f));
+                    if needs_json_conversion {
+                        for batch in batches.iter_mut() {
+                            *batch = convert_json_columns(batch).map_err(Error::from)?;
+                        }
+                    }
+
                     if data_storage_version == LanceFileVersion::Legacy {
                         // Need to match the existing batch size exactly, otherwise
                         // we'll get errors.
@@ -1055,9 +1289,16 @@ impl MergeInsertJob {
                     // will be the original source data, and all subsequent batches
                     // will be updates.
                     let mut source_batches = Vec::with_capacity(batches.len() + 1);
-                    source_batches.push(batches[0].clone()); // placeholder for source data
+                    // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) so every
+                    // batch is in physical format, matching what the updater reads from the
+                    // fragment. `convert_json_columns` is a no-op clone when there is nothing
+                    // to convert, so it can be applied unconditionally. The first entry is a
+                    // placeholder for the source data (overwritten each iteration below); it
+                    // must be converted too, otherwise its schema would diverge from the rest.
+                    source_batches.push(convert_json_columns(&batches[0]).map_err(Error::from)?);
                     for batch in &batches {
-                        source_batches.push(batch.drop_column(ROW_ADDR)?);
+                        let dropped = batch.drop_column(ROW_ADDR)?;
+                        source_batches.push(convert_json_columns(&dropped).map_err(Error::from)?);
                     }
 
                     // This function is here to help rustc with lifetimes.
@@ -1163,6 +1404,7 @@ impl MergeInsertJob {
                 batches: Vec<RecordBatch>,
                 new_fragments: Arc<Mutex<Vec<Fragment>>>,
                 reservation_size: usize,
+                target_bases_info: Arc<Option<Vec<TargetBaseInfo>>>,
             ) -> Result<usize> {
                 // Batches still have _rowaddr (used elsewhere to merge with existing data)
                 // We need to remove it before writing to Lance files.
@@ -1194,7 +1436,7 @@ impl MergeInsertJob {
                     write_schema,
                     stream,
                     Default::default(), // TODO: support write params.
-                    None,               // Merge insert doesn't use target_bases
+                    (*target_bases_info).clone(),
                 )
                 .await?;
 
@@ -1222,15 +1464,26 @@ impl MergeInsertJob {
                 }
 
                 if let Some(res) = tasks.join_next().await {
-                    let size = res??;
-                    reservation.shrink(size);
+                    match res.map_err(Error::from).and_then(|size| size) {
+                        Ok(size) => reservation.shrink(size),
+                        Err(e) => {
+                            cleanup_on_failure(
+                                &dataset,
+                                &target_bases_info,
+                                &new_fragments,
+                                &mut tasks,
+                            )
+                            .await;
+                            return Err(e);
+                        }
+                    }
                 }
             }
 
             match frag_id.first() {
                 Some(ScalarValue::UInt64(Some(frag_id))) => {
                     let frag_id = *frag_id;
-                    let fragment = dataset.get_fragment(frag_id as usize).ok_or_else(|| {
+                    let Some(fragment) = dataset.get_fragment(frag_id as usize) else {
                         error!(
                             fragment_id = frag_id,
                             dataset_uri = %dataset.uri(),
@@ -1239,15 +1492,22 @@ impl MergeInsertJob {
                             branch = ?dataset.manifest().branch,
                             "Non-existent fragment id returned from merge result",
                         );
-                        Error::internal(format!(
+                        cleanup_on_failure(
+                            &dataset,
+                            &target_bases_info,
+                            &new_fragments,
+                            &mut tasks,
+                        )
+                        .await;
+                        return Err(Error::internal(format!(
                             "Got non-existent fragment id from merge result: {} (uri={}, version={}, manifest={}, branch={})",
                             frag_id,
                             dataset.uri(),
                             dataset.manifest().version,
                             dataset.manifest_location().path,
                             dataset.manifest().branch.as_deref().unwrap_or("main"),
-                        ))
-                    })?;
+                        )));
+                    };
                     let metadata = fragment.metadata.clone();
 
                     let fut = handle_fragment(
@@ -1267,10 +1527,13 @@ impl MergeInsertJob {
                         batches,
                         new_fragments.clone(),
                         memory_size,
+                        target_bases_info.clone(),
                     );
                     tasks.spawn(fut);
                 }
                 _ => {
+                    cleanup_on_failure(&dataset, &target_bases_info, &new_fragments, &mut tasks)
+                        .await;
                     return Err(Error::internal(format!(
                         "Got non-fragment id from merge result: {:?}",
                         frag_id
@@ -1280,8 +1543,14 @@ impl MergeInsertJob {
         }
 
         while let Some(res) = tasks.join_next().await {
-            let size = res??;
-            reservation.shrink(size);
+            match res.map_err(Error::from).and_then(|size| size) {
+                Ok(size) => reservation.shrink(size),
+                Err(e) => {
+                    cleanup_on_failure(&dataset, &target_bases_info, &new_fragments, &mut tasks)
+                        .await;
+                    return Err(e);
+                }
+            }
         }
         let mut updated_fragments = Arc::try_unwrap(updated_fragments)
             .unwrap()
@@ -1518,8 +1787,7 @@ impl MergeInsertJob {
 
         // Extract merge stats from the execution plan
         let (stats, transaction, affected_rows, inserted_rows_filter) = if let Some(full_exec) =
-            plan.as_any()
-                .downcast_ref::<exec::FullSchemaMergeInsertExec>()
+            plan.downcast_ref::<exec::FullSchemaMergeInsertExec>()
         {
             let stats = full_exec.merge_stats().ok_or_else(|| {
                 Error::internal("Merge stats not available - execution may not have completed")
@@ -1530,10 +1798,7 @@ impl MergeInsertJob {
             let affected_rows = full_exec.affected_rows().map(RowAddrTreeMap::from);
             let inserted_rows_filter = full_exec.inserted_rows_filter();
             (stats, transaction, affected_rows, inserted_rows_filter)
-        } else if let Some(delete_exec) = plan
-            .as_any()
-            .downcast_ref::<exec::DeleteOnlyMergeInsertExec>()
-        {
+        } else if let Some(delete_exec) = plan.downcast_ref::<exec::DeleteOnlyMergeInsertExec>() {
             let stats = delete_exec.merge_stats().ok_or_else(|| {
                 Error::internal("Merge stats not available - execution may not have completed")
             })?;
@@ -1613,12 +1878,26 @@ impl MergeInsertJob {
             }
         }
 
+        // A partial-schema source that both deletes matched rows and inserts
+        // unmatched rows cannot be expressed by the indexed-scan delete path
+        // (the delete cannot be folded into a partial write). Keep it off the
+        // scalar-index route so it falls through to the v2 plan, which handles
+        // delete + insert directly.
+        let is_partial_delete_with_insert = is_subset_schema
+            && self.params.insert_not_matched
+            && matches!(self.params.when_matched, WhenMatched::Delete);
+
         let would_use_scalar_index = if self.params.use_index
+            && !is_partial_delete_with_insert
             && matches!(
                 self.params.delete_not_matched_by_source,
                 WhenNotMatchedBySource::Keep
             ) {
-            self.join_key_as_scalar_index().await?.is_some()
+            // Only when EVERY join column is indexed. A partially-indexed
+            // composite key cannot be fully resolved by the index probe and the
+            // indexed-scan path then under-matches (e.g. a delete silently
+            // no-ops), so it must fall through to the correct full path.
+            self.indexed_join_keys().await?.len() == self.params.on.len()
         } else {
             false
         };
@@ -1675,6 +1954,8 @@ impl MergeInsertJob {
             });
         }
 
+        let target_bases_info = resolve_target_bases(&self.dataset, &self.params).await?;
+
         let source_schema = source.schema();
         let lance_schema = lance_core::datatypes::Schema::try_from(source_schema.as_ref())?;
         let full_schema = self.dataset.schema();
@@ -1703,7 +1984,66 @@ impl MergeInsertJob {
             .try_flatten();
         let stream = RecordBatchStreamAdapter::new(merger_schema, stream);
 
-        let (operation, affected_rows) = if !is_full_schema {
+        // A partial-schema source can patch columns or delete matched rows, but
+        // not both in one commit: the writer rejects subschema rows up front and
+        // the delete cannot be folded into the write. Reject that combination.
+        if !is_full_schema
+            && matches!(self.params.when_matched, WhenMatched::Delete)
+            && self.params.insert_not_matched
+        {
+            return Err(Error::not_supported_source("Combining when_matched(Delete) with inserts from a partial-schema source is not supported; provide the full target schema in the source".into()));
+        }
+
+        // The commit strategy follows what the merge does to matched rows. A
+        // pure delete (no inserts) writes nothing: the merger emits no batches
+        // and only records the matched row ids. This holds for any source schema
+        // width, so it is keyed on the operation rather than `is_full_schema`.
+        let is_delete_only = matches!(self.params.when_matched, WhenMatched::Delete)
+            && !self.params.insert_not_matched;
+
+        let (operation, affected_rows) = if is_delete_only {
+            // Consume the stream so the merger records the matched row ids in
+            // `deleted_rows`; it produces no batches.
+            let drained: Vec<RecordBatch> = Box::pin(stream).try_collect().await?;
+            debug_assert!(drained.is_empty(), "delete-only merge must not emit rows");
+
+            let removed_row_ids = Arc::into_inner(deleted_rows).unwrap().into_inner().unwrap();
+            let removed_row_addr_vec =
+                if let Some(row_id_index) = get_row_id_index(&self.dataset).await? {
+                    removed_row_ids
+                        .iter()
+                        .filter_map(|id| row_id_index.get(*id).map(|address| address.into()))
+                        .collect::<Vec<_>>()
+                } else {
+                    removed_row_ids
+                };
+            let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec);
+
+            let (updated_fragments, removed_fragment_ids) =
+                Self::apply_deletions(&self.dataset, &removed_row_addrs).await?;
+
+            let operation = Operation::Update {
+                removed_fragment_ids,
+                updated_fragments,
+                new_fragments: vec![],
+                fields_modified: vec![],
+                merged_generations: self.params.merged_generations.clone(),
+                fields_for_preserving_frag_bitmap: full_schema
+                    .fields
+                    .iter()
+                    .map(|f| f.id as u32)
+                    .collect(),
+                update_mode: Some(RewriteRows),
+                inserted_rows_filter: None, // not implemented for v1
+                updated_fragment_offsets: None,
+            };
+
+            let affected_rows = Some(RowAddrTreeMap::from(removed_row_addrs));
+            (operation, affected_rows)
+        } else if !is_full_schema {
+            // Non-delete partial-schema merge: patch the provided columns into
+            // existing fragments in place. (Delete is handled above; a wider
+            // full source takes the row-rewrite branch below.)
             if !matches!(
                 self.params.delete_not_matched_by_source,
                 WhenNotMatchedBySource::Keep
@@ -1717,6 +2057,7 @@ impl MergeInsertJob {
                 self.dataset.clone(),
                 Box::pin(stream),
                 self.dataset.manifest.version + 1,
+                target_bases_info,
             )
             .await?;
 
@@ -1735,6 +2076,7 @@ impl MergeInsertJob {
             // we can't use affected rows here.
             (operation, None)
         } else {
+            let cleanup_bases = target_bases_info.clone();
             let (mut new_fragments, _) = write_fragments_internal(
                 Some(&self.dataset),
                 self.dataset.object_store.clone(),
@@ -1742,48 +2084,66 @@ impl MergeInsertJob {
                 self.dataset.schema().clone(),
                 Box::pin(stream),
                 WriteParams::default(),
-                None, // Merge insert doesn't use target_bases
+                target_bases_info,
             )
             .await?;
 
-            if let Some(row_id_sequence) = updating_row_ids.lock().unwrap().row_id_sequence() {
-                let fragment_sizes = new_fragments
-                    .iter()
-                    .map(|f| f.physical_rows.unwrap() as u64);
+            // The new data files exist but are not committed yet; clean them up
+            // (including files routed to target bases) if any later step fails.
+            let post_write_result: Result<RoaringTreemap> = async {
+                if let Some(row_id_sequence) = updating_row_ids.lock().unwrap().row_id_sequence() {
+                    let fragment_sizes = new_fragments
+                        .iter()
+                        .map(|f| f.physical_rows.unwrap() as u64);
 
-                let sequences = lance_table::rowids::rechunk_sequences(
-                    [row_id_sequence.clone()],
-                    fragment_sizes,
-                    true,
-                )
-                .map_err(|e| {
-                    Error::internal(format!(
-                        "Captured row ids not equal to number of rows written: {}",
-                        e
-                    ))
-                })?;
+                    let sequences = lance_table::rowids::rechunk_sequences(
+                        [row_id_sequence.clone()],
+                        fragment_sizes,
+                        true,
+                    )
+                    .map_err(|e| {
+                        Error::internal(format!(
+                            "Captured row ids not equal to number of rows written: {}",
+                            e
+                        ))
+                    })?;
 
-                for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) {
-                    let serialized = lance_table::rowids::write_row_ids(&sequence);
-                    fragment.row_id_meta = Some(RowIdMeta::Inline(serialized));
+                    for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) {
+                        let serialized = lance_table::rowids::write_row_ids(&sequence);
+                        fragment.row_id_meta = Some(RowIdMeta::Inline(serialized));
+                    }
                 }
-            }
 
-            // Apply deletions
-            let removed_row_ids = Arc::into_inner(deleted_rows).unwrap().into_inner().unwrap();
+                // Apply deletions
+                let removed_row_ids = Arc::into_inner(deleted_rows).unwrap().into_inner().unwrap();
 
-            let removed_row_addr_vec =
-                if let Some(row_id_index) = get_row_id_index(&self.dataset).await? {
-                    let addresses: Vec<u64> = removed_row_ids
-                        .iter()
-                        .filter_map(|id| row_id_index.get(*id).map(|address| address.into()))
-                        .collect::<Vec<_>>();
-                    addresses
-                } else {
-                    removed_row_ids
-                };
+                let removed_row_addr_vec =
+                    if let Some(row_id_index) = get_row_id_index(&self.dataset).await? {
+                        let addresses: Vec<u64> = removed_row_ids
+                            .iter()
+                            .filter_map(|id| row_id_index.get(*id).map(|address| address.into()))
+                            .collect::<Vec<_>>();
+                        addresses
+                    } else {
+                        removed_row_ids
+                    };
 
-            let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec.into_iter());
+                Ok(RoaringTreemap::from_iter(removed_row_addr_vec))
+            }
+            .await;
+            let removed_row_addrs = match post_write_result {
+                Ok(removed_row_addrs) => removed_row_addrs,
+                Err(e) => {
+                    cleanup_data_fragments(
+                        &self.dataset.object_store,
+                        &self.dataset.base,
+                        cleanup_bases.as_deref(),
+                        &new_fragments,
+                    )
+                    .await;
+                    return Err(e);
+                }
+            };
 
             let deletions_result = Self::apply_deletions(&self.dataset, &removed_row_addrs).await;
             let (old_fragments, removed_fragment_ids) = match deletions_result {
@@ -1792,6 +2152,7 @@ impl MergeInsertJob {
                     cleanup_data_fragments(
                         &self.dataset.object_store,
                         &self.dataset.base,
+                        cleanup_bases.as_deref(),
                         &new_fragments,
                     )
                     .await;
@@ -2029,6 +2390,10 @@ impl RetryExecutor for MergeInsertJobWithIterator {
         // Update stats with the current attempt count
         data.stats.num_attempts = self.attempt_count.load(Ordering::SeqCst);
 
+        // The dataset argument is the refreshed per-attempt dataset (the same
+        // manifest execute_impl resolved against); keep a handle so conflict
+        // cleanup resolves bases added between attempts.
+        let cleanup_dataset = dataset.clone();
         let mut commit_builder =
             CommitBuilder::new(dataset).with_skip_auto_cleanup(self.job.params.skip_auto_cleanup);
         if let Some(commit_retries) = self.job.params.commit_retries {
@@ -2037,9 +2402,36 @@ impl RetryExecutor for MergeInsertJobWithIterator {
         if let Some(affected_rows) = data.affected_rows {
             commit_builder = commit_builder.with_affected_rows(affected_rows);
         }
-        let new_dataset = commit_builder.execute(data.transaction).await?;
 
-        Ok((Arc::new(new_dataset), data.stats))
+        let new_fragments = match &data.transaction.operation {
+            Operation::Update { new_fragments, .. } => new_fragments.clone(),
+            _ => Vec::new(),
+        };
+        match commit_builder.execute(data.transaction).await {
+            Ok(new_dataset) => Ok((Arc::new(new_dataset), data.stats)),
+            Err(e) => {
+                // A retryable conflict discards this attempt and re-executes it,
+                // so its data files are provably uncommitted; remove them
+                // (including files routed to target bases, which version cleanup
+                // never scans). Other commit errors may be ambiguous about
+                // whether the manifest was written, so leave the files alone.
+                if matches!(e, Error::RetryableCommitConflict { .. }) && !new_fragments.is_empty() {
+                    let target_bases_info =
+                        resolve_target_bases(&cleanup_dataset, &self.job.params)
+                            .await
+                            .ok()
+                            .flatten();
+                    cleanup_data_fragments(
+                        &cleanup_dataset.object_store,
+                        &cleanup_dataset.base,
+                        target_bases_info.as_deref(),
+                        &new_fragments,
+                    )
+                    .await;
+                }
+                Err(e)
+            }
+        }
     }
 
     fn update_dataset(&mut self, dataset: Arc<Dataset>) {
@@ -2149,18 +2541,13 @@ impl Merger {
         &self.output_schema
     }
 
-    // Retrieves a bitmap of rows where at least one of the columns in the range
-    // col_offset..coll_offset+num_cols is not null.
-    //
-    fn not_all_null(
-        batch: &RecordBatch,
-        col_offset: usize,
-        num_cols: usize,
-    ) -> Result<BooleanArray> {
+    // Retrieves a bitmap of rows where at least one of the given columns is
+    // not null.
+    fn not_all_null(batch: &RecordBatch, cols: &[usize]) -> Result<BooleanArray> {
         // For our purposes we know there is always at least 1 on key
-        debug_assert_ne!(num_cols, 0);
-        let mut at_least_one_valid = arrow::compute::is_not_null(batch.column(col_offset))?;
-        for idx in col_offset + 1..col_offset + num_cols {
+        debug_assert!(!cols.is_empty());
+        let mut at_least_one_valid = arrow::compute::is_not_null(batch.column(cols[0]))?;
+        for &idx in &cols[1..] {
             let is_valid = arrow::compute::is_not_null(batch.column(idx))?;
             at_least_one_valid = arrow::compute::or(&at_least_one_valid, &is_valid)?;
         }
@@ -2188,8 +2575,37 @@ impl Merger {
         right_offset: usize,
         num_keys: usize,
     ) -> Result<(BooleanArray, BooleanArray, BooleanArray)> {
-        let in_left = Self::not_all_null(combined_batch, 0, num_keys)?;
-        let in_right = Self::not_all_null(combined_batch, right_offset, num_keys)?;
+        // The outer join distinguishes its three cases by which side's join
+        // keys were NULL-padded: a present row always has non-null keys, while
+        // the absent side is filled with NULLs. We therefore test the *key*
+        // columns, located by name. They are NOT necessarily the first
+        // `num_keys` columns — a partial-schema source can place a payload
+        // column (e.g. an all-null vector) at position 0, and checking
+        // positions [0, num_keys) there misreads an all-null leading payload
+        // column as an absent join side, silently dropping every matched row
+        // (https://github.com/lancedb/lancedb/issues/3515). The target half
+        // carries the same columns in the same order, offset by `right_offset`.
+        let source_key_cols = self
+            .params
+            .on
+            .iter()
+            .map(|key| {
+                combined_batch.schema().index_of(key).map_err(|_| {
+                    Error::internal(format!(
+                        "merge insert key column '{}' not found in joined batch",
+                        key
+                    ))
+                })
+            })
+            .collect::<Result<Vec<_>>>()?;
+        debug_assert_eq!(source_key_cols.len(), num_keys);
+        let target_key_cols = source_key_cols
+            .iter()
+            .map(|c| c + right_offset)
+            .collect::<Vec<_>>();
+
+        let in_left = Self::not_all_null(combined_batch, &source_key_cols)?;
+        let in_right = Self::not_all_null(combined_batch, &target_key_cols)?;
         let in_both = arrow::compute::and(&in_left, &in_right)?;
         let left_only = arrow::compute::and(&in_left, &arrow::compute::not(&in_right)?)?;
         let right_only = arrow::compute::and(&arrow::compute::not(&in_left)?, &in_right)?;
@@ -2235,40 +2651,27 @@ impl Merger {
         // borrow checker (the stream needs to be `sync` since it crosses an await point)
         let mut deleted_row_ids = self.deleted_rows.lock().unwrap();
 
-        if self.params.when_matched != WhenMatched::DoNothing {
-            let mut matched = arrow::compute::filter_record_batch(&batch, &in_both)?;
-
-            if let Some(match_filter) = self.match_filter_expr {
-                let unzipped = unzip_batch(&matched, &self.schema);
-                let filtered = match_filter.evaluate(&unzipped)?;
-                match filtered {
-                    ColumnarValue::Array(mask) => {
-                        // Some rows matched, filter down and replace those rows
-                        matched = arrow::compute::filter_record_batch(&matched, mask.as_boolean())?;
-                    }
-                    ColumnarValue::Scalar(scalar) => {
-                        if let ScalarValue::Boolean(Some(true)) = scalar {
-                            // All rows matched, go ahead and replace the whole batch
-                        } else {
-                            // Nothing matched, replace nothing
-                            matched = RecordBatch::new_empty(matched.schema());
-                        }
-                    }
-                }
-            }
-
-            merge_statistics.num_updated_rows += matched.num_rows() as u64;
-
-            // If the filter eliminated all rows then its important we don't try and write
-            // the batch at all.  Writing an empty batch currently panics
-            if matched.num_rows() > 0 {
+        // Each `WhenMatched` variant handles the matched rows (`in_both`)
+        // differently.
+        let match_filter_expr = self.match_filter_expr;
+        match &self.params.when_matched {
+            WhenMatched::DoNothing => {}
+            WhenMatched::Delete => {
+                // Matched rows are removed, not rewritten: record their row ids
+                // for the commit to delete and emit no replacement batch. A
+                // source with duplicate keys matches the same target row more
+                // than once; apply the same `source_dedupe_behavior` policy as
+                // updates so a duplicate either aborts (`Fail`) or is skipped
+                // and counted once (`FirstSeen`) — the commit deletes the row a
+                // single time regardless.
+                let matched = arrow::compute::filter_record_batch(&batch, &in_both)?;
                 let row_ids = matched.column(row_id_col).as_primitive::<UInt64Type>();
 
                 let mut processed_row_ids = self.processed_row_ids.lock().unwrap();
-                let mut keep_indices: Vec<u32> = Vec::with_capacity(matched.num_rows());
                 for (row_idx, &row_id) in row_ids.values().iter().enumerate() {
                     if processed_row_ids.insert(row_id) {
-                        keep_indices.push(row_idx as u32);
+                        merge_statistics.num_deleted_rows += 1;
+                        deleted_row_ids.push(row_id);
                     } else {
                         match self.params.source_dedupe_behavior {
                             SourceDedupeBehavior::Fail => {
@@ -2279,54 +2682,115 @@ impl Merger {
                                 ));
                             }
                             SourceDedupeBehavior::FirstSeen => {
-                                // Skip this duplicate row (don't add to keep_indices)
+                                merge_statistics.num_skipped_duplicates += 1;
                             }
                         }
                     }
                 }
-                drop(processed_row_ids);
-
-                // Filter out duplicate rows if any were skipped
-                let num_skipped = matched.num_rows() - keep_indices.len();
-                if num_skipped > 0 {
-                    merge_statistics.num_skipped_duplicates += num_skipped as u64;
-                    merge_statistics.num_updated_rows -= num_skipped as u64;
-
-                    let indices = UInt32Array::from(keep_indices);
-                    matched = take_record_batch(&matched, &indices)?;
+            }
+            WhenMatched::Fail => {
+                // Any matched row aborts the whole operation.
+                if let Some(row_idx) = (0..in_both.len()).find(|&i| in_both.value(i)) {
+                    return Err(DataFusionError::Execution(format!(
+                        "Merge insert failed: found matching row with key values: {}",
+                        format_key_values_on_columns(&batch, row_idx, &self.params.on)
+                    )));
+                }
+            }
+            WhenMatched::UpdateAll | WhenMatched::UpdateIf(_) | WhenMatched::UpdateIfExpr(_) => {
+                let mut matched = arrow::compute::filter_record_batch(&batch, &in_both)?;
+
+                if let Some(match_filter) = match_filter_expr {
+                    let unzipped = unzip_batch(&matched, &self.schema);
+                    let filtered = match_filter.evaluate(&unzipped)?;
+                    match filtered {
+                        ColumnarValue::Array(mask) => {
+                            // Some rows matched, filter down and replace those rows
+                            matched =
+                                arrow::compute::filter_record_batch(&matched, mask.as_boolean())?;
+                        }
+                        ColumnarValue::Scalar(scalar) => {
+                            if let ScalarValue::Boolean(Some(true)) = scalar {
+                                // All rows matched, go ahead and replace the whole batch
+                            } else {
+                                // Nothing matched, replace nothing
+                                matched = RecordBatch::new_empty(matched.schema());
+                            }
+                        }
+                    }
                 }
 
-                // Only process and write if there are remaining rows after filtering duplicates
+                merge_statistics.num_updated_rows += matched.num_rows() as u64;
+
+                // If the filter eliminated all rows then its important we don't try and write
+                // the batch at all.  Writing an empty batch currently panics
                 if matched.num_rows() > 0 {
-                    // Get row_ids again after filtering (if any duplicates were removed)
                     let row_ids = matched.column(row_id_col).as_primitive::<UInt64Type>();
-                    deleted_row_ids.extend(row_ids.values());
-                    if self.enable_stable_row_ids {
-                        self.updating_row_ids
-                            .lock()
-                            .unwrap()
-                            .capture(row_ids.values())?;
+
+                    let mut processed_row_ids = self.processed_row_ids.lock().unwrap();
+                    let mut keep_indices: Vec<u32> = Vec::with_capacity(matched.num_rows());
+                    for (row_idx, &row_id) in row_ids.values().iter().enumerate() {
+                        if processed_row_ids.insert(row_id) {
+                            keep_indices.push(row_idx as u32);
+                        } else {
+                            match self.params.source_dedupe_behavior {
+                                SourceDedupeBehavior::Fail => {
+                                    return Err(create_duplicate_row_error(
+                                        &matched,
+                                        row_idx,
+                                        &self.params.on,
+                                    ));
+                                }
+                                SourceDedupeBehavior::FirstSeen => {
+                                    // Skip this duplicate row (don't add to keep_indices)
+                                }
+                            }
+                        }
                     }
+                    drop(processed_row_ids);
 
-                    let projection = if let Some(row_addr_col) = row_addr_col {
-                        let mut cols = Vec::from_iter(left_cols.iter().cloned());
-                        cols.push(row_addr_col);
-                        cols
-                    } else {
-                        #[allow(clippy::redundant_clone)]
-                        left_cols.clone()
-                    };
-                    let matched = matched.project(&projection)?;
-                    // The payload columns of an outer join are always nullable.  We need to restore
-                    // non-nullable to columns that were originally non-nullable.  This should be safe
-                    // since the not_matched rows should all be valid on the right_cols
-                    //
-                    // Sadly we can't use with_schema because it doesn't let you toggle nullability
-                    let matched = RecordBatch::try_new(
-                        self.output_schema.clone(),
-                        Vec::from_iter(matched.columns().iter().cloned()),
-                    )?;
-                    batches.push(Ok(matched));
+                    // Filter out duplicate rows if any were skipped
+                    let num_skipped = matched.num_rows() - keep_indices.len();
+                    if num_skipped > 0 {
+                        merge_statistics.num_skipped_duplicates += num_skipped as u64;
+                        merge_statistics.num_updated_rows -= num_skipped as u64;
+
+                        let indices = UInt32Array::from(keep_indices);
+                        matched = take_record_batch(&matched, &indices)?;
+                    }
+
+                    // Only process and write if there are remaining rows after filtering duplicates
+                    if matched.num_rows() > 0 {
+                        // Get row_ids again after filtering (if any duplicates were removed)
+                        let row_ids = matched.column(row_id_col).as_primitive::<UInt64Type>();
+                        deleted_row_ids.extend(row_ids.values());
+                        if self.enable_stable_row_ids {
+                            self.updating_row_ids
+                                .lock()
+                                .unwrap()
+                                .capture(row_ids.values())?;
+                        }
+
+                        let projection = if let Some(row_addr_col) = row_addr_col {
+                            let mut cols = Vec::from_iter(left_cols.iter().cloned());
+                            cols.push(row_addr_col);
+                            cols
+                        } else {
+                            #[allow(clippy::redundant_clone)]
+                            left_cols.clone()
+                        };
+                        let matched = matched.project(&projection)?;
+                        // The payload columns of an outer join are always nullable.  We need to restore
+                        // non-nullable to columns that were originally non-nullable.  This should be safe
+                        // since the not_matched rows should all be valid on the right_cols
+                        //
+                        // Sadly we can't use with_schema because it doesn't let you toggle nullability
+                        let matched = RecordBatch::try_new(
+                            self.output_schema.clone(),
+                            Vec::from_iter(matched.columns().iter().cloned()),
+                        )?;
+                        batches.push(Ok(matched));
+                    }
                 }
             }
         }
@@ -2435,41 +2899,162 @@ mod tests {
         t
     }
 
-    async fn check_then_refresh_dataset(
-        new_data: RecordBatch,
-        mut job: MergeInsertJob,
-        keys_from_left: &[u32],
-        keys_from_right: &[u32],
-        stats: &[u64],
-    ) -> Arc<Dataset> {
-        let mut dataset = (*job.dataset).clone();
-        dataset.restore().await.unwrap();
-        job.dataset = Arc::new(dataset);
-
-        let schema = new_data.schema();
-        let new_reader = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone()));
-        let new_stream = reader_to_stream(new_reader);
-
-        let (merged_dataset, merge_stats) = job.execute(new_stream).boxed().await.unwrap();
-
-        let batches = merged_dataset
-            .scan()
-            .try_into_stream()
-            .await
+    // An update-style merge_insert leaves the source and new fragments with
+    // overlapping id ranges; a scattered delete punches holes in that range. A
+    // filtered `with_row_id` scan must still resolve every id (round-tripped via take).
+    #[tokio::test(flavor = "multi_thread")]
+    async fn merge_insert_then_delete_resolves_overlapping_row_ids() {
+        use arrow_array::ArrayRef;
+        use arrow_array::types::UInt64Type;
+        let dir = TempStrDir::default();
+        let uri = dir.as_str();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("slug", DataType::Utf8, false),
+            Field::new("title", DataType::Utf8, false),
+            Field::new("category", DataType::Utf8, false),
+        ]));
+        let mk = |slugs: Vec<String>, titles: Vec<String>| {
+            let cats: Vec<String> = (0..slugs.len())
+                .map(|i| ["A", "B", "C", "D", "E"][i % 5].to_string())
+                .collect();
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(StringArray::from(slugs)) as ArrayRef,
+                    Arc::new(StringArray::from(titles)) as ArrayRef,
+                    Arc::new(StringArray::from(cats)) as ArrayRef,
+                ],
+            )
             .unwrap()
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        let merged = concat_batches(&schema, &batches).unwrap();
+        };
 
-        let keyvals = merged
-            .column(0)
-            .as_primitive::<UInt32Type>()
-            .values()
-            .iter()
-            .zip(
-                merged
+        // Empty dataset with stable row ids; write-seed 40 rows (ids 0..40).
+        let params = WriteParams {
+            mode: WriteMode::Create,
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(mk(vec![], vec![]))], schema.clone()),
+            uri,
+            Some(params),
+        )
+        .await
+        .unwrap();
+        ds.append(
+            RecordBatchIterator::new(
+                vec![Ok(mk(
+                    (1..=40).map(|i| format!("t{i}")).collect(),
+                    (1..=40).map(|i| format!("r{i}")).collect(),
+                ))],
+                schema.clone(),
+            ),
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Update every other row (so the new fragment's ids interleave with -- and
+        // its range overlaps -- the source's) plus a few inserts.
+        let mut slugs: Vec<String> = (1..=40).step_by(2).map(|i| format!("t{i}")).collect();
+        let mut titles: Vec<String> = (1..=40).step_by(2).map(|i| format!("e{i}")).collect();
+        for i in 41..=45 {
+            slugs.push(format!("t{i}"));
+            titles.push(format!("e{i}"));
+        }
+        let mut b = MergeInsertBuilder::try_new(Arc::new(ds), vec!["slug".into()]).unwrap();
+        b.when_matched(WhenMatched::UpdateAll);
+        b.when_not_matched(WhenNotMatched::InsertAll);
+        let (ds, _) = b
+            .try_build()
+            .unwrap()
+            .execute_reader(RecordBatchIterator::new(
+                vec![Ok(mk(slugs, titles))],
+                schema.clone(),
+            ))
+            .await
+            .unwrap();
+
+        // Scattered delete -> interior holes in the overlapped id range.
+        let mut ds = (*ds).clone();
+        let ds = (*ds.delete("category = 'A'").await.unwrap().new_dataset).clone();
+
+        // The `with_row_id` scan builds the RowIdIndex; every scanned id must
+        // round-trip through take_rows.
+        let batches: Vec<RecordBatch> = ds
+            .scan()
+            .with_row_id()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let row_ids: Vec<u64> = batches
+            .iter()
+            .flat_map(|b| {
+                b.column_by_name(ROW_ID)
+                    .unwrap()
+                    .as_primitive::<UInt64Type>()
+                    .values()
+                    .to_vec()
+            })
+            .collect();
+        let scanned_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        let taken = ds.take_rows(&row_ids, ds.schema().clone()).await.unwrap();
+        assert_eq!(taken.num_rows(), scanned_rows);
+
+        // A point lookup on a surviving row resolves to exactly one row.
+        let filtered: Vec<RecordBatch> = ds
+            .scan()
+            .with_row_id()
+            .filter("slug = 't30'")
+            .unwrap()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect()
+            .await
+            .unwrap();
+        let n: usize = filtered.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(n, 1, "expected exactly one row for slug='t30'");
+    }
+
+    async fn check_then_refresh_dataset(
+        new_data: RecordBatch,
+        mut job: MergeInsertJob,
+        keys_from_left: &[u32],
+        keys_from_right: &[u32],
+        stats: &[u64],
+    ) -> Arc<Dataset> {
+        let mut dataset = (*job.dataset).clone();
+        dataset.restore().await.unwrap();
+        job.dataset = Arc::new(dataset);
+
+        let schema = new_data.schema();
+        let new_reader = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone()));
+        let new_stream = reader_to_stream(new_reader);
+
+        let (merged_dataset, merge_stats) = job.execute(new_stream).boxed().await.unwrap();
+
+        let batches = merged_dataset
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let merged = concat_batches(&schema, &batches).unwrap();
+
+        let keyvals = merged
+            .column(0)
+            .as_primitive::<UInt32Type>()
+            .values()
+            .iter()
+            .zip(
+                merged
                     .column(1)
                     .as_primitive::<UInt32Type>()
                     .values()
@@ -3442,6 +4027,116 @@ mod tests {
         }
     }
 
+    /// Reproduces https://github.com/lancedb/lancedb/issues/3515:
+    /// a partial-schema `merge_insert` with a scalar index on the join key,
+    /// where every fragment is covered by the index (no unindexed data),
+    /// silently updates 0 rows instead of the expected matches.
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn test_repro_3515_partial_schema_fully_indexed(
+        #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)]
+        version: LanceFileVersion,
+    ) {
+        const N: usize = 1000;
+        const UPD: usize = 128;
+        let vec_field = Field::new(
+            "vector",
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
+            true,
+        );
+        let full_schema = Arc::new(Schema::new(vec![
+            vec_field.clone(),
+            Field::new("path", DataType::Utf8, false),
+            Field::new("status", DataType::Utf8, true),
+            Field::new("file_size", DataType::Int64, true),
+        ]));
+
+        // 1000 rows: vector all-null, path "/img/{i}.jpg", status "pending".
+        let paths = StringArray::from((0..N).map(|i| format!("/img/{i}.jpg")).collect::<Vec<_>>());
+        let statuses = StringArray::from(vec!["pending"; N]);
+        let file_sizes = Int64Array::from((0..N as i64).map(|i| 1000 + i).collect::<Vec<_>>());
+        let null_vectors = arrow_array::new_null_array(vec_field.data_type(), N);
+        let batch = RecordBatch::try_new(
+            full_schema.clone(),
+            vec![
+                null_vectors,
+                Arc::new(paths),
+                Arc::new(statuses),
+                Arc::new(file_sizes),
+            ],
+        )
+        .unwrap();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new([Ok(batch)], full_schema.clone()),
+            "memory://",
+            Some(WriteParams {
+                data_storage_version: Some(version),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Scalar index on the merge key, covering every fragment.
+        ds.create_index(
+            &["path"],
+            IndexType::Scalar,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+        let ds = Arc::new(ds);
+
+        // Partial-schema source (no `file_size`): update the first 128 rows.
+        let upd_schema = Arc::new(Schema::new(vec![
+            vec_field,
+            Field::new("path", DataType::Utf8, false),
+            Field::new("status", DataType::Utf8, true),
+        ]));
+        let upd_paths = StringArray::from(
+            (0..UPD)
+                .map(|i| format!("/img/{i}.jpg"))
+                .collect::<Vec<_>>(),
+        );
+        let upd_vectors =
+            FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.1f32; 4 * UPD]), 4)
+                .unwrap();
+        let upd_statuses = StringArray::from(vec!["indexed"; UPD]);
+        let updates = RecordBatch::try_new(
+            upd_schema.clone(),
+            vec![
+                Arc::new(upd_vectors),
+                Arc::new(upd_paths),
+                Arc::new(upd_statuses),
+            ],
+        )
+        .unwrap();
+
+        let (ds, stats) = MergeInsertBuilder::try_new(ds.clone(), vec!["path".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .try_build()
+            .unwrap()
+            .execute_reader(RecordBatchIterator::new([Ok(updates)], upd_schema))
+            .await
+            .unwrap();
+
+        assert_eq!(
+            stats.num_updated_rows, UPD as u64,
+            "expected {UPD} updated rows on {version:?}, got {}",
+            stats.num_updated_rows
+        );
+        let n_indexed = ds
+            .count_rows(Some("status = 'indexed'".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(n_indexed, UPD, "expected {UPD} rows flipped to 'indexed'");
+    }
+
     #[tokio::test]
     async fn test_indexed_merge_insert() {
         let test_dir = TempStrDir::default();
@@ -3461,107 +4156,1088 @@ mod tests {
             .await
             .unwrap();
 
-        // Create some new (unindexed) data
-        let data = lance_datagen::gen_batch()
-            .with_seed(Seed::from(2))
-            .col("value", array::step::<UInt32Type>())
-            .col("key", array::rand_pseudo_uuid_hex());
-        let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(8));
-        let ds = Dataset::write(
-            data,
-            test_uri,
+        // Create some new (unindexed) data
+        let data = lance_datagen::gen_batch()
+            .with_seed(Seed::from(2))
+            .col("value", array::step::<UInt32Type>())
+            .col("key", array::rand_pseudo_uuid_hex());
+        let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(8));
+        let ds = Dataset::write(
+            data,
+            test_uri,
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let ds = Arc::new(ds);
+
+        let just_index_col = Schema::new(vec![Field::new("key", DataType::Utf8, false)]);
+
+        // Sample 2048 random indices and then paste on a column of 9999999's
+        let some_indices = ds
+            .sample(2048, &(&just_index_col).try_into().unwrap(), None)
+            .await
+            .unwrap();
+        let some_indices = some_indices.column(0).clone();
+        let some_vals = lance_datagen::gen_batch()
+            .anon_col(array::fill::<UInt32Type>(9999999))
+            .into_batch_rows(RowCount::from(2048))
+            .unwrap();
+        let some_vals = some_vals.column(0).clone();
+        let source_batch =
+            RecordBatch::try_new(schema.clone(), vec![some_vals, some_indices]).unwrap();
+        // To make things more interesting, lets make the input a stream of four batches
+        let source_batches = vec![
+            source_batch.slice(0, 512),
+            source_batch.slice(512, 512),
+            source_batch.slice(1024, 512),
+            source_batch.slice(1536, 512),
+        ];
+        let source = Box::new(RecordBatchIterator::new(
+            source_batches.clone().into_iter().map(Ok),
+            schema.clone(),
+        ));
+
+        // Run merge_insert
+        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
+            .unwrap()
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .when_matched(WhenMatched::UpdateAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(source)
+            .await
+            .unwrap();
+
+        // Check that the data is as expected
+        let updated = ds
+            .count_rows(Some("value = 9999999".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(updated, 2048);
+
+        // Make sure we don't use an indexed scan if there is a delete criteria
+        let source = Box::new(RecordBatchIterator::new(
+            source_batches.clone().into_iter().map(Ok),
+            schema.clone(),
+        ));
+        // Run merge_insert
+        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
+            .unwrap()
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .when_matched(WhenMatched::UpdateAll)
+            .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
+            .try_build()
+            .unwrap()
+            .execute_reader(source)
+            .await
+            .unwrap();
+
+        // Check that the data is as expected
+        assert_eq!(ds.count_rows(None).await.unwrap(), 2048);
+
+        let source = Box::new(RecordBatchIterator::new(
+            source_batches.clone().into_iter().map(Ok),
+            schema.clone(),
+        ));
+        // Run merge_insert one last time.  The index is now completely out of date.  Every
+        // row it points to is a deleted row.  Make sure that doesn't break.
+        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
+            .unwrap()
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .when_matched(WhenMatched::UpdateAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(source)
+            .await
+            .unwrap();
+
+        assert_eq!(ds.count_rows(None).await.unwrap(), 2048);
+    }
+
+    /// Multi-column (composite key) merge_insert when one or more join
+    /// columns have a scalar index.  Before this change the indexed path
+    /// was hard-gated to single-column joins; composite-key merges fell
+    /// through to a full target scan even when every key column was
+    /// indexed.  Now each indexed column contributes an AND-ed `IsIn`
+    /// probe inside one `MapIndexExec`, and the downstream hash join trims
+    /// the candidates to the exact composite-key matches — rows that
+    /// happen to match one key column but not the other must NOT be
+    /// touched.
+    #[rstest::rstest]
+    #[case::index_on_first(true, false)]
+    #[case::index_on_second(false, true)]
+    #[case::index_on_both(true, true)]
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key(
+        #[case] index_on_a: bool,
+        #[case] index_on_b: bool,
+    ) {
+        // Target rows: every (a, b) combination from {1,2} x {10,20}.
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2, 2]),
+            ("b", Int32, [10, 20, 10, 20]),
+            ("value", Int32, [100, 200, 300, 400])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::default();
+        if index_on_a {
+            ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+                .await
+                .unwrap();
+        }
+        if index_on_b {
+            ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+                .await
+                .unwrap();
+        }
+
+        // Update (1, 10) and insert (3, 30).  A naive single-column probe
+        // on `a` would also pull (1, 20) into the candidate set; the
+        // composite hash join must keep (1, 20) untouched.
+        let source = record_batch!(
+            ("a", Int32, [1, 3]),
+            ("b", Int32, [10, 30]),
+            ("value", Int32, [999, 333])
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 5);
+
+        let untouched = updated_ds
+            .count_rows(Some("a = 1 AND b = 20 AND value = 200".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(
+            untouched, 1,
+            "(1, 20) must not be clobbered by an `a`-only probe"
+        );
+
+        let updated = updated_ds
+            .count_rows(Some("a = 1 AND b = 10 AND value = 999".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(updated, 1);
+
+        let inserted = updated_ds
+            .count_rows(Some("a = 3 AND b = 30 AND value = 333".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(inserted, 1);
+    }
+
+    /// Composite key merge_insert with no scalar index on any join column
+    /// must keep working via the full-scan fallback.  Guards against the
+    /// indexed path becoming a hard requirement after this optimization.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_no_index() {
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2]),
+            ("b", Int32, [10, 20, 10]),
+            ("value", Int32, [100, 200, 300])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let source = record_batch!(
+            ("a", Int32, [1]),
+            ("b", Int32, [20]),
+            ("value", Int32, [999])
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_updated_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 0);
+        let count = updated_ds
+            .count_rows(Some("a = 1 AND b = 20 AND value = 999".to_string()))
+            .await
+            .unwrap();
+        assert_eq!(count, 1);
+    }
+
+    /// Composite-key merge_insert must use standard SQL NULL semantics
+    /// (NULL != NULL) on the post-filter hash join so its behavior is
+    /// identical to the full-scan path; otherwise enabling the indexed
+    /// path for multi-column joins would silently change semantics.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_null_semantics() {
+        let initial = record_batch!(
+            ("a", Int32, [Some(1)]),
+            ("b", Utf8, [Option::<&str>::None]),
+            ("value", Int32, [Some(10)])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        ds.create_index(
+            &["a"],
+            IndexType::Scalar,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        let source = record_batch!(
+            ("a", Int32, [Some(1)]),
+            ("b", Utf8, [Option::<&str>::None]),
+            ("value", Int32, [Some(99)])
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_inserted_rows, 1);
+        assert_eq!(stats.num_updated_rows, 0);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 2);
+    }
+
+    /// Composite-key merge_insert where new (unindexed) fragments are
+    /// appended after the indices were built.  The indexed take only sees
+    /// fragments covered by every chosen index, so the unindexed remainder
+    /// must be unioned in via a full scan — otherwise updates to rows
+    /// that live in those fragments are silently dropped.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_unindexed_fragments() {
+        let first = record_batch!(
+            ("a", Int32, [1, 2]),
+            ("b", Int32, [10, 20]),
+            ("value", Int32, [100, 200])
+        )
+        .unwrap();
+        let schema = first.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(first.clone())], schema.clone()),
+            "memory://",
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::default();
+        ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+        ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Append a fragment AFTER both indices are built.  The new (3, 30)
+        // row lives in a fragment neither index covers, so the indexed
+        // take alone would miss it.
+        let appended = record_batch!(
+            ("a", Int32, [3]),
+            ("b", Int32, [30]),
+            ("value", Int32, [300])
+        )
+        .unwrap();
+        ds.append(
+            RecordBatchIterator::new(vec![Ok(appended.clone())], appended.schema()),
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Source updates one row in the indexed fragment AND one row in
+        // the appended (unindexed) fragment.
+        let source = record_batch!(
+            ("a", Int32, [1, 3]),
+            ("b", Int32, [10, 30]),
+            ("value", Int32, [999, 333])
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(
+            stats.num_updated_rows, 2,
+            "row in the unindexed fragment must also be updated"
+        );
+        assert_eq!(stats.num_inserted_rows, 0);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 3);
+    }
+
+    /// Composite-key delete-only merge_insert (`when_matched(Delete)`,
+    /// `when_not_matched_by_source(Keep)`) removes the matched rows for every
+    /// combination of which join columns carry a scalar index, including when
+    /// every column is indexed.
+    #[rstest::rstest]
+    #[case::index_on_both(true, true)]
+    #[case::index_on_first(true, false)]
+    #[case::index_on_second(false, true)]
+    #[case::no_index(false, false)]
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_delete(
+        #[case] index_on_a: bool,
+        #[case] index_on_b: bool,
+    ) {
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2, 2]),
+            ("b", Int32, [10, 20, 10, 20]),
+            ("value", Int32, [100, 200, 300, 400])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::default();
+        if index_on_a {
+            ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+                .await
+                .unwrap();
+        }
+        if index_on_b {
+            ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+                .await
+                .unwrap();
+        }
+
+        // Delete (1, 10) by composite key.  Only key columns in the source.
+        let source = record_batch!(("a", Int32, [1]), ("b", Int32, [10])).unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::Delete)
+                .when_not_matched(WhenNotMatched::DoNothing)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 1, "matched row must be deleted");
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 3);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1 AND b = 10".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "(1, 10) must be gone after the delete"
+        );
+        // The sibling key (1, 20) must remain untouched.
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1 AND b = 20 AND value = 200".to_string()))
+                .await
+                .unwrap(),
+            1,
+        );
+    }
+
+    /// Delete-only merge_insert on a single indexed key removes the matched
+    /// rows (the indexed path is not exclusive to composite keys).
+    #[tokio::test]
+    async fn test_indexed_merge_insert_single_key_delete() {
+        let initial = record_batch!(
+            ("id", Int32, [1, 2, 3, 4]),
+            ("value", Int32, [10, 20, 30, 40])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        ds.create_index(
+            &["id"],
+            IndexType::Scalar,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+
+        let source = record_batch!(("id", Int32, [2, 4])).unwrap();
+
+        let (updated_ds, stats) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::Delete)
+            .when_not_matched(WhenNotMatched::DoNothing)
+            .try_build()
+            .unwrap()
+            .execute_reader(Box::new(RecordBatchIterator::new(
+                vec![Ok(source.clone())],
+                source.schema(),
+            )))
+            .await
+            .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 2);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 2);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("id = 2 OR id = 4".to_string()))
+                .await
+                .unwrap(),
+            0,
+        );
+    }
+
+    /// `when_matched(Fail)` on an indexed key aborts the operation when a
+    /// source row matches an existing key, and inserts cleanly when none do.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_when_matched_fail() {
+        let initial =
+            record_batch!(("id", Int32, [1, 2, 3]), ("value", Int32, [10, 20, 30])).unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        ds.create_index(
+            &["id"],
+            IndexType::Scalar,
+            None,
+            &ScalarIndexParams::default(),
+            false,
+        )
+        .await
+        .unwrap();
+        let ds = Arc::new(ds);
+
+        // A source row matching an existing key must fail the operation.
+        let matching = record_batch!(("id", Int32, [2]), ("value", Int32, [999])).unwrap();
+        let err = MergeInsertBuilder::try_new(ds.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::Fail)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(Box::new(RecordBatchIterator::new(
+                vec![Ok(matching.clone())],
+                matching.schema(),
+            )))
+            .await
+            .unwrap_err();
+        let msg = err.to_string();
+        assert!(msg.contains("Merge insert failed"), "got: {msg}");
+        assert!(msg.contains("found matching row"), "got: {msg}");
+
+        // A source with no matching key inserts without failing.
+        let new_rows = record_batch!(("id", Int32, [4]), ("value", Int32, [40])).unwrap();
+        let (updated_ds, stats) = MergeInsertBuilder::try_new(ds.clone(), vec!["id".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::Fail)
+            .when_not_matched(WhenNotMatched::InsertAll)
+            .try_build()
+            .unwrap()
+            .execute_reader(Box::new(RecordBatchIterator::new(
+                vec![Ok(new_rows.clone())],
+                new_rows.schema(),
+            )))
+            .await
+            .unwrap();
+        assert_eq!(stats.num_inserted_rows, 1);
+        assert_eq!(stats.num_updated_rows, 0);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 4);
+    }
+
+    /// Fully-indexed composite-key `when_matched(Delete)` combined with
+    /// `when_not_matched(InsertAll)` must both delete matched rows and write
+    /// the inserted rows.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_delete_with_insert() {
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2, 2]),
+            ("b", Int32, [10, 20, 10, 20]),
+            ("value", Int32, [100, 200, 300, 400])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::default();
+        ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+        ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Source matches (1, 10) -> delete, and (3, 30) is new -> insert.
+        let source = record_batch!(
+            ("a", Int32, [1, 3]),
+            ("b", Int32, [10, 30]),
+            ("value", Int32, [999, 333])
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::Delete)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+        // 4 - 1 deleted + 1 inserted = 4.
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 4);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1 AND b = 10".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "matched row must be deleted, not updated"
+        );
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 3 AND b = 30 AND value = 333".to_string()))
+                .await
+                .unwrap(),
+            1,
+            "unmatched source row must be inserted"
+        );
+    }
+
+    /// A delete whose source contains duplicate keys matching the same target
+    /// row applies `source_dedupe_behavior` on the indexed-scan path, exactly
+    /// like an update: the default `Fail` aborts (naming the ambiguous key),
+    /// while `FirstSeen` removes and counts the row once and reports the extra
+    /// match as a skipped duplicate.
+    #[rstest::rstest]
+    #[case::fail(SourceDedupeBehavior::Fail)]
+    #[case::first_seen(SourceDedupeBehavior::FirstSeen)]
+    #[tokio::test]
+    async fn test_indexed_merge_insert_delete_source_duplicates(
+        #[case] behavior: SourceDedupeBehavior,
+    ) {
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2, 2]),
+            ("b", Int32, [10, 20, 10, 20]),
+            ("value", Int32, [100, 200, 300, 400])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Index every join column so the merge takes the indexed-scan delete path.
+        let params = ScalarIndexParams::default();
+        ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+        ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Two source rows collide on the same target key (1, 10).
+        let source = record_batch!(("a", Int32, [1, 1]), ("b", Int32, [10, 10])).unwrap();
+
+        let result =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::Delete)
+                .when_not_matched(WhenNotMatched::DoNothing)
+                .source_dedupe_behavior(behavior)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await;
+
+        if behavior == SourceDedupeBehavior::Fail {
+            let err = result.unwrap_err().to_string();
+            assert!(
+                err.contains("Ambiguous merge inserts") && err.contains("a = 1"),
+                "Fail must abort naming the ambiguous key, got: {err}"
+            );
+            return;
+        }
+
+        let (updated_ds, stats) = result.unwrap();
+        assert_eq!(stats.num_deleted_rows, 1);
+        assert_eq!(stats.num_skipped_duplicates, 1);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 3);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1 AND b = 10".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "the matched row must be removed exactly once"
+        );
+    }
+
+    /// The v2 plans apply the same `source_dedupe_behavior` to deletes when the
+    /// source has duplicate keys matching one target row — covering both
+    /// `FullSchemaMergeInsertExec` (`Delete + InsertAll`) and
+    /// `DeleteOnlyMergeInsertExec` (pure delete). No scalar index, so routing
+    /// stays on the v2 path.
+    #[rstest::rstest]
+    #[case::full_schema_fail(true, SourceDedupeBehavior::Fail)]
+    #[case::full_schema_first_seen(true, SourceDedupeBehavior::FirstSeen)]
+    #[case::delete_only_fail(false, SourceDedupeBehavior::Fail)]
+    #[case::delete_only_first_seen(false, SourceDedupeBehavior::FirstSeen)]
+    #[tokio::test]
+    async fn test_v2_merge_insert_delete_source_duplicates(
+        #[case] with_insert: bool,
+        #[case] behavior: SourceDedupeBehavior,
+    ) {
+        let initial =
+            record_batch!(("a", Int32, [1, 2, 3]), ("value", Int32, [10, 20, 30])).unwrap();
+        let schema = initial.schema();
+
+        let ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Two source rows collide on target key a=1. With insert, a=4 is new.
+        let (source, when_not_matched, expected_inserted, expected_total) = if with_insert {
+            (
+                record_batch!(("a", Int32, [1, 1, 4]), ("value", Int32, [99, 99, 40])).unwrap(),
+                WhenNotMatched::InsertAll,
+                1,
+                3, // 3 - 1 deleted + 1 inserted
+            )
+        } else {
+            (
+                record_batch!(("a", Int32, [1, 1])).unwrap(),
+                WhenNotMatched::DoNothing,
+                0,
+                2, // 3 - 1 deleted
+            )
+        };
+
+        let result = MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string()])
+            .unwrap()
+            .when_matched(WhenMatched::Delete)
+            .when_not_matched(when_not_matched)
+            .source_dedupe_behavior(behavior)
+            .try_build()
+            .unwrap()
+            .execute_reader(Box::new(RecordBatchIterator::new(
+                vec![Ok(source.clone())],
+                source.schema(),
+            )))
+            .await;
+
+        if behavior == SourceDedupeBehavior::Fail {
+            let err = result.unwrap_err().to_string();
+            assert!(
+                err.contains("Ambiguous merge inserts") && err.contains("a = 1"),
+                "Fail must abort naming the ambiguous key, got: {err}"
+            );
+            return;
+        }
+
+        let (updated_ds, stats) = result.unwrap();
+        assert_eq!(stats.num_deleted_rows, 1, "the matched row is removed once");
+        assert_eq!(stats.num_skipped_duplicates, 1);
+        assert_eq!(stats.num_inserted_rows, expected_inserted);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), expected_total);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "the matched row must be removed exactly once"
+        );
+    }
+
+    /// A partial-schema source that combines `when_matched(Delete)` with
+    /// `when_not_matched(InsertAll)` must succeed even when every join key is
+    /// indexed. The indexed-scan delete path cannot fold a delete into a
+    /// partial write, so this case routes to the v2 plan (which fills omitted
+    /// nullable target columns) instead of being rejected.
+    #[tokio::test]
+    async fn test_indexed_merge_insert_partial_schema_delete_with_insert() {
+        // Target carries two nullable non-key columns; the source omits `note`.
+        let full_schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+            Field::new("value", DataType::Int32, true),
+            Field::new("note", DataType::Utf8, true),
+        ]));
+        let full_batch = RecordBatch::try_new(
+            full_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 1, 2, 2])),
+                Arc::new(Int32Array::from(vec![10, 20, 10, 20])),
+                Arc::new(Int32Array::from(vec![100, 200, 300, 400])),
+                Arc::new(StringArray::from(vec!["w", "x", "y", "z"])),
+            ],
+        )
+        .unwrap();
+
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(full_batch)], full_schema.clone()),
+            "memory://",
+            None,
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::default();
+        ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+        ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Source deletes matched (1, 10) and inserts new (3, 30), omitting `note`.
+        let partial_schema = Arc::new(Schema::new(vec![
+            Field::new("a", DataType::Int32, false),
+            Field::new("b", DataType::Int32, false),
+            Field::new("value", DataType::Int32, true),
+        ]));
+        let source = RecordBatch::try_new(
+            partial_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 3])),
+                Arc::new(Int32Array::from(vec![10, 30])),
+                Arc::new(Int32Array::from(vec![999, 333])),
+            ],
+        )
+        .unwrap();
+
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::Delete)
+                .when_not_matched(WhenNotMatched::InsertAll)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
+
+        assert_eq!(stats.num_deleted_rows, 1);
+        assert_eq!(stats.num_inserted_rows, 1);
+        // 4 - 1 deleted + 1 inserted = 4.
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 4);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 1 AND b = 10".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "matched row must be deleted, not updated"
+        );
+        // Inserted row carries the omitted `note` column as NULL.
+        assert_eq!(
+            updated_ds
+                .count_rows(Some(
+                    "a = 3 AND b = 30 AND value = 333 AND note IS NULL".to_string()
+                ))
+                .await
+                .unwrap(),
+            1,
+            "unmatched source row must be inserted with omitted column NULL-filled"
+        );
+    }
+
+    /// Fully-indexed composite-key delete across multiple fragments, with
+    /// stable row ids on/off.  Exercises the indexed-scan delete commit
+    /// path: matched row ids are resolved to addresses (via the row-id
+    /// index when stable) and removed without rewriting any fragments.
+    /// Also covers an appended fragment that neither index covers, so the
+    /// delete must reach rows via the unindexed-remainder union too.
+    #[rstest::rstest]
+    #[case(true)]
+    #[case(false)]
+    #[tokio::test]
+    async fn test_indexed_merge_insert_composite_key_delete_multi_fragment(
+        #[case] enable_stable_row_ids: bool,
+    ) {
+        let initial = record_batch!(
+            ("a", Int32, [1, 1, 2, 2]),
+            ("b", Int32, [10, 20, 10, 20]),
+            ("value", Int32, [100, 200, 300, 400])
+        )
+        .unwrap();
+        let schema = initial.schema();
+
+        // One row per fragment so the delete spans multiple fragments.
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(initial.clone())], schema.clone()),
+            "memory://",
             Some(WriteParams {
-                mode: WriteMode::Append,
+                max_rows_per_file: 1,
+                enable_stable_row_ids,
                 ..Default::default()
             }),
         )
         .await
         .unwrap();
 
-        let ds = Arc::new(ds);
-
-        let just_index_col = Schema::new(vec![Field::new("key", DataType::Utf8, false)]);
-
-        // Sample 2048 random indices and then paste on a column of 9999999's
-        let some_indices = ds
-            .sample(2048, &(&just_index_col).try_into().unwrap(), None)
+        let params = ScalarIndexParams::default();
+        ds.create_index(&["a"], IndexType::Scalar, None, &params, false)
             .await
             .unwrap();
-        let some_indices = some_indices.column(0).clone();
-        let some_vals = lance_datagen::gen_batch()
-            .anon_col(array::fill::<UInt32Type>(9999999))
-            .into_batch_rows(RowCount::from(2048))
-            .unwrap();
-        let some_vals = some_vals.column(0).clone();
-        let source_batch =
-            RecordBatch::try_new(schema.clone(), vec![some_vals, some_indices]).unwrap();
-        // To make things more interesting, lets make the input a stream of four batches
-        let source_batches = vec![
-            source_batch.slice(0, 512),
-            source_batch.slice(512, 512),
-            source_batch.slice(1024, 512),
-            source_batch.slice(1536, 512),
-        ];
-        let source = Box::new(RecordBatchIterator::new(
-            source_batches.clone().into_iter().map(Ok),
-            schema.clone(),
-        ));
-
-        // Run merge_insert
-        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
-            .unwrap()
-            .when_not_matched(WhenNotMatched::DoNothing)
-            .when_matched(WhenMatched::UpdateAll)
-            .try_build()
-            .unwrap()
-            .execute_reader(source)
+        ds.create_index(&["b"], IndexType::Scalar, None, &params, false)
             .await
             .unwrap();
 
-        // Check that the data is as expected
-        let updated = ds
-            .count_rows(Some("value = 9999999".to_string()))
-            .await
-            .unwrap();
-        assert_eq!(updated, 2048);
+        // Append a row AFTER the indices are built so it lives in a fragment
+        // neither index covers.  The delete must still reach it.
+        let appended = record_batch!(
+            ("a", Int32, [3]),
+            ("b", Int32, [30]),
+            ("value", Int32, [500])
+        )
+        .unwrap();
+        ds.append(
+            RecordBatchIterator::new(vec![Ok(appended.clone())], appended.schema()),
+            None,
+        )
+        .await
+        .unwrap();
 
-        // Make sure we don't use an indexed scan if there is a delete criteria
-        let source = Box::new(RecordBatchIterator::new(
-            source_batches.clone().into_iter().map(Ok),
-            schema.clone(),
-        ));
-        // Run merge_insert
-        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
-            .unwrap()
-            .when_not_matched(WhenNotMatched::DoNothing)
-            .when_matched(WhenMatched::UpdateAll)
-            .when_not_matched_by_source(WhenNotMatchedBySource::Delete)
-            .try_build()
-            .unwrap()
-            .execute_reader(source)
-            .await
-            .unwrap();
+        // Delete an indexed row (1, 10) and the unindexed appended row (3, 30).
+        let source = record_batch!(("a", Int32, [1, 3]), ("b", Int32, [10, 30])).unwrap();
 
-        // Check that the data is as expected
-        assert_eq!(ds.count_rows(None).await.unwrap(), 2048);
+        let (updated_ds, stats) =
+            MergeInsertBuilder::try_new(Arc::new(ds), vec!["a".to_string(), "b".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::Delete)
+                .when_not_matched(WhenNotMatched::DoNothing)
+                .try_build()
+                .unwrap()
+                .execute_reader(Box::new(RecordBatchIterator::new(
+                    vec![Ok(source.clone())],
+                    source.schema(),
+                )))
+                .await
+                .unwrap();
 
-        let source = Box::new(RecordBatchIterator::new(
-            source_batches.clone().into_iter().map(Ok),
-            schema.clone(),
+        assert_eq!(stats.num_deleted_rows, 2);
+        assert_eq!(updated_ds.count_rows(None).await.unwrap(), 3);
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("(a = 1 AND b = 10) OR (a = 3 AND b = 30)".to_string()))
+                .await
+                .unwrap(),
+            0,
+            "both matched rows must be gone"
+        );
+        // Untouched rows survive with their original values.
+        assert_eq!(
+            updated_ds
+                .count_rows(Some("a = 2 AND b = 20 AND value = 400".to_string()))
+                .await
+                .unwrap(),
+            1,
+        );
+    }
+
+    /// Composite-key `MapIndexExec` formats its Display so plans expose
+    /// every probed column, and `with_new_children` round-trips the full
+    /// lookup list rather than collapsing back to a single-column form.
+    /// Lives here (not in scalar_index.rs's tests) so the new lines don't
+    /// pile up in a file with pending upstream conflicts.
+    #[test]
+    fn map_index_exec_multi_lookup_plan_shape() {
+        use crate::io::exec::scalar_index::{IndexLookup, MapIndexExec, ScalarIndexExec};
+        use crate::utils::test::NoContextTestFixture;
+        use datafusion::physical_plan::{ExecutionPlan, displayable};
+        use datafusion::scalar::ScalarValue;
+        use lance_index::scalar::{
+            SargableQuery,
+            expression::{ScalarIndexExpr, ScalarIndexSearch},
+        };
+        use lance_select::result::IndexExprResultWireFormat;
+
+        let fixture = NoContextTestFixture::new();
+        let dataset = Arc::new(fixture.dataset);
+
+        let dummy_input: Arc<dyn ExecutionPlan> = Arc::new(ScalarIndexExec::new(
+            dataset.clone(),
+            ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: "ordered".to_string(),
+                index_name: "ordered_idx".to_string(),
+                index_type: "BTree".to_string(),
+                query: Arc::new(SargableQuery::Equals(ScalarValue::UInt64(Some(1)))),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            }),
+            IndexExprResultWireFormat::default(),
         ));
-        // Run merge_insert one last time.  The index is now completely out of date.  Every
-        // row it points to is a deleted row.  Make sure that doesn't break.
-        let (ds, _) = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()])
-            .unwrap()
-            .when_not_matched(WhenNotMatched::DoNothing)
-            .when_matched(WhenMatched::UpdateAll)
-            .try_build()
-            .unwrap()
-            .execute_reader(source)
-            .await
-            .unwrap();
 
-        assert_eq!(ds.count_rows(None).await.unwrap(), 2048);
+        let lookups = vec![
+            IndexLookup::new("a", "a_idx"),
+            IndexLookup::new("b", "b_idx"),
+        ];
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(MapIndexExec::new_multi(
+            dataset.clone(),
+            lookups,
+            dummy_input.clone(),
+        ));
+
+        let rendered = format!("{}", displayable(plan.as_ref()).indent(false));
+        assert!(
+            rendered.contains("IndexedLookup [a, b]"),
+            "multi-lookup Display must list every probed column, got: {rendered}",
+        );
+
+        let rebuilt = plan
+            .with_new_children(vec![dummy_input.clone()])
+            .expect("with_new_children must accept exactly one child");
+        let rebuilt_rendered = format!("{}", displayable(rebuilt.as_ref()).indent(false));
+        assert!(
+            rebuilt_rendered.contains("IndexedLookup [a, b]"),
+            "with_new_children must preserve every lookup, got: {rebuilt_rendered}",
+        );
+
+        // The single-lookup convenience constructor still renders without
+        // the column list, so existing EXPLAIN output is unchanged for
+        // single-column joins.
+        let single: Arc<dyn ExecutionPlan> = Arc::new(MapIndexExec::new(
+            dataset,
+            "ordered".to_string(),
+            "ordered_idx".to_string(),
+            dummy_input,
+        ));
+        let single_rendered = format!("{}", displayable(single.as_ref()).indent(false));
+        assert!(
+            single_rendered.contains("IndexedLookup")
+                && !single_rendered.contains("IndexedLookup ["),
+            "single-lookup Display must not include the column list, got: {single_rendered}",
+        );
     }
 
     mod subcols {
@@ -6240,7 +7916,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
         let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
         assert!(plan.contains("HashJoinExec"));
         assert!(plan.contains("join_type=Full"));
-        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.lines().any(|line| line.contains("HashJoinExec")
+                && line.contains("projection=[")
+                && line.contains("_rowid")),
+            "join should push down a projection that retains _rowid: {plan}"
+        );
         assert!(
             plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
             "target-side scan should prune the FSL payload from the join build side: {plan}"
@@ -6308,7 +7989,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
         let plan = merge_insert_job.explain_plan(None, false).await.unwrap();
         assert!(plan.contains("HashJoinExec"));
         assert!(plan.contains("join_type=Full"));
-        assert!(plan.contains("projection=[_rowid"));
+        assert!(
+            plan.lines().any(|line| line.contains("HashJoinExec")
+                && line.contains("projection=[")
+                && line.contains("_rowid")),
+            "join should push down a projection that retains _rowid: {plan}"
+        );
         assert!(
             plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"),
             "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}"
@@ -9487,4 +11173,285 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n
             "Newly written merge-insert data files should be cleaned up on apply_deletions failure"
         );
     }
+
+    #[tokio::test]
+    async fn test_merge_insert_full_fragment_rewrite_with_json_columns() {
+        // This test verifies the "all rows updated" fast path in handle_fragment
+        // correctly converts Arrow JSON (Utf8) to Lance JSON (LargeBinary/JSONB)
+        // before writing. Without conversion, the file would have Utf8 data (i32
+        // offsets) but schema says LargeBinary (i64 offsets), causing decoder panic
+        // on subsequent reads.
+        //
+        // To trigger the fast path we need:
+        // 1. Subschema update (not all columns) → forces v1 update_fragments path
+        // 2. ALL rows in a fragment updated → triggers the fast path
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::{ARROW_JSON_EXT_NAME, is_arrow_json_field};
+
+        let test_dir = TempStrDir::default();
+        let mut json_metadata = HashMap::new();
+        json_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            ARROW_JSON_EXT_NAME.to_string(),
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("name", DataType::Utf8, true),
+            Field::new("score", DataType::Int64, true),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()),
+        ]));
+        // Small fragment so ALL rows will be updated
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3])),
+                Arc::new(StringArray::from(vec!["a", "b", "c"])),
+                Arc::new(Int64Array::from(vec![10, 20, 30])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"x":1}"#,
+                    r#"{"x":2}"#,
+                    r#"{"x":3}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let write_params = WriteParams {
+            data_storage_version: Some(LanceFileVersion::V2_2),
+            ..Default::default()
+        };
+        let dataset = Arc::new(
+            Dataset::write(reader, test_dir.as_ref(), Some(write_params))
+                .await
+                .unwrap(),
+        );
+        assert_eq!(dataset.get_fragments().len(), 1);
+
+        // Subschema update: only provide [id, meta] (missing "name" and "score")
+        // This forces the v1 path (update_fragments) instead of v2 (create_plan).
+        // Update ALL rows → triggers the "all rows updated" fast path.
+        let update_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata),
+        ]));
+        let update_batch = RecordBatch::try_new(
+            update_schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3])), // all rows
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true,"id":1}"#,
+                    r#"{"updated":true,"id":2}"#,
+                    r#"{"updated":true,"id":3}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let update_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(update_batch)],
+            update_schema,
+        ));
+        let stream = reader_to_stream(update_reader);
+
+        // Execute merge_insert with subschema
+        let mut builder =
+            MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap();
+        builder.when_matched(WhenMatched::UpdateAll);
+        builder.when_not_matched(WhenNotMatched::DoNothing);
+        let job = builder.try_build().unwrap();
+        let (updated_dataset, stats) = job.execute(stream).await.unwrap();
+
+        assert_eq!(stats.num_updated_rows, 3);
+
+        // Critical: read the data back. Without the fix, this would PANIC with:
+        // "the offset of the new Buffer cannot exceed the existing Length:
+        //  slice offset=0 Length=N selfLen=N/2"
+        let batches = updated_dataset
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let result = concat_batches(&batches[0].schema(), &batches).unwrap();
+        assert_eq!(result.num_rows(), 3);
+
+        // Verify JSON column is in Arrow JSON format (Utf8) on read
+        let result_schema = result.schema();
+        let meta_field = result_schema.field_with_name("meta").unwrap();
+        assert!(
+            is_arrow_json_field(meta_field),
+            "Expected Arrow JSON (Utf8 + arrow.json), got {:?}",
+            meta_field
+        );
+
+        // Verify data correctness
+        let metas = result
+            .column_by_name("meta")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .expect("meta should be StringArray after read conversion");
+        for i in 0..3 {
+            let val = metas.value(i);
+            assert!(
+                val.contains("updated"),
+                "row {} should have updated meta, got: {}",
+                i,
+                val
+            );
+        }
+
+        // Verify non-updated columns are preserved
+        let scores = result
+            .column_by_name("score")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        assert_eq!(scores.values(), &[10, 20, 30]);
+
+        // Also verify via take (exercises the take read conversion path)
+        let take_result = updated_dataset
+            .take(&[0, 1, 2], updated_dataset.schema().clone())
+            .await
+            .unwrap();
+        let take_schema = take_result.schema();
+        let take_meta_field = take_schema.field_with_name("meta").unwrap();
+        assert!(
+            is_arrow_json_field(take_meta_field),
+            "take() should return Arrow JSON, got {:?}",
+            take_meta_field
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_insert_subschema_with_json_columns() {
+        use lance_arrow::ARROW_EXT_NAME_KEY;
+        use lance_arrow::json::ARROW_JSON_EXT_NAME;
+
+        // Create a dataset with an Arrow JSON extension column
+        let test_dir = TempStrDir::default();
+        let mut json_metadata = HashMap::new();
+        json_metadata.insert(
+            ARROW_EXT_NAME_KEY.to_string(),
+            ARROW_JSON_EXT_NAME.to_string(),
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("name", DataType::Utf8, true),
+            Field::new("score", DataType::Int64, true),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])),
+                Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])),
+                Arc::new(Int64Array::from(vec![10, 20, 30, 40, 50])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"x":1}"#,
+                    r#"{"x":2}"#,
+                    r#"{"x":3}"#,
+                    r#"{"x":4}"#,
+                    r#"{"x":5}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let dataset = Arc::new(
+            Dataset::write(reader, test_dir.as_ref(), None)
+                .await
+                .unwrap(),
+        );
+
+        // Perform a subschema merge_insert: only update "meta" column (JSON type)
+        // This exercises the update_fragments path with interleave_batches
+        let update_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata),
+        ]));
+        let update_batch = RecordBatch::try_new(
+            update_schema.clone(),
+            vec![
+                Arc::new(Int64Array::from(vec![2, 4])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true,"id":2}"#,
+                    r#"{"updated":true,"id":4}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let update_reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
+            vec![Ok(update_batch)],
+            update_schema,
+        ));
+        let stream = reader_to_stream(update_reader);
+
+        // Execute merge_insert with subschema (only id + meta columns)
+        let mut builder =
+            MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap();
+        builder.when_matched(WhenMatched::UpdateAll);
+        builder.when_not_matched(WhenNotMatched::DoNothing);
+        let job = builder.try_build().unwrap();
+        let (updated_dataset, stats) = job.execute(stream).await.unwrap();
+
+        // Verify: the merge should not fail with type mismatch
+        assert_eq!(stats.num_updated_rows, 2);
+
+        // Read back and verify the JSON column was updated correctly
+        let batches = updated_dataset
+            .scan()
+            .try_into_stream()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let result = concat_batches(&batches[0].schema(), &batches).unwrap();
+        assert_eq!(result.num_rows(), 5);
+
+        // Verify the "score" column (not in update) is preserved, and "meta" updated
+        let ids = result
+            .column_by_name("id")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        let scores = result
+            .column_by_name("score")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<Int64Array>()
+            .unwrap();
+        let metas = result
+            .column_by_name("meta")
+            .unwrap()
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+        for i in 0..5 {
+            let id = ids.value(i);
+            let score = scores.value(i);
+            let meta = metas.value(i);
+            // score = id * 10, regardless of row order
+            assert_eq!(score, id * 10, "id={} score mismatch", id);
+            if id == 2 || id == 4 {
+                assert!(
+                    meta.contains("updated"),
+                    "id={} should have updated meta, got: {}",
+                    id,
+                    meta
+                );
+            } else {
+                assert!(
+                    meta.contains("\"x\""),
+                    "id={} should have original meta, got: {}",
+                    id,
+                    meta
+                );
+            }
+        }
+    }
 }
diff --git a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
index be7897e2441..120557549a2 100644
--- a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
+++ b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs
@@ -22,7 +22,10 @@ use roaring::RoaringTreemap;
 use crate::Dataset;
 use crate::dataset::transaction::{Operation, Transaction};
 use crate::dataset::write::merge_insert::assign_action::Action;
-use crate::dataset::write::merge_insert::{MERGE_ACTION_COLUMN, MergeInsertParams, MergeStats};
+use crate::dataset::write::merge_insert::{
+    MERGE_ACTION_COLUMN, MergeInsertParams, MergeStats, SourceDedupeBehavior,
+    create_duplicate_row_error, resolve_target_bases,
+};
 
 use super::{MergeInsertMetrics, apply_deletions};
 
@@ -101,6 +104,8 @@ impl DeleteOnlyMergeInsertExec {
     async fn collect_deletions(
         mut input_stream: SendableRecordBatchStream,
         metrics: MergeInsertMetrics,
+        source_dedupe_behavior: SourceDedupeBehavior,
+        on_columns: &[String],
     ) -> DFResult<RoaringTreemap> {
         let schema = input_stream.schema();
 
@@ -156,8 +161,25 @@ impl DeleteOnlyMergeInsertExec {
 
                 if action == Action::Delete && !row_addr_array.is_null(row_idx) {
                     let row_addr = row_addr_array.value(row_idx);
-                    delete_row_addrs.insert(row_addr);
-                    metrics.num_deleted_rows.add(1);
+                    // The treemap dedupes addresses, so a repeat insert signals
+                    // a duplicate source row matching the same target; apply the
+                    // same dedupe policy as updates. (Delete-only never carries
+                    // `delete_not_matched_by_source`, so every delete here is a
+                    // source match.)
+                    if delete_row_addrs.insert(row_addr) {
+                        metrics.num_deleted_rows.add(1);
+                    } else {
+                        match source_dedupe_behavior {
+                            SourceDedupeBehavior::Fail => {
+                                return Err(create_duplicate_row_error(
+                                    &batch, row_idx, on_columns,
+                                ));
+                            }
+                            SourceDedupeBehavior::FirstSeen => {
+                                metrics.num_skipped_duplicates.add(1);
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -194,10 +216,6 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
         "DeleteOnlyMergeInsertExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         Arc::new(arrow_schema::Schema::empty())
     }
@@ -257,13 +275,24 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
         let input_stream = self.input.execute(partition, context)?;
 
         let dataset = self.dataset.clone();
+        let params = self.params.clone();
         let merge_stats_holder = self.merge_stats.clone();
         let transaction_holder = self.transaction.clone();
         let affected_rows_holder = self.affected_rows.clone();
         let merged_generations = self.params.merged_generations.clone();
+        let source_dedupe_behavior = self.params.source_dedupe_behavior;
+        let on_columns = self.params.on.clone();
 
         let result_stream = futures::stream::once(async move {
-            let delete_row_addrs = Self::collect_deletions(input_stream, metrics).await?;
+            // Delete-only merges write no data files, but still validate any
+            // requested target bases so unknown bases fail on every path.
+            resolve_target_bases(&dataset, &params).await?;
+            // `metrics` is moved into `collect_deletions`; keep a handle on the
+            // skipped-duplicate counter so it can be folded into the stats below.
+            let skipped_duplicates = metrics.num_skipped_duplicates.clone();
+            let delete_row_addrs =
+                Self::collect_deletions(input_stream, metrics, source_dedupe_behavior, &on_columns)
+                    .await?;
 
             let (updated_fragments, removed_fragment_ids) =
                 apply_deletions(&dataset, &delete_row_addrs)
@@ -297,7 +326,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec {
                 bytes_written: 0,
                 num_files_written: 0,
                 num_attempts: 1,
-                num_skipped_duplicates: 0,
+                num_skipped_duplicates: skipped_duplicates.value() as u64,
             };
 
             if let Ok(mut transaction_guard) = transaction_holder.lock() {
diff --git a/rust/lance/src/dataset/write/merge_insert/exec/write.rs b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
index 8c57ebc9df3..e1f41408147 100644
--- a/rust/lance/src/dataset/write/merge_insert/exec/write.rs
+++ b/rust/lance/src/dataset/write/merge_insert/exec/write.rs
@@ -30,14 +30,14 @@ use crate::dataset::write::merge_insert::inserted_rows::{
 };
 use crate::dataset::write::merge_insert::{
     MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, create_duplicate_row_error,
-    format_key_values_on_columns,
+    format_key_values_on_columns, resolve_target_bases,
 };
 use crate::{
     Dataset,
     dataset::{
         transaction::{Operation, Transaction},
         write::{
-            WriteParams,
+            WriteParams, cleanup_data_fragments,
             merge_insert::{
                 MERGE_ACTION_COLUMN, MergeInsertParams, MergeStats, assign_action::Action,
                 exec::MergeInsertMetrics,
@@ -103,6 +103,29 @@ impl MergeState {
                 // Delete action - only delete, don't write back
                 if !row_addr_array.is_null(row_idx) {
                     let row_addr = row_addr_array.value(row_idx);
+                    let row_id = row_id_array.value(row_idx);
+
+                    // A source with duplicate keys matches the same target row
+                    // more than once; apply the same dedupe policy as updates.
+                    // (Target-only deletes from `delete_not_matched_by_source`
+                    // also reach here but never duplicate, so they never trip
+                    // `Fail`.)
+                    if !self.processed_row_ids.insert(row_id) {
+                        match self.source_dedupe_behavior {
+                            SourceDedupeBehavior::Fail => {
+                                return Err(create_duplicate_row_error(
+                                    batch,
+                                    row_idx,
+                                    &self.on_columns,
+                                ));
+                            }
+                            SourceDedupeBehavior::FirstSeen => {
+                                self.metrics.num_skipped_duplicates.add(1);
+                                return Ok(None); // Skip this duplicate row
+                            }
+                        }
+                    }
+
                     self.delete_row_addrs.insert(row_addr);
                     self.metrics.num_deleted_rows.add(1);
                 }
@@ -774,10 +797,6 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
         "FullSchemaMergeInsertExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         Arc::new(arrow_schema::Schema::empty())
     }
@@ -866,6 +885,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
 
         // Use flat_map to handle the async write operation
         let dataset = self.dataset.clone();
+        let params = self.params.clone();
         let merge_stats_holder = self.merge_stats.clone();
         let transaction_holder = self.transaction.clone();
         let affected_rows_holder = self.affected_rows.clone();
@@ -879,6 +899,9 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
 
         let result_stream = stream::once(async move {
             // Step 2: Write new fragments using the filtered data (inserts + updates)
+            let target_bases_info = resolve_target_bases(&dataset, &params).await?;
+            // Keep a copy so failures after the write can clean up routed files.
+            let cleanup_bases = target_bases_info.clone();
             let (mut new_fragments, _) = write_fragments_internal(
                 Some(&dataset),
                 dataset.object_store.clone(),
@@ -886,31 +909,44 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
                 dataset.schema().clone(),
                 write_data_stream,
                 WriteParams::default(),
-                None, // Merge insert doesn't use target_bases
+                target_bases_info,
             )
             .await?;
 
-            if let Some(row_id_sequence) = updating_row_ids.lock().unwrap().row_id_sequence() {
-                let fragment_sizes = new_fragments
-                    .iter()
-                    .map(|f| f.physical_rows.unwrap() as u64);
-
-                let sequences = lance_table::rowids::rechunk_sequences(
-                    [row_id_sequence.clone()],
-                    fragment_sizes,
-                    true,
-                )
-                .map_err(|e| {
-                    Error::internal(format!(
-                        "Captured row ids not equal to number of rows written: {}",
-                        e
-                    ))
-                })?;
-
-                for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) {
-                    let serialized = lance_table::rowids::write_row_ids(&sequence);
-                    fragment.row_id_meta = Some(RowIdMeta::Inline(serialized));
+            let row_id_result: lance_core::Result<()> = (|| {
+                if let Some(row_id_sequence) = updating_row_ids.lock().unwrap().row_id_sequence() {
+                    let fragment_sizes = new_fragments
+                        .iter()
+                        .map(|f| f.physical_rows.unwrap() as u64);
+
+                    let sequences = lance_table::rowids::rechunk_sequences(
+                        [row_id_sequence.clone()],
+                        fragment_sizes,
+                        true,
+                    )
+                    .map_err(|e| {
+                        Error::internal(format!(
+                            "Captured row ids not equal to number of rows written: {}",
+                            e
+                        ))
+                    })?;
+
+                    for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) {
+                        let serialized = lance_table::rowids::write_row_ids(&sequence);
+                        fragment.row_id_meta = Some(RowIdMeta::Inline(serialized));
+                    }
                 }
+                Ok(())
+            })();
+            if let Err(e) = row_id_result {
+                cleanup_data_fragments(
+                    &dataset.object_store,
+                    &dataset.base,
+                    cleanup_bases.as_deref(),
+                    &new_fragments,
+                )
+                .await;
+                return Err(e.into());
             }
 
             // Step 2.5: Calculate write metrics from new fragments
@@ -932,7 +968,21 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
             };
 
             let (updated_fragments, removed_fragment_ids) =
-                apply_deletions(&dataset, &delete_row_addrs_clone).await?;
+                match apply_deletions(&dataset, &delete_row_addrs_clone).await {
+                    Ok(result) => result,
+                    Err(e) => {
+                        // The new data files are not committed; remove them (including
+                        // ones routed to target bases) before surfacing the error.
+                        cleanup_data_fragments(
+                            &dataset.object_store,
+                            &dataset.base,
+                            cleanup_bases.as_deref(),
+                            &new_fragments,
+                        )
+                        .await;
+                        return Err(e.into());
+                    }
+                };
 
             // Step 4: Create the transaction operation
             let operation = Operation::Update {
@@ -941,10 +991,14 @@ impl ExecutionPlan for FullSchemaMergeInsertExec {
                 new_fragments,
                 fields_modified: vec![], // No fields are modified in schema for upsert
                 merged_generations,
+                // Use the full pre-order field list (not just top-level `fields`) so
+                // that nested leaf field ids are included. A merge_insert rewrites whole
+                // rows, so every field is potentially modified; omitting nested ids would
+                // let `register_pure_rewrite_rows_update_frags_in_indices` wrongly extend a
+                // nested-field index over the rewritten fragment, silently dropping rows.
                 fields_for_preserving_frag_bitmap: dataset
                     .schema()
-                    .fields
-                    .iter()
+                    .fields_pre_order()
                     .map(|f| f.id as u32)
                     .collect(),
                 update_mode: Some(RewriteRows),
diff --git a/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs b/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs
index 89ec893705e..805073e75e2 100644
--- a/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs
+++ b/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs
@@ -13,9 +13,9 @@ use arrow_array::{
     StringArray, StructArray,
 };
 use arrow_schema::DataType;
-use deepsize::DeepSizeOf;
 use lance_core::Result;
-use lance_index::scalar::bloomfilter::sbbf::{Sbbf, SbbfBuilder};
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder};
 use lance_table::format::pb;
 
 // Default bloom filter config: 8192 items @ 0.00057 fpp -> 16KiB filter
diff --git a/rust/lance/src/dataset/write/update.rs b/rust/lance/src/dataset/write/update.rs
index 6672c58db1f..a87b3a4260b 100644
--- a/rust/lance/src/dataset/write/update.rs
+++ b/rust/lance/src/dataset/write/update.rs
@@ -25,6 +25,7 @@ use datafusion::prelude::Expr;
 use datafusion::scalar::ScalarValue;
 use futures::StreamExt;
 use lance_arrow::RecordBatchExt;
+use lance_core::datatypes::BlobHandling;
 use lance_core::error::{InvalidInputSnafu, box_error};
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::{ROW_ADDR_FIELD, ROW_ID_FIELD, ROW_OFFSET_FIELD};
@@ -34,6 +35,16 @@ use lance_table::format::{Fragment, RowIdMeta};
 use roaring::RoaringTreemap;
 use snafu::ResultExt;
 
+/// Collect a field id and all of its descendant field ids (pre-order). A struct
+/// column update rewrites the whole subtree, so an index on any descendant must be
+/// treated as modified.
+fn collect_subtree_field_ids(field: &lance_core::datatypes::Field, out: &mut Vec<u32>) {
+    out.push(field.id as u32);
+    for child in &field.children {
+        collect_subtree_field_ids(child, out);
+    }
+}
+
 /// Build an update operation.
 ///
 /// This operation is similar to SQL's UPDATE statement. It allows you to change
@@ -270,6 +281,7 @@ impl UpdateJob {
     async fn execute_impl(self) -> Result<UpdateData> {
         let mut scanner = self.dataset.scan();
         scanner.with_row_id();
+        scanner.blob_handling(BlobHandling::AllBinary);
 
         if let Some(expr) = &self.condition {
             scanner.filter_expr(expr.clone());
@@ -359,6 +371,7 @@ impl UpdateJob {
                 cleanup_data_fragments(
                     &self.dataset.object_store,
                     &self.dataset.base,
+                    None,
                     &new_fragments,
                 )
                 .await;
@@ -386,10 +399,14 @@ impl UpdateJob {
         dataset: Arc<Dataset>,
         update_data: UpdateData,
     ) -> Result<UpdateResult> {
+        // Updated columns are top-level (nested references are rejected by `set`), but a
+        // struct-column update rewrites all of its descendants. Collect the full field
+        // subtree so an index on a nested child field is recognized as modified and not
+        // wrongly extended over the rewritten fragment.
         let mut fields_for_preserving_frag_bitmap = Vec::new();
         for column_name in self.updates.keys() {
-            if let Ok(field_id) = dataset.schema().field_id(column_name) {
-                fields_for_preserving_frag_bitmap.push(field_id as u32);
+            if let Some(field) = dataset.schema().field(column_name) {
+                collect_subtree_field_ids(field, &mut fields_for_preserving_frag_bitmap);
             }
         }
 
@@ -1687,4 +1704,74 @@ mod tests {
             "Rewritten data files should be cleaned up on apply_deletions failure"
         );
     }
+
+    #[tokio::test]
+    async fn test_update_with_blob() {
+        use arrow_array::LargeBinaryArray;
+        use arrow_schema::Field;
+        use lance_arrow::BLOB_META_KEY;
+
+        let test_dir = TempStrDir::default();
+        let blob_meta = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]);
+        let schema = Arc::new(ArrowSchema::new(vec![
+            Field::new("blobs", DataType::LargeBinary, true).with_metadata(blob_meta),
+            Field::new("id", DataType::Int64, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(LargeBinaryArray::from(vec![
+                    Some(b"foo".as_slice()),
+                    Some(b"bar".as_slice()),
+                    Some(b"baz".as_slice()),
+                ])),
+                Arc::new(Int64Array::from(vec![0, 1, 2])),
+            ],
+        )
+        .unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let dataset = Dataset::write(
+            reader,
+            &test_dir,
+            Some(WriteParams {
+                data_storage_version: Some(LanceFileVersion::V2_1),
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // Perform an update: update the "blobs" column where id = 1
+        let dataset = Arc::new(dataset);
+        let updated_dataset = UpdateBuilder::new(dataset)
+            .update_where("id = 1")
+            .unwrap()
+            .set("blobs", "arrow_cast('updated_bar', 'LargeBinary')")
+            .unwrap()
+            .build()
+            .unwrap()
+            .execute()
+            .await
+            .unwrap()
+            .new_dataset;
+
+        // Verify the updated value
+        let mut scanner = updated_dataset.scan();
+        // Read as binary to assert actual value
+        scanner.blob_handling(BlobHandling::AllBinary);
+        let batches = scanner.try_into_batch().await.unwrap();
+        let blobs = batches.column_by_name("blobs").unwrap().as_binary::<i64>();
+        let ids = batches
+            .column_by_name("id")
+            .unwrap()
+            .as_primitive::<Int64Type>();
+
+        // Find the index of id = 1
+        let idx = ids.values().iter().position(|&x| x == 1).unwrap();
+        assert_eq!(blobs.value(idx), b"updated_bar");
+
+        let idx_foo = ids.values().iter().position(|&x| x == 0).unwrap();
+        assert_eq!(blobs.value(idx_foo), b"foo");
+    }
 }
diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs
index eaa3dc6119d..78b900c76ea 100644
--- a/rust/lance/src/index.rs
+++ b/rust/lance/src/index.rs
@@ -4,8 +4,10 @@
 //! Secondary Index
 //!
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, OnceLock};
+use std::time::Instant;
 
 use arrow_schema::DataType;
 use async_trait::async_trait;
@@ -15,7 +17,6 @@ use itertools::Itertools;
 use lance_core::cache::CacheKey;
 use lance_core::datatypes::Field;
 use lance_core::datatypes::Schema as LanceSchema;
-use lance_core::utils::address::RowAddress;
 use lance_core::utils::parse::parse_env_as_bool;
 use lance_core::utils::tracing::{
     IO_TYPE_OPEN_FRAG_REUSE, IO_TYPE_OPEN_MEM_WAL, IO_TYPE_OPEN_VECTOR, TRACE_IO_EVENTS,
@@ -24,18 +25,16 @@ use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::reader::FileReaderOptions;
 use lance_index::INDEX_METADATA_SCHEMA_KEY;
 pub use lance_index::IndexParams;
-use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex};
-use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex};
+use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex, FragReuseIndexHandle};
+use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexHandle};
 use lance_index::optimize::OptimizeOptions;
 use lance_index::pb::index::Implementation;
 pub use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
-use lance_index::scalar::expression::{
-    IndexInformationProvider, MultiQueryParser, ScalarQueryParser,
-};
+use lance_index::scalar::expression::{IndexInformationProvider, MultiQueryParser};
 use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexPlugin};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering};
-use lance_index::scalar::{CreatedIndex, ScalarIndex};
+use lance_index::scalar::{CreatedIndex, ScalarIndex, index_files_to_table, table_files_to_index};
 use lance_index::vector::bq::builder::RabitQuantizer;
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HNSW;
@@ -47,6 +46,8 @@ use lance_index::{INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector:
 use lance_index::{
     IndexCriteria, is_system_index,
     metrics::{MetricsCollector, NoOpMetricsCollector},
+    registry::display_type_from_url,
+    scalar::btree::BTREE_LOOKUP_NAME,
 };
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::traits::Reader;
@@ -55,12 +56,12 @@ use lance_io::utils::{
     read_version,
 };
 use lance_table::format::{Fragment, SelfDescribingFileReader};
-use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
+use lance_table::format::{IndexFile, IndexMetadata, list_index_files_with_sizes};
 use lance_table::io::manifest::read_manifest_indexes;
 use roaring::RoaringBitmap;
 use scalar::index_matches_criteria;
 use serde_json::json;
-use tracing::{info, instrument};
+use tracing::{info, instrument, warn};
 use uuid::Uuid;
 use vector::details::{
     derive_vector_index_type, infer_missing_vector_details, vector_details_as_json,
@@ -164,7 +165,8 @@ pub(crate) async fn build_index_metadata_from_segments(
     let mut new_indices = Vec::with_capacity(segments.len());
     for segment in segments {
         let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts();
-        if index_details.type_url.ends_with("InvertedIndexDetails") {
+        let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails");
+        if is_inverted_index {
             let metadata = IndexMetadata {
                 uuid,
                 name: index_name.to_string(),
@@ -181,7 +183,10 @@ pub(crate) async fn build_index_metadata_from_segments(
                 .await?;
         }
         let index_dir = dataset.indices_dir().clone().join(uuid.to_string());
-        let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
+        let mut files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
+        if is_inverted_index {
+            retain_committed_inverted_files(&mut files);
+        }
         new_indices.push(IndexMetadata {
             uuid,
             name: index_name.to_string(),
@@ -199,6 +204,231 @@ pub(crate) async fn build_index_metadata_from_segments(
     Ok(new_indices)
 }
 
+fn retain_committed_inverted_files(files: &mut Vec<IndexFile>) {
+    files.retain(|file| !file.path.starts_with("staging/"));
+}
+
+async fn prewarm_opened_index(
+    index: Arc<dyn Index>,
+    options: Option<&PrewarmOptions>,
+) -> Result<()> {
+    match options {
+        None => index.prewarm().await,
+        Some(PrewarmOptions::Fts(fts_options)) => {
+            let inverted = index
+                .as_any()
+                .downcast_ref::<InvertedIndex>()
+                .ok_or_else(|| {
+                    Error::invalid_input(format!(
+                        "FTS prewarm options are only supported for inverted indices, got {:?}",
+                        index.index_type()
+                    ))
+                })?;
+            inverted.prewarm_with_options(fts_options).await
+        }
+        Some(_) => Err(Error::not_supported(
+            "unsupported prewarm options for this lance version".to_owned(),
+        )),
+    }
+}
+
+fn total_index_segment_size_bytes(indices: &[IndexMetadata]) -> Option<u64> {
+    let mut total = 0u64;
+    for index_meta in indices {
+        total += index_meta.total_size_bytes()?;
+    }
+    Some(total)
+}
+
+fn cache_size_delta(after: usize, before: usize) -> i64 {
+    after.saturating_sub(before) as i64 - before.saturating_sub(after) as i64
+}
+
+fn prewarm_options_fields(options: Option<&PrewarmOptions>) -> (&'static str, bool) {
+    match options {
+        None => ("default", false),
+        Some(PrewarmOptions::Fts(fts_options)) => ("fts", fts_options.with_position),
+        Some(_) => ("unsupported", false),
+    }
+}
+
+async fn prewarm_index_segments_by_metadata(
+    dataset: &Dataset,
+    name: &str,
+    indices: Vec<IndexMetadata>,
+    options: Option<&PrewarmOptions>,
+    available_segment_count: usize,
+    requested_segment_count: Option<usize>,
+) -> Result<()> {
+    let request_started = Instant::now();
+    let selected_segment_count = indices.len();
+    let selected_size_bytes = total_index_segment_size_bytes(&indices);
+    let (prewarm_options, fts_with_position) = prewarm_options_fields(options);
+    let cache_stats_before = dataset.session.index_cache_stats().await;
+    info!(
+        index_name = name,
+        selected_segment_count,
+        available_segment_count,
+        requested_segment_count = requested_segment_count.unwrap_or(0),
+        segment_filter = requested_segment_count.is_some(),
+        selected_size_bytes = selected_size_bytes.unwrap_or(0),
+        selected_size_bytes_known = selected_size_bytes.is_some(),
+        index_cache_entries_before = cache_stats_before.num_entries,
+        index_cache_size_bytes_before = cache_stats_before.size_bytes,
+        prewarm_options,
+        fts_with_position,
+        "prewarm index segments started"
+    );
+
+    let result = futures::future::try_join_all(indices.into_iter().map(|index_meta| async move {
+        let index_uuid = index_meta.uuid;
+        let size_bytes = index_meta.total_size_bytes();
+        let fragment_count = index_meta
+            .fragment_bitmap
+            .as_ref()
+            .map(|bitmap| bitmap.len());
+        let segment_started = Instant::now();
+        info!(
+            index_name = name,
+            %index_uuid,
+            index_version = index_meta.index_version,
+            dataset_version = index_meta.dataset_version,
+            fragment_count = fragment_count.unwrap_or(0),
+            fragment_count_known = fragment_count.is_some(),
+            size_bytes = size_bytes.unwrap_or(0),
+            size_bytes_known = size_bytes.is_some(),
+            "prewarm index segment started"
+        );
+
+        let index = match dataset
+            .open_generic_index(name, &index_uuid, &NoOpMetricsCollector)
+            .await
+        {
+            Ok(index) => {
+                info!(
+                    index_name = name,
+                    %index_uuid,
+                    index_type = ?index.index_type(),
+                    elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                    "opened index segment for prewarm"
+                );
+                index
+            }
+            Err(err) => {
+                warn!(
+                    index_name = name,
+                    %index_uuid,
+                    error = %err,
+                    elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                    "failed to open index segment for prewarm"
+                );
+                return Err(err);
+            }
+        };
+
+        if let Err(err) = prewarm_opened_index(index, options).await {
+            warn!(
+                index_name = name,
+                %index_uuid,
+                error = %err,
+                elapsed_ms = segment_started.elapsed().as_millis() as u64,
+                "prewarm index segment failed"
+            );
+            return Err(err);
+        }
+
+        info!(
+            index_name = name,
+            %index_uuid,
+            elapsed_ms = segment_started.elapsed().as_millis() as u64,
+            "prewarm index segment finished"
+        );
+        Ok(())
+    }))
+    .await;
+
+    match result {
+        Ok(_) => {
+            let cache_stats_after = dataset.session.index_cache_stats().await;
+            info!(
+                index_name = name,
+                selected_segment_count,
+                index_cache_entries_after = cache_stats_after.num_entries,
+                index_cache_entries_delta = cache_size_delta(
+                    cache_stats_after.num_entries,
+                    cache_stats_before.num_entries,
+                ),
+                index_cache_size_bytes_after = cache_stats_after.size_bytes,
+                index_cache_size_bytes_delta =
+                    cache_size_delta(cache_stats_after.size_bytes, cache_stats_before.size_bytes,),
+                elapsed_ms = request_started.elapsed().as_millis() as u64,
+                "prewarm index segments finished"
+            );
+            Ok(())
+        }
+        Err(err) => {
+            let cache_stats_after = dataset.session.index_cache_stats().await;
+            warn!(
+                index_name = name,
+                selected_segment_count,
+                index_cache_entries_after = cache_stats_after.num_entries,
+                index_cache_entries_delta = cache_size_delta(
+                    cache_stats_after.num_entries,
+                    cache_stats_before.num_entries,
+                ),
+                index_cache_size_bytes_after = cache_stats_after.size_bytes,
+                index_cache_size_bytes_delta = cache_size_delta(
+                    cache_stats_after.size_bytes,
+                    cache_stats_before.size_bytes,
+                ),
+                error = %err,
+                elapsed_ms = request_started.elapsed().as_millis() as u64,
+                "prewarm index segments failed"
+            );
+            Err(err)
+        }
+    }
+}
+
+fn filter_index_segments_by_ids(
+    name: &str,
+    indices: Vec<IndexMetadata>,
+    segment_ids: &[Uuid],
+) -> Result<Vec<IndexMetadata>> {
+    if segment_ids.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let requested = segment_ids.iter().copied().collect::<HashSet<_>>();
+    let mut matched = HashSet::new();
+    let filtered = indices
+        .into_iter()
+        .filter(|index_meta| {
+            if requested.contains(&index_meta.uuid) {
+                matched.insert(index_meta.uuid);
+                true
+            } else {
+                false
+            }
+        })
+        .collect::<Vec<_>>();
+
+    if matched.len() != requested.len() {
+        let mut missing = requested
+            .difference(&matched)
+            .map(ToString::to_string)
+            .collect::<Vec<_>>();
+        missing.sort();
+        return Err(Error::index_not_found(format!(
+            "name={}, segment_ids=[{}]",
+            name,
+            missing.join(", ")
+        )));
+    }
+
+    Ok(filtered)
+}
+
 fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> {
     let mut type_url = None::<&str>;
     for segment in segments {
@@ -250,15 +480,56 @@ fn segment_has_inverted_details(segment: &IndexMetadata) -> bool {
         .is_some_and(|details| details.type_url.ends_with("InvertedIndexDetails"))
 }
 
+fn segment_has_bitmap_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails"))
+}
+
+/// Detect BTree segments, preserving a legacy pre-details fallback.
+fn segment_has_btree_details(segment: &IndexMetadata) -> bool {
+    segment.index_details.as_ref().map_or_else(
+        || {
+            segment
+                .files
+                .as_ref()
+                .is_some_and(|files| files.iter().any(|file| file.path == BTREE_LOOKUP_NAME))
+        },
+        |details| details.type_url.ends_with("BTreeIndexDetails"),
+    )
+}
+
+fn segment_has_zonemap_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("ZoneMapIndexDetails"))
+}
+
+fn segment_has_fmindex_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("FMIndexDetails"))
+}
+
+fn segment_has_label_list_details(segment: &IndexMetadata) -> bool {
+    segment
+        .index_details
+        .as_ref()
+        .is_some_and(|details| details.type_url.ends_with("LabelListIndexDetails"))
+}
+
 // Cache keys for different index types
 #[derive(Debug, Clone)]
 pub(crate) struct LegacyVectorIndexCacheKey<'a> {
-    uuid: &'a str,
+    uuid: &'a Uuid,
     fri_uuid: Option<&'a Uuid>,
 }
 
 impl<'a> LegacyVectorIndexCacheKey<'a> {
-    fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
+    fn new(uuid: &'a Uuid, fri_uuid: Option<&'a Uuid>) -> Self {
         Self { uuid, fri_uuid }
     }
 }
@@ -270,7 +541,7 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> {
         if let Some(fri_uuid) = self.fri_uuid {
             format!("{}-{}", self.uuid, fri_uuid).into()
         } else {
-            self.uuid.into()
+            self.uuid.to_string().into()
         }
     }
 
@@ -286,12 +557,12 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> {
 /// Legacy indices use `LegacyVectorIndexCacheKey` instead (in-memory only).
 #[derive(Debug, Clone)]
 pub(crate) struct IvfIndexStateCacheKey<'a> {
-    uuid: &'a str,
+    uuid: &'a Uuid,
     fri_uuid: Option<&'a Uuid>,
 }
 
 impl<'a> IvfIndexStateCacheKey<'a> {
-    fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
+    fn new(uuid: &'a Uuid, fri_uuid: Option<&'a Uuid>) -> Self {
         Self { uuid, fri_uuid }
     }
 }
@@ -307,7 +578,7 @@ impl CacheKey for IvfIndexStateCacheKey<'_> {
         if let Some(fri_uuid) = self.fri_uuid {
             format!("{}-{}", self.uuid, fri_uuid).into()
         } else {
-            self.uuid.into()
+            self.uuid.to_string().into()
         }
     }
 
@@ -318,17 +589,17 @@ impl CacheKey for IvfIndexStateCacheKey<'_> {
 
 /// Wrapper that stores a live VectorIndex in the cache.
 /// Used for v0.1/v0.2 indices that don't support serializable caching.
-#[derive(Debug, deepsize::DeepSizeOf)]
+#[derive(Debug, lance_core::deepsize::DeepSizeOf)]
 pub(crate) struct CachedLegacyVectorIndex(Arc<dyn VectorIndex>);
 
 #[derive(Debug, Clone)]
 pub struct FragReuseIndexCacheKey<'a> {
-    pub uuid: &'a str,
+    pub uuid: &'a Uuid,
     pub fri_uuid: Option<&'a Uuid>,
 }
 
 impl<'a> FragReuseIndexCacheKey<'a> {
-    pub fn new(uuid: &'a str, fri_uuid: Option<&'a Uuid>) -> Self {
+    pub fn new(uuid: &'a Uuid, fri_uuid: Option<&'a Uuid>) -> Self {
         Self { uuid, fri_uuid }
     }
 }
@@ -340,7 +611,7 @@ impl CacheKey for FragReuseIndexCacheKey<'_> {
         if let Some(fri_uuid) = self.fri_uuid {
             format!("{}-{}", self.uuid, fri_uuid).into()
         } else {
-            self.uuid.into()
+            self.uuid.to_string().into()
         }
     }
 
@@ -416,6 +687,7 @@ fn legacy_type_name(index_uri: &str, index_type_hint: Option<&str>) -> String {
         "BloomFilter" => IndexType::BloomFilter.to_string(),
         "RTree" => IndexType::RTree.to_string(),
         "Inverted" => IndexType::Inverted.to_string(),
+        "FMIndex" | "FM" => IndexType::Fm.to_string(),
         "Json" => IndexType::Scalar.to_string(),
         "Flat" | "Vector" => IndexType::Vector.to_string(),
         other if other.contains("Vector") => IndexType::Vector.to_string(),
@@ -434,7 +706,7 @@ pub trait IndexBuilder {
 pub(crate) async fn remap_index(
     dataset: &Dataset,
     index_id: &Uuid,
-    row_id_map: &HashMap<u64, Option<u64>>,
+    row_id_map: &RowAddrRemap,
 ) -> Result<RemapResult> {
     // Load indices from the disk.
     let indices = dataset.load_indices().await?;
@@ -449,20 +721,14 @@ pub(crate) async fn remap_index(
         ));
     }
 
-    if row_id_map.values().all(|v| v.is_none()) {
-        let deleted_bitmap = RoaringBitmap::from_iter(
-            row_id_map
-                .keys()
-                .map(|row_id| RowAddress::new_from_u64(*row_id))
-                .map(|addr| addr.fragment_id()),
-        );
-        if Some(deleted_bitmap) == matched.fragment_bitmap {
-            // If remap deleted all rows, we can just return the same index ID.
-            // This can happen if there is a bug where the index is covering empty
-            // fragment that haven't been cleaned up. They should be cleaned up
-            // outside of this function.
-            return Ok(RemapResult::Keep(*index_id));
-        }
+    if let Some(deleted_bitmap) = row_id_map.fully_deleted_fragments()
+        && Some(deleted_bitmap) == matched.fragment_bitmap
+    {
+        // If remap deleted all rows, we can just return the same index ID.
+        // This can happen if there is a bug where the index is covering empty
+        // fragment that haven't been cleaned up. They should be cleaned up
+        // outside of this function.
+        return Ok(RemapResult::Keep(*index_id));
     }
 
     let field_id = matched
@@ -474,7 +740,7 @@ pub(crate) async fn remap_index(
     let new_id = Uuid::new_v4();
 
     let generic = match dataset
-        .open_generic_index(&field_path, &index_id.to_string(), &NoOpMetricsCollector)
+        .open_generic_index(&field_path, index_id, &NoOpMetricsCollector)
         .await
     {
         Ok(g) => g,
@@ -492,10 +758,10 @@ pub(crate) async fn remap_index(
 
     let created_index = match generic.index_type() {
         it if it.is_scalar() => {
-            let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_id.to_string())?;
+            let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_id)?;
 
             let scalar_index = dataset
-                .open_scalar_index(&field_path, &index_id.to_string(), &NoOpMetricsCollector)
+                .open_scalar_index(&field_path, index_id, &NoOpMetricsCollector)
                 .await?;
             if !scalar_index.can_remap() {
                 return Ok(RemapResult::Drop);
@@ -545,7 +811,7 @@ pub(crate) async fn remap_index(
                     matched.index_version, matched.name
                 ))
             })?;
-            remap_vector_index(
+            let files = remap_vector_index(
                 Arc::new(dataset.clone()),
                 &field_path,
                 index_id,
@@ -555,17 +821,13 @@ pub(crate) async fn remap_index(
             )
             .await?;
 
-            // Capture file sizes for the vector index
-            let index_dir = dataset.indices_dir().join(new_id.to_string());
-            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
-
             CreatedIndex {
                 index_details: prost_types::Any::from_msg(
                     &lance_index::pb::VectorIndexDetails::default(),
                 )
                 .unwrap(),
                 index_version,
-                files: Some(files),
+                files: table_files_to_index(files),
             }
         }
         _ => {
@@ -581,20 +843,54 @@ pub(crate) async fn remap_index(
         new_id,
         index_details: created_index.index_details,
         index_version: created_index.index_version,
-        files: created_index.files,
+        files: Some(index_files_to_table(created_index.files)),
     }))
 }
 
+/// Snapshot of every scalar index on a dataset, captured at planning time
+/// and consumed by the scalar/aggregate pushdown machinery.
+///
+/// Built once per planner invocation by walking the manifest's `IndexMetadata`
+/// entries; thereafter all lookups are synchronous, so optimizer rules and the
+/// filter parser can interrogate it without needing an async context.
 #[derive(Debug)]
 pub struct ScalarIndexInfo {
+    /// Per-column dispatch table for [`apply_scalar_indices`]: keyed by the
+    /// full dotted field path (e.g. `"x"`, `"metadata.status.code"`), the same
+    /// string callers use when referring to columns in filter expressions.
+    ///
+    /// The value pairs the column's data type with a [`MultiQueryParser`]
+    /// that fans out to every per-index parser registered for that column.
+    /// When a column carries more than one index (e.g. BTree + bitmap), the
+    /// `MultiQueryParser` tries each in order and the first match wins; the
+    /// resulting [`crate::scalar::expression::ScalarIndexSearch`] records
+    /// which specific index was chosen. So *which* index served the query is
+    /// an output of parsing, not an input — that's why this map is keyed only
+    /// by column.
+    ///
+    /// `fragment_bitmaps`, by contrast, *is* keyed by `(column, index_name)`,
+    /// because by the time the optimizer needs the bitmap the index name is
+    /// already pinned in the parsed leaf.
     indexed_columns: HashMap<String, (DataType, Box<MultiQueryParser>)>,
+    /// `(column, index_name) → fragment_bitmap` taken straight off each
+    /// [`IndexMetadata`] at construction time. Used by the optimizer rule for
+    /// aggregate pushdown to reason about index coverage synchronously.
+    /// Indices that omit `fragment_bitmap` (legacy or unsupported) simply
+    /// don't appear here and so report coverage as unknown.
+    fragment_bitmaps: HashMap<(String, String), RoaringBitmap>,
 }
 
 impl IndexInformationProvider for ScalarIndexInfo {
-    fn get_index(&self, col: &str) -> Option<(&DataType, &dyn ScalarQueryParser)> {
+    fn get_index(&self, col: &str) -> Option<(&DataType, &MultiQueryParser)> {
         self.indexed_columns
             .get(col)
-            .map(|(ty, parser)| (ty, parser.as_ref() as &dyn ScalarQueryParser))
+            .map(|(ty, parser)| (ty, parser.as_ref()))
+    }
+
+    fn fragment_bitmap(&self, column: &str, index_name: &str) -> Option<RoaringBitmap> {
+        self.fragment_bitmaps
+            .get(&(column.to_string(), index_name.to_string()))
+            .cloned()
     }
 }
 
@@ -617,7 +913,10 @@ struct IndexDescriptionImpl {
     field_ids: Vec<u32>,
     segments: Vec<IndexMetadata>,
     index_type: String,
-    details: IndexDetails,
+    /// Index details, or `None` for indices created before details were
+    /// persisted in the manifest. Such indices are still described on a
+    /// best-effort basis rather than rejected.
+    details: Option<IndexDetails>,
     rows_indexed: u64,
 }
 
@@ -645,57 +944,52 @@ impl IndexDescriptionImpl {
         }
         let field_ids_vec: Vec<u32> = field_ids.iter().map(|id| *id as u32).collect();
 
-        // This should not fail as we have already filtered out indexes without index details.
-        let index_details = example_metadata.index_details.as_ref().ok_or_else(|| {
-            let fields = field_ids
-                .iter()
-                .map(|id| {
-                    dataset
-                        .schema()
-                        .field_by_id(*id)
-                        .map(|f| format!("{}({})", f.name, id))
-                        .unwrap_or_else(|| format!("<unknown>({})", id))
-                })
-                .collect::<Vec<_>>()
-                .join(", ");
-
-            Error::index(format!(
-                "Index details are required for index description. This index must be retrained to support this method. (index_name={}, uuid={}, fields=[{}])",
-                name,
-                example_metadata.uuid,
-                fields
-            ))
-        })?;
-        let type_url = &index_details.type_url;
-        if !segments.iter().all(|shard| {
-            shard
-                .index_details
-                .as_ref()
-                .map(|d| d.type_url == *type_url)
-                .unwrap_or(false)
-        }) {
-            return Err(Error::index(
-                "Index type URL should be present and identical across all segments".to_string(),
-            ));
+        // Index details may be absent on indices created before details were
+        // persisted in the manifest. We describe such indices on a best-effort
+        // basis rather than erroring, so callers can still see they exist.
+        let details = example_metadata.index_details.clone().map(IndexDetails);
+        if let Some(details) = details.as_ref() {
+            let type_url = &details.0.type_url;
+            if !segments.iter().all(|shard| {
+                shard
+                    .index_details
+                    .as_ref()
+                    .map(|d| d.type_url == *type_url)
+                    .unwrap_or(false)
+            }) {
+                return Err(Error::index(
+                    "Index type URL should be present and identical across all segments"
+                        .to_string(),
+                ));
+            }
         }
 
-        let details = IndexDetails(index_details.clone());
-
-        let index_type = if details.is_vector() {
-            derive_vector_index_type(index_details)
-        } else if let Some(system_type) = lance_index::infer_system_index_type(example_metadata) {
-            // System indices (frag-reuse, mem-wal) are identified by name, not
-            // by a plugin entry, so the plugin lookup below would return
-            // "Unknown" otherwise.
-            system_type.to_string()
-        } else {
-            // We attempted to infer the index type when we loaded the indices,
-            // so if we hit this branch the index type is truly unknown.
-            details
-                .get_plugin()
-                .map(|p| p.name().to_string())
-                .unwrap_or_else(|_| "Unknown".to_string())
-        };
+        let index_type =
+            if let Some(system_type) = lance_index::infer_system_index_type(example_metadata) {
+                // System indices (frag-reuse, mem-wal) are identified by name, not
+                // by index details, so this must be checked before the plugin lookup.
+                system_type.to_string()
+            } else if let Some(details) = details.as_ref() {
+                if details.is_vector() {
+                    derive_vector_index_type(&details.0)
+                } else {
+                    // Fall back to a name derived from the type URL when no plugin
+                    // is registered, so a known type URL is never reported as the
+                    // opaque "Unknown".
+                    details
+                        .get_plugin()
+                        .map(|p| p.name().to_string())
+                        .unwrap_or_else(|_| {
+                            display_type_from_url(details.0.type_url.as_str()).to_string()
+                        })
+                }
+            } else if segment_has_vector_details(example_metadata) {
+                // Legacy vector indices predate VectorIndexDetails and are
+                // recognized by their monolithic index file name.
+                "Vector".to_string()
+            } else {
+                "Unknown".to_string()
+            };
 
         let mut fragment_rows = HashMap::with_capacity(dataset.manifest.fragments.len());
         for fragment in dataset.iter_fragments() {
@@ -709,10 +1003,15 @@ impl IndexDescriptionImpl {
         let mut missing_fragment_refs = 0u64;
 
         for shard in &segments {
-            let fragment_bitmap = shard
-            .fragment_bitmap
-            .as_ref()
-            .ok_or_else(|| Error::index("Fragment bitmap is required for index description.  This index must be retrained to support this method.".to_string()))?;
+            let Some(fragment_bitmap) = shard.fragment_bitmap.as_ref() else {
+                // A system index (e.g. __mem_wal) indexes no fragments, so a
+                // missing bitmap means zero indexed rows. For a data index it
+                // means unknown coverage — reject rather than fabricate a count.
+                if is_system_index(shard) {
+                    continue;
+                }
+                return Err(Error::index("Fragment bitmap is required for index description.  This index must be retrained to support this method.".to_string()));
+            };
 
             indexed_fragment_refs += fragment_bitmap.len();
             for fragment_id in fragment_bitmap.iter() {
@@ -772,7 +1071,10 @@ impl IndexDescription for IndexDescriptionImpl {
     }
 
     fn type_url(&self) -> &str {
-        self.details.0.type_url.as_str()
+        self.details
+            .as_ref()
+            .map(|d| d.0.type_url.as_str())
+            .unwrap_or("")
     }
 
     fn rows_indexed(&self) -> u64 {
@@ -780,13 +1082,14 @@ impl IndexDescription for IndexDescriptionImpl {
     }
 
     fn details(&self) -> Result<String> {
-        if self.details.is_vector() {
-            vector_details_as_json(&self.details.0)
+        let Some(details) = self.details.as_ref() else {
+            return Ok("{}".to_string());
+        };
+        if details.is_vector() {
+            vector_details_as_json(&details.0)
         } else {
-            let plugin = self.details.get_plugin()?;
-            plugin
-                .details_as_json(&self.details.0)
-                .map(|v| v.to_string())
+            let plugin = details.get_plugin()?;
+            plugin.details_as_json(&details.0).map(|v| v.to_string())
         }
     }
 
@@ -803,6 +1106,159 @@ impl IndexDescription for IndexDescriptionImpl {
     }
 }
 
+impl Dataset {
+    /// Build, without committing, the transaction that publishes one or more
+    /// existing physical index segments as a logical index.
+    ///
+    /// This stages the same manifest update as
+    /// [`commit_existing_index_segments`](DatasetIndexExt::commit_existing_index_segments)
+    /// but does not advance the dataset version. Use
+    /// [`CommitBuilder`](crate::dataset::CommitBuilder) to commit the returned
+    /// [`Transaction`].
+    ///
+    /// The transaction is a snapshot built against the current dataset version.
+    /// Conflicting concurrent changes are rejected at commit time: creating or
+    /// dropping a same-name index between staging and commit fails the commit
+    /// with a retryable conflict, and a schema change that drops or re-ids the
+    /// indexed column (e.g. [`Dataset::drop_columns`] or an
+    /// [`Dataset::alter_columns`] type cast) fails it as incompatible. A
+    /// concurrent compaction/rewrite that defers index remapping is not
+    /// conflict-checked; the committed segments may then cover
+    /// already-compacted fragments until the deferred remap catches up, so
+    /// commit promptly.
+    ///
+    /// # Side effects
+    ///
+    /// For most index types this only reads the segment directories. For inverted
+    /// (full-text) segments it also finalizes the segment's on-disk files within
+    /// the segment's UUID directory before returning. Finalization is idempotent,
+    /// and any files left behind if the returned transaction is never committed
+    /// are reclaimed by `cleanup_old_versions` like other unreferenced index
+    /// files.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// # use std::sync::Arc;
+    /// # use lance::Result;
+    /// # use lance::dataset::{CommitBuilder, Dataset};
+    /// # use lance::index::IndexSegment;
+    /// # async fn example(dataset: Arc<Dataset>, segments: Vec<IndexSegment>) -> Result<()> {
+    /// let transaction = dataset
+    ///     .build_existing_index_segments_transaction("vector_idx", "vector", segments)
+    ///     .await?;
+    /// CommitBuilder::new(dataset).execute(transaction).await?;
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub async fn build_existing_index_segments_transaction(
+        &self,
+        index_name: &str,
+        column: &str,
+        segments: Vec<impl IntoIndexSegment + Send>,
+    ) -> Result<Transaction> {
+        let Some(field) = self.schema().field(column) else {
+            return Err(Error::index(format!(
+                "CreateIndex: column '{column}' does not exist"
+            )));
+        };
+
+        let segments = segments
+            .into_iter()
+            .map(IntoIndexSegment::into_index_segment)
+            .collect::<Result<Vec<_>>>()?;
+        let new_indices =
+            build_index_metadata_from_segments(self, index_name, field.id, segments).await?;
+        validate_segment_metadata(index_name, &new_indices)?;
+        validate_segment_index_details(index_name, &new_indices)?;
+
+        let incoming_type_url = new_indices[0]
+            .index_details
+            .as_ref()
+            .map(|details| details.type_url.clone());
+        let dataset_fragments = self.fragment_bitmap.as_ref().clone();
+        let mut incoming_fragments = RoaringBitmap::new();
+        for segment in &new_indices {
+            if segment.fields != [field.id] {
+                return Err(Error::invalid_input(format!(
+                    "CreateIndex: segment {} was built for fields {:?}, expected [{}]",
+                    segment.uuid, segment.fields, field.id
+                )));
+            }
+            if let Some(fragment_bitmap) = &segment.fragment_bitmap {
+                incoming_fragments |= fragment_bitmap.clone();
+            }
+        }
+
+        let existing_named_indices = self.load_indices_by_name(index_name).await?;
+        if existing_named_indices
+            .iter()
+            .any(|idx| idx.fields != [field.id])
+        {
+            return Err(Error::index(format!(
+                "Index name '{index_name}' already exists with different fields, \
+                please specify a different name"
+            )));
+        }
+        let removed_indices = existing_named_indices
+            .into_iter()
+            .filter(|idx| {
+                idx.index_details
+                    .as_ref()
+                    .zip(incoming_type_url.as_deref())
+                    .is_none_or(|(details, expected)| details.type_url == expected)
+            })
+            .map(|idx| -> Result<Option<IndexMetadata>> {
+                let Some(existing_fragments) = idx.effective_fragment_bitmap(&dataset_fragments)
+                else {
+                    if incoming_fragments != dataset_fragments {
+                        return Err(Error::invalid_input(format!(
+                            "CreateIndex: cannot replace legacy index segment {} for '{}' with partial fragment coverage; rebuild all fragments in one commit",
+                            idx.uuid, index_name
+                        )));
+                    }
+                    return Ok(Some(idx));
+                };
+
+                // A zero-fragment segment can be used to create an index while
+                // deferring the actual build. Such a segment is disjoint from every
+                // other segment but should still be removed.
+                if existing_fragments.is_empty() {
+                    return Ok(Some(idx));
+                }
+
+                if existing_fragments.is_disjoint(&incoming_fragments) {
+                    return Ok(None);
+                }
+
+                let uncovered = existing_fragments - &incoming_fragments;
+                if !uncovered.is_empty() {
+                    return Err(Error::invalid_input(format!(
+                        "CreateIndex: incoming segments for '{}' would orphan fragments {:?} from existing segment {}",
+                        index_name,
+                        uncovered.iter().collect::<Vec<_>>(),
+                        idx.uuid
+                    )));
+                }
+
+                Ok(Some(idx))
+            })
+            .collect::<Result<Vec<_>>>()?
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+
+        Ok(Transaction::new(
+            self.manifest.version,
+            Operation::CreateIndex {
+                new_indices,
+                removed_indices,
+            },
+            None,
+        ))
+    }
+}
+
 #[async_trait]
 impl DatasetIndexExt for Dataset {
     type IndexBuilder<'a> = CreateIndexBuilder<'a>;
@@ -898,14 +1354,9 @@ impl DatasetIndexExt for Dataset {
             return Err(Error::index_not_found(format!("name={}", name)));
         }
 
-        for index_meta in indices {
-            let index = self
-                .open_generic_index(name, &index_meta.uuid.to_string(), &NoOpMetricsCollector)
-                .await?;
-            index.prewarm().await?;
-        }
-
-        Ok(())
+        let available_segment_count = indices.len();
+        prewarm_index_segments_by_metadata(self, name, indices, None, available_segment_count, None)
+            .await
     }
 
     async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> {
@@ -914,33 +1365,59 @@ impl DatasetIndexExt for Dataset {
             return Err(Error::index_not_found(format!("name={}", name)));
         }
 
-        for index_meta in indices {
-            let index = self
-                .open_generic_index(name, &index_meta.uuid.to_string(), &NoOpMetricsCollector)
-                .await?;
+        let available_segment_count = indices.len();
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            Some(options),
+            available_segment_count,
+            None,
+        )
+        .await
+    }
 
-            match options {
-                PrewarmOptions::Fts(fts_options) => {
-                    let inverted = index
-                        .as_any()
-                        .downcast_ref::<InvertedIndex>()
-                        .ok_or_else(|| {
-                            Error::invalid_input(format!(
-                                "FTS prewarm options are only supported for inverted indices, got {:?}",
-                                index.index_type()
-                            ))
-                        })?;
-                    inverted.prewarm_with_options(fts_options).await?;
-                }
-                _ => {
-                    return Err(Error::not_supported(
-                        "unsupported prewarm options for this lance version".to_owned(),
-                    ));
-                }
-            }
+    async fn prewarm_index_segments(&self, name: &str, segment_ids: &[Uuid]) -> Result<()> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            return Err(Error::index_not_found(format!("name={}", name)));
         }
+        let available_segment_count = indices.len();
+        let indices = filter_index_segments_by_ids(name, indices, segment_ids)?;
 
-        Ok(())
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            None,
+            available_segment_count,
+            Some(segment_ids.len()),
+        )
+        .await
+    }
+
+    async fn prewarm_index_segments_with_options(
+        &self,
+        name: &str,
+        segment_ids: &[Uuid],
+        options: &PrewarmOptions,
+    ) -> Result<()> {
+        let indices = self.load_indices_by_name(name).await?;
+        if indices.is_empty() {
+            return Err(Error::index_not_found(format!("name={}", name)));
+        }
+        let available_segment_count = indices.len();
+        let indices = filter_index_segments_by_ids(name, indices, segment_ids)?;
+
+        prewarm_index_segments_by_metadata(
+            self,
+            name,
+            indices,
+            Some(options),
+            available_segment_count,
+            Some(segment_ids.len()),
+        )
+        .await
     }
 
     async fn describe_indices<'a, 'b>(
@@ -1026,8 +1503,9 @@ impl DatasetIndexExt for Dataset {
         if let Some(frag_reuse_index_meta) =
             indices.iter().find(|idx| idx.name == FRAG_REUSE_INDEX_NAME)
         {
-            let uuid = frag_reuse_index_meta.uuid.to_string();
-            let fri_key = FragReuseIndexKey { uuid: &uuid };
+            let fri_key = FragReuseIndexKey {
+                uuid: &frag_reuse_index_meta.uuid,
+            };
             let frag_reuse_index = self
                 .index_cache
                 .get_or_insert_with_key(fri_key, || async move {
@@ -1069,7 +1547,19 @@ impl DatasetIndexExt for Dataset {
         }
         let all_vector = source_segments.iter().all(segment_has_vector_details);
         let all_inverted = source_segments.iter().all(segment_has_inverted_details);
-        if !all_vector && !all_inverted {
+        let all_bitmap = source_segments.iter().all(segment_has_bitmap_details);
+        let all_btree = source_segments.iter().all(segment_has_btree_details);
+        let all_fmindex = source_segments.iter().all(segment_has_fmindex_details);
+        let all_zonemap = source_segments.iter().all(segment_has_zonemap_details);
+        let all_label_list = source_segments.iter().all(segment_has_label_list_details);
+        if !all_vector
+            && !all_inverted
+            && !all_bitmap
+            && !all_btree
+            && !all_fmindex
+            && !all_zonemap
+            && !all_label_list
+        {
             return Err(Error::invalid_input(
                 "merge_existing_index_segments requires all segments to have the same supported index type"
                     .to_string(),
@@ -1083,8 +1573,18 @@ impl DatasetIndexExt for Dataset {
                 source_segments,
             )
             .await?
-        } else {
+        } else if all_inverted {
             crate::index::scalar::inverted::merge_segments(self, source_segments).await?
+        } else if all_fmindex {
+            crate::index::scalar::fmindex::merge_segments(self, source_segments).await?
+        } else if all_bitmap {
+            crate::index::scalar::bitmap::merge_segments(self, source_segments).await?
+        } else if all_label_list {
+            crate::index::scalar::label_list::merge_segments(self, source_segments).await?
+        } else if all_zonemap {
+            crate::index::scalar::zonemap::merge_segments(self, source_segments).await?
+        } else {
+            crate::index::scalar::btree::merge_segments(self, source_segments).await?
         };
         merged_segment.dataset_version = self.manifest.version;
         merged_segment.fields = vec![field_id];
@@ -1097,97 +1597,9 @@ impl DatasetIndexExt for Dataset {
         column: &str,
         segments: Vec<impl IntoIndexSegment + Send>,
     ) -> Result<()> {
-        let Some(field) = self.schema().field(column) else {
-            return Err(Error::index(format!(
-                "CreateIndex: column '{column}' does not exist"
-            )));
-        };
-
-        let segments = segments
-            .into_iter()
-            .map(IntoIndexSegment::into_index_segment)
-            .collect::<Result<Vec<_>>>()?;
-        let new_indices =
-            build_index_metadata_from_segments(self, index_name, field.id, segments).await?;
-        validate_segment_metadata(index_name, &new_indices)?;
-        validate_segment_index_details(index_name, &new_indices)?;
-
-        let incoming_type_url = new_indices[0]
-            .index_details
-            .as_ref()
-            .map(|details| details.type_url.clone());
-        let dataset_fragments = self.fragment_bitmap.as_ref().clone();
-        let mut incoming_fragments = RoaringBitmap::new();
-        for segment in &new_indices {
-            if segment.fields != [field.id] {
-                return Err(Error::invalid_input(format!(
-                    "CreateIndex: segment {} was built for fields {:?}, expected [{}]",
-                    segment.uuid, segment.fields, field.id
-                )));
-            }
-            if let Some(fragment_bitmap) = &segment.fragment_bitmap {
-                incoming_fragments |= fragment_bitmap.clone();
-            }
-        }
-
-        let existing_named_indices = self.load_indices_by_name(index_name).await?;
-        if existing_named_indices
-            .iter()
-            .any(|idx| idx.fields != [field.id])
-        {
-            return Err(Error::index(format!(
-                "Index name '{index_name}' already exists with different fields, \
-                please specify a different name"
-            )));
-        }
-        let removed_indices = existing_named_indices
-            .into_iter()
-            .filter(|idx| {
-                idx.index_details
-                    .as_ref()
-                    .zip(incoming_type_url.as_deref())
-                    .is_none_or(|(details, expected)| details.type_url == expected)
-            })
-            .map(|idx| -> Result<Option<IndexMetadata>> {
-                let Some(existing_fragments) = idx.fragment_bitmap.as_ref() else {
-                    if incoming_fragments != dataset_fragments {
-                        return Err(Error::invalid_input(format!(
-                            "CreateIndex: cannot replace legacy index segment {} for '{}' with partial fragment coverage; rebuild all fragments in one commit",
-                            idx.uuid, index_name
-                        )));
-                    }
-                    return Ok(Some(idx));
-                };
-
-                if existing_fragments.is_disjoint(&incoming_fragments) {
-                    return Ok(None);
-                }
-
-                let uncovered = existing_fragments - &incoming_fragments;
-                if !uncovered.is_empty() {
-                    return Err(Error::invalid_input(format!(
-                        "CreateIndex: incoming segments for '{}' would orphan fragments {:?} from existing segment {}",
-                        index_name,
-                        uncovered.iter().collect::<Vec<_>>(),
-                        idx.uuid
-                    )));
-                }
-
-                Ok(Some(idx))
-            })
-            .collect::<Result<Vec<_>>>()?
-            .into_iter()
-            .flatten()
-            .collect::<Vec<_>>();
-
-        let transaction = Transaction::new(
-            self.manifest.version,
-            Operation::CreateIndex {
-                new_indices,
-                removed_indices,
-            },
-            None,
-        );
+        let transaction = self
+            .build_existing_index_segments_transaction(index_name, column, segments)
+            .await?;
 
         self.apply_commit(transaction, &Default::default(), &Default::default())
             .await?;
@@ -1311,7 +1723,7 @@ impl DatasetIndexExt for Dataset {
                 index_version: res.new_index_version,
                 created_at: Some(chrono::Utc::now()),
                 base_id: None, // New merged index file locates in the cloned dataset.
-                files: res.files,
+                files: Some(res.files),
             };
             removed_indices.extend(res.removed_indices.iter().map(|&idx| idx.clone()));
             new_indices.push(new_idx);
@@ -1459,7 +1871,7 @@ async fn index_statistics_frag_reuse(ds: &Dataset) -> Result<String> {
         .open_frag_reuse_index(&NoOpMetricsCollector)
         .await?
         .expect("FragmentReuse index does not exist");
-    serialize_index_statistics(&index.statistics()?)
+    serialize_index_statistics(&FragReuseIndexHandle(index).statistics()?)
 }
 
 async fn index_statistics_mem_wal(ds: &Dataset) -> Result<String> {
@@ -1467,7 +1879,7 @@ async fn index_statistics_mem_wal(ds: &Dataset) -> Result<String> {
         .open_mem_wal_index(&NoOpMetricsCollector)
         .await?
         .expect("MemWal index does not exist");
-    serialize_index_statistics(&index.statistics()?)
+    serialize_index_statistics(&MemWalIndexHandle(index).statistics()?)
 }
 
 async fn index_statistics_scalar(
@@ -1549,7 +1961,7 @@ async fn collect_regular_indices_statistics(
         }
 
         let index = ds
-            .open_generic_index(field_path, &meta.uuid.to_string(), &NoOpMetricsCollector)
+            .open_generic_index(field_path, &meta.uuid, &NoOpMetricsCollector)
             .await?;
 
         indices_stats.push(index.statistics()?);
@@ -1642,21 +2054,21 @@ pub trait DatasetIndexInternalExt: DatasetIndexExt {
     async fn open_generic_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn Index>>;
     /// Opens the requested scalar index
     async fn open_scalar_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn ScalarIndex>>;
     /// Opens the requested vector index
     async fn open_vector_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn VectorIndex>>;
     /// Opens all segments for one logical vector index and returns a materialized snapshot.
@@ -1703,7 +2115,7 @@ impl DatasetIndexInternalExt for Dataset {
     async fn open_generic_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn Index>> {
         // Checking for cache existence is cheap so we just check the vector caches.
@@ -1727,7 +2139,7 @@ impl DatasetIndexInternalExt for Dataset {
 
         let frag_reuse_cache_key = FragReuseIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref());
         if let Some(index) = self.index_cache.get_with_key(&frag_reuse_cache_key).await {
-            return Ok(index.as_index());
+            return Ok(Arc::new(FragReuseIndexHandle(index)).as_index());
         }
 
         // Sometimes we want to open an index and we don't care if it is a scalar or vector index.
@@ -1748,7 +2160,10 @@ impl DatasetIndexInternalExt for Dataset {
         } else {
             // Fall back to file existence check for older indices without file metadata
             let index_dir = self.indice_files_dir(&index_meta)?;
-            let index_file = index_dir.clone().join(uuid).join(INDEX_FILE_NAME);
+            let index_file = index_dir
+                .clone()
+                .join(uuid.to_string())
+                .join(INDEX_FILE_NAME);
             let object_store = self.object_store_for_index(&index_meta).await?;
             object_store.exists(&index_file).await?
         };
@@ -1766,7 +2181,7 @@ impl DatasetIndexInternalExt for Dataset {
     async fn open_scalar_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn ScalarIndex>> {
         // Caching (including the choice of in-memory vs. serializable state) is
@@ -1782,7 +2197,7 @@ impl DatasetIndexInternalExt for Dataset {
     async fn open_vector_index(
         &self,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         metrics: &dyn MetricsCollector,
     ) -> Result<Arc<dyn VectorIndex>> {
         let frag_reuse_uuid = self.frag_reuse_index_uuid().await;
@@ -1797,9 +2212,15 @@ impl DatasetIndexInternalExt for Dataset {
         if let Some(entry) = self.index_cache.get_with_key(&state_key).await {
             log::debug!("Found IvfIndexState in cache uuid: {}", uuid);
             let partition_cache = self.index_cache.with_key_prefix(&state_key.key());
+            let frag_reuse_index = self.open_frag_reuse_index(metrics).await?;
             return entry
                 .0
-                .reconstruct(object_store, self.metadata_cache.as_ref(), partition_cache)
+                .reconstruct(
+                    object_store,
+                    self.metadata_cache.as_ref(),
+                    partition_cache,
+                    frag_reuse_index,
+                )
                 .await;
         }
 
@@ -1811,8 +2232,19 @@ impl DatasetIndexInternalExt for Dataset {
 
         let frag_reuse_index = self.open_frag_reuse_index(metrics).await?;
         let index_dir = self.indice_files_dir(&index_meta)?;
-        let index_file = index_dir.clone().join(uuid).join(INDEX_FILE_NAME);
-        let reader: Arc<dyn Reader> = object_store.open(&index_file).await?.into();
+        let index_file = index_dir
+            .clone()
+            .join(uuid.to_string())
+            .join(INDEX_FILE_NAME);
+        let file_sizes = index_meta.file_size_map();
+        let reader: Arc<dyn Reader> = vector::open_index_file(
+            object_store.as_ref(),
+            &index_file,
+            INDEX_FILE_NAME,
+            &file_sizes,
+        )
+        .await?
+        .into();
 
         let tailing_bytes = read_last_block(reader.as_ref()).await?;
         let (major_version, minor_version) = read_version(&tailing_bytes)?;
@@ -1835,7 +2267,7 @@ impl DatasetIndexInternalExt for Dataset {
             minor_version,
         ) {
             (0, 1) | (0, 0) => {
-                info!(target: TRACE_IO_EVENTS, index_uuid=uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.1", index_type="IVF_PQ");
+                info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.1", index_type="IVF_PQ");
                 let proto = open_index_proto(reader.as_ref()).await?;
                 match &proto.implementation {
                     Some(Implementation::VectorIndex(vector_index)) => {
@@ -1857,7 +2289,7 @@ impl DatasetIndexInternalExt for Dataset {
             }
 
             (0, 2) => {
-                info!(target: TRACE_IO_EVENTS, index_uuid=uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.2", index_type="IVF_PQ");
+                info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.2", index_type="IVF_PQ");
                 let reader = PreviousFileReader::try_new_self_described_from_reader(
                     reader.clone(),
                     Some(&self.metadata_cache.file_metadata_cache(&index_file)),
@@ -1879,7 +2311,6 @@ impl DatasetIndexInternalExt for Dataset {
                     self.object_store.clone(),
                     SchedulerConfig::max_bandwidth(&self.object_store),
                 );
-                let file_sizes = index_meta.file_size_map();
                 let cached_size = file_sizes
                     .get(INDEX_FILE_NAME)
                     .map(|&size| CachedFileSize::new(size))
@@ -1906,7 +2337,7 @@ impl DatasetIndexInternalExt for Dataset {
 
                 let (_, element_type) = get_vector_type(self.schema(), &field_path)?;
 
-                info!(target: TRACE_IO_EVENTS, index_uuid=uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.3", index_type=index_metadata.index_type);
+                info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.3", index_type=index_metadata.index_type);
 
                 match index_metadata.index_type.as_str() {
                     "IVF_FLAT" => match element_type {
@@ -2054,6 +2485,15 @@ impl DatasetIndexInternalExt for Dataset {
         };
         let (index, ivf_entry) = result?;
         metrics.record_index_load();
+        // Attribute the one-time index-open I/O (file footers, IVF centroids,
+        // quantization metadata) to this query's metrics.  This runs only on a
+        // real open; cache hits return earlier, so a warm query reports zero
+        // index-open I/O.
+        if let Some(io_stats) = metrics.io_stats()
+            && let Some(open_stats) = index.open_io_stats()
+        {
+            io_stats.add_scan_stats(&open_stats);
+        }
         if let Some(ivf_entry) = ivf_entry {
             let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref());
             self.index_cache
@@ -2091,7 +2531,7 @@ impl DatasetIndexInternalExt for Dataset {
         let mut segments = Vec::with_capacity(metadatas.len());
         for metadata in metadatas {
             let index = self
-                .open_vector_index(column, &metadata.uuid.to_string(), &NoOpMetricsCollector)
+                .open_vector_index(column, &metadata.uuid, &NoOpMetricsCollector)
                 .await?;
             segments.push((metadata, index));
         }
@@ -2104,19 +2544,27 @@ impl DatasetIndexInternalExt for Dataset {
         metrics: &dyn MetricsCollector,
     ) -> Result<Option<Arc<FragReuseIndex>>> {
         if let Some(frag_reuse_index_meta) = self.load_index_by_name(FRAG_REUSE_INDEX_NAME).await? {
-            let uuid = frag_reuse_index_meta.uuid.to_string();
-            let frag_reuse_key = FragReuseIndexKey { uuid: &uuid };
-            let uuid_clone = uuid.clone();
+            let frag_reuse_uuid = frag_reuse_index_meta.uuid;
+            let frag_reuse_key = FragReuseIndexKey {
+                uuid: &frag_reuse_uuid,
+            };
 
             let index = self
                 .index_cache
                 .get_or_insert_with_key(frag_reuse_key, || async move {
-                    let index_meta = self.load_index(&uuid_clone).await?.ok_or_else(|| Error::index(format!("Index with id {} does not exist", uuid_clone)))?;
+                    let index_meta =
+                        self.load_index(&frag_reuse_uuid).await?.ok_or_else(|| {
+                            Error::index(format!(
+                                "Index with id {} does not exist",
+                                frag_reuse_uuid
+                            ))
+                        })?;
                     let index_details = load_frag_reuse_index_details(self, &index_meta).await?;
                     let index =
-                        open_frag_reuse_index(frag_reuse_index_meta.uuid, index_details.as_ref()).await?;
+                        open_frag_reuse_index(frag_reuse_index_meta.uuid, index_details.as_ref())
+                            .await?;
 
-                    info!(target: TRACE_IO_EVENTS, index_uuid=uuid_clone, r#type=IO_TYPE_OPEN_FRAG_REUSE);
+                    info!(target: TRACE_IO_EVENTS, index_uuid=%frag_reuse_uuid, r#type=IO_TYPE_OPEN_FRAG_REUSE);
                     metrics.record_index_load();
 
                     Ok(index)
@@ -2144,7 +2592,7 @@ impl DatasetIndexInternalExt for Dataset {
             return Ok(Some(index));
         }
 
-        let uuid = mem_wal_meta.uuid.to_string();
+        let uuid = mem_wal_meta.uuid;
 
         let index_meta = self
             .load_index(&uuid)
@@ -2152,7 +2600,7 @@ impl DatasetIndexInternalExt for Dataset {
             .ok_or_else(|| Error::index(format!("Index with id {} does not exist", uuid)))?;
         let index = open_mem_wal_index(index_meta)?;
 
-        info!(target: TRACE_IO_EVENTS, index_uuid=uuid, r#type=IO_TYPE_OPEN_MEM_WAL);
+        info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_MEM_WAL);
         metrics.record_index_load();
 
         self.index_cache
@@ -2177,6 +2625,12 @@ impl DatasetIndexInternalExt for Dataset {
         let indices = self.load_indices().await?;
         let schema = self.schema();
         let mut indexed_fields = Vec::new();
+        // (column, index_name) → union of every contributing IndexMetadata's
+        // fragment_bitmap. Multiple entries can land here for delta-merged
+        // indices that share a name. We only insert when every contributing
+        // entry has a bitmap; if any are missing, we leave the entry absent
+        // so the optimizer treats coverage as unknown.
+        let mut fragment_bitmaps: HashMap<(String, String), Option<RoaringBitmap>> = HashMap::new();
         for index in indices.iter().filter(|idx| {
             let idx_schema = schema.project_by_ids(idx.fields.as_slice(), true);
             let is_vector_index = idx_schema
@@ -2235,6 +2689,23 @@ impl DatasetIndexInternalExt for Dataset {
             let query_parser = plugin.new_query_parser(index.name.clone(), &index_details.0);
 
             if let Some(query_parser) = query_parser {
+                // Union the per-segment fragment bitmap into this
+                // (column, index_name) entry. If any segment is missing a
+                // bitmap, downgrade the entry to None so callers know
+                // coverage is partial/unknown.
+                let key = (field_path.clone(), index.name.clone());
+                fragment_bitmaps
+                    .entry(key)
+                    .and_modify(|entry| {
+                        if let (Some(acc), Some(seg)) =
+                            (entry.as_mut(), index.fragment_bitmap.as_ref())
+                        {
+                            *acc |= seg;
+                        } else {
+                            *entry = None;
+                        }
+                    })
+                    .or_insert_with(|| index.fragment_bitmap.clone());
                 indexed_fields.push((field_path, (field.data_type(), query_parser)));
             }
         }
@@ -2259,8 +2730,14 @@ impl DatasetIndexInternalExt for Dataset {
                     )
                 });
         }
+        // Drop entries we couldn't pin to a known bitmap.
+        let fragment_bitmaps = fragment_bitmaps
+            .into_iter()
+            .filter_map(|(k, v)| v.map(|bm| (k, bm)))
+            .collect();
         Ok(ScalarIndexInfo {
             indexed_columns: index_info_map,
+            fragment_bitmaps,
         })
     }
 
@@ -2486,7 +2963,7 @@ mod tests {
         kmeans::{KMeansParams, train_kmeans},
         sq::builder::SQBuildParams,
     };
-    use lance_io::{assert_io_eq, assert_io_lt};
+    use lance_io::{assert_io_eq, assert_io_lt, utils::tracking_store::IoStats};
     use lance_linalg::distance::{DistanceType, MetricType};
     use lance_testing::datagen::generate_random_array;
     use object_store::ObjectStoreExt;
@@ -2528,6 +3005,20 @@ mod tests {
         }
     }
 
+    fn list_io_stats(stats: &IoStats) -> IoStats {
+        let requests = stats
+            .requests
+            .iter()
+            .filter(|request| request.method == "list")
+            .cloned()
+            .collect::<Vec<_>>();
+        IoStats {
+            read_iops: requests.len() as u64,
+            requests,
+            ..Default::default()
+        }
+    }
+
     fn segment_from_metadata(metadata: &IndexMetadata) -> IndexSegment {
         IndexSegment::new(
             metadata.uuid,
@@ -3847,7 +4338,7 @@ mod tests {
             .unwrap();
         let indices = dataset.load_indices().await.unwrap();
         let index = dataset
-            .open_generic_index("tag", &indices[0].uuid.to_string(), &NoOpMetricsCollector)
+            .open_generic_index("tag", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         assert_eq!(index.index_type(), IndexType::Bitmap);
@@ -3895,7 +4386,9 @@ mod tests {
         assert_io_eq!(stats, read_bytes, 0);
         assert_eq!(indices.len(), 1);
 
-        session.index_cache.clear().await; // Clear the cache
+        // Clear the cache
+        session.index_cache.clear().await;
+        session.file_metadata_cache().clear().await;
 
         let dataset2 = DatasetBuilder::from_uri(test_uri)
             .with_session(session.clone())
@@ -3935,7 +4428,7 @@ mod tests {
         let remap_to_empty = (0..dataset.count_all_rows().await.unwrap())
             .map(|i| (i as u64, None))
             .collect::<HashMap<_, _>>();
-        let new_uuid = remap_index(&dataset, &index_uuid, &remap_to_empty)
+        let new_uuid = remap_index(&dataset, &index_uuid, &RowAddrRemap::direct(remap_to_empty))
             .await
             .unwrap();
         assert_eq!(new_uuid, RemapResult::Keep(index_uuid));
@@ -4372,6 +4865,81 @@ mod tests {
         assert_eq!(descriptions[0].index_type(), inferred_type);
     }
 
+    #[tokio::test]
+    async fn test_describe_indices_tolerates_missing_index_details() {
+        // An index whose manifest entry has no index details (e.g. created
+        // before details were persisted) is still described on a best-effort
+        // basis rather than causing describe_indices to error.
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+        let field_id = dataset.schema().field("id").unwrap().id;
+
+        let metadata = IndexMetadata {
+            uuid: Uuid::new_v4(),
+            name: "mystery_idx".to_string(),
+            fields: vec![field_id],
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: Some(std::iter::once(0_u32).collect()),
+            index_details: None,
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        };
+
+        let desc = IndexDescriptionImpl::try_new(vec![metadata], &dataset)
+            .await
+            .unwrap();
+        assert_eq!(desc.index_type(), "Unknown");
+        assert_eq!(desc.type_url(), "");
+        assert_eq!(desc.details().unwrap(), "{}");
+        assert_eq!(desc.rows_indexed(), 10);
+    }
+
+    #[tokio::test]
+    async fn test_describe_indices_derives_type_from_url_without_plugin() {
+        // When index details exist but no plugin is registered for the type
+        // URL, the index type is derived from the type URL rather than being
+        // reported as the opaque "Unknown".
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+        let field_id = dataset.schema().field("id").unwrap().id;
+
+        let metadata = IndexMetadata {
+            uuid: Uuid::new_v4(),
+            name: "mystery_idx".to_string(),
+            fields: vec![field_id],
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: Some(std::iter::once(0_u32).collect()),
+            index_details: Some(Arc::new(prost_types::Any {
+                type_url: "/lance.index.pb.MysteryIndexDetails".to_string(),
+                value: Vec::new(),
+            })),
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        };
+
+        let desc = IndexDescriptionImpl::try_new(vec![metadata], &dataset)
+            .await
+            .unwrap();
+        assert_eq!(desc.index_type(), "Mystery");
+        assert_eq!(desc.type_url(), "/lance.index.pb.MysteryIndexDetails");
+    }
+
     #[rstest]
     #[case::btree("i", IndexType::BTree, Box::new(ScalarIndexParams::default()))]
     #[case::bitmap("i", IndexType::Bitmap, Box::new(ScalarIndexParams::default()))]
@@ -6456,155 +7024,744 @@ mod tests {
         let test_dir = tempfile::tempdir().unwrap();
         let test_uri = test_dir.path().to_str().unwrap();
 
-        let reader = lance_datagen::gen_batch()
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let err = dataset
+            .commit_existing_index_segments("vector_idx", "vector", Vec::<IndexSegment>::new())
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("at least one index segment"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32, 1_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [1_u32],
+            b"seg1",
+        )
+        .await;
+
+        let err = dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)],
+            )
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("overlapping fragment coverage"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_mixed_index_detail_types() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = IndexMetadata {
+            uuid: Uuid::new_v4(),
+            name: "vector_idx".to_string(),
+            fields: vec![field_id],
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: Some(std::iter::once(1_u32).collect()),
+            index_details: Some(Arc::new(
+                prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap(),
+            )),
+            index_version: IndexType::BTree.version(),
+            created_at: Some(chrono::Utc::now()),
+            base_id: None,
+            files: seg0.files.clone(),
+        };
+
+        let err = dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![seg0, seg1])
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("mixes incompatible index detail types")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_rejects_partial_replacement_of_wider_segment() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 20,
+                max_rows_per_group: 20,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(dataset.get_fragments().len(), 2);
+
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let original = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32, 1_u32],
+            b"original",
+        )
+        .await;
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![original])
+            .await
+            .unwrap();
+
+        let replacement = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"replacement",
+        )
+        .await;
+
+        let err = dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![replacement])
+            .await
+            .unwrap_err();
+        assert!(err.to_string().contains("would orphan fragments"));
+    }
+
+    #[tokio::test]
+    async fn test_commit_existing_index_segments_removes_empty_segment() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let uuid = Uuid::new_v4();
+
+        // Commit a 0-fragment segment, then a real segment covering the dataset.
+        let empty = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            uuid,
+            std::iter::empty::<u32>(),
+            b"empty",
+        )
+        .await;
+        dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&empty)],
+            )
+            .await
+            .unwrap();
+        let seg = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg",
+        )
+        .await;
+        dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg)],
+            )
+            .await
+            .unwrap();
+
+        // The real segment covers the dataset, so the redundant empty one is removed.
+        let committed = dataset.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(
+            committed.iter().map(|i| i.uuid).collect::<HashSet<_>>(),
+            HashSet::from([seg.uuid]),
+            "empty segment should be removed once a real segment covers the dataset",
+        );
+    }
+
+    #[tokio::test]
+    async fn test_build_existing_index_segments_transaction_does_not_commit() {
+        use crate::dataset::CommitBuilder;
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(2));
+
+        let dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 10,
+                max_rows_per_group: 10,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        // 20 rows with max_rows_per_file=10 yields two single-fragment files.
+        assert_eq!(dataset.get_fragments().len(), 2);
+
+        let read_version = dataset.manifest.version;
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [1_u32],
+            b"seg1",
+        )
+        .await;
+
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)],
+            )
+            .await
+            .unwrap();
+
+        // Building the transaction must not publish the index.
+        assert!(
+            dataset
+                .load_indices_by_name("vector_idx")
+                .await
+                .unwrap()
+                .is_empty(),
+            "building a transaction must not publish the index"
+        );
+        assert_eq!(transaction.read_version, read_version);
+        let Operation::CreateIndex {
+            new_indices,
+            removed_indices,
+        } = &transaction.operation
+        else {
+            panic!("expected index creation transaction");
+        };
+        assert_eq!(
+            new_indices.iter().map(|i| i.uuid).collect::<HashSet<_>>(),
+            HashSet::from([seg0.uuid, seg1.uuid]),
+        );
+        assert!(removed_indices.is_empty());
+
+        // The returned transaction can be committed via CommitBuilder.
+        let committed = CommitBuilder::new(Arc::new(dataset))
+            .execute(transaction)
+            .await
+            .unwrap();
+        let indices = committed.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(
+            indices.iter().map(|i| i.uuid).collect::<HashSet<_>>(),
+            HashSet::from([seg0.uuid, seg1.uuid]),
+        );
+    }
+
+    #[tokio::test]
+    async fn test_build_existing_index_segments_transaction_removes_empty_segment() {
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
+        let field_id = dataset.schema().field("vector").unwrap().id;
+
+        // Commit a 0-fragment placeholder segment.
+        let empty = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            std::iter::empty::<u32>(),
+            b"empty",
+        )
+        .await;
+        dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&empty)],
+            )
+            .await
+            .unwrap();
+
+        // Staging a real segment that covers the dataset must mark the placeholder
+        // for removal at build time, exercising the zero-fragment guard on the
+        // staged path (not just the committed path).
+        let seg = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg",
+        )
+        .await;
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg)],
+            )
+            .await
+            .unwrap();
+        let Operation::CreateIndex {
+            new_indices,
+            removed_indices,
+        } = &transaction.operation
+        else {
+            panic!("expected index creation transaction");
+        };
+        assert_eq!(
+            new_indices.iter().map(|i| i.uuid).collect::<HashSet<_>>(),
+            HashSet::from([seg.uuid]),
+        );
+        assert_eq!(
+            removed_indices
+                .iter()
+                .map(|i| i.uuid)
+                .collect::<HashSet<_>>(),
+            HashSet::from([empty.uuid]),
+            "the zero-fragment placeholder must be staged for removal",
+        );
+    }
+
+    #[tokio::test]
+    async fn test_build_existing_index_segments_transaction_commits_after_version_advances() {
+        use crate::dataset::CommitBuilder;
+        use lance_datagen::{BatchCount, RowCount, array};
+
+        let test_dir = tempfile::tempdir().unwrap();
+        let test_uri = test_dir.path().to_str().unwrap();
+
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(2));
+        let dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 10,
+                max_rows_per_group: 10,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let read_version = dataset.manifest.version;
+        let field_id = dataset.schema().field("vector").unwrap().id;
+        let seg0 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg0",
+        )
+        .await;
+        let seg1 = write_vector_segment_metadata(
+            &dataset,
+            "vector_idx",
+            field_id,
+            Uuid::new_v4(),
+            [1_u32],
+            b"seg1",
+        )
+        .await;
+
+        // Stage the transaction at `read_version`.
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)],
+            )
+            .await
+            .unwrap();
+        assert_eq!(transaction.read_version, read_version);
+
+        // Advance the dataset with an unrelated append, moving HEAD past read_version.
+        let more = lance_datagen::gen_batch()
             .col("id", array::step::<arrow_array::types::Int32Type>())
             .col(
                 "vector",
                 array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
             )
             .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let dataset = Dataset::write(
+            more,
+            test_uri,
+            Some(WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert!(dataset.manifest.version > read_version);
 
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
-
-        let err = dataset
-            .commit_existing_index_segments("vector_idx", "vector", Vec::<IndexSegment>::new())
+        // The staged transaction still commits cleanly against the advanced HEAD.
+        let committed = CommitBuilder::new(Arc::new(dataset))
+            .execute(transaction)
             .await
-            .unwrap_err();
-        assert!(err.to_string().contains("at least one index segment"));
+            .unwrap();
+        let indices = committed.load_indices_by_name("vector_idx").await.unwrap();
+        assert_eq!(
+            indices.iter().map(|i| i.uuid).collect::<HashSet<_>>(),
+            HashSet::from([seg0.uuid, seg1.uuid]),
+        );
     }
 
     #[tokio::test]
-    async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() {
+    async fn test_build_existing_index_segments_transaction_conflicts_with_index_drop() {
+        use crate::dataset::CommitBuilder;
         use lance_datagen::{BatchCount, RowCount, array};
 
         let test_dir = tempfile::tempdir().unwrap();
-        let test_uri = test_dir.path().to_str().unwrap();
-
         let reader = lance_datagen::gen_batch()
             .col("id", array::step::<arrow_array::types::Int32Type>())
             .col(
                 "vector",
                 array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
             )
-            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
-
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
-
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
         let field_id = dataset.schema().field("vector").unwrap().id;
-        let seg0 = write_vector_segment_metadata(
+
+        // Publish an initial index covering the whole dataset.
+        let initial = write_vector_segment_metadata(
             &dataset,
             "vector_idx",
             field_id,
             Uuid::new_v4(),
-            [0_u32, 1_u32],
-            b"seg0",
+            [0_u32],
+            b"initial",
         )
         .await;
-        let seg1 = write_vector_segment_metadata(
+        dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&initial)],
+            )
+            .await
+            .unwrap();
+
+        // Stage a replacement for the published index.
+        let replacement = write_vector_segment_metadata(
             &dataset,
             "vector_idx",
             field_id,
             Uuid::new_v4(),
-            [1_u32],
-            b"seg1",
+            [0_u32],
+            b"replacement",
         )
         .await;
-
-        let err = dataset
-            .commit_existing_index_segments(
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
                 "vector_idx",
                 "vector",
-                vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)],
+                vec![segment_from_metadata(&replacement)],
             )
             .await
-            .unwrap_err();
-        assert!(err.to_string().contains("overlapping fragment coverage"));
+            .unwrap();
+
+        // The index is dropped before the staged transaction commits.
+        dataset.drop_index("vector_idx").await.unwrap();
+
+        // Committing the staged transaction must not resurrect the dropped index.
+        let err = CommitBuilder::new(Arc::new(dataset.clone()))
+            .execute(transaction)
+            .await
+            .expect_err("staged index commit over a concurrent drop must conflict");
+        assert!(
+            matches!(err, Error::RetryableCommitConflict { .. }),
+            "expected a retryable commit conflict, got: {err}"
+        );
+        assert!(
+            err.to_string()
+                .contains("preempted by concurrent transaction CreateIndex"),
+            "conflict message must identify the concurrent CreateIndex removal, got: {err}"
+        );
+        assert!(
+            dataset
+                .load_indices_by_name("vector_idx")
+                .await
+                .unwrap()
+                .is_empty(),
+            "the dropped index must stay dropped"
+        );
     }
 
     #[tokio::test]
-    async fn test_commit_existing_index_segments_rejects_mixed_index_detail_types() {
+    async fn test_build_existing_index_segments_transaction_conflicts_with_column_drop() {
+        use crate::dataset::CommitBuilder;
         use lance_datagen::{BatchCount, RowCount, array};
 
         let test_dir = tempfile::tempdir().unwrap();
-        let test_uri = test_dir.path().to_str().unwrap();
-
         let reader = lance_datagen::gen_batch()
             .col("id", array::step::<arrow_array::types::Int32Type>())
             .col(
                 "vector",
                 array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
             )
-            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
-
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
-
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
         let field_id = dataset.schema().field("vector").unwrap().id;
-        let seg0 = write_vector_segment_metadata(
+
+        let seg = write_vector_segment_metadata(
             &dataset,
             "vector_idx",
             field_id,
             Uuid::new_v4(),
             [0_u32],
-            b"seg0",
+            b"seg",
         )
         .await;
-        let seg1 = IndexMetadata {
-            uuid: Uuid::new_v4(),
-            name: "vector_idx".to_string(),
-            fields: vec![field_id],
-            dataset_version: dataset.manifest.version,
-            fragment_bitmap: Some(std::iter::once(1_u32).collect()),
-            index_details: Some(Arc::new(
-                prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap(),
-            )),
-            index_version: IndexType::BTree.version(),
-            created_at: Some(chrono::Utc::now()),
-            base_id: None,
-            files: seg0.files.clone(),
-        };
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&seg)],
+            )
+            .await
+            .unwrap();
 
-        let err = dataset
-            .commit_existing_index_segments("vector_idx", "vector", vec![seg0, seg1])
+        // The indexed column is dropped before the staged transaction commits.
+        dataset.drop_columns(&["vector"]).await.unwrap();
+
+        // Committing the staged transaction must not publish index metadata for
+        // a field that no longer exists.
+        let err = CommitBuilder::new(Arc::new(dataset))
+            .execute(transaction)
             .await
-            .unwrap_err();
+            .expect_err("staged index commit for a dropped column must fail");
+        assert!(
+            matches!(err, Error::IncompatibleTransaction { .. }),
+            "expected an incompatible transaction error, got: {err}"
+        );
         assert!(
             err.to_string()
-                .contains("mixes incompatible index detail types")
+                .contains("incompatible with concurrent transaction Project"),
+            "conflict message must identify the concurrent Project, got: {err}"
         );
     }
 
     #[tokio::test]
-    async fn test_commit_existing_index_segments_rejects_partial_replacement_of_wider_segment() {
+    async fn test_build_existing_index_segments_transaction_conflicts_with_column_cast() {
+        use crate::dataset::{ColumnAlteration, CommitBuilder};
         use lance_datagen::{BatchCount, RowCount, array};
 
         let test_dir = tempfile::tempdir().unwrap();
-        let test_uri = test_dir.path().to_str().unwrap();
-
         let reader = lance_datagen::gen_batch()
             .col("id", array::step::<arrow_array::types::Int32Type>())
             .col(
                 "vector",
                 array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
             )
-            .into_reader_rows(RowCount::from(20), BatchCount::from(2));
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
+        let field_id = dataset.schema().field("id").unwrap().id;
 
-        let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap();
+        let seg = write_vector_segment_metadata(
+            &dataset,
+            "id_idx",
+            field_id,
+            Uuid::new_v4(),
+            [0_u32],
+            b"seg",
+        )
+        .await;
+        let transaction = dataset
+            .build_existing_index_segments_transaction(
+                "id_idx",
+                "id",
+                vec![segment_from_metadata(&seg)],
+            )
+            .await
+            .unwrap();
+
+        // A type cast commits Operation::Merge and assigns the column a new
+        // field id, so the staged transaction's field no longer exists.
+        dataset
+            .alter_columns(&[
+                ColumnAlteration::new("id".into()).cast_to(arrow_schema::DataType::Int64)
+            ])
+            .await
+            .unwrap();
+        assert!(dataset.schema().field_by_id(field_id).is_none());
+
+        let err = CommitBuilder::new(Arc::new(dataset))
+            .execute(transaction)
+            .await
+            .expect_err("staged index commit for a re-identified column must fail");
+        assert!(
+            matches!(err, Error::IncompatibleTransaction { .. }),
+            "expected an incompatible transaction error, got: {err}"
+        );
+        assert!(
+            err.to_string().contains("no longer exists in the schema"),
+            "conflict message must identify the missing field, got: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_drop_index_conflicts_with_concurrent_replacement() {
+        use lance_datagen::{BatchCount, RowCount, array};
 
+        let test_dir = tempfile::tempdir().unwrap();
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "vector",
+                array::rand_vec::<arrow_array::types::Float32Type>(8.into()),
+            )
+            .into_reader_rows(RowCount::from(10), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None)
+            .await
+            .unwrap();
         let field_id = dataset.schema().field("vector").unwrap().id;
-        let original = write_vector_segment_metadata(
+
+        let initial = write_vector_segment_metadata(
             &dataset,
             "vector_idx",
             field_id,
             Uuid::new_v4(),
-            [0_u32, 1_u32],
-            b"original",
+            [0_u32],
+            b"initial",
         )
         .await;
         dataset
-            .commit_existing_index_segments("vector_idx", "vector", vec![original])
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&initial)],
+            )
             .await
             .unwrap();
 
+        // A stale handle prepares to drop while the index is replaced.
+        let mut stale = dataset.clone();
         let replacement = write_vector_segment_metadata(
             &dataset,
             "vector_idx",
@@ -6614,12 +7771,37 @@ mod tests {
             b"replacement",
         )
         .await;
+        dataset
+            .commit_existing_index_segments(
+                "vector_idx",
+                "vector",
+                vec![segment_from_metadata(&replacement)],
+            )
+            .await
+            .unwrap();
 
-        let err = dataset
-            .commit_existing_index_segments("vector_idx", "vector", vec![replacement])
+        // The stale drop must conflict instead of silently no-op'ing (its
+        // removal targets a UUID the replacement already removed).
+        let err = stale
+            .drop_index("vector_idx")
             .await
-            .unwrap_err();
-        assert!(err.to_string().contains("would orphan fragments"));
+            .expect_err("a drop racing a same-name replacement must conflict");
+        assert!(
+            matches!(err, Error::RetryableCommitConflict { .. }),
+            "expected a retryable commit conflict, got: {err}"
+        );
+
+        // Retrying against the latest version drops the replacement for real.
+        stale.checkout_latest().await.unwrap();
+        stale.drop_index("vector_idx").await.unwrap();
+        assert!(
+            stale
+                .load_indices_by_name("vector_idx")
+                .await
+                .unwrap()
+                .is_empty(),
+            "the retried drop must remove the replaced index"
+        );
     }
 
     #[tokio::test]
@@ -7094,6 +8276,103 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_scalar_index_create_does_not_list_files() {
+        let test_dir = TempStrDir::default();
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("category", DataType::Int32, false),
+        ]));
+        let ids = Int32Array::from_iter_values(0..128);
+        let categories = Int32Array::from_iter_values((0..128).map(|value| value % 8));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)])
+            .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+        let io_tracker = dataset.object_store.as_ref().io_tracker().clone();
+
+        io_tracker.incremental_stats();
+        dataset
+            .create_index(
+                &["category"],
+                IndexType::Bitmap,
+                Some("category_bitmap".to_string()),
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        let stats = io_tracker.incremental_stats();
+        let list_stats = list_io_stats(&stats);
+        assert_io_eq!(
+            list_stats,
+            read_iops,
+            0,
+            "new scalar index files should be reported by writer return values"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_index_create_does_not_list_files() {
+        let test_dir = TempStrDir::default();
+        let dimension = 8;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new(
+                "vector",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dimension,
+                ),
+                false,
+            ),
+        ]));
+        let ids = Int32Array::from_iter_values(0..256);
+        let vectors = (0..256)
+            .map(|row| {
+                Some(
+                    (0..dimension)
+                        .map(|dim| Some((row * dimension + dim) as f32))
+                        .collect::<Vec<_>>(),
+                )
+            })
+            .collect::<Vec<_>>();
+        let vector_array =
+            FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(vectors, dimension);
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)])
+                .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), None)
+            .await
+            .unwrap();
+        let io_tracker = dataset.object_store.as_ref().io_tracker().clone();
+
+        io_tracker.incremental_stats();
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("vector_ivf_flat".to_string()),
+                &VectorIndexParams::ivf_flat(4, MetricType::L2),
+                true,
+            )
+            .await
+            .unwrap();
+
+        let stats = io_tracker.incremental_stats();
+        let list_stats = list_io_stats(&stats);
+        assert_io_eq!(
+            list_stats,
+            read_iops,
+            0,
+            "new V3 vector index files should be reported by builder return values"
+        );
+    }
+
     #[tokio::test]
     async fn test_index_file_sizes_through_lifecycle() {
         use crate::dataset::WriteDestination;
@@ -7439,12 +8718,16 @@ mod tests {
             .unwrap();
         assert!(results.num_rows() > 0);
 
-        // Verify IOPs
+        // Verify IOPs. The deferred DocSet loads per-doc num_tokens/row_ids on
+        // first use rather than eagerly at index open, so a cold (un-prewarmed)
+        // query opens the docs file on demand — a couple more IOPs than the
+        // eager path, but constant and only on the first query (prewarm or a
+        // warm cache serve it with zero IO).
         let stats = dataset.object_store.as_ref().io_stats_incremental();
         assert_io_lt!(
             stats,
             read_iops,
-            15,
+            18,
             "Inverted index query should use minimal IOPs"
         );
     }
diff --git a/rust/lance/src/index/api.rs b/rust/lance/src/index/api.rs
index 0119db7ac01..54c710f42ea 100644
--- a/rust/lance/src/index/api.rs
+++ b/rust/lance/src/index/api.rs
@@ -167,6 +167,25 @@ pub trait DatasetIndexExt {
         ))
     }
 
+    /// Prewarm selected physical segments of an index by name.
+    async fn prewarm_index_segments(&self, _name: &str, _segment_ids: &[Uuid]) -> Result<()> {
+        Err(Error::not_supported(
+            "segment-level prewarm is not supported by this dataset implementation".to_owned(),
+        ))
+    }
+
+    /// Prewarm selected physical segments of an index by name with additional options.
+    async fn prewarm_index_segments_with_options(
+        &self,
+        _name: &str,
+        _segment_ids: &[Uuid],
+        _options: &PrewarmOptions,
+    ) -> Result<()> {
+        Err(Error::not_supported(
+            "prewarm options are not supported by this dataset implementation".to_owned(),
+        ))
+    }
+
     /// Read all indices of this Dataset version.
     ///
     /// The indices are lazy loaded and cached in memory within the `Dataset` instance.
@@ -177,13 +196,10 @@ pub trait DatasetIndexExt {
     ///
     /// Note that it is possible to have multiple indices with the same UUID,
     /// as they are the deltas of the same index.
-    async fn load_index(&self, uuid: &str) -> Result<Option<IndexMetadata>> {
-        self.load_indices().await.map(|indices| {
-            indices
-                .iter()
-                .find(|idx| idx.uuid.to_string() == uuid)
-                .cloned()
-        })
+    async fn load_index(&self, uuid: &Uuid) -> Result<Option<IndexMetadata>> {
+        self.load_indices()
+            .await
+            .map(|indices| indices.iter().find(|idx| idx.uuid == *uuid).cloned())
     }
 
     /// Loads a specific index with the given index name.
diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs
index 4398928d3e2..3a6b0e42a44 100644
--- a/rust/lance/src/index/append.rs
+++ b/rust/lance/src/index/append.rs
@@ -11,11 +11,12 @@ use lance_index::{
     optimize::OptimizeOptions,
     progress::NoopIndexBuildProgress,
     scalar::{
-        CreatedIndex, OldIndexDataFilter, inverted::InvertedIndex, lance_format::LanceIndexStore,
+        CreatedIndex, OldIndexDataFilter, ScalarIndex, index_files_to_table,
+        inverted::InvertedIndex, lance_format::LanceIndexStore, table_files_to_index,
     },
 };
 use lance_select::{RowAddrTreeMap, RowSetOps};
-use lance_table::format::{Fragment, IndexMetadata, list_index_files_with_sizes};
+use lance_table::format::{Fragment, IndexMetadata};
 use roaring::RoaringBitmap;
 use uuid::Uuid;
 
@@ -36,7 +37,7 @@ pub struct IndexMergeResults<'a> {
     pub new_index_version: i32,
     pub new_index_details: prost_types::Any,
     /// List of files and their sizes for the merged index
-    pub files: Option<Vec<lance_table::format::IndexFile>>,
+    pub files: Vec<lance_table::format::IndexFile>,
 }
 
 async fn build_stable_row_id_filter(
@@ -64,16 +65,341 @@ async fn build_stable_row_id_filter(
         .try_collect::<Vec<_>>()
         .await?;
 
-    let row_id_maps = row_id_sequences
-        .iter()
-        .map(|(_, seq)| RowAddrTreeMap::from(seq.as_ref()))
-        .collect::<Vec<_>>();
+    let frag_by_id: std::collections::HashMap<u32, _> = dataset
+        .get_fragments()
+        .into_iter()
+        .map(|f| (f.id() as u32, f))
+        .collect();
+
+    let mut row_id_maps = Vec::with_capacity(row_id_sequences.len());
+    for (frag_id, seq) in &row_id_sequences {
+        row_id_maps.push(live_row_ids(frag_by_id.get(frag_id), seq).await?);
+    }
     let row_id_map_refs = row_id_maps.iter().collect::<Vec<_>>();
 
     // Merge all fragment-local row-id sets into one exact membership structure.
     Ok(<RowAddrTreeMap as RowSetOps>::union_all(&row_id_map_refs))
 }
 
+/// The fragment's live row ids: its persisted row-id sequence minus the rows
+/// its deletion vector marks gone. A persisted sequence covers every row the
+/// fragment ever held, so a row whose old copy was deleted (e.g. rewritten by an
+/// update under the same stable row id) would otherwise be retained as a stale
+/// old-index entry.
+async fn live_row_ids(
+    fragment: Option<&crate::dataset::fragment::FileFragment>,
+    seq: &lance_table::rowids::RowIdSequence,
+) -> Result<RowAddrTreeMap> {
+    // Propagate a deletion-vector read failure rather than swallowing it: a
+    // swallowed error would fall through to the "no deletions" branch below,
+    // putting the deleted rows back into the allow-list as stale entries.
+    let deletion_vector = match fragment {
+        Some(f) if f.metadata().deletion_file.is_some() => f.get_deletion_vector().await?,
+        _ => None,
+    };
+    Ok(match deletion_vector {
+        Some(dv) => seq
+            .iter()
+            .enumerate()
+            .filter(|(offset, _)| !dv.contains(*offset as u32))
+            .map(|(_, row_id)| row_id)
+            .collect(),
+        None => RowAddrTreeMap::from(seq),
+    })
+}
+
+/// Build the [`OldIndexDataFilter`] that must be applied to existing index
+/// rows when their owning fragments have been pruned by compaction or
+/// deletions.
+pub async fn build_old_data_filter(
+    dataset: &Dataset,
+    effective_old_frags: &RoaringBitmap,
+    deleted_old_frags: &RoaringBitmap,
+) -> Result<Option<OldIndexDataFilter>> {
+    if dataset.manifest.uses_stable_row_ids() {
+        let valid_old_row_ids = build_stable_row_id_filter(dataset, effective_old_frags).await?;
+        Ok(Some(OldIndexDataFilter::RowIds(valid_old_row_ids)))
+    } else {
+        Ok(Some(OldIndexDataFilter::Fragments {
+            to_keep: effective_old_frags.clone(),
+            to_remove: deleted_old_frags.clone(),
+        }))
+    }
+}
+
+/// Split the stored fragment coverage of `segments` into fragments still live in
+/// `dataset` (`effective`) and fragments that compaction or deletion has already
+/// retired (`deleted`).
+pub fn split_segment_coverage<'a>(
+    dataset: &Dataset,
+    segments: impl IntoIterator<Item = &'a IndexMetadata>,
+) -> (RoaringBitmap, RoaringBitmap) {
+    let mut effective = RoaringBitmap::new();
+    let mut deleted = RoaringBitmap::new();
+    for segment in segments {
+        if let Some(eff) = segment.effective_fragment_bitmap(&dataset.fragment_bitmap) {
+            effective |= eff;
+        }
+        if let Some(del) = segment.deleted_fragment_bitmap(&dataset.fragment_bitmap) {
+            deleted |= del;
+        }
+    }
+    (effective, deleted)
+}
+
+/// Build one [`OldIndexDataFilter`] per segment, each derived from that segment's
+/// *own* effective (still-live) and retired fragment coverage, plus the union of
+/// every segment's still-live coverage.
+pub async fn build_per_segment_filters(
+    dataset: &Dataset,
+    segments: &[&IndexMetadata],
+) -> Result<(RoaringBitmap, Vec<Option<OldIndexDataFilter>>)> {
+    let mut effective_union = RoaringBitmap::new();
+    let mut filters = Vec::with_capacity(segments.len());
+    for segment in segments {
+        if segment.fragment_bitmap.is_none() {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing fragment coverage",
+                segment.uuid
+            )));
+        }
+        let effective = segment
+            .effective_fragment_bitmap(&dataset.fragment_bitmap)
+            .unwrap_or_default();
+        let deleted = segment
+            .deleted_fragment_bitmap(&dataset.fragment_bitmap)
+            .unwrap_or_default();
+        effective_union |= &effective;
+        filters.push(build_old_data_filter(dataset, &effective, &deleted).await?);
+    }
+    Ok((effective_union, filters))
+}
+
+async fn load_unindexed_training_data(
+    dataset: &Dataset,
+    field_path: &str,
+    update_criteria: &lance_index::scalar::UpdateCriteria,
+    unindexed: &[Fragment],
+) -> Result<datafusion::execution::SendableRecordBatchStream> {
+    let fragments = if update_criteria.requires_old_data {
+        None
+    } else {
+        Some(unindexed.to_vec())
+    };
+    load_training_data(
+        dataset,
+        field_path,
+        &update_criteria.data_criteria,
+        fragments,
+        true,
+        None,
+    )
+    .await
+}
+
+/// Build a fresh, canonical (non-sharded) scalar index over `fragment_ids`,
+/// reusing `reference_index`'s params and training criteria.
+async fn rebuild_scalar_segment(
+    dataset: &Dataset,
+    reference_index: &Arc<dyn ScalarIndex>,
+    field_path: &str,
+    column_name: &str,
+    uuid: Uuid,
+    fragment_ids: Vec<u32>,
+) -> Result<CreatedIndex> {
+    let params = reference_index.derive_index_params()?;
+    let update_criteria = reference_index.update_criteria();
+    let training_data = load_training_data(
+        dataset,
+        field_path,
+        &update_criteria.data_criteria,
+        None,
+        true,
+        Some(fragment_ids),
+    )
+    .await?;
+    super::scalar::build_scalar_index(
+        dataset,
+        column_name,
+        uuid,
+        &params,
+        true,
+        None,
+        Some(training_data),
+        Arc::new(NoopIndexBuildProgress),
+    )
+    .await
+}
+
+/// The index segments to rewrite in this optimize pass.
+///
+/// Normally the trailing `num_indices_to_merge` segments. Under stable row ids,
+/// any *older* segment that still covers a fragment carrying deletions is added
+/// too: an update deletes a row's old copy (leaving a deletion vector) and
+/// rewrites it under the same row id, so its stale old-value postings survive
+/// until that segment is rewritten and filtered. Only the segments that actually
+/// cover a deleted-from fragment are pulled in -- clean segments in between are
+/// left untouched -- so an edit to old data does not force a full reindex.
+///
+/// The deletion check is conservative (any current deletion vector on a covered
+/// fragment), so a segment built after those deletions may be rewritten as a
+/// harmless no-op; it never leaves a stale segment behind (PR #7359).
+fn select_segments_to_merge<'a>(
+    dataset: &Dataset,
+    old_indices: &[&'a IndexMetadata],
+    options: &OptimizeOptions,
+) -> Vec<&'a IndexMetadata> {
+    let num_to_merge = options
+        .num_indices_to_merge
+        .unwrap_or(1)
+        .min(old_indices.len());
+    let tail_start = old_indices.len() - num_to_merge;
+
+    // Address-style row ids mask stale postings at search time, and append mode
+    // (num_to_merge == 0) defers cleanup to a real merge; both keep the plain tail.
+    if num_to_merge == 0 || !dataset.manifest.uses_stable_row_ids() {
+        return old_indices[tail_start..].to_vec();
+    }
+
+    let deleted_frags: RoaringBitmap = dataset
+        .get_fragments()
+        .iter()
+        .filter(|f| f.metadata().deletion_file.is_some())
+        .map(|f| f.id() as u32)
+        .collect();
+    if deleted_frags.is_empty() {
+        return old_indices[tail_start..].to_vec();
+    }
+
+    let mut selected = Vec::new();
+    for (i, idx) in old_indices.iter().enumerate() {
+        let covers_deleted = idx
+            .effective_fragment_bitmap(&dataset.fragment_bitmap)
+            .is_some_and(|eff| !eff.is_disjoint(&deleted_frags));
+        if i >= tail_start || covers_deleted {
+            selected.push(*idx);
+        }
+    }
+    selected
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn merge_scalar_indices<'a>(
+    dataset: Arc<Dataset>,
+    old_indices: &[&'a IndexMetadata],
+    unindexed: &[Fragment],
+    options: &OptimizeOptions,
+    index_type: IndexType,
+    field_path: &str,
+    column_name: &str,
+    base_unindexed_bitmap: RoaringBitmap,
+) -> Result<Option<(Uuid, Vec<&'a IndexMetadata>, RoaringBitmap, CreatedIndex)>> {
+    if old_indices.is_empty() {
+        return Err(Error::index(
+            "merge_scalar_indices: no previous index found".to_string(),
+        ));
+    }
+
+    let selected_old_indices = select_segments_to_merge(dataset.as_ref(), old_indices, options);
+
+    // No new data + ≤1 old selected = rewriting one segment to itself.
+    if unindexed.is_empty() && selected_old_indices.len() <= 1 {
+        return Ok(None);
+    }
+
+    // For the delta case (`selected` empty) the reference is purely
+    // for reading params; fall back to the last old index then.
+    let reference_idx = selected_old_indices
+        .first()
+        .copied()
+        .unwrap_or(old_indices[old_indices.len() - 1]);
+    let reference_index = dataset
+        .open_scalar_index(field_path, &reference_idx.uuid, &NoOpMetricsCollector)
+        .await?;
+    let update_criteria = reference_index.update_criteria();
+
+    // Effective = bitmap ∩ live fragments; deleted = bitmap \ live fragments.
+    let (effective_old_frags, deleted_old_frags) =
+        split_segment_coverage(dataset.as_ref(), selected_old_indices.iter().copied());
+
+    let mut frag_bitmap = base_unindexed_bitmap.clone();
+    frag_bitmap |= &effective_old_frags;
+    let new_uuid = Uuid::new_v4();
+
+    // Scalar Index that expos an N:1 segment-merge primitive reachable without
+    // rescanning the dataset
+    let has_segment_merge_primitive = matches!(index_type, IndexType::BTree);
+
+    // Merge new data into the existing segment(s) without rebuilding from
+    // scratch, when all hold:
+    //   - `effective_old_frags`: the selected segments' coverage intersected
+    //     with live fragments is non-empty, i.e. there is old data worth keeping.
+    //   - `update_criteria` only requires the newly appended data. Indexes that
+    //     need old data must rebuild over `frag_bitmap` so the scanned rows
+    //     exactly match the segment coverage being committed.
+    //   - `has_segment_merge_primitive` (Indices supports N:1 segments merge) OR
+    //     `selected_old_indices.len() == 1` (any scalar type can `update` one).
+    // Otherwise (e.g. ≥2 selected segments of a type without an N:1 merge
+    // primitive) the index is rebuilt from scratch over `frag_bitmap`.
+    let can_merge_segments = !effective_old_frags.is_empty()
+        && !update_criteria.requires_old_data
+        && (has_segment_merge_primitive || selected_old_indices.len() == 1);
+
+    let created_index = if !can_merge_segments {
+        rebuild_scalar_segment(
+            dataset.as_ref(),
+            &reference_index,
+            field_path,
+            column_name,
+            new_uuid,
+            frag_bitmap.iter().collect(),
+        )
+        .await?
+    } else {
+        let new_data_stream =
+            load_unindexed_training_data(dataset.as_ref(), field_path, &update_criteria, unindexed)
+                .await?;
+        let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?;
+
+        match index_type {
+            IndexType::BTree => {
+                let (_, old_data_filters) =
+                    build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?;
+                crate::index::scalar::btree::open_and_merge_segments(
+                    dataset.as_ref(),
+                    field_path,
+                    &selected_old_indices,
+                    new_data_stream,
+                    &new_store,
+                    &old_data_filters,
+                )
+                .await?
+            }
+            // NOTE: IndexType::Inverted never reaches here -- it is handled by the
+            // dedicated arm in merge_indices_with_unindexed_frags before this
+            // function is called.
+            _ => {
+                let old_data_filter = build_old_data_filter(
+                    dataset.as_ref(),
+                    &effective_old_frags,
+                    &deleted_old_frags,
+                )
+                .await?;
+                reference_index
+                    .update(new_data_stream, &new_store, old_data_filter)
+                    .await?
+            }
+        }
+    };
+
+    Ok(Some((
+        new_uuid,
+        selected_old_indices.to_vec(),
+        frag_bitmap,
+        created_index,
+    )))
+}
+
 async fn metadata_is_vector_index(dataset: &Dataset, index: &IndexMetadata) -> Result<bool> {
     if let Some(files) = &index.files {
         return Ok(files.iter().any(|file| file.path == INDEX_FILE_NAME));
@@ -238,7 +564,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 vec![(selected_metadata, selected_index)],
             )?;
             let selected_ivf_view = selected_logical_index.as_ivf()?;
-            let (new_uuid, indices_merged) = Box::pin(optimize_vector_indices(
+            let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices(
                 dataset.as_ref().clone(),
                 Option::<
                     lance_io::stream::RecordBatchStreamAdapter<
@@ -254,8 +580,6 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 return Ok(None);
             }
 
-            let index_dir = dataset.indices_dir().join(new_uuid.to_string());
-            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
             let new_fragment_bitmap = removed_segment
                 .effective_fragment_bitmap(&dataset.fragment_bitmap)
                 .or_else(|| removed_segment.fragment_bitmap.clone())
@@ -268,7 +592,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 CreatedIndex {
                     index_details: vector_index_details_default(),
                     index_version: lance_index::IndexType::Vector.version() as u32,
-                    files: Some(files),
+                    files: table_files_to_index(files),
                 },
             ))
         } else {
@@ -290,7 +614,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 Some(scanner.try_into_stream().await?)
             };
 
-            let (new_uuid, indices_merged) = optimize_vector_indices(
+            let (new_uuid, indices_merged, files) = optimize_vector_indices(
                 dataset.as_ref().clone(),
                 new_data_stream,
                 &field_path,
@@ -321,9 +645,6 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 .map(|d| d.as_ref().clone())
                 .unwrap_or_else(vector_index_details_default);
 
-            let index_dir = dataset.indices_dir().join(new_uuid.to_string());
-            let files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?;
-
             Ok((
                 new_uuid,
                 removed_indices,
@@ -334,16 +655,15 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     // index_version <= our max supported version, so we can safely
                     // write the current library's version for this index type.
                     index_version: lance_index::IndexType::Vector.version() as u32,
-                    files: Some(files),
+                    files: table_files_to_index(files),
                 },
             ))
         }
     } else {
-        let mut frag_bitmap = base_unindexed_bitmap.clone();
         let mut indices = Vec::with_capacity(old_indices.len());
         for idx in old_indices {
             match dataset
-                .open_generic_index(&field_path, &idx.uuid.to_string(), &NoOpMetricsCollector)
+                .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector)
                 .await
             {
                 Ok(index) => indices.push(index),
@@ -372,26 +692,17 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
         let index_type = indices[0].index_type();
         match index_type {
             IndexType::Inverted => {
-                let num_to_merge = options
-                    .num_indices_to_merge
-                    .unwrap_or(1)
-                    .min(old_indices.len());
-                if unindexed.is_empty() && num_to_merge <= 1 {
+                let selected_old_indices =
+                    select_segments_to_merge(dataset.as_ref(), old_indices, options);
+                if unindexed.is_empty() && selected_old_indices.len() <= 1 {
                     return Ok(None);
                 }
-
-                let selected_start = old_indices.len().saturating_sub(num_to_merge);
-                let selected_old_indices = &old_indices[selected_start..];
                 let reference_idx = selected_old_indices
                     .first()
                     .copied()
                     .unwrap_or(old_indices[old_indices.len() - 1]);
                 let reference_index = dataset
-                    .open_scalar_index(
-                        &field_path,
-                        &reference_idx.uuid.to_string(),
-                        &NoOpMetricsCollector,
-                    )
+                    .open_scalar_index(&field_path, &reference_idx.uuid, &NoOpMetricsCollector)
                     .await?;
                 let update_criteria = reference_index.update_criteria();
                 if update_criteria.requires_old_data {
@@ -409,7 +720,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                     let created_index = super::scalar::build_scalar_index(
                         dataset.as_ref(),
                         column.name.as_str(),
-                        &new_uuid.to_string(),
+                        new_uuid,
                         &params,
                         true,
                         None,
@@ -423,7 +734,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                         new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(),
                         new_index_version: created_index.index_version as i32,
                         new_index_details: created_index.index_details,
-                        files: created_index.files,
+                        files: index_files_to_table(created_index.files),
                     }));
                 }
 
@@ -441,18 +752,14 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 let mut frag_bitmap = base_unindexed_bitmap;
                 let mut effective_old_frags = RoaringBitmap::new();
                 let mut selected_indices = Vec::with_capacity(selected_old_indices.len());
-                for idx in selected_old_indices {
+                for idx in &selected_old_indices {
                     if let Some(effective) = idx.effective_fragment_bitmap(&dataset.fragment_bitmap)
                     {
                         frag_bitmap |= &effective;
                         effective_old_frags |= &effective;
                     }
                     let scalar_index = dataset
-                        .open_scalar_index(
-                            &field_path,
-                            &idx.uuid.to_string(),
-                            &NoOpMetricsCollector,
-                        )
+                        .open_scalar_index(&field_path, &idx.uuid, &NoOpMetricsCollector)
                         .await?;
                     let inverted_index = scalar_index
                         .as_any()
@@ -481,14 +788,13 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 };
 
                 let new_uuid = Uuid::new_v4();
-                let new_store =
-                    LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid.to_string())?;
+                let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?;
                 let created_index = if selected_indices.is_empty() {
                     let params = reference_index.derive_index_params()?;
                     super::scalar::build_scalar_index(
                         dataset.as_ref(),
                         column.name.as_str(),
-                        &new_uuid.to_string(),
+                        new_uuid,
                         &params,
                         true,
                         None,
@@ -515,105 +821,21 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
                 ))
             }
             it if it.is_scalar() => {
-                let num_to_merge = options
-                    .num_indices_to_merge
-                    .unwrap_or(1)
-                    .min(old_indices.len());
-                if unindexed.is_empty() && num_to_merge <= 1 {
-                    return Ok(None);
-                }
-
-                // Use effective bitmap (intersected with existing dataset fragments)
-                // to avoid carrying stale data from pruned indices.
-                let effective_old_frags: RoaringBitmap = old_indices
-                    .iter()
-                    .filter_map(|idx| idx.effective_fragment_bitmap(&dataset.fragment_bitmap))
-                    .fold(RoaringBitmap::new(), |mut acc, b| {
-                        acc |= &b;
-                        acc
-                    });
-                let deleted_old_frags: RoaringBitmap = old_indices
-                    .iter()
-                    .filter_map(|idx| idx.deleted_fragment_bitmap(&dataset.fragment_bitmap))
-                    .fold(RoaringBitmap::new(), |mut acc, b| {
-                        acc |= &b;
-                        acc
-                    });
-                frag_bitmap |= &effective_old_frags;
-
-                let index = dataset
-                    .open_scalar_index(
-                        &field_path,
-                        &old_indices[0].uuid.to_string(),
-                        &NoOpMetricsCollector,
-                    )
-                    .await?;
-
-                let update_criteria = index.update_criteria();
-
-                let fragments = if update_criteria.requires_old_data {
-                    None
-                } else {
-                    Some(unindexed.to_vec())
-                };
-                let new_data_stream = load_training_data(
-                    dataset.as_ref(),
+                let Some(result) = merge_scalar_indices(
+                    dataset.clone(),
+                    old_indices,
+                    unindexed,
+                    options,
+                    it,
                     &field_path,
-                    &update_criteria.data_criteria,
-                    fragments,
-                    true,
-                    None,
+                    column.name.as_str(),
+                    base_unindexed_bitmap,
                 )
-                .await?;
-
-                let new_uuid = Uuid::new_v4();
-
-                let created_index = if effective_old_frags.is_empty() {
-                    // Old data is fully stale (bitmap pruned to empty). Rebuild
-                    // from scratch instead of merging stale entries.
-                    let params = index.derive_index_params()?;
-                    super::scalar::build_scalar_index(
-                        dataset.as_ref(),
-                        column.name.as_str(),
-                        &new_uuid.to_string(),
-                        &params,
-                        true,
-                        None,
-                        Some(new_data_stream),
-                        Arc::new(NoopIndexBuildProgress),
-                    )
-                    .await?
-                } else {
-                    let new_store =
-                        LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid.to_string())?;
-                    let old_data_filter = if dataset.manifest.uses_stable_row_ids() {
-                        // Stable row IDs are opaque IDs, so fragment-bit filtering on
-                        // (row_id >> 32) is invalid. Build an exact allow-list from retained
-                        // fragments' row-id sequences and use precise filtering.
-                        let valid_old_row_ids =
-                            build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags)
-                                .await?;
-                        Some(OldIndexDataFilter::RowIds(valid_old_row_ids))
-                    } else {
-                        // Address-style row IDs encode fragment_id in high 32 bits.
-                        // Fragment bitmap filtering is valid and cheaper in this mode.
-                        Some(OldIndexDataFilter::Fragments {
-                            to_keep: effective_old_frags,
-                            to_remove: deleted_old_frags,
-                        })
-                    };
-                    index
-                        .update(new_data_stream, &new_store, old_data_filter)
-                        .await?
+                .await?
+                else {
+                    return Ok(None);
                 };
-
-                // TODO: don't hard-code index version
-                Ok((
-                    new_uuid,
-                    vec![old_indices[old_indices.len() - 1]],
-                    frag_bitmap,
-                    created_index,
-                ))
+                Ok(result)
             }
             _ => Err(Error::index(format!(
                 "Append index: invalid index type: {:?}",
@@ -628,7 +850,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>(
         new_fragment_bitmap,
         new_index_version: created_index.index_version as i32,
         new_index_details: created_index.index_details,
-        files: created_index.files,
+        files: index_files_to_table(created_index.files),
     }))
 }
 
@@ -641,8 +863,10 @@ mod tests {
     use arrow::datatypes::{Float32Type, UInt32Type};
     use arrow_array::cast::AsArray;
     use arrow_array::{
-        FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray, UInt32Array,
+        ArrayRef, FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray,
+        UInt32Array,
     };
+    use arrow_buffer::{BooleanBufferBuilder, NullBuffer};
     use arrow_schema::{DataType, Field, Schema};
     use futures::TryStreamExt;
     use lance_arrow::FixedSizeListArrayExt;
@@ -653,7 +877,7 @@ mod tests {
     use lance_index::vector::sq::builder::SQBuildParams;
     use lance_index::{
         IndexType,
-        scalar::ScalarIndexParams,
+        scalar::{BuiltinIndexType, ScalarIndexParams, SearchResult, TextQuery},
         vector::{ivf::IvfBuildParams, pq::PQBuildParams},
     };
     use lance_linalg::distance::MetricType;
@@ -661,8 +885,8 @@ mod tests {
     use rstest::rstest;
 
     use crate::dataset::builder::DatasetBuilder;
-    use crate::dataset::optimize::compact_files;
-    use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteParams};
+    use crate::dataset::optimize::{CompactionOptions, compact_files};
+    use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams};
     use crate::index::vector::VectorIndexParams;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
 
@@ -775,11 +999,7 @@ mod tests {
         let mut num_rows = 0;
         for index in indices.iter() {
             let index = dataset
-                .open_vector_index(
-                    "vector",
-                    index.uuid.to_string().as_str(),
-                    &NoOpMetricsCollector,
-                )
+                .open_vector_index("vector", &index.uuid, &NoOpMetricsCollector)
                 .await
                 .unwrap();
             num_rows += index.num_rows();
@@ -787,6 +1007,118 @@ mod tests {
         assert_eq!(num_rows, 2000);
     }
 
+    #[tokio::test]
+    async fn test_optimize_append_preserves_case_sensitive_nullable_vector_column() {
+        const DIM: usize = 64;
+        const ROWS: usize = 1000;
+
+        fn make_vectors(rows: usize, dim: usize, include_null: bool) -> FixedSizeListArray {
+            if include_null {
+                let mut nulls_builder = BooleanBufferBuilder::new(rows);
+                for row_idx in 0..rows {
+                    nulls_builder.append(row_idx != 0);
+                }
+                let nulls = NullBuffer::new(nulls_builder.finish());
+                FixedSizeListArray::try_new(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    dim as i32,
+                    Arc::new(generate_random_array(rows * dim)),
+                    Some(nulls),
+                )
+                .unwrap()
+            } else {
+                FixedSizeListArray::try_new_from_values(
+                    generate_random_array(rows * dim),
+                    dim as i32,
+                )
+                .unwrap()
+            }
+        }
+
+        fn make_batch(
+            schema: Arc<Schema>,
+            start_id: u32,
+            vectors: Arc<FixedSizeListArray>,
+        ) -> RecordBatch {
+            let columns: Vec<ArrayRef> = vec![
+                Arc::new(UInt32Array::from_iter_values(
+                    start_id..start_id + ROWS as u32,
+                )) as ArrayRef,
+                vectors as ArrayRef,
+            ];
+            RecordBatch::try_new(schema, columns).unwrap()
+        }
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let vector_type = DataType::FixedSizeList(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            DIM as i32,
+        );
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new("VECTOR", vector_type, true),
+        ]));
+
+        let initial_vectors = Arc::new(make_vectors(ROWS, DIM, false));
+        let initial_batch = make_batch(schema.clone(), 0, initial_vectors);
+        let batches = RecordBatchIterator::new(std::iter::once(Ok(initial_batch)), schema.clone());
+        let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap();
+
+        let params = VectorIndexParams::with_ivf_pq_params(
+            MetricType::L2,
+            IvfBuildParams::new(2),
+            PQBuildParams {
+                num_sub_vectors: 2,
+                ..Default::default()
+            },
+        );
+        dataset
+            .create_index(&["VECTOR"], IndexType::Vector, None, &params, true)
+            .await
+            .unwrap();
+
+        let appended_vectors = Arc::new(make_vectors(ROWS, DIM, true));
+        let query = appended_vectors.value(5);
+        let appended_batch = make_batch(schema.clone(), ROWS as u32, appended_vectors);
+        let batches = RecordBatchIterator::new(std::iter::once(Ok(appended_batch)), schema);
+        dataset.append(batches, None).await.unwrap();
+
+        let index_name = dataset.load_indices().await.unwrap()[0].name.clone();
+        assert!(
+            !dataset
+                .unindexed_fragments(&index_name)
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        assert!(
+            dataset
+                .unindexed_fragments(&index_name)
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        let mut scanner = dataset.scan();
+        scanner
+            .nearest("VECTOR", query.as_primitive::<Float32Type>(), 10)
+            .unwrap();
+        let results = scanner.try_into_batch().await.unwrap();
+        assert_eq!(
+            results.num_rows(),
+            10,
+            "expected the requested k=10 nearest-neighbor results"
+        );
+    }
+
     /// Regression: a second `OptimizeOptions::append()` call on a steady-state
     /// vector index used to fall through to `optimize_vector_indices` and write
     /// a new UUID directory + manifest even though nothing had changed. The
@@ -995,7 +1327,23 @@ mod tests {
         assert_eq!(results.num_rows(), 2);
         let mut id_arr = results["id"].as_primitive::<UInt32Type>().values().to_vec();
         id_arr.sort();
-        assert_eq!(id_arr, vec![0, 1000]);
+        // For exact indexes (e.g. IvfFlat) the top-2 nearest neighbors of the
+        // query vector are deterministic (id 0 in the first delta and id 1000
+        // in the second delta, since the same vector is duplicated across
+        // the two fragments). For approximate indexes (e.g. IvfPq, IvfHnswSq)
+        // the returned ids are not guaranteed to be exactly [0, 1000]; the key
+        // property this test verifies is that both delta indices are queried
+        // (i.e. one result comes from each delta), so we only assert that.
+        let is_approximate = !matches!(index_params.index_type(), IndexType::IvfFlat);
+        if is_approximate {
+            assert!(
+                id_arr[0] < TOTAL as u32 && id_arr[1] >= TOTAL as u32,
+                "expected one result from each delta index, got {:?}",
+                id_arr
+            );
+        } else {
+            assert_eq!(id_arr, vec![0, 1000]);
+        }
     }
 
     #[tokio::test]
@@ -1220,7 +1568,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_optimize_btree_keeps_rows_with_stable_row_ids_after_compaction() {
+    async fn test_optimize_btree_multi_segment_optimize_default() {
         async fn query_id_count(dataset: &Dataset, id: &str) -> usize {
             dataset
                 .scan()
@@ -1238,69 +1586,965 @@ mod tests {
         let test_uri = test_dir.as_str();
 
         let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
-        let ids = StringArray::from_iter_values((0..256).map(|i| format!("song-{i}")));
-        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids)]).unwrap();
-        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let make_batch = |start: i32, end: i32| {
+            let ids = StringArray::from_iter_values((start..end).map(|i| format!("song-{i}")));
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids)]).unwrap()
+        };
+
+        // Three fragments of 64 rows each; each commits as its own BTree
+        // segment so optimize sees a multi-segment scalar logical index.
+        let reader = RecordBatchIterator::new(
+            vec![
+                Ok(make_batch(0, 64)),
+                Ok(make_batch(64, 128)),
+                Ok(make_batch(128, 192)),
+            ],
+            schema.clone(),
+        );
         let mut dataset = Dataset::write(
             reader,
             test_uri,
             Some(WriteParams {
                 max_rows_per_file: 64,
-                enable_stable_row_ids: true,
                 ..Default::default()
             }),
         )
         .await
         .unwrap();
 
-        dataset
-            .create_index(
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 3);
+
+        let mut staged_segments = Vec::new();
+        for fragment in &fragments {
+            let segment = crate::index::create::CreateIndexBuilder::new(
+                &mut dataset,
                 &["id"],
                 IndexType::BTree,
-                Some("id_idx".into()),
-                &ScalarIndexParams::default(),
-                true,
+                &params,
             )
+            .name("id_idx".into())
+            .fragments(vec![fragment.id() as u32])
+            .execute_uncommitted()
             .await
             .unwrap();
+            staged_segments.push(segment);
+        }
+        dataset
+            .commit_existing_index_segments("id_idx", "id", staged_segments)
+            .await
+            .unwrap();
+        assert_eq!(
+            dataset.load_indices_by_name("id_idx").await.unwrap().len(),
+            3
+        );
 
-        assert_eq!(query_id_count(&dataset, "song-42").await, 1);
-
-        compact_files(
-            &mut dataset,
-            crate::dataset::optimize::CompactionOptions {
-                target_rows_per_fragment: 512,
+        let appended = RecordBatchIterator::new(vec![Ok(make_batch(192, 256))], schema.clone());
+        let mut dataset = Dataset::write(
+            appended,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Append,
                 ..Default::default()
-            },
-            None,
+            }),
         )
         .await
         .unwrap();
-
-        let frags = dataset.get_fragments();
-        assert!(!frags.is_empty());
-        assert!(frags.iter().all(|frag| frag.id() > 0));
-        assert!(
-            dataset
-                .unindexed_fragments("id_idx")
-                .await
-                .unwrap()
-                .is_empty()
-        );
+        assert_eq!(dataset.get_fragments().len(), 4);
 
         dataset
             .optimize_indices(&OptimizeOptions::default())
             .await
             .unwrap();
 
+        // Reload from disk to ensure we're reading committed manifest state.
         let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
-        assert_eq!(query_id_count(&dataset, "song-42").await, 1);
-    }
-
-    #[tokio::test]
-    async fn test_optimize_scalar_no_unindexed_fragments() {
-        let test_dir = TempStrDir::default();
-        let test_uri = test_dir.as_str();
+
+        // Each of these IDs lives in a distinct old segment / fragment.
+        // song-10 lives in fragment 0, song-80 in fragment 1, song-160 in
+        // fragment 2, and song-200 in the appended fragment. After optimize
+        // every row must still be reachable through the logical index,
+        // regardless of which segment absorbed the new data.
+        for id in ["song-10", "song-80", "song-160", "song-200"] {
+            assert_eq!(
+                query_id_count(&dataset, id).await,
+                1,
+                "expected exactly one row for {id} after multi-segment optimize"
+            );
+        }
+
+        // `OptimizeOptions::default()` (= num_indices_to_merge: None) merges
+        // the newest segment with the unindexed fragment, like the
+        // inverted/vector default. The three old segments minus the merged one
+        // plus the new delta means three segments remain, and together they
+        // must still cover every dataset fragment without overlap.
+        let segments_after = dataset.load_indices_by_name("id_idx").await.unwrap();
+        assert_eq!(
+            segments_after.len(),
+            3,
+            "default optimize must merge one delta, not all segments, got {segments_after:?}"
+        );
+        let mut covered = RoaringBitmap::new();
+        for segment in &segments_after {
+            let bitmap = segment
+                .fragment_bitmap
+                .as_ref()
+                .expect("each segment should carry fragment coverage");
+            assert!(
+                covered.is_disjoint(bitmap),
+                "post-optimize segments must not overlap, got {segments_after:?}"
+            );
+            covered |= bitmap;
+        }
+        let mut expected = RoaringBitmap::new();
+        for frag in dataset.get_fragments() {
+            expected.insert(frag.id() as u32);
+        }
+        assert_eq!(
+            covered, expected,
+            "post-optimize segments should cover every dataset fragment"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_optimize_fmindex_default_rebuilds_old_and_new_rows() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
+        let make_batch = |values: &[&str]| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(StringArray::from_iter_values(
+                    values.iter().copied(),
+                ))],
+            )
+            .unwrap()
+        };
+
+        let reader = RecordBatchIterator::new(
+            vec![Ok(make_batch(&["old alpha needle", "old beta"]))],
+            schema.clone(),
+        );
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                max_rows_per_file: 2,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        dataset
+            .create_index(
+                &["text"],
+                IndexType::Fm,
+                Some("text_fmindex".to_string()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        let appended = RecordBatchIterator::new(
+            vec![Ok(make_batch(&["new gamma needle", "new delta"]))],
+            schema.clone(),
+        );
+        dataset.append(appended, None).await.unwrap();
+
+        assert!(
+            !dataset
+                .unindexed_fragments("text_fmindex")
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        assert!(
+            dataset
+                .unindexed_fragments("text_fmindex")
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        let committed = dataset.load_indices_by_name("text_fmindex").await.unwrap();
+        assert_eq!(committed.len(), 1);
+        assert_eq!(
+            committed[0]
+                .fragment_bitmap
+                .as_ref()
+                .expect("FMIndex segment should carry fragment coverage")
+                .len(),
+            2
+        );
+
+        let logical = crate::index::scalar_logical::open_named_scalar_index(
+            &dataset,
+            "text",
+            "text_fmindex",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+
+        for (pattern, expected) in [("old alpha", 1), ("new gamma", 1), ("needle", 2)] {
+            let query = TextQuery::StringContains(pattern.to_string());
+            let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+            let row_addrs = match result {
+                SearchResult::Exact(row_addrs) => row_addrs,
+                other => panic!("expected exact result for {pattern}, got {other:?}"),
+            };
+            let count = row_addrs.true_rows().row_addrs().unwrap().count();
+            assert_eq!(
+                count, expected,
+                "expected {expected} matches for {pattern}, got {count}"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_optimize_btree_optimize_append() {
+        async fn query_id_count(dataset: &Dataset, id: &str) -> usize {
+            dataset
+                .scan()
+                .filter(&format!("id = '{}'", id))
+                .unwrap()
+                .project(&["id"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap()
+                .num_rows()
+        }
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
+        let make_batch = |start: i32, end: i32| {
+            let ids = StringArray::from_iter_values((start..end).map(|i| format!("song-{i}")));
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids)]).unwrap()
+        };
+
+        // Start with two fragments + two committed BTree segments.
+        let reader = RecordBatchIterator::new(
+            vec![Ok(make_batch(0, 64)), Ok(make_batch(64, 128))],
+            schema.clone(),
+        );
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let original_segment_uuids: Vec<_> = {
+            let mut staged = Vec::new();
+            for fragment in dataset.get_fragments() {
+                let segment = crate::index::create::CreateIndexBuilder::new(
+                    &mut dataset,
+                    &["id"],
+                    IndexType::BTree,
+                    &params,
+                )
+                .name("id_idx".into())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+                staged.push(segment);
+            }
+            let uuids = staged.iter().map(|s| s.uuid).collect::<Vec<_>>();
+            dataset
+                .commit_existing_index_segments("id_idx", "id", staged)
+                .await
+                .unwrap();
+            uuids
+        };
+        assert_eq!(original_segment_uuids.len(), 2);
+
+        // Append a third fragment, leave it unindexed, then run append-mode optimize.
+        let appended = RecordBatchIterator::new(vec![Ok(make_batch(128, 192))], schema.clone());
+        let mut dataset = Dataset::write(
+            appended,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        // Read fresh from disk to make sure we're inspecting committed state.
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+
+        // append() must preserve every original old segment unchanged and add
+        // exactly one new segment covering only the newly appended fragments.
+        let committed = dataset.load_indices_by_name("id_idx").await.unwrap();
+        let committed_uuids: std::collections::HashSet<_> =
+            committed.iter().map(|idx| idx.uuid).collect();
+        for original in &original_segment_uuids {
+            assert!(
+                committed_uuids.contains(original),
+                "append() must not remove pre-existing segment {original}, \
+                 but the committed UUIDs are {committed_uuids:?}"
+            );
+        }
+        assert_eq!(
+            committed.len(),
+            original_segment_uuids.len() + 1,
+            "append() should add exactly one new delta segment, got {committed:?}"
+        );
+        let new_segment = committed
+            .iter()
+            .find(|idx| !original_segment_uuids.contains(&idx.uuid))
+            .expect("append() must add a new delta segment");
+        let new_segment_frags: Vec<_> = new_segment
+            .fragment_bitmap
+            .as_ref()
+            .unwrap()
+            .iter()
+            .collect();
+        // The appended fragment should be the only one covered by the new delta;
+        // old segments retain their own coverage.
+        assert_eq!(new_segment_frags.len(), 1);
+
+        // Sanity check: queries across all fragments still return their rows.
+        for id in ["song-10", "song-100", "song-160"] {
+            assert_eq!(query_id_count(&dataset, id).await, 1, "missing row {id}");
+        }
+    }
+
+    #[tokio::test]
+    async fn test_optimize_bitmap_index_append() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "category",
+            DataType::Utf8,
+            false,
+        )]));
+        let make_batch = |labels: &[&str]| {
+            let arr = StringArray::from_iter_values(labels.iter().copied());
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(arr)]).unwrap()
+        };
+
+        // One fragment + one Bitmap segment.
+        let reader =
+            RecordBatchIterator::new(vec![Ok(make_batch(&["a", "b", "a", "c"]))], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 4,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Bitmap);
+        dataset
+            .create_index(
+                &["category"],
+                IndexType::Bitmap,
+                Some("cat_idx".into()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+        let original_uuid = {
+            let committed = dataset.load_indices_by_name("cat_idx").await.unwrap();
+            assert_eq!(committed.len(), 1);
+            committed[0].uuid
+        };
+
+        // Append a second fragment, leave it unindexed, then optimize with
+        // `append()` (= num_indices_to_merge: Some(0)).
+        let appended =
+            RecordBatchIterator::new(vec![Ok(make_batch(&["b", "d", "d", "a"]))], schema.clone());
+        let mut dataset = Dataset::write(
+            appended,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 4,
+                mode: WriteMode::Append,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+
+        // append() (= num_indices_to_merge: Some(0)) is now honored uniformly:
+        // Bitmap, like BTree, must keep the original segment untouched and add
+        // exactly one delta segment covering only the appended fragment.
+        let committed = dataset.load_indices_by_name("cat_idx").await.unwrap();
+        assert_eq!(
+            committed.len(),
+            2,
+            "Bitmap optimize append() must add a delta segment, not merge, got {committed:?}"
+        );
+        assert!(
+            committed.iter().any(|idx| idx.uuid == original_uuid),
+            "append() must preserve the pre-existing segment {original_uuid}, got {committed:?}"
+        );
+        let new_segment = committed
+            .iter()
+            .find(|idx| idx.uuid != original_uuid)
+            .expect("append() must add a new delta segment");
+        let new_segment_frags: std::collections::BTreeSet<u32> = new_segment
+            .fragment_bitmap
+            .as_ref()
+            .expect("delta Bitmap should carry fragment coverage")
+            .iter()
+            .collect();
+        assert_eq!(
+            new_segment_frags,
+            [1u32].into_iter().collect(),
+            "the delta segment must cover only the appended fragment"
+        );
+
+        // Data correctness: a value that lives only in the appended fragment
+        // is queryable through the (now multi-segment) index.
+        let rows = dataset
+            .scan()
+            .filter("category = 'd'")
+            .unwrap()
+            .project(&["category"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(rows, 2, "value 'd' lives in appended fragment");
+    }
+
+    #[tokio::test]
+    async fn test_optimize_btree_keeps_rows_with_stable_row_ids_after_compaction() {
+        async fn query_id_count(dataset: &Dataset, id: &str) -> usize {
+            dataset
+                .scan()
+                .filter(&format!("id = '{}'", id))
+                .unwrap()
+                .project(&["id"])
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap()
+                .num_rows()
+        }
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
+        let ids = StringArray::from_iter_values((0..256).map(|i| format!("song-{i}")));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids)]).unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset
+            .create_index(
+                &["id"],
+                IndexType::BTree,
+                Some("id_idx".into()),
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(query_id_count(&dataset, "song-42").await, 1);
+
+        compact_files(
+            &mut dataset,
+            crate::dataset::optimize::CompactionOptions {
+                target_rows_per_fragment: 512,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        let frags = dataset.get_fragments();
+        assert!(!frags.is_empty());
+        assert!(frags.iter().all(|frag| frag.id() > 0));
+        assert!(
+            dataset
+                .unindexed_fragments("id_idx")
+                .await
+                .unwrap()
+                .is_empty()
+        );
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        assert_eq!(query_id_count(&dataset, "song-42").await, 1);
+    }
+
+    /// Under stable row ids, updating an indexed column and then calling
+    /// `optimize_indices` must not leave stale entries (old value -> updated row)
+    /// in the scalar index. An update deletes the old copy of each row and
+    /// rewrites it under the same stable row id, so the old index entry is stale
+    /// and must be dropped on merge. Covers BTree, Bitmap, and Inverted (FTS),
+    /// which take three different merge paths.
+    #[tokio::test]
+    async fn test_optimize_scalar_index_drops_stale_rows_after_update() {
+        use crate::dataset::UpdateBuilder;
+        use arrow_array::Int32Array;
+        use lance_index::scalar::FullTextSearchQuery;
+        use lance_index::scalar::inverted::InvertedIndexParams;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        // 100 rows: num == id; cat = "A" for id<50 else "B"; body = "alpha" for
+        // id<50 else "beta".
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("num", DataType::Int32, false),
+            Field::new("cat", DataType::Utf8, false),
+            Field::new("body", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(StringArray::from_iter_values(
+                    (0..100).map(|i| if i < 50 { "A" } else { "B" }),
+                )),
+                Arc::new(StringArray::from_iter_values(
+                    (0..100).map(|i| if i < 50 { "alpha" } else { "beta" }),
+                )),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset
+            .create_index(
+                &["num"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["cat"],
+                IndexType::Bitmap,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["body"],
+                IndexType::Inverted,
+                None,
+                &InvertedIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Update the first 25 rows (id < 25): num -> -1, cat -> 'B', body -> 'beta'.
+        let res = UpdateBuilder::new(Arc::new(dataset.clone()))
+            .update_where("id < 25")
+            .unwrap()
+            .set("num", "-1")
+            .unwrap()
+            .set("cat", "'B'")
+            .unwrap()
+            .set("body", "'beta'")
+            .unwrap()
+            .build()
+            .unwrap()
+            .execute()
+            .await
+            .unwrap();
+        dataset = res.new_dataset.as_ref().clone();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+
+        // BTree: `num >= 0` matches ids 25..99 (75 rows); the 25 updated rows
+        // hold num = -1 and must not appear.
+        let btree_count = dataset
+            .scan()
+            .filter("num >= 0")
+            .unwrap()
+            .count_rows()
+            .await
+            .unwrap();
+        assert_eq!(btree_count, 75, "btree returned stale/incorrect rows");
+
+        // Bitmap: only the 25 rows (ids 25..49) that still carry cat = 'A' match;
+        // the 25 rows updated to 'B' must not.
+        let bitmap_count = dataset
+            .scan()
+            .filter("cat = 'A'")
+            .unwrap()
+            .count_rows()
+            .await
+            .unwrap();
+        assert_eq!(bitmap_count, 25, "bitmap returned stale rows");
+
+        // FTS: only the 25 rows (ids 25..49) whose body still reads "alpha" match;
+        // the 25 rows updated to "beta" must not.
+        let mut scan = dataset.scan();
+        scan.full_text_search(FullTextSearchQuery::new("alpha".to_owned()))
+            .unwrap();
+        let fts_count = scan.count_rows().await.unwrap();
+        assert_eq!(fts_count, 25, "FTS index returned stale rows");
+    }
+
+    /// Multi-segment variant (Jack Ye's repro, PR #7359): with one BTree segment
+    /// per fragment, default optimize merges only the tail segment. A stable-row-id
+    /// update to a row in an older segment's fragment must still drop that
+    /// segment's stale postings -- the merge has to reach back to cover it.
+    #[tokio::test]
+    async fn test_optimize_btree_drops_stale_rows_across_segments_after_update() {
+        use crate::dataset::UpdateBuilder;
+        use crate::index::CreateIndexBuilder;
+        use arrow_array::Int32Array;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("num", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(Int32Array::from_iter_values(0..100)),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        // Two fragments (0..49, 50..99) -> one BTree segment each.
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                max_rows_per_file: 50,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
+        let fragments = dataset.get_fragments();
+        let mut segments = Vec::new();
+        for fragment in &fragments {
+            segments.push(
+                CreateIndexBuilder::new(&mut dataset, &["num"], IndexType::BTree, &params)
+                    .name("num_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("num_idx", "num", segments)
+            .await
+            .unwrap();
+
+        // Update the first 25 rows (in the first/older segment's fragment).
+        let res = UpdateBuilder::new(Arc::new(dataset.clone()))
+            .update_where("id < 25")
+            .unwrap()
+            .set("num", "-1")
+            .unwrap()
+            .build()
+            .unwrap()
+            .execute()
+            .await
+            .unwrap();
+        dataset = res.new_dataset.as_ref().clone();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+
+        assert_eq!(
+            dataset
+                .scan()
+                .filter("num = 0")
+                .unwrap()
+                .count_rows()
+                .await
+                .unwrap(),
+            0,
+            "stale entry leaked from the older, unmerged segment"
+        );
+        assert_eq!(
+            dataset
+                .scan()
+                .filter("num >= 0")
+                .unwrap()
+                .count_rows()
+                .await
+                .unwrap(),
+            75
+        );
+    }
+
+    /// Same multi-segment gap for FTS, which takes the separate Inverted dispatch
+    /// path. One Inverted segment per fragment; an update to the older segment's
+    /// fragment must not leave its old-token postings searchable.
+    #[tokio::test]
+    async fn test_optimize_fts_drops_stale_rows_across_segments_after_update() {
+        use crate::dataset::UpdateBuilder;
+        use crate::index::CreateIndexBuilder;
+        use arrow_array::Int32Array;
+        use lance_index::scalar::FullTextSearchQuery;
+        use lance_index::scalar::inverted::InvertedIndexParams;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("body", DataType::Utf8, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(StringArray::from_iter_values(
+                    (0..100).map(|i| if i < 50 { "alpha" } else { "beta" }),
+                )),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                max_rows_per_file: 50,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params = InvertedIndexParams::default();
+        let fragments = dataset.get_fragments();
+        let mut segments = Vec::new();
+        for fragment in &fragments {
+            segments.push(
+                CreateIndexBuilder::new(&mut dataset, &["body"], IndexType::Inverted, &params)
+                    .name("body_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("body_idx", "body", segments)
+            .await
+            .unwrap();
+
+        // Update the first 25 rows (older segment's fragment): body -> "beta".
+        let res = UpdateBuilder::new(Arc::new(dataset.clone()))
+            .update_where("id < 25")
+            .unwrap()
+            .set("body", "'beta'")
+            .unwrap()
+            .build()
+            .unwrap()
+            .execute()
+            .await
+            .unwrap();
+        dataset = res.new_dataset.as_ref().clone();
+
+        dataset
+            .optimize_indices(&OptimizeOptions::default())
+            .await
+            .unwrap();
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+
+        let mut scan = dataset.scan();
+        scan.full_text_search(FullTextSearchQuery::new("alpha".to_owned()))
+            .unwrap();
+        assert_eq!(
+            scan.count_rows().await.unwrap(),
+            25,
+            "FTS stale rows leaked from the older, unmerged segment"
+        );
+    }
+
+    /// `optimize_indices` builds the stable-row-id allow-list by subtracting each
+    /// fragment's deletion vector. If a deletion vector cannot be read, the merge
+    /// must fail loudly: swallowing the error (treating the load as "no
+    /// deletions") would put every deleted row back into the allow-list and
+    /// silently reintroduce the stale entries this fix removes. Simulate an
+    /// unreadable deletion vector by deleting the file the manifest still
+    /// references, then assert optimize errors instead of succeeding.
+    #[tokio::test]
+    async fn test_optimize_errors_when_deletion_vector_unreadable() {
+        use crate::dataset::UpdateBuilder;
+        use arrow_array::Int32Array;
+        use lance_table::io::deletion::deletion_file_path;
+
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("num", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from_iter_values(0..100)),
+                Arc::new(Int32Array::from_iter_values(0..100)),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        dataset
+            .create_index(
+                &["num"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Update rewrites the first 25 rows under the same stable row ids,
+        // leaving a deletion vector on the original fragment.
+        UpdateBuilder::new(Arc::new(dataset.clone()))
+            .update_where("id < 25")
+            .unwrap()
+            .set("num", "-1")
+            .unwrap()
+            .build()
+            .unwrap()
+            .execute()
+            .await
+            .unwrap();
+
+        // Reload cold (nothing has cached the deletion vector), then remove the
+        // deletion file the manifest still references so the next read fails.
+        let mut dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        let mut removed = 0;
+        for fragment in dataset.get_fragments() {
+            if let Some(deletion_file) = fragment.metadata().deletion_file.clone() {
+                let path =
+                    deletion_file_path(&dataset.base, fragment.metadata().id, &deletion_file);
+                dataset.object_store.delete(&path).await.unwrap();
+                removed += 1;
+            }
+        }
+        assert_eq!(
+            removed, 1,
+            "update should have left exactly one deletion file"
+        );
+
+        let result = dataset.optimize_indices(&OptimizeOptions::default()).await;
+        assert!(
+            result.is_err(),
+            "optimize must fail when a deletion vector cannot be read, not \
+             silently keep the deleted rows in the index"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_optimize_scalar_no_unindexed_fragments() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
 
         let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
         let ids = StringArray::from_iter_values((0..32).map(|i| format!("song-{i}")));
@@ -1353,4 +2597,205 @@ mod tests {
         assert_eq!(after_default[0].uuid, original_uuid);
         assert_eq!(dataset.manifest.version, original_version);
     }
+
+    #[rstest]
+    #[case::address_row_ids(false)]
+    #[case::stable_row_ids(true)]
+    #[tokio::test]
+    async fn test_optimize_btree_no_duplicate_row_addr(#[case] use_stable_row_ids: bool) {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("payload", DataType::Int32, false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1])),
+                Arc::new(Int32Array::from(vec![10])),
+            ],
+        )
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let write_params = WriteParams {
+            enable_stable_row_ids: use_stable_row_ids,
+            ..Default::default()
+        };
+        let mut dataset = Dataset::write(reader, test_uri, Some(write_params))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
+        dataset
+            .create_index(
+                &["id"],
+                IndexType::BTree,
+                Some("id_idx".into()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Reordered source columns (payload, id) force the partial-schema
+        // RewriteColumns path instead of a full row rewrite.
+        let source_schema = Arc::new(Schema::new(vec![
+            Field::new("payload", DataType::Int32, false),
+            Field::new("id", DataType::Int32, false),
+        ]));
+        let source_batch = RecordBatch::try_new(
+            source_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![100])),
+                Arc::new(Int32Array::from(vec![1])),
+            ],
+        )
+        .unwrap();
+        let merge_job =
+            MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()])
+                .unwrap()
+                .when_matched(WhenMatched::UpdateAll)
+                .try_build()
+                .unwrap();
+        let source_reader = Box::new(RecordBatchIterator::new(
+            [Ok(source_batch)],
+            source_schema.clone(),
+        ));
+        merge_job
+            .execute(reader_to_stream(source_reader))
+            .await
+            .unwrap();
+
+        // Build a delta BTree segment over the now-unindexed fragment.
+        let mut dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+        assert_eq!(
+            dataset.load_indices_by_name("id_idx").await.unwrap().len(),
+            2,
+            "append must create a delta segment over the rewritten fragment"
+        );
+
+        // Force the old segment + delta segment to merge.
+        dataset
+            .optimize_indices(&OptimizeOptions::merge(2))
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        let rows = dataset
+            .scan()
+            .filter("id = 1")
+            .unwrap()
+            .project(&["id"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(rows, 1, "id = 1 must return exactly one row after merge");
+    }
+
+    #[tokio::test]
+    async fn test_optimize_btree_merge_remaps_deferred_compaction() {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let make = |range: std::ops::Range<i32>| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(Int32Array::from_iter_values(range))],
+            )
+            .unwrap()
+        };
+
+        // Two fragments: [0, 50) and [50, 100).
+        let reader =
+            RecordBatchIterator::new(vec![Ok(make(0..50)), Ok(make(50..100))], schema.clone());
+        let mut dataset = Dataset::write(
+            reader,
+            test_uri,
+            Some(WriteParams {
+                max_rows_per_file: 50,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        assert_eq!(dataset.get_fragments().len(), 2);
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
+        dataset
+            .create_index(
+                &["id"],
+                IndexType::BTree,
+                Some("id_idx".into()),
+                &params,
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Deferred-remap compaction fuses the two fragments into one and leaves a
+        // pending FragReuseIndex; the index segment is not eagerly remapped.
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                defer_index_remap: true,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        // Append a third fragment, left unindexed.
+        let mut dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        dataset
+            .append(
+                RecordBatchIterator::new(vec![Ok(make(100..150))], schema.clone()),
+                None,
+            )
+            .await
+            .unwrap();
+
+        // Merge the deferred-remapped old segment with the new delta.
+        dataset
+            .optimize_indices(&OptimizeOptions::merge(2))
+            .await
+            .unwrap();
+
+        let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap();
+        // A value from the compacted fragments must still be found via the index.
+        let hit = dataset
+            .scan()
+            .filter("id = 25")
+            .unwrap()
+            .project(&["id"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(
+            hit, 1,
+            "compacted-then-merged row must remain queryable via the index"
+        );
+        let total = dataset
+            .scan()
+            .filter("id >= 0")
+            .unwrap()
+            .project(&["id"])
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert_eq!(total, 150, "no rows may be lost across compaction + merge");
+    }
 }
diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs
index af7ea7ce19c..7e5fe98bbf0 100644
--- a/rust/lance/src/index/create.rs
+++ b/rust/lance/src/index/create.rs
@@ -10,21 +10,24 @@ use crate::{
     index::{
         DatasetIndexExt, DatasetIndexInternalExt, IntoIndexSegment,
         build_index_metadata_from_segments,
-        scalar::build_scalar_index,
+        scalar::{build_bitmap_index_segment, build_scalar_index},
         vector::{
-            LANCE_VECTOR_INDEX, VectorIndexParams, build_distributed_vector_index,
-            build_empty_vector_index, build_vector_index,
+            LANCE_VECTOR_INDEX, StageParams, VectorIndexParams, build_distributed_vector_index,
+            build_empty_vector_index, build_filtered_vector_index, build_vector_index,
         },
         vector_index_details, vector_index_details_default,
     },
 };
-use futures::future::BoxFuture;
+use futures::{FutureExt, future::BoxFuture};
 use lance_core::datatypes::format_field_path;
 use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress};
 use lance_index::{IndexParams, IndexType, scalar::CreatedIndex};
 use lance_index::{
     metrics::NoOpMetricsCollector,
-    scalar::{LANCE_SCALAR_INDEX, ScalarIndexParams, inverted::tokenizer::InvertedIndexParams},
+    scalar::{
+        LANCE_SCALAR_INDEX, ScalarIndexParams, index_files_to_table,
+        inverted::tokenizer::InvertedIndexParams, table_files_to_index,
+    },
 };
 use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
 use std::{collections::HashMap, future::IntoFuture, sync::Arc};
@@ -53,7 +56,7 @@ pub struct CreateIndexBuilder<'a> {
     replace: bool,
     train: bool,
     fragments: Option<Vec<u32>>,
-    index_uuid: Option<String>,
+    index_uuid: Option<Uuid>,
     preprocessed_data: Option<Box<dyn RecordBatchReader + Send + 'static>>,
     progress: Arc<dyn IndexBuildProgress>,
     /// Transaction properties to store with this commit.
@@ -103,7 +106,7 @@ impl<'a> CreateIndexBuilder<'a> {
         self
     }
 
-    pub fn index_uuid(mut self, uuid: String) -> Self {
+    pub fn index_uuid(mut self, uuid: Uuid) -> Self {
         self.index_uuid = Some(uuid);
         self
     }
@@ -133,6 +136,11 @@ impl<'a> CreateIndexBuilder<'a> {
 
     #[instrument(skip_all)]
     pub async fn execute_uncommitted(&mut self) -> Result<IndexMetadata> {
+        self.execute_uncommitted_impl().await
+    }
+
+    fn execute_uncommitted_impl(&mut self) -> BoxFuture<'_, Result<IndexMetadata>> {
+        async move {
         if self.columns.len() != 1 {
             return Err(Error::index(
                 "Only support building index on 1 column at the moment".to_string(),
@@ -153,12 +161,16 @@ impl<'a> CreateIndexBuilder<'a> {
         let quoted_column: String = format_field_path(&names);
         let column = quoted_column.as_str();
 
-        // If train is true but dataset is empty, automatically set train to false
-        let train = if self.train {
-            self.dataset.count_rows(None).await? > 0
-        } else {
-            false
-        };
+        let vector_fragments_for_validation =
+            is_builtin_vector_index(self.index_type, self.params)
+                .then_some(self.fragments.as_deref())
+                .flatten();
+        let train = should_train_index(
+            self.dataset,
+            self.train,
+            vector_fragments_for_validation,
+        )
+        .await?;
 
         // Load indices from the disk.
         let indices = self.dataset.load_indices().await?;
@@ -207,14 +219,10 @@ impl<'a> CreateIndexBuilder<'a> {
             self.index_type,
             self.params,
             self.fragments.as_ref(),
-            self.index_uuid.as_deref(),
+            self.index_uuid.as_ref(),
         )?;
 
-        let index_id = match &self.index_uuid {
-            Some(uuid_str) => Uuid::parse_str(uuid_str)
-                .map_err(|e| Error::index(format!("Invalid UUID string provided: {}", e)))?,
-            None => Uuid::new_v4(),
-        };
+        let index_id = self.index_uuid.unwrap_or_else(Uuid::new_v4);
         let mut output_index_uuid = index_id;
         let created_index = match (self.index_type, self.params.index_name()) {
             (
@@ -222,6 +230,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 | IndexType::BTree
                 | IndexType::Inverted
                 | IndexType::NGram
+                | IndexType::Fm
                 | IndexType::ZoneMap
                 | IndexType::BloomFilter
                 | IndexType::LabelList
@@ -258,17 +267,44 @@ impl<'a> CreateIndexBuilder<'a> {
                     .preprocessed_data
                     .take()
                     .map(|reader| lance_datafusion::utils::reader_to_stream(Box::new(reader)));
-                build_scalar_index(
-                    self.dataset,
-                    column,
-                    &index_id.to_string(),
-                    &params,
-                    train,
-                    self.fragments.clone(),
-                    preprocesssed_data,
-                    self.progress.clone(),
-                )
-                .await?
+                if self.index_type == IndexType::Bitmap && self.fragments.is_some() {
+                    if !train {
+                        return Err(Error::invalid_input(
+                            "canonical bitmap segment build requires train=true".to_string(),
+                        ));
+                    }
+                    if preprocesssed_data.is_some() {
+                        return Err(Error::invalid_input(
+                            "canonical bitmap segment build does not accept preprocessed data"
+                                .to_string(),
+                        ));
+                    }
+                    let fragments = self.fragments.clone().ok_or_else(|| {
+                        Error::invalid_input(
+                            "canonical bitmap segment build requires fragment ids".to_string(),
+                        )
+                    })?;
+                    build_bitmap_index_segment(
+                        self.dataset,
+                        column,
+                        index_id,
+                        fragments,
+                        self.progress.clone(),
+                    )
+                    .await?
+                } else {
+                    build_scalar_index(
+                        self.dataset,
+                        column,
+                        index_id,
+                        &params,
+                        train,
+                        self.fragments.clone(),
+                        preprocesssed_data,
+                        self.progress.clone(),
+                    )
+                    .await?
+                }
             }
             (IndexType::Scalar, LANCE_SCALAR_INDEX) => {
                 // Guess the index type
@@ -282,7 +318,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 build_scalar_index(
                     self.dataset,
                     column,
-                    &index_id.to_string(),
+                    index_id,
                     params,
                     train,
                     self.fragments.clone(),
@@ -308,7 +344,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 build_scalar_index(
                     self.dataset,
                     column,
-                    &index_id.to_string(),
+                    index_id,
                     &params,
                     train,
                     self.fragments.clone(),
@@ -338,35 +374,52 @@ impl<'a> CreateIndexBuilder<'a> {
                     })?;
                 let index_version = vec_params.index_type().version() as u32;
 
-                if train {
+                let effective_fragments =
+                    effective_vector_fragments(self.dataset, self.fragments.as_deref());
+                let files = if train {
                     // Check if this is distributed indexing (fragment-level)
-                    if let Some(fragments) = &self.fragments {
-                        // For distributed indexing, build only on specified fragments
-                        // This creates temporary index metadata without committing
-                        let segment_uuid = Box::pin(build_distributed_vector_index(
-                            self.dataset,
-                            column,
-                            &index_name,
-                            &index_id.to_string(),
-                            vec_params,
-                            fri,
-                            fragments,
-                            self.progress.clone(),
-                        ))
-                        .await?;
-                        output_index_uuid = segment_uuid;
+                    if let Some(fragments) = effective_fragments.as_deref() {
+                        if vector_params_have_precomputed_ivf(vec_params) {
+                            // For distributed indexing, build only on specified fragments
+                            // This creates temporary index metadata without committing
+                            let (segment_uuid, files) = Box::pin(build_distributed_vector_index(
+                                self.dataset,
+                                column,
+                                &index_name,
+                                index_id,
+                                vec_params,
+                                fri,
+                                fragments,
+                                self.progress.clone(),
+                            ))
+                            .await?;
+                            output_index_uuid = segment_uuid;
+                            files
+                        } else {
+                            Box::pin(build_filtered_vector_index(
+                                self.dataset,
+                                column,
+                                &index_name,
+                                index_id,
+                                vec_params,
+                                fri,
+                                fragments,
+                                self.progress.clone(),
+                            ))
+                            .await?
+                        }
                     } else {
                         // Standard full dataset indexing
                         Box::pin(build_vector_index(
                             self.dataset,
                             column,
                             &index_name,
-                            &index_id.to_string(),
+                            index_id,
                             vec_params,
                             fri,
                             self.progress.clone(),
                         ))
-                        .await?;
+                        .await?
                     }
                 } else {
                     // Create empty vector index
@@ -374,22 +427,15 @@ impl<'a> CreateIndexBuilder<'a> {
                         self.dataset,
                         column,
                         &index_name,
-                        &index_id.to_string(),
+                        index_id,
                         vec_params,
                     )
-                    .await?;
-                }
-                // Capture file sizes after vector index creation
-                let index_dir = self
-                    .dataset
-                    .indices_dir()
-                    .join(output_index_uuid.to_string());
-                let files =
-                    list_index_files_with_sizes(&self.dataset.object_store, &index_dir).await?;
+                    .await?
+                };
                 CreatedIndex {
                     index_details: vector_index_details(vec_params),
                     index_version,
-                    files: Some(files),
+                    files: table_files_to_index(files),
                 }
             }
             // Can't use if let Some(...) here because it's not stable yet.
@@ -416,7 +462,7 @@ impl<'a> CreateIndexBuilder<'a> {
                     ))?;
 
                 if train {
-                    ext.create_index(self.dataset, column, &index_id.to_string(), self.params)
+                    ext.create_index(self.dataset, column, &index_id, self.params)
                         .await?;
                 } else {
                     todo!("create empty vector index when train=false");
@@ -428,7 +474,7 @@ impl<'a> CreateIndexBuilder<'a> {
                 CreatedIndex {
                     index_details: vector_index_details_default(),
                     index_version: self.index_type.version() as u32,
-                    files: Some(files),
+                    files: table_files_to_index(files),
                 }
             }
             (IndexType::FragmentReuse, _) => {
@@ -461,12 +507,22 @@ impl<'a> CreateIndexBuilder<'a> {
             index_version: created_index.index_version as i32,
             created_at: Some(chrono::Utc::now()),
             base_id: None,
-            files: created_index.files,
+            files: Some(index_files_to_table(created_index.files)),
         })
+        }
+        .boxed()
     }
 
     #[instrument(skip_all)]
     async fn execute(mut self) -> Result<IndexMetadata> {
+        // Multi-segment FM-Index path: when num_segments > 1, build one segment
+        // per fragment group and commit them all atomically.
+        if let Some(num_segments) = self.fmindex_num_segments()
+            && num_segments > 1
+        {
+            return self.execute_multi_segment_fmindex(num_segments).await;
+        }
+
         let new_idx = self.execute_uncommitted().await?;
         let index_uuid = new_idx.uuid;
         let removed_indices = if self.replace {
@@ -480,23 +536,12 @@ impl<'a> CreateIndexBuilder<'a> {
         } else {
             vec![]
         };
-        let transaction = if uses_segment_commit_path(self.index_type, &new_idx.name, self.params) {
-            let field_id = *new_idx.fields.first().ok_or_else(|| {
-                Error::internal(format!(
-                    "Index '{}' is missing field ids after build",
-                    new_idx.name
-                ))
-            })?;
-            let index_name = new_idx.name.clone();
+        let transaction = if uses_segment_commit_path(self.index_type, self.params) {
             let dataset_version = new_idx.dataset_version;
-            let segments = vec![new_idx.into_index_segment()?];
-            let new_indices =
-                build_index_metadata_from_segments(self.dataset, &index_name, field_id, segments)
-                    .await?;
             TransactionBuilder::new(
                 dataset_version,
                 Operation::CreateIndex {
-                    new_indices,
+                    new_indices: vec![new_idx],
                     removed_indices,
                 },
             )
@@ -532,6 +577,275 @@ impl<'a> CreateIndexBuilder<'a> {
                 ))
             })
     }
+    /// Extract `num_segments` from FM-Index params if this is an FM-Index build.
+    fn fmindex_num_segments(&self) -> Option<u32> {
+        if self.index_type != IndexType::Fm {
+            return None;
+        }
+        let scalar_params = self.params.as_any().downcast_ref::<ScalarIndexParams>()?;
+        let params_json = scalar_params.params.as_deref()?;
+        let json: serde_json::Value = serde_json::from_str(params_json).ok()?;
+        json.get("num_segments")?.as_u64().map(|n| n as u32)
+    }
+
+    /// Build FM-Index with multiple segments, each covering a subset of fragments.
+    async fn execute_multi_segment_fmindex(&mut self, num_segments: u32) -> Result<IndexMetadata> {
+        // Validate column count: same check as execute_uncommitted
+        if self.columns.len() != 1 {
+            return Err(Error::index(
+                "Only support building index on 1 column at the moment".to_string(),
+            ));
+        }
+
+        let column_input = &self.columns[0];
+        let Some(field_path) = self.dataset.schema().resolve_case_insensitive(column_input) else {
+            return Err(Error::index(format!(
+                "CreateIndex: column '{column_input}' does not exist"
+            )));
+        };
+        let field = *field_path.last().unwrap();
+        let names: Vec<&str> = field_path.iter().map(|f| f.name.as_str()).collect();
+        let column = format_field_path(&names);
+
+        let train = if self.train {
+            self.dataset.count_rows(None).await? > 0
+        } else {
+            false
+        };
+
+        let indices = self.dataset.load_indices().await?;
+        let index_name = if let Some(name) = self.name.take() {
+            name
+        } else {
+            let column_path = default_index_name(&names);
+            let base_name = format!("{column_path}_idx");
+            let mut candidate = base_name.clone();
+            let mut counter = 2;
+            while indices
+                .iter()
+                .any(|idx| idx.name == candidate && idx.fields != [field.id])
+            {
+                candidate = format!("{base_name}_{counter}");
+                counter += 1;
+            }
+            candidate
+        };
+        let existing_named_indices = indices
+            .iter()
+            .filter(|idx| idx.name == index_name)
+            .collect::<Vec<_>>();
+        if existing_named_indices
+            .iter()
+            .any(|idx| idx.fields != [field.id])
+        {
+            return Err(Error::index(format!(
+                "Index name '{index_name}' already exists with different fields, \
+                please specify a different name"
+            )));
+        }
+        if !existing_named_indices.is_empty() && !self.replace {
+            return Err(Error::index(format!(
+                "Index name '{index_name}' already exists, \
+                please specify a different name or use replace=True"
+            )));
+        }
+
+        let all_fragment_ids: Vec<u32> = self.dataset.fragment_bitmap.as_ref().iter().collect();
+        if !train || all_fragment_ids.is_empty() {
+            let segment_uuid = Uuid::new_v4();
+            let created_index = build_scalar_index(
+                self.dataset,
+                &column,
+                segment_uuid,
+                &ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Fm),
+                false,
+                None,
+                None,
+                self.progress.clone(),
+            )
+            .await?;
+            let metadata = IndexMetadata {
+                uuid: segment_uuid,
+                name: index_name.clone(),
+                fields: vec![field.id],
+                dataset_version: self.dataset.manifest.version,
+                fragment_bitmap: Some(roaring::RoaringBitmap::new()),
+                index_details: Some(Arc::new(created_index.index_details)),
+                index_version: created_index.index_version as i32,
+                created_at: Some(chrono::Utc::now()),
+                base_id: None,
+                files: Some(index_files_to_table(created_index.files)),
+            };
+            let segments = vec![metadata.into_index_segment()?];
+            let new_indices =
+                build_index_metadata_from_segments(self.dataset, &index_name, field.id, segments)
+                    .await?;
+
+            // Collect all same-name indices for removal when replace is set
+            let removed_indices = if self.replace {
+                existing_named_indices
+                    .into_iter()
+                    .cloned()
+                    .collect::<Vec<_>>()
+            } else {
+                vec![]
+            };
+
+            let transaction = TransactionBuilder::new(
+                self.dataset.manifest.version,
+                Operation::CreateIndex {
+                    new_indices,
+                    removed_indices,
+                },
+            )
+            .transaction_properties(self.transaction_properties.clone())
+            .build();
+
+            self.dataset
+                .apply_commit(transaction, &Default::default(), &Default::default())
+                .await?;
+
+            let indices = self.dataset.load_indices_by_name(&index_name).await?;
+            return indices.into_iter().next().ok_or_else(|| {
+                Error::internal(format!(
+                    "FM-Index segments for '{}' not found after commit",
+                    index_name
+                ))
+            });
+        }
+
+        let num_segments = (num_segments as usize).min(all_fragment_ids.len()).max(1);
+        let chunk_size = all_fragment_ids.len().div_ceil(num_segments);
+
+        let mut segment_metadatas = Vec::with_capacity(num_segments);
+        for chunk in all_fragment_ids.chunks(chunk_size) {
+            let fragment_ids = chunk.to_vec();
+            let segment_uuid = Uuid::new_v4();
+            let created_index = build_scalar_index(
+                self.dataset,
+                &column,
+                segment_uuid,
+                &ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Fm),
+                true,
+                Some(fragment_ids.clone()),
+                None,
+                self.progress.clone(),
+            )
+            .await?;
+
+            segment_metadatas.push(IndexMetadata {
+                uuid: segment_uuid,
+                name: index_name.clone(),
+                fields: vec![field.id],
+                dataset_version: self.dataset.manifest.version,
+                fragment_bitmap: Some(fragment_ids.into_iter().collect()),
+                index_details: Some(Arc::new(created_index.index_details)),
+                index_version: created_index.index_version as i32,
+                created_at: Some(chrono::Utc::now()),
+                base_id: None,
+                files: Some(index_files_to_table(created_index.files)),
+            });
+        }
+
+        // Convert to IndexSegments and build proper transaction metadata
+        let segments = segment_metadatas
+            .into_iter()
+            .map(IntoIndexSegment::into_index_segment)
+            .collect::<Result<Vec<_>>>()?;
+        let new_indices =
+            build_index_metadata_from_segments(self.dataset, &index_name, field.id, segments)
+                .await?;
+
+        // Collect all same-name indices for removal when replace is set,
+        // matching the standard execute() path behavior.
+        let removed_indices = if self.replace {
+            existing_named_indices
+                .into_iter()
+                .cloned()
+                .collect::<Vec<_>>()
+        } else {
+            vec![]
+        };
+
+        let transaction = TransactionBuilder::new(
+            self.dataset.manifest.version,
+            Operation::CreateIndex {
+                new_indices,
+                removed_indices,
+            },
+        )
+        .transaction_properties(self.transaction_properties.clone())
+        .build();
+
+        self.dataset
+            .apply_commit(transaction, &Default::default(), &Default::default())
+            .await?;
+
+        let indices = self.dataset.load_indices_by_name(&index_name).await?;
+        indices.into_iter().next().ok_or_else(|| {
+            Error::internal(format!(
+                "FM-Index segments for '{}' not found after commit",
+                index_name
+            ))
+        })
+    }
+}
+
+fn is_btree_scalar_params(params: &dyn IndexParams) -> bool {
+    params
+        .as_any()
+        .downcast_ref::<ScalarIndexParams>()
+        .is_some_and(|p| p.index_type.eq_ignore_ascii_case("btree"))
+}
+
+fn is_builtin_vector_index(index_type: IndexType, params: &dyn IndexParams) -> bool {
+    params.index_name() == LANCE_VECTOR_INDEX
+        && matches!(
+            index_type,
+            IndexType::Vector
+                | IndexType::IvfPq
+                | IndexType::IvfSq
+                | IndexType::IvfFlat
+                | IndexType::IvfRq
+                | IndexType::IvfHnswFlat
+                | IndexType::IvfHnswPq
+                | IndexType::IvfHnswSq
+        )
+        && params.as_any().is::<VectorIndexParams>()
+}
+
+async fn should_train_index(
+    dataset: &Dataset,
+    train: bool,
+    vector_fragments: Option<&[u32]>,
+) -> Result<bool> {
+    if !train {
+        return Ok(false);
+    }
+
+    if dataset.fragment_bitmap.is_empty() {
+        return Ok(false);
+    }
+
+    if let Some(fragment_ids) = vector_fragments {
+        dataset.get_fragments_from_ids(fragment_ids)?;
+        return Ok(true);
+    }
+
+    Ok(dataset.count_rows(None).await? > 0)
+}
+
+fn vector_params_have_precomputed_ivf(params: &VectorIndexParams) -> bool {
+    matches!(
+        params.stages.first(),
+        Some(StageParams::Ivf(ivf_params)) if ivf_params.centroids.is_some()
+    )
+}
+
+fn effective_vector_fragments(dataset: &Dataset, fragments: Option<&[u32]>) -> Option<Vec<u32>> {
+    let fragments = Dataset::normalize_fragment_ids(fragments?);
+    let fragment_bitmap: roaring::RoaringBitmap = fragments.iter().copied().collect();
+    (fragment_bitmap != *dataset.fragment_bitmap).then_some(fragments)
 }
 
 /// Validate that a user-supplied `index_uuid` is permitted for this build.
@@ -539,7 +853,7 @@ fn ensure_index_uuid_allowed(
     index_type: IndexType,
     params: &dyn IndexParams,
     fragments: Option<&Vec<u32>>,
-    index_uuid: Option<&str>,
+    index_uuid: Option<&Uuid>,
 ) -> Result<()> {
     let is_btree = index_type == IndexType::BTree
         || params
@@ -560,26 +874,35 @@ fn ensure_index_uuid_allowed(
     Ok(())
 }
 
-fn uses_segment_commit_path(
-    index_type: IndexType,
-    index_name: &str,
-    params: &dyn IndexParams,
-) -> bool {
-    if index_name != LANCE_VECTOR_INDEX {
-        return false;
+fn uses_segment_commit_path(index_type: IndexType, params: &dyn IndexParams) -> bool {
+    let params_family = params.index_name();
+
+    if params_family == LANCE_VECTOR_INDEX
+        && matches!(
+            index_type,
+            IndexType::Vector
+                | IndexType::IvfPq
+                | IndexType::IvfSq
+                | IndexType::IvfFlat
+                | IndexType::IvfRq
+                | IndexType::IvfHnswFlat
+                | IndexType::IvfHnswPq
+                | IndexType::IvfHnswSq
+        )
+        && params.as_any().is::<VectorIndexParams>()
+    {
+        return true;
     }
 
-    matches!(
-        index_type,
-        IndexType::Vector
-            | IndexType::IvfPq
-            | IndexType::IvfSq
-            | IndexType::IvfFlat
-            | IndexType::IvfRq
-            | IndexType::IvfHnswFlat
-            | IndexType::IvfHnswPq
-            | IndexType::IvfHnswSq
-    ) && params.as_any().is::<VectorIndexParams>()
+    if params_family == LANCE_SCALAR_INDEX {
+        match index_type {
+            IndexType::BTree => return true,
+            IndexType::Scalar if is_btree_scalar_params(params) => return true,
+            _ => {}
+        }
+    }
+
+    false
 }
 
 impl<'a> IntoFuture for CreateIndexBuilder<'a> {
@@ -597,9 +920,9 @@ mod tests {
     use crate::dataset::{WriteMode, WriteParams};
     use crate::index::{DatasetIndexExt, IndexSegment};
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
-    use arrow::datatypes::{Float32Type, Int32Type};
+    use arrow::datatypes::{Float32Type, Int32Type, Int64Type};
     use arrow_array::cast::AsArray;
-    use arrow_array::{FixedSizeListArray, RecordBatchIterator};
+    use arrow_array::{Array, FixedSizeListArray, ListArray, RecordBatchIterator};
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
     use datafusion::common::ScalarValue;
@@ -615,7 +938,6 @@ mod tests {
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
     use lance_linalg::distance::{DistanceType, MetricType};
-    use serde_json::json;
     use std::{collections::BTreeSet, ops::Bound, sync::Arc};
     use uuid::Uuid;
 
@@ -903,6 +1225,50 @@ mod tests {
         IvfBuildParams::try_with_centroids(4, centroids).unwrap()
     }
 
+    async fn write_vector_fragment_dataset(uri: &str) -> Dataset {
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .col(
+                "vector",
+                lance_datagen::array::rand_vec::<Float32Type>(lance_datagen::Dimension::from(16)),
+            )
+            .into_reader_rows(
+                lance_datagen::RowCount::from(256),
+                lance_datagen::BatchCount::from(4),
+            );
+        Dataset::write(
+            reader,
+            uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn test_get_frags_from_ordered_ids_accepts_unsorted_duplicates() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let first = fragments[0].id() as u32;
+        let second = fragments[1].id() as u32;
+
+        let resolved = dataset.get_frags_from_ordered_ids(&[second, first, second, u32::MAX]);
+
+        assert_eq!(resolved.len(), 4);
+        assert_eq!(resolved[0].as_ref().unwrap().id() as u32, second);
+        assert_eq!(resolved[1].as_ref().unwrap().id() as u32, first);
+        assert_eq!(resolved[2].as_ref().unwrap().id() as u32, second);
+        assert!(resolved[3].is_none());
+    }
+
     #[tokio::test]
     async fn test_execute_uncommitted() {
         // Test the complete workflow that covers the user's specified code pattern:
@@ -1014,7 +1380,7 @@ mod tests {
         let params = InvertedIndexParams::default();
         let fragments = dataset.get_fragments();
         let fragment_ids: Vec<u32> = fragments.iter().map(|f| f.id() as u32).collect();
-        let shared_uuid = Uuid::new_v4().to_string();
+        let shared_uuid = Uuid::new_v4();
         let build_progress = Arc::new(RecordingProgress::default());
 
         for &fragment_id in &fragment_ids {
@@ -1022,11 +1388,11 @@ mod tests {
                 CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Inverted, &params)
                     .name("distributed_index".to_string())
                     .fragments(vec![fragment_id])
-                    .index_uuid(shared_uuid.clone())
+                    .index_uuid(shared_uuid)
                     .progress(build_progress.clone());
 
             let index_metadata = builder.execute_uncommitted().await.unwrap();
-            assert_eq!(index_metadata.uuid.to_string(), shared_uuid);
+            assert_eq!(index_metadata.uuid, shared_uuid);
             assert_eq!(index_metadata.name, "distributed_index");
 
             let fragment_bitmap = index_metadata.fragment_bitmap.as_ref().unwrap();
@@ -1135,7 +1501,7 @@ mod tests {
 
         let err = dataset
             .merge_index_metadata(
-                &Uuid::new_v4().to_string(),
+                &Uuid::new_v4(),
                 IndexType::BTree,
                 None,
                 Arc::new(NoopIndexBuildProgress),
@@ -1286,7 +1652,7 @@ mod tests {
             let err = CreateIndexBuilder::new(&mut dataset, &["value"], index_type, &params)
                 .name("value_btree_segments".to_string())
                 .fragments(vec![fragment_id])
-                .index_uuid(Uuid::new_v4().to_string())
+                .index_uuid(Uuid::new_v4())
                 .execute_uncommitted()
                 .await
                 .unwrap_err();
@@ -1389,10 +1755,8 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_distributed_build_bitmap() {
-        use datafusion::common::ScalarValue;
-        use lance_index::scalar::{SargableQuery, SearchResult, bitmap::BITMAP_LOOKUP_NAME};
-        use lance_select::RowSetOps;
+    async fn test_bitmap_execute_uncommitted_writes_canonical_segment() {
+        use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME;
 
         let tmpdir = TempStrDir::default();
         let dataset_uri = format!("file://{}", tmpdir.as_str());
@@ -1432,69 +1796,15 @@ mod tests {
             ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Bitmap);
         let fragments = dataset.get_fragments();
         let fragment_ids: Vec<u32> = fragments.iter().map(|f| f.id() as u32).collect();
-        let shared_uuid = Uuid::new_v4().to_string();
-        let mut shard_metadata = None;
-        let shard_groups = fragment_ids.chunks(2).collect::<Vec<_>>();
-
-        for (shard_id, fragment_group) in shard_groups.iter().enumerate() {
-            let params = base_params
-                .clone()
-                .with_params(&json!({ "shard_id": shard_id as u32 }));
-            let index_metadata =
-                CreateIndexBuilder::new(&mut dataset, &["category"], IndexType::Bitmap, &params)
-                    .name("distributed_bitmap".to_string())
-                    .fragments(fragment_group.to_vec())
-                    .index_uuid(shared_uuid.clone())
-                    .execute_uncommitted()
-                    .await
-                    .unwrap();
-            if shard_metadata.is_none() {
-                shard_metadata = Some(index_metadata);
-            }
-        }
+        let selected_fragments = fragment_ids[..2].to_vec();
+        let index =
+            CreateIndexBuilder::new(&mut dataset, &["category"], IndexType::Bitmap, &base_params)
+                .name("bitmap_segment".to_string())
+                .fragments(selected_fragments.clone())
+                .execute_uncommitted()
+                .await
+                .unwrap();
 
-        dataset
-            .merge_index_metadata(
-                &shared_uuid,
-                IndexType::Bitmap,
-                None,
-                Arc::new(NoopIndexBuildProgress),
-            )
-            .await
-            .unwrap();
-
-        let mut committed_index_metadata = shard_metadata.unwrap();
-        committed_index_metadata.fragment_bitmap = Some(fragment_ids.iter().copied().collect());
-        committed_index_metadata.files = Some(
-            list_index_files_with_sizes(
-                dataset.object_store.as_ref(),
-                &dataset.indices_dir().clone().join(shared_uuid.clone()),
-            )
-            .await
-            .unwrap(),
-        );
-        committed_index_metadata.dataset_version = dataset.manifest.version;
-
-        let transaction = TransactionBuilder::new(
-            dataset.manifest.version,
-            Operation::CreateIndex {
-                new_indices: vec![committed_index_metadata],
-                removed_indices: vec![],
-            },
-        )
-        .build();
-        dataset
-            .apply_commit(transaction, &Default::default(), &Default::default())
-            .await
-            .unwrap();
-
-        let dataset = Dataset::open(&dataset_uri).await.unwrap();
-        let indices = dataset
-            .load_indices_by_name("distributed_bitmap")
-            .await
-            .unwrap();
-        assert_eq!(indices.len(), 1);
-        let index = &indices[0];
         assert_eq!(
             index
                 .fragment_bitmap
@@ -1502,37 +1812,15 @@ mod tests {
                 .unwrap()
                 .iter()
                 .collect::<Vec<_>>(),
-            fragment_ids
+            selected_fragments
         );
 
         let files = index.files.as_ref().unwrap();
         assert!(files.iter().any(|file| file.path == BITMAP_LOOKUP_NAME));
         assert!(
             files.iter().all(|file| !file.path.starts_with("part_")),
-            "committed bitmap index should only reference merged files"
+            "staged bitmap segment should only reference canonical files"
         );
-
-        let scalar_index = crate::index::scalar::open_scalar_index(
-            &dataset,
-            "category",
-            index,
-            &NoOpMetricsCollector,
-        )
-        .await
-        .unwrap();
-        assert_eq!(scalar_index.index_type(), IndexType::Bitmap);
-
-        let query_result = scalar_index
-            .search(
-                &SargableQuery::Equals(ScalarValue::Int32(Some(2))),
-                &NoOpMetricsCollector,
-            )
-            .await
-            .unwrap();
-        let SearchResult::Exact(query_rows) = query_result else {
-            panic!("expected exact bitmap result");
-        };
-        assert_eq!(query_rows.true_rows().len(), Some(2));
     }
 
     #[tokio::test]
@@ -1638,6 +1926,218 @@ mod tests {
         assert!(result.num_rows() > 0);
     }
 
+    #[tokio::test]
+    async fn test_vector_explicit_all_fragments_uses_full_build_without_precomputed_ivf() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragment_ids = dataset
+            .get_fragments()
+            .iter()
+            .map(|fragment| fragment.id() as u32)
+            .collect::<Vec<_>>();
+        assert!(fragment_ids.len() >= 2);
+
+        let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10);
+        params.version(crate::index::vector::IndexFileVersion::Legacy);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(fragment_ids)
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert_eq!(
+            segment.fragment_bitmap.as_ref().unwrap(),
+            dataset.fragment_bitmap.as_ref()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_precomputed_ivf_num_partitions_mismatch_errors() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await;
+        let centroid_count = ivf_params.centroids.as_ref().unwrap().len();
+        ivf_params.num_partitions = Some(centroid_count + 1);
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params);
+
+        let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+            .name("vector_idx".to_string())
+            .execute_uncommitted()
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.to_string().contains(&format!(
+                "num_partitions {} does not match precomputed IVF centroids length {}",
+                centroid_count + 1,
+                centroid_count
+            )),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_subset_legacy_ivf_pq_rejects_filtered_build() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10);
+        params.version(crate::index::vector::IndexFileVersion::Legacy);
+
+        let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+            .name("vector_idx".to_string())
+            .fragments(vec![fragments[0].id() as u32])
+            .execute_uncommitted()
+            .await
+            .unwrap_err();
+
+        assert!(
+            err.to_string()
+                .contains("filtered IVF_PQ builds do not support legacy format"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_subset_fragments_train_without_precomputed_ivf() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 3);
+        let selected = vec![
+            fragments[1].id() as u32,
+            fragments[0].id() as u32,
+            fragments[1].id() as u32,
+        ];
+        let expected_bitmap = selected.iter().copied().collect::<roaring::RoaringBitmap>();
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(selected)
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert_eq!(segment.fragment_bitmap.as_ref().unwrap(), &expected_bitmap);
+    }
+
+    #[tokio::test]
+    async fn test_vector_merge_rejects_independently_trained_fragment_segments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let mut segments = Vec::new();
+        for fragment in fragments.iter().take(2) {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            segments.push(segment);
+        }
+
+        let err = dataset
+            .merge_existing_index_segments(segments)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("IVF centroids mismatch across shards"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_optimize_rejects_independently_trained_fragment_segments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+
+        let params = VectorIndexParams::ivf_flat(2, DistanceType::L2);
+        let mut segments = Vec::new();
+        for fragment in fragments.iter().take(2) {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            segments.push(segment);
+        }
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", segments)
+            .await
+            .unwrap();
+
+        let err = dataset
+            .optimize_indices(&OptimizeOptions::merge(2))
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("vector index segments do not share IVF centroids"),
+            "unexpected error: {err}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_vector_empty_fragments_with_precomputed_ivf_builds_empty_segment() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let mut dataset = write_vector_fragment_dataset(&dataset_uri).await;
+
+        let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await;
+        let expected_partitions = ivf_params.centroids.as_ref().unwrap().len();
+        ivf_params.num_partitions = None;
+        let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(vec![])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        assert!(segment.fragment_bitmap.as_ref().unwrap().is_empty());
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![segment])
+            .await
+            .unwrap();
+        let logical_index = dataset
+            .open_logical_vector_index("vector", "vector_idx")
+            .await
+            .unwrap();
+        let metadata = logical_index.metadatas().next().unwrap();
+        assert_eq!(
+            logical_index.as_ivf().unwrap().num_partitions_per_segment(),
+            vec![(metadata.uuid, expected_partitions)]
+        );
+    }
+
     #[tokio::test]
     async fn test_commit_existing_index_segments_vector_commits_multi_segment_logical_index() {
         let tmpdir = TempStrDir::default();
@@ -1829,6 +2329,23 @@ mod tests {
         let segments = input_segments.clone();
         assert_eq!(segments.len(), input_segments.len());
 
+        crate::index::scalar::inverted::finalize_segment_files_if_needed(
+            &dataset,
+            &input_segments[0],
+        )
+        .await
+        .unwrap();
+        let stale_staging_path = dataset
+            .indices_dir()
+            .join(input_segments[0].uuid.to_string())
+            .join("staging")
+            .join("orphan.lance");
+        dataset
+            .object_store
+            .put(&stale_staging_path, b"stale")
+            .await
+            .unwrap();
+
         dataset
             .commit_existing_index_segments("text_idx", "text", segments)
             .await
@@ -1852,6 +2369,19 @@ mod tests {
 
         let indices = dataset.load_indices_by_name("text_idx").await.unwrap();
         assert_eq!(indices.len(), input_segments.len());
+        let finalized_segment = indices
+            .iter()
+            .find(|index| index.uuid == input_segments[0].uuid)
+            .expect("finalized segment should be committed");
+        assert!(
+            finalized_segment
+                .files
+                .as_ref()
+                .expect("committed segment should track files")
+                .iter()
+                .all(|file| !file.path.starts_with("staging/")),
+            "stale staging files must not be committed in IndexMetadata.files"
+        );
     }
 
     #[tokio::test]
@@ -1933,6 +2463,450 @@ mod tests {
         assert_eq!(results.num_rows(), 20);
     }
 
+    #[tokio::test]
+    async fn test_btree_merge_existing_index_segments() {
+        use datafusion::common::ScalarValue;
+        use lance_index::scalar::{SargableQuery, SearchResult};
+        use std::ops::Bound;
+
+        // Open `segment` and count rows whose `id` falls in `[lo, hi)`.
+        async fn count_in_range(
+            dataset: &Dataset,
+            segment: &IndexMetadata,
+            lo: i32,
+            hi: i32,
+        ) -> usize {
+            let field_path = dataset.schema().field_path(segment.fields[0]).unwrap();
+            let index = crate::index::scalar::open_scalar_index(
+                dataset,
+                &field_path,
+                segment,
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+            let query = SargableQuery::Range(
+                Bound::Included(ScalarValue::Int32(Some(lo))),
+                Bound::Excluded(ScalarValue::Int32(Some(hi))),
+            );
+            match index.search(&query, &NoOpMetricsCollector).await.unwrap() {
+                SearchResult::Exact(row_addrs) => {
+                    row_addrs.true_rows().row_addrs().unwrap().count()
+                }
+                other => panic!("expected exact result, got {other:?}"),
+            }
+        }
+
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        // 128 rows across two 64-row fragments. Stable row ids so the
+        // retired-fragment filter below exercises the exact row-id allow-list.
+        let reader = gen_batch()
+            .col("id", lance_datagen::array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(64),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        // One staged BTree segment per fragment, committed as a multi-segment
+        // logical index.
+        let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree);
+        let mut staged = Vec::new();
+        for fragment in dataset.get_fragments() {
+            staged.push(
+                CreateIndexBuilder::new(&mut dataset, &["id"], IndexType::BTree, &params)
+                    .name("id_btree".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("id_btree", "id", staged)
+            .await
+            .unwrap();
+
+        // Phase 1 — healthy merge: the two per-fragment segments consolidate
+        // into a single canonical segment covering both fragments, and a range
+        // spanning both (ids 50..100) returns every matching row.
+        let merged = dataset
+            .merge_existing_index_segments(dataset.load_indices_by_name("id_btree").await.unwrap())
+            .await
+            .unwrap();
+        assert_eq!(
+            merged.fragment_bitmap.as_ref().unwrap(),
+            &roaring::RoaringBitmap::from_iter([0u32, 1])
+        );
+        assert!(
+            merged
+                .index_details
+                .as_ref()
+                .unwrap()
+                .type_url
+                .ends_with("BTreeIndexDetails")
+        );
+        assert_eq!(count_in_range(&dataset, &merged, 50, 100).await, 50);
+
+        // Phase 2 — retire fragment 0: delete >10% of its rows so compaction
+        // rewrites only frag 0 (frag 1 has no deletions and is at target size).
+        // The committed per-fragment segment now claims a fragment the dataset
+        // no longer has.
+        dataset.delete("id < 16").await.unwrap();
+        crate::dataset::optimize::compact_files(
+            &mut dataset,
+            crate::dataset::optimize::CompactionOptions {
+                target_rows_per_fragment: 64,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        let live_frags: roaring::RoaringBitmap = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect();
+        assert!(!live_frags.contains(0), "compaction should retire frag 0");
+
+        // Filtered merge: coverage drops the retired fragment but keeps the
+        // live one, and the merged page data does not leak the retired row ids
+        // (ids < 16 lived only in frag 0, so the range now returns nothing).
+        let merged = dataset
+            .merge_existing_index_segments(dataset.load_indices_by_name("id_btree").await.unwrap())
+            .await
+            .unwrap();
+        let coverage = merged.fragment_bitmap.as_ref().unwrap();
+        assert!(!coverage.contains(0), "must drop retired frag 0");
+        assert!(coverage.contains(1), "must keep live frag 1");
+        assert_eq!(
+            count_in_range(&dataset, &merged, 0, 16).await,
+            0,
+            "must filter retired-fragment row ids"
+        );
+    }
+
+    // Distributed LabelList build: one segment per fragment via
+    // `execute_uncommitted`, then `merge_existing_index_segments` consolidates them
+    // into a single canonical segment that answers `array_has_any` across all rows.
+    #[tokio::test]
+    async fn test_label_list_merge_existing_index_segments() {
+        use lance_index::scalar::{LabelListQuery, SearchResult};
+
+        // Open `segment` and count rows whose `labels` list contains `label`.
+        async fn count_has_any(dataset: &Dataset, segment: &IndexMetadata, label: i64) -> usize {
+            let field_path = dataset.schema().field_path(segment.fields[0]).unwrap();
+            let index = crate::index::scalar::open_scalar_index(
+                dataset,
+                &field_path,
+                segment,
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+            let query = LabelListQuery::HasAnyLabel(vec![ScalarValue::Int64(Some(label))]);
+            match index.search(&query, &NoOpMetricsCollector).await.unwrap() {
+                SearchResult::Exact(row_addrs) => {
+                    row_addrs.true_rows().row_addrs().unwrap().count()
+                }
+                other => panic!("expected exact result, got {other:?}"),
+            }
+        }
+
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+
+        // 4000 rows across two 2000-row fragments; each `labels` list cycles over 1..=5.
+        let mut dataset = gen_batch()
+            .col(
+                "labels",
+                lance_datagen::array::rand_list_any(
+                    lance_datagen::array::cycle::<arrow::datatypes::Int64Type>(vec![1, 2, 3, 4, 5]),
+                    false,
+                ),
+            )
+            .into_dataset(
+                &dataset_uri,
+                FragmentCount::from(2),
+                FragmentRowCount::from(2000),
+            )
+            .await
+            .unwrap();
+
+        // Ground truth via a full scan before any index exists.
+        let expected = dataset
+            .scan()
+            .project(&["labels"])
+            .unwrap()
+            .filter("array_has_any(labels, [3])")
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap()
+            .num_rows();
+        assert!(
+            expected > 0,
+            "test dataset must contain at least one row whose labels include 3"
+        );
+
+        // One LabelList segment per fragment, committed as a multi-segment index.
+        let params =
+            ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::LabelList);
+        let mut staged = Vec::new();
+        for fragment in dataset.get_fragments() {
+            staged.push(
+                CreateIndexBuilder::new(&mut dataset, &["labels"], IndexType::LabelList, &params)
+                    .name("labels_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("labels_idx", "labels", staged)
+            .await
+            .unwrap();
+
+        // Merge the two per-fragment segments into a single segment covering both.
+        let merged = dataset
+            .merge_existing_index_segments(
+                dataset.load_indices_by_name("labels_idx").await.unwrap(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(
+            merged.fragment_bitmap.as_ref().unwrap(),
+            &roaring::RoaringBitmap::from_iter([0u32, 1])
+        );
+        assert!(
+            merged
+                .index_details
+                .as_ref()
+                .unwrap()
+                .type_url
+                .ends_with("LabelListIndexDetails")
+        );
+        // The merged segment returns every row whose labels include 3 across both
+        // fragments — i.e. the per-fragment bitmaps and null sets were unioned.
+        assert_eq!(count_has_any(&dataset, &merged, 3).await, expected);
+    }
+
+    fn label_list_batch(labels: Vec<Option<Vec<Option<i64>>>>) -> (Arc<ArrowSchema>, RecordBatch) {
+        let labels = ListArray::from_iter_primitive::<Int64Type, _, _>(labels);
+        let row_count = labels.len();
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("id", DataType::Int32, false),
+            ArrowField::new("labels", labels.data_type().clone(), true),
+        ]));
+        let ids = Int32Array::from_iter_values((0..row_count).map(|id| id as i32));
+        let batch =
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(labels)]).unwrap();
+        (schema, batch)
+    }
+
+    #[tokio::test]
+    async fn test_label_list_merge_existing_index_segments_drops_retired_fragments() {
+        use lance_index::scalar::{LabelListQuery, SearchResult};
+
+        async fn search_counts(
+            dataset: &Dataset,
+            segment: &IndexMetadata,
+            label: i64,
+        ) -> (usize, usize) {
+            let field_path = dataset.schema().field_path(segment.fields[0]).unwrap();
+            let index = crate::index::scalar::open_scalar_index(
+                dataset,
+                &field_path,
+                segment,
+                &NoOpMetricsCollector,
+            )
+            .await
+            .unwrap();
+            let query = LabelListQuery::HasAnyLabel(vec![ScalarValue::Int64(Some(label))]);
+            match index.search(&query, &NoOpMetricsCollector).await.unwrap() {
+                SearchResult::Exact(row_addrs) => (
+                    row_addrs.true_rows().row_addrs().unwrap().count(),
+                    row_addrs.null_rows().row_addrs().unwrap().count(),
+                ),
+                other => panic!("expected exact result, got {other:?}"),
+            }
+        }
+
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let (schema, batch) = label_list_batch(vec![
+            Some(vec![Some(1)]),
+            None,
+            Some(vec![Some(1)]),
+            Some(vec![Some(1)]),
+            Some(vec![Some(2)]),
+            Some(vec![Some(2)]),
+            Some(vec![Some(2)]),
+            Some(vec![Some(2)]),
+        ]);
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 4,
+                mode: WriteMode::Overwrite,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params =
+            ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::LabelList);
+        let mut staged = Vec::new();
+        for fragment in dataset.get_fragments() {
+            staged.push(
+                CreateIndexBuilder::new(&mut dataset, &["labels"], IndexType::LabelList, &params)
+                    .name("labels_retired_idx".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("labels_retired_idx", "labels", staged)
+            .await
+            .unwrap();
+
+        dataset.delete("id < 4").await.unwrap();
+        crate::dataset::optimize::compact_files(
+            &mut dataset,
+            crate::dataset::optimize::CompactionOptions {
+                target_rows_per_fragment: 4,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        assert!(
+            !dataset.fragment_bitmap.contains(0),
+            "compaction should retire fragment 0"
+        );
+
+        let merged = dataset
+            .merge_existing_index_segments(
+                dataset
+                    .load_indices_by_name("labels_retired_idx")
+                    .await
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        let coverage = merged.fragment_bitmap.as_ref().unwrap();
+        assert!(!coverage.contains(0), "must drop retired frag 0");
+        assert!(coverage.contains(1), "must keep live indexed frag 1");
+
+        let (retired_true, _) = search_counts(&dataset, &merged, 1).await;
+        assert_eq!(
+            retired_true, 0,
+            "must filter value bitmaps from retired fragments"
+        );
+
+        let (live_true, null_rows) = search_counts(&dataset, &merged, 2).await;
+        assert_eq!(live_true, 4, "must keep live fragment rows");
+        assert_eq!(
+            null_rows, 0,
+            "must filter list_nulls from retired fragments"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_label_list_merge_rejects_nullable_segment_missing_null_metadata() {
+        use crate::dataset::index::LanceIndexStoreExt;
+        use lance_index::scalar::IndexStore;
+        use lance_index::scalar::label_list::{
+            BITMAP_LOOKUP_NAME, LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION,
+        };
+        use lance_index::scalar::lance_format::LanceIndexStore;
+
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let (schema, batch) = label_list_batch(vec![
+            Some(vec![Some(1)]),
+            None,
+            Some(vec![Some(2)]),
+            Some(vec![Some(3)]),
+        ]);
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                mode: WriteMode::Overwrite,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+
+        let params =
+            ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::LabelList);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["labels"], IndexType::LabelList, &params)
+                .name("labels_legacy_idx".to_string())
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+        let source_store = LanceIndexStore::from_dataset_for_existing(&dataset, &segment)
+            .await
+            .unwrap();
+        let reader = source_store
+            .open_index_file(BITMAP_LOOKUP_NAME)
+            .await
+            .unwrap();
+        let batch = reader.read_range(0..reader.num_rows(), None).await.unwrap();
+        let source_schema: ArrowSchema = reader.schema().into();
+        let schema_without_metadata = Arc::new(ArrowSchema::new(source_schema.fields().clone()));
+
+        let legacy_uuid = Uuid::new_v4();
+        let legacy_store = LanceIndexStore::from_dataset_for_new(&dataset, &legacy_uuid).unwrap();
+        let mut writer = legacy_store
+            .new_index_file(BITMAP_LOOKUP_NAME, schema_without_metadata)
+            .await
+            .unwrap();
+        writer.write_record_batch(batch).await.unwrap();
+        let legacy_file = writer.finish().await.unwrap();
+
+        let mut legacy_segment = segment.clone();
+        legacy_segment.uuid = legacy_uuid;
+        legacy_segment.index_version = LABEL_LIST_NULLS_MIN_VERSION;
+        legacy_segment.files = Some(index_files_to_table(vec![legacy_file]));
+
+        let err = dataset
+            .merge_existing_index_segments(vec![legacy_segment])
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains(LABEL_LIST_NULLS_METADATA_KEY),
+            "unexpected error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_commit_existing_index_supports_local_hnsw_segments() {
         let tmpdir = TempStrDir::default();
@@ -1969,7 +2943,7 @@ mod tests {
 
         CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, &params)
             .name("vector_idx".to_string())
-            .index_uuid(uuid.to_string())
+            .index_uuid(uuid)
             .execute_uncommitted()
             .await
             .unwrap();
@@ -2194,39 +3168,38 @@ mod tests {
         // Load indices after optimization
         let indices_after = dataset.load_indices().await.unwrap();
 
-        // There should be 3 indices:
-        // 1. one scalar index with name "id_idx", and the bitmap is [0,1]
-        // 2. one delta vector index with name "vector_idx", and the bitmap is [0]
-        // 3. one delta vector index with name "vector_idx", and the bitmap is [1]
-        assert_eq!(indices_after.len(), 3, "{:?}", indices_after);
-        let id_idx = indices_after
+        // After unifying scalar optimize, `OptimizeOptions::append()` honors
+        // `Some(0)` for BTree the same way it does for vector: keep the old
+        // segment, add a delta for the unindexed fragment. So we now expect:
+        //   1. id_idx old segment, bitmap [0]
+        //   2. id_idx delta segment, bitmap [1]
+        //   3. vector_idx old segment, bitmap [0]
+        //   4. vector_idx delta segment, bitmap [1]
+        // Previously BTree silently merged into 1 segment because legacy
+        // scalar ignored `num_indices_to_merge`.
+        assert_eq!(indices_after.len(), 4, "{:?}", indices_after);
+        let id_indices = indices_after
             .iter()
-            .find(|idx| idx.name == "id_idx")
-            .unwrap();
+            .filter(|idx| idx.name == "id_idx")
+            .collect::<Vec<_>>();
         let vector_indices = indices_after
             .iter()
             .filter(|idx| idx.name == "vector_idx")
             .collect::<Vec<_>>();
-        assert!(
-            id_idx
-                .fragment_bitmap
-                .as_ref()
-                .unwrap()
-                .contains_range(0..2)
-                && id_idx.fragment_bitmap.as_ref().unwrap().len() == 2
-        );
-        assert_eq!(vector_indices.len(), 2);
-        assert!(
-            vector_indices
-                .iter()
-                .any(|idx| idx.fragment_bitmap.as_ref().unwrap().contains(0)
-                    && idx.fragment_bitmap.as_ref().unwrap().len() == 1)
-        );
-        assert!(
-            vector_indices
-                .iter()
-                .any(|idx| idx.fragment_bitmap.as_ref().unwrap().contains(1)
-                    && idx.fragment_bitmap.as_ref().unwrap().len() == 1)
-        );
+        for indices in [&id_indices, &vector_indices] {
+            assert_eq!(indices.len(), 2);
+            assert!(
+                indices
+                    .iter()
+                    .any(|idx| idx.fragment_bitmap.as_ref().unwrap().contains(0)
+                        && idx.fragment_bitmap.as_ref().unwrap().len() == 1)
+            );
+            assert!(
+                indices
+                    .iter()
+                    .any(|idx| idx.fragment_bitmap.as_ref().unwrap().contains(1)
+                        && idx.fragment_bitmap.as_ref().unwrap().len() == 1)
+            );
+        }
     }
 }
diff --git a/rust/lance/src/index/mem_wal.rs b/rust/lance/src/index/mem_wal.rs
index de2c70b62d2..84c9ae25d86 100644
--- a/rust/lance/src/index/mem_wal.rs
+++ b/rust/lance/src/index/mem_wal.rs
@@ -490,4 +490,77 @@ mod tests {
 
         assert!(indices.is_empty());
     }
+
+    /// Regression: a committed `__mem_wal` (legitimately `fragment_bitmap:
+    /// None`) must not break `describe_indices` — the path behind lancedb's
+    /// `list_indices`/`wait_for_index`. It's described as zero indexed rows,
+    /// like `__frag_reuse`.
+    #[tokio::test]
+    async fn test_describe_indices_includes_mem_wal_system_index() {
+        use crate::index::DatasetIndexExt;
+        use lance_index::IndexType;
+        use lance_index::scalar::ScalarIndexParams;
+
+        let mut dataset = test_dataset().await;
+
+        // A real user index that describe_indices must keep returning.
+        dataset
+            .create_index(
+                &["a"],
+                IndexType::Scalar,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+
+        // Commit a __mem_wal index, as WAL provisioning does in production.
+        let shard = Uuid::new_v4();
+        let txn = Transaction::new(
+            dataset.manifest.version,
+            Operation::UpdateMemWalState {
+                merged_generations: vec![MergedGeneration::new(shard, 1)],
+            },
+            None,
+        );
+        let dataset = CommitBuilder::new(Arc::new(dataset))
+            .execute(txn)
+            .await
+            .unwrap();
+
+        // The system index is present with no fragment_bitmap (by design).
+        let mem_wal = dataset
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .find(|i| i.name == MEM_WAL_INDEX_NAME)
+            .unwrap()
+            .clone();
+        assert!(mem_wal.fragment_bitmap.is_none());
+
+        // describe_indices describes the bitmap-less __mem_wal alongside the
+        // real index instead of erroring.
+        let descriptions = dataset.describe_indices(None).await.unwrap();
+        let mem_wal_desc = descriptions
+            .iter()
+            .find(|d| d.name() == MEM_WAL_INDEX_NAME)
+            .expect("__mem_wal must be described, not skipped");
+        assert_eq!(
+            mem_wal_desc.index_type(),
+            "MemWal",
+            "system index type must resolve via infer_system_index_type"
+        );
+        assert_eq!(
+            mem_wal_desc.rows_indexed(),
+            0,
+            "a bitmap-less system index indexes zero rows"
+        );
+        assert_eq!(
+            descriptions.len(),
+            2,
+            "both the real scalar index and __mem_wal must be listed"
+        );
+    }
 }
diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs
index 18c218ef4f7..2f346163974 100644
--- a/rust/lance/src/index/scalar.rs
+++ b/rust/lance/src/index/scalar.rs
@@ -4,12 +4,21 @@
 //! Utilities for integrating scalar indices with datasets
 //!
 
+pub(crate) mod bitmap;
+pub(crate) mod btree;
+pub(crate) mod fmindex;
 pub(crate) mod inverted;
+pub(crate) mod label_list;
+pub(crate) mod zonemap;
 
-pub use inverted::{load_segment_details, load_segments};
+pub use inverted::{load_segment_details, load_segment_params, load_segments};
+
+pub use crate::index::scalar_logical::{LogicalScalarIndex, load_named_scalar_segments};
 
 use std::sync::{Arc, LazyLock};
 
+use uuid::Uuid;
+
 use crate::index::DatasetIndexExt;
 use crate::index::DatasetIndexInternalExt;
 use crate::session::index_caches::ProstAny;
@@ -26,6 +35,7 @@ use lance_core::datatypes::Field;
 use lance_core::utils::tracing::{IO_TYPE_OPEN_SCALAR, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ADDR, ROW_ID, Result};
 use lance_datafusion::exec::LanceExecutionOptions;
+use lance_index::frag_reuse::FragReuseIndexHandle;
 use lance_index::metrics::{MetricsCollector, NoOpMetricsCollector};
 use lance_index::pbold::{
     BTreeIndexDetails, BitmapIndexDetails, InvertedIndexDetails, LabelListIndexDetails,
@@ -38,12 +48,12 @@ use lance_index::scalar::label_list::{
     LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION,
 };
 use lance_index::scalar::registry::{
-    ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, VALUE_COLUMN_NAME,
+    ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, VALUE_COLUMN_NAME,
 };
-use lance_index::scalar::{CreatedIndex, InvertedIndexParams};
+use lance_index::scalar::{BuiltinIndexType, CreatedIndex, InvertedIndexParams};
 use lance_index::scalar::{
-    ScalarIndex, ScalarIndexParams, bitmap::BITMAP_LOOKUP_NAME, inverted::INVERT_LIST_FILE,
-    lance_format::LanceIndexStore,
+    RowIdRemapper, ScalarIndex, ScalarIndexParams, bitmap::BITMAP_LOOKUP_NAME,
+    inverted::INVERT_LIST_FILE, lance_format::LanceIndexStore,
 };
 use lance_index::{IndexCriteria, IndexType};
 use lance_table::format::{Fragment, IndexMetadata};
@@ -270,7 +280,7 @@ impl IndexDetails {
 pub(super) async fn build_scalar_index(
     dataset: &Dataset,
     column: &str,
-    uuid: &str,
+    uuid: Uuid,
     params: &ScalarIndexParams,
     train: bool,
     fragment_ids: Option<Vec<u32>>,
@@ -285,11 +295,16 @@ pub(super) async fn build_scalar_index(
         ))?;
     let field: arrow_schema::Field = field.into();
 
-    let index_store = LanceIndexStore::from_dataset_for_new(dataset, uuid)?;
+    let index_store = LanceIndexStore::from_dataset_for_new(dataset, &uuid)?;
 
     let plugin = SCALAR_INDEX_PLUGIN_REGISTRY.get_plugin_by_name(&params.index_type)?;
+    let trainer = plugin.basic_trainer().ok_or_else(|| {
+        Error::invalid_input_source(
+            format!("The '{}' index type does not support basic training, please refer to the index's documentation for more details on how to create this index.", params.index_type).into(),
+        )
+    })?;
     let training_request =
-        plugin.new_training_request(params.params.as_deref().unwrap_or("{}"), &field)?;
+        trainer.new_training_request(params.params.as_deref().unwrap_or("{}"), &field)?;
 
     progress.stage_start("load_data", None, "rows").await?;
     let training_data = match preprocessed_data {
@@ -308,7 +323,7 @@ pub(super) async fn build_scalar_index(
     };
     progress.stage_complete("load_data").await?;
 
-    let created_index = plugin
+    let created_index = trainer
         .train_index(
             training_data,
             &index_store,
@@ -321,6 +336,56 @@ pub(super) async fn build_scalar_index(
     Ok(created_index)
 }
 
+/// Build a canonical bitmap index segment over a caller-selected fragment set.
+///
+/// This is intentionally separate from `build_scalar_index(..., fragment_ids=Some(...))`.
+/// The latter is the legacy distributed scalar-index shard path. Here fragment ids only
+/// restrict the scanned rows; the bitmap plugin receives no shard id and writes the
+/// canonical bitmap layout for the staged segment root.
+#[instrument(level = "debug", skip_all)]
+pub(super) async fn build_bitmap_index_segment(
+    dataset: &Dataset,
+    column: &str,
+    uuid: Uuid,
+    fragment_ids: Vec<u32>,
+    progress: Arc<dyn IndexBuildProgress>,
+) -> Result<CreatedIndex> {
+    let field = dataset
+        .schema()
+        .field(column)
+        .ok_or(Error::invalid_input_source(
+            format!("No column with name {}", column).into(),
+        ))?;
+    let field: arrow_schema::Field = field.into();
+
+    let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap);
+    let plugin = SCALAR_INDEX_PLUGIN_REGISTRY.get_plugin_by_name(&params.index_type)?;
+    let trainer = plugin.basic_trainer().ok_or_else(|| {
+        Error::invalid_input_source(
+            format!("The '{}' index type does not support basic training, please refer to the index's documentation for more details on how to create this index.", params.index_type).into(),
+        )
+    })?;
+    let training_request =
+        trainer.new_training_request(params.params.as_deref().unwrap_or("{}"), &field)?;
+    let criteria = training_request.criteria();
+
+    progress.stage_start("load_data", None, "rows").await?;
+    let training_data =
+        load_training_data(dataset, column, criteria, None, true, Some(fragment_ids)).await?;
+    progress.stage_complete("load_data").await?;
+
+    let index_store = LanceIndexStore::from_dataset_for_new(dataset, &uuid)?;
+    trainer
+        .train_index(
+            training_data,
+            &index_store,
+            training_request,
+            None,
+            progress,
+        )
+        .await
+}
+
 /// Fetches the scalar index plugin for a given index metadata
 ///
 /// The fast path, on newer datasets, is just a plugin lookup by the type URL of the index details.
@@ -383,7 +448,7 @@ pub async fn open_scalar_index(
     index: &IndexMetadata,
     metrics: &dyn MetricsCollector,
 ) -> Result<Arc<dyn ScalarIndex>> {
-    let uuid_str = index.uuid.to_string();
+    let index_uuid = index.uuid;
     let index_store = Arc::new(LanceIndexStore::from_dataset_for_existing(dataset, index).await?);
 
     let index_details = fetch_index_details(dataset, column, index).await?;
@@ -393,31 +458,37 @@ pub async fn open_scalar_index(
 
     let index_cache = dataset
         .index_cache
-        .for_index(&uuid_str, frag_reuse_index.as_ref().map(|f| &f.uuid));
-
-    if let Some(index) = plugin
-        .get_from_cache(index_store.clone(), frag_reuse_index.clone(), &index_cache)
-        .await?
-    {
-        // Compatibility check is only needed on first load; a cache hit means
-        // the index was already validated when it was originally opened in
-        // this session, so we can skip the extra `open_index_file` IOP.
-        return Ok(index);
-    }
-
-    if index_details.type_url.ends_with("LabelListIndexDetails") {
-        validate_label_list_index_compatibility(dataset, column, index, &index_store).await?;
-    }
+        .for_index(&index.uuid, frag_reuse_index.as_ref().map(|f| &f.uuid));
+
+    let frag_reuse_index: Option<Arc<dyn RowIdRemapper>> =
+        frag_reuse_index.map(|f| Arc::new(FragReuseIndexHandle(f)) as Arc<dyn RowIdRemapper>);
+
+    // Runs only on a cold miss, and at most once even under concurrent opens
+    // (the plugin coalesces). The compat check lives here because a warm hit was
+    // already validated this session, saving the extra `open_index_file` IOP.
+    let load: ScalarIndexLoad = Box::pin({
+        let index_store = index_store.clone();
+        let frag_reuse_index = frag_reuse_index.clone();
+        let index_cache = index_cache.clone();
+        async move {
+            if index_details.type_url.ends_with("LabelListIndexDetails") {
+                validate_label_list_index_compatibility(dataset, column, index, &index_store)
+                    .await?;
+            }
 
-    let index = plugin
-        .load_index(index_store, &index_details, frag_reuse_index, &index_cache)
-        .await?;
+            let index = plugin
+                .load_index(index_store, &index_details, frag_reuse_index, &index_cache)
+                .await?;
 
-    tracing::info!(target: TRACE_IO_EVENTS, index_uuid = uuid_str, r#type = IO_TYPE_OPEN_SCALAR, index_type = index.index_type().to_string());
-    metrics.record_index_load();
+            tracing::info!(target: TRACE_IO_EVENTS, index_uuid = %index_uuid, r#type = IO_TYPE_OPEN_SCALAR, index_type = index.index_type().to_string());
+            metrics.record_index_load();
+            Ok(index)
+        }
+    });
 
-    plugin.put_in_cache(&index_cache, index.clone()).await?;
-    Ok(index)
+    plugin
+        .get_or_insert_in_cache(index_store, frag_reuse_index, &index_cache, load)
+        .await
 }
 
 pub(crate) async fn infer_scalar_index_details(
@@ -425,13 +496,14 @@ pub(crate) async fn infer_scalar_index_details(
     column: &str,
     index: &IndexMetadata,
 ) -> Result<Arc<prost_types::Any>> {
-    let uuid = index.uuid.to_string();
-    let type_key = crate::session::index_caches::ScalarIndexDetailsKey { uuid: &uuid };
+    let type_key = crate::session::index_caches::ScalarIndexDetailsKey { uuid: &index.uuid };
     if let Some(index_details) = dataset.index_cache.get_with_key(&type_key).await {
         return Ok(index_details.0.clone());
     }
 
-    let index_dir = dataset.indice_files_dir(index)?.join(uuid.clone());
+    let index_dir = dataset
+        .indice_files_dir(index)?
+        .join(index.uuid.to_string());
     let col = dataset
         .schema()
         .field(column)
@@ -564,11 +636,7 @@ pub async fn initialize_scalar_index(
     let column_name = field_names[0];
 
     let source_scalar_index = source_dataset
-        .open_scalar_index(
-            column_name,
-            &source_index.uuid.to_string(),
-            &NoOpMetricsCollector,
-        )
+        .open_scalar_index(column_name, &source_index.uuid, &NoOpMetricsCollector)
         .await?;
 
     let params = source_scalar_index.derive_index_params()?;
@@ -988,6 +1056,51 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_open_scalar_index_coalesces_concurrent_cold_opens() {
+        use crate::dataset::builder::DatasetBuilder;
+        use arrow::datatypes::Int64Type;
+        use futures::future::try_join_all;
+        use lance_index::metrics::LocalMetricsCollector;
+        use std::sync::atomic::Ordering;
+
+        let dir = TempStrDir::default();
+        let uri = dir.as_ref();
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int64Type>())
+            .into_dataset(uri, FragmentCount::from(1), FragmentRowCount::from(200))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree);
+        ds.create_index(&["id"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Reopen so the index cache is cold for the concurrent opens below.
+        let ds = DatasetBuilder::from_uri(uri).load().await.unwrap();
+        let id_field = ds.schema().field("id").unwrap().id;
+        let index_meta = ds
+            .load_indices()
+            .await
+            .unwrap()
+            .iter()
+            .find(|idx| idx.fields == [id_field])
+            .cloned()
+            .expect("btree index on `id`");
+
+        // Eight concurrent cold opens; single-flight means the loader (which calls
+        // `record_index_load`) runs exactly once, not once per open.
+        let metrics = LocalMetricsCollector::default();
+        let indices =
+            try_join_all((0..8).map(|_| open_scalar_index(&ds, "id", &index_meta, &metrics)))
+                .await
+                .unwrap();
+
+        assert_eq!(indices.len(), 8);
+        assert_eq!(metrics.index_loads.load(Ordering::Relaxed), 1);
+    }
+
     #[tokio::test]
     async fn test_load_training_data_addr_sort() {
         // Create test data using lance_datagen
@@ -1111,11 +1224,7 @@ mod tests {
 
         // Verify the index type is correct
         let target_scalar_index = target_dataset
-            .open_scalar_index(
-                "id",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_scalar_index("id", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -1189,7 +1298,7 @@ mod tests {
 
         // Verify the index type is correct
         let scalar_index = dataset
-            .open_scalar_index("id", &indices[0].uuid.to_string(), &NoOpMetricsCollector)
+            .open_scalar_index("id", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -1234,7 +1343,7 @@ mod tests {
         );
 
         let scalar_index = dataset
-            .open_scalar_index("id", &indices[0].uuid.to_string(), &NoOpMetricsCollector)
+            .open_scalar_index("id", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -1417,11 +1526,7 @@ mod tests {
 
         // Verify the index type is correct
         let target_scalar_index = target_dataset
-            .open_scalar_index(
-                "text",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_scalar_index("text", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -1555,11 +1660,7 @@ mod tests {
 
         // Verify the index type is correct
         let target_scalar_index = target_dataset
-            .open_scalar_index(
-                "value",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_scalar_index("value", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -1736,6 +1837,122 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_dataset_column_value_range() {
+        use crate::index::DatasetIndexExt;
+        use arrow::datatypes::Int64Type;
+        use datafusion::scalar::ScalarValue;
+        use lance_datagen::array;
+        use lance_index::IndexType;
+        use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+        // 2 fragments x 5 rows: `id` and `other` both step 0..9.
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int64Type>())
+            .col("other", array::step::<Int64Type>())
+            .into_ram_dataset(FragmentCount::from(2), FragmentRowCount::from(5))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        ds.create_index(&["id"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Folded straight from the ZoneMap summaries: global [0, 9].
+        assert_eq!(
+            ds.statistics().column_value_range("id").await.unwrap(),
+            Some((ScalarValue::Int64(Some(0)), ScalarValue::Int64(Some(9))))
+        );
+        // `other` has no ZoneMap index -> no precomputed range.
+        assert_eq!(
+            ds.statistics().column_value_range("other").await.unwrap(),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn test_column_value_range_none_when_index_misses_fragments() {
+        use crate::index::DatasetIndexExt;
+        use arrow::datatypes::Int64Type;
+        use lance_datagen::{BatchCount, RowCount, array};
+        use lance_index::IndexType;
+        use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+        // Index covers the initial 2 fragments.
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int64Type>())
+            .into_ram_dataset(FragmentCount::from(2), FragmentRowCount::from(5))
+            .await
+            .unwrap();
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        ds.create_index(&["id"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+        assert!(
+            ds.statistics()
+                .column_value_range("id")
+                .await
+                .unwrap()
+                .is_some()
+        );
+
+        // Append a fragment the index doesn't cover. Its rows may hold values
+        // outside the indexed range, so any non-None range would be a subset
+        // unsound to prune with -> must return None until the index is rebuilt.
+        let reader = lance_datagen::gen_batch()
+            .col("id", array::step::<Int64Type>())
+            .into_reader_rows(RowCount::from(5), BatchCount::from(1));
+        ds.append(reader, None).await.unwrap();
+
+        assert_eq!(
+            ds.statistics().column_value_range("id").await.unwrap(),
+            None
+        );
+    }
+
+    #[tokio::test]
+    async fn test_column_value_range_folds_multiple_segments() {
+        use crate::index::DatasetIndexExt;
+        use arrow::datatypes::Int64Type;
+        use datafusion::scalar::ScalarValue;
+        use lance_datagen::array;
+        use lance_index::IndexType;
+        use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+        // 2 fragments x 5 rows (`id` steps 0..9). Build one ZoneMap segment per
+        // fragment, then commit them as a single multi-segment logical index.
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int64Type>())
+            .into_ram_dataset(FragmentCount::from(2), FragmentRowCount::from(5))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        let columns = ["id"];
+        let mut segments = Vec::new();
+        for fragment_id in [0_u32, 1_u32] {
+            let mut builder = ds
+                .create_index_builder(&columns, IndexType::Scalar, &params)
+                .name("id_idx".to_string())
+                .fragments(vec![fragment_id]);
+            segments.push(builder.execute_uncommitted().await.unwrap());
+        }
+        // execute_uncommitted yields ready IndexMetadata segments (IntoIndexSegment);
+        // build_all is vector-only, so commit the per-fragment segments directly.
+        ds.commit_existing_index_segments("id_idx", "id", segments)
+            .await
+            .unwrap();
+
+        // Two disjoint segments jointly cover every live fragment -> the fold
+        // spans both and yields the global range, not None.
+        assert_eq!(ds.load_indices_by_name("id_idx").await.unwrap().len(), 2);
+        assert_eq!(
+            ds.statistics().column_value_range("id").await.unwrap(),
+            Some((ScalarValue::Int64(Some(0)), ScalarValue::Int64(Some(9))))
+        );
+    }
+
     #[tokio::test]
     async fn test_zonemap_index_then_deletion() {
         // Tests the opposite scenario: create index FIRST, then perform deletions
@@ -2088,4 +2305,70 @@ mod tests {
             "Should have 0 rows with value='banana' after deletion"
         );
     }
+
+    // End-to-end: create index → delete a whole fragment → search (via index) →
+    // update index → search again.  No deleted rows should ever appear.
+    #[tokio::test]
+    async fn test_zonemap_search_with_deleted_fragment_before_and_after_update() {
+        use arrow::datatypes::Int32Type;
+        use lance_datagen::array;
+        use lance_index::IndexType;
+        use lance_index::optimize::OptimizeOptions;
+        use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+        // 3 fragments × 10 rows: id 0-9 (frag 0), 10-19 (frag 1), 20-29 (frag 2).
+        let mut ds = lance_datagen::gen_batch()
+            .col("id", array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(10))
+            .await
+            .unwrap();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        ds.create_index(&["id"], IndexType::Scalar, None, &params, false)
+            .await
+            .unwrap();
+
+        // Delete the middle fragment entirely.
+        ds.delete("id >= 10 AND id < 20").await.unwrap();
+
+        // Helper: run a filter scan and return the sorted id values.
+        async fn live_ids(ds: &crate::Dataset) -> Vec<i32> {
+            let batch = ds
+                .scan()
+                .filter("id >= 0")
+                .unwrap()
+                .try_into_batch()
+                .await
+                .unwrap();
+            let mut ids: Vec<i32> = batch["id"]
+                .as_any()
+                .downcast_ref::<arrow_array::Int32Array>()
+                .unwrap()
+                .values()
+                .to_vec();
+            ids.sort_unstable();
+            ids
+        }
+
+        // --- Before index update ---
+        let ids = live_ids(&ds).await;
+        assert_eq!(ids.len(), 20, "expected 20 live rows before index update");
+        assert!(
+            ids.iter().all(|&id| !(10..20).contains(&id)),
+            "deleted fragment rows (id 10-19) must not appear before index update; got: {:?}",
+            ids
+        );
+
+        // Update the zone map index to reflect the deletion.
+        ds.optimize_indices(&OptimizeOptions::new()).await.unwrap();
+
+        // --- After index update ---
+        let ids = live_ids(&ds).await;
+        assert_eq!(ids.len(), 20, "expected 20 live rows after index update");
+        assert!(
+            ids.iter().all(|&id| !(10..20).contains(&id)),
+            "deleted fragment rows (id 10-19) must not appear after index update; got: {:?}",
+            ids
+        );
+    }
 }
diff --git a/rust/lance/src/index/scalar/bitmap.rs b/rust/lance/src/index/scalar/bitmap.rs
new file mode 100644
index 00000000000..84c8b6a8b91
--- /dev/null
+++ b/rust/lance/src/index/scalar/bitmap.rs
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::bitmap::BitmapIndex;
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_table::format::IndexMetadata;
+use roaring::RoaringBitmap;
+use std::sync::Arc;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+/// Merge one caller-defined group of source bitmap segments into a single segment.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+
+    let mut source_indices = Vec::with_capacity(segments.len());
+    let mut fragment_bitmap = RoaringBitmap::new();
+    for segment in &segments {
+        fragment_bitmap |= segment.fragment_bitmap.as_ref().cloned().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing fragment coverage",
+                segment.uuid
+            ))
+        })?;
+        let scalar_index =
+            super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?;
+        let bitmap_index = scalar_index
+            .as_any()
+            .downcast_ref::<BitmapIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "merge_existing_index_segments: expected bitmap segment {}, got {:?}",
+                    segment.uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push(Arc::new(bitmap_index.clone()));
+    }
+
+    let new_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
+    let created_index = lance_index::scalar::bitmap::merge_bitmap_indices(
+        &source_indices,
+        &new_store,
+        lance_index::progress::noop_progress(),
+    )
+    .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version: dataset.manifest.version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar/btree.rs b/rust/lance/src/index/scalar/btree.rs
new file mode 100644
index 00000000000..7089997721c
--- /dev/null
+++ b/rust/lance/src/index/scalar/btree.rs
@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+#![allow(clippy::redundant_pub_crate)]
+
+//! BTree-specific helpers for the segmented index workflow.
+use std::sync::Arc;
+
+use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
+use datafusion::execution::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use lance_core::ROW_ID;
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::pbold::BTreeIndexDetails;
+use lance_index::scalar::btree::BTreeIndex;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_index::scalar::registry::VALUE_COLUMN_NAME;
+use lance_index::scalar::{CreatedIndex, OldIndexDataFilter, index_files_to_table};
+use lance_table::format::IndexMetadata;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+/// Build a row-empty `new_data` stream for the BTree merge API.
+fn empty_btree_update_stream(
+    dataset: &Dataset,
+    field_id: i32,
+) -> Result<SendableRecordBatchStream> {
+    let field = dataset.schema().field_by_id(field_id).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "merge_existing_index_segments: field id {} does not exist",
+            field_id
+        ))
+    })?;
+    let schema = Arc::new(ArrowSchema::new(vec![
+        ArrowField::new(VALUE_COLUMN_NAME, field.data_type(), true),
+        ArrowField::new(ROW_ID, arrow_schema::DataType::UInt64, false),
+    ]));
+    Ok(Box::pin(RecordBatchStreamAdapter::new(
+        schema,
+        futures::stream::empty(),
+    )))
+}
+
+fn ensure_btree_details(segment: &IndexMetadata) -> Result<()> {
+    if let Some(details) = segment.index_details.as_ref()
+        && !details.type_url.ends_with("BTreeIndexDetails")
+    {
+        return Err(Error::invalid_input(format!(
+            "Segment '{}' is not a BTree segment (details type_url = '{}')",
+            segment.uuid, details.type_url
+        )));
+    }
+    Ok(())
+}
+
+/// Open the given BTree `segments` and k-way merge their already-sorted page
+/// data, together with `new_data`, into a single canonical BTree written to
+/// `new_store`.
+pub(crate) async fn open_and_merge_segments(
+    dataset: &Dataset,
+    field_path: &str,
+    segments: &[&IndexMetadata],
+    new_data: SendableRecordBatchStream,
+    new_store: &LanceIndexStore,
+    old_data_filters: &[Option<OldIndexDataFilter>],
+) -> Result<CreatedIndex> {
+    let mut source_indices = Vec::with_capacity(segments.len());
+    for &segment in segments {
+        let scalar_index =
+            super::open_scalar_index(dataset, field_path, segment, &NoOpMetricsCollector).await?;
+        let btree = scalar_index
+            .as_any()
+            .downcast_ref::<BTreeIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "BTree merge: expected BTree segment {}, got {:?}",
+                    segment.uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push(Arc::new(btree.clone()));
+    }
+    BTreeIndex::merge_segments(&source_indices, new_data, new_store, old_data_filters).await
+}
+
+/// Merge one caller-defined group of source BTree segments into a single
+/// physical segment.
+pub(crate) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    for segment in &segments {
+        ensure_btree_details(segment)?;
+    }
+
+    // All source segments must belong to the same column.
+    let reference_fields = segments[0].fields.as_slice();
+    for segment in segments.iter().skip(1) {
+        if segment.fields.as_slice() != reference_fields {
+            return Err(Error::invalid_input(format!(
+                "BTree merge_segments: segment {} has fields {:?}, expected {:?}",
+                segment.uuid, segment.fields, reference_fields,
+            )));
+        }
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+
+    let segment_refs: Vec<&IndexMetadata> = segments.iter().collect();
+    let (fragment_bitmap, old_data_filters) =
+        crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?;
+
+    let output_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &output_uuid)?;
+    // Pure segment consolidation: no dataset scan, so `new_data` is an empty
+    // stream and the merge is driven entirely by the source page data.
+    let empty_new_data = empty_btree_update_stream(dataset, field_id)?;
+    let created_index = open_and_merge_segments(
+        dataset,
+        &field_path,
+        &segment_refs,
+        empty_new_data,
+        &new_store,
+        &old_data_filters,
+    )
+    .await?;
+
+    if !created_index
+        .index_details
+        .type_url
+        .ends_with("BTreeIndexDetails")
+    {
+        return Err(Error::internal(format!(
+            "merge_existing_index_segments: BTree merge produced unexpected details type_url '{}'",
+            created_index.index_details.type_url
+        )));
+    }
+    debug_assert_eq!(
+        created_index.index_details,
+        prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap(),
+    );
+
+    Ok(IndexMetadata {
+        uuid: output_uuid,
+        name: segments[0].name.clone(),
+        fields: vec![field_id],
+        dataset_version: dataset.manifest.version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+    })
+}
diff --git a/rust/lance/src/index/scalar/fmindex.rs b/rust/lance/src/index/scalar/fmindex.rs
new file mode 100644
index 00000000000..9c1baadbbb6
--- /dev/null
+++ b/rust/lance/src/index/scalar/fmindex.rs
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use lance_index::scalar::index_files_to_table;
+use lance_table::format::IndexMetadata;
+use roaring::RoaringBitmap;
+use std::sync::Arc;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result};
+
+/// Merge one caller-defined group of source FM-Index segments into a single segment.
+///
+/// FM-Index merge requires rebuilding from source text — there is no cheap way
+/// to combine two BWT structures. This function re-reads text data from the
+/// dataset for all fragments covered by the source segments and builds a fresh
+/// FM-Index over the combined data.
+///
+/// As an exception, a single source segment whose fragment coverage is still
+/// fully live is returned as-is: rebuilding it would produce equivalent content
+/// over the same fragments, which makes distributed builds (one uncommitted
+/// segment per worker, merged 1:1 into final segments) pay for every segment
+/// twice.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let column = dataset.schema().field_path(field_id)?;
+
+    let mut fragment_bitmap = RoaringBitmap::new();
+    for segment in &segments {
+        fragment_bitmap |= segment.fragment_bitmap.as_ref().cloned().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing fragment coverage",
+                segment.uuid
+            ))
+        })?;
+    }
+
+    // Intersect with the dataset's current live fragments to drop retired/compacted
+    // fragments, mirroring the btree merge behavior.
+    fragment_bitmap &= dataset.fragment_bitmap.as_ref();
+
+    if fragment_bitmap.is_empty() {
+        // All covered fragments have been retired; produce an empty index.
+        let new_uuid = Uuid::new_v4();
+        let created_index = super::build_scalar_index(
+            dataset,
+            &column,
+            new_uuid,
+            &lance_index::scalar::ScalarIndexParams::for_builtin(
+                lance_index::scalar::BuiltinIndexType::Fm,
+            ),
+            false,
+            None,
+            None,
+            Arc::new(lance_index::progress::NoopIndexBuildProgress),
+        )
+        .await?;
+
+        return Ok(IndexMetadata {
+            uuid: new_uuid,
+            fields: vec![field_id],
+            dataset_version: dataset.manifest.version,
+            fragment_bitmap: Some(fragment_bitmap),
+            index_details: Some(Arc::new(created_index.index_details)),
+            index_version: created_index.index_version as i32,
+            created_at: Some(chrono::Utc::now()),
+            base_id: None,
+            files: Some(index_files_to_table(created_index.files)),
+            ..segments[0].clone()
+        });
+    }
+
+    if segments.len() == 1 && segments[0].fragment_bitmap.as_ref() == Some(&fragment_bitmap) {
+        return Ok(segments.into_iter().next().unwrap());
+    }
+
+    let fragment_ids: Vec<u32> = fragment_bitmap.iter().collect();
+    let new_uuid = Uuid::new_v4();
+
+    let created_index = super::build_scalar_index(
+        dataset,
+        &column,
+        new_uuid,
+        &lance_index::scalar::ScalarIndexParams::for_builtin(
+            lance_index::scalar::BuiltinIndexType::Fm,
+        ),
+        true,
+        Some(fragment_ids),
+        None,
+        Arc::new(lance_index::progress::NoopIndexBuildProgress),
+    )
+    .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version: dataset.manifest.version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs
index 44cf6ff2e08..426d104c912 100644
--- a/rust/lance/src/index/scalar/inverted.rs
+++ b/rust/lance/src/index/scalar/inverted.rs
@@ -11,7 +11,8 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use lance_core::ROW_ID;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::pbold::InvertedIndexDetails;
-use lance_index::scalar::inverted::InvertedIndex;
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexParams};
 use lance_index::scalar::lance_format::LanceIndexStore;
 use lance_index::scalar::registry::VALUE_COLUMN_NAME;
 use lance_table::format::IndexMetadata;
@@ -64,7 +65,7 @@ pub(crate) async fn finalize_segment_files_if_needed(
 
     let store = Arc::new(LanceIndexStore::from_dataset_for_new(
         dataset,
-        &segment.uuid.to_string(),
+        &segment.uuid,
     )?);
     lance_index::scalar::inverted::builder::merge_index_files(
         dataset.object_store.as_ref(),
@@ -118,7 +119,7 @@ pub(crate) async fn merge_segments(
     }
 
     let new_uuid = Uuid::new_v4();
-    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid.to_string())?;
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
     let created_index = InvertedIndex::merge_segments(
         &source_indices,
         empty_inverted_update_stream(dataset, field_id)?,
@@ -137,7 +138,7 @@ pub(crate) async fn merge_segments(
         index_version: created_index.index_version as i32,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: created_index.files,
+        files: Some(index_files_to_table(created_index.files)),
         ..segments[0].clone()
     })
 }
@@ -219,6 +220,15 @@ pub async fn load_segment_details(
     })
 }
 
+/// Read one segment's [`InvertedIndexParams`]
+pub async fn load_segment_params(
+    dataset: &Dataset,
+    segment: &IndexMetadata,
+) -> Result<InvertedIndexParams> {
+    let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?;
+    InvertedIndex::load_params(&store).await
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/rust/lance/src/index/scalar/label_list.rs b/rust/lance/src/index/scalar/label_list.rs
new file mode 100644
index 00000000000..884346f0d6b
--- /dev/null
+++ b/rust/lance/src/index/scalar/label_list.rs
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::IndexStore;
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::label_list::{
+    BITMAP_LOOKUP_NAME, LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, LabelListIndex,
+};
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_table::format::IndexMetadata;
+use roaring::RoaringBitmap;
+use std::sync::Arc;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+async fn validate_nullable_segment_for_merge(
+    dataset: &Dataset,
+    field_id: i32,
+    segment: &IndexMetadata,
+) -> Result<()> {
+    let field = dataset.schema().field_by_id(field_id).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "merge_existing_index_segments: field id {} does not exist",
+            field_id
+        ))
+    })?;
+
+    if !field.nullable {
+        return Ok(());
+    }
+
+    if segment.index_version < LABEL_LIST_NULLS_MIN_VERSION {
+        return Err(Error::invalid_input(format!(
+            "Cannot merge nullable LabelList segment {} because it was created before list-null metadata was required. Rebuild the segment instead.",
+            segment.uuid
+        )));
+    }
+
+    let index_store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?;
+    let reader = index_store.open_index_file(BITMAP_LOOKUP_NAME).await?;
+    if !reader
+        .schema()
+        .metadata
+        .contains_key(LABEL_LIST_NULLS_METADATA_KEY)
+    {
+        return Err(Error::invalid_input(format!(
+            "Cannot merge nullable LabelList segment {} because it is missing required metadata key {}. Rebuild the segment instead.",
+            segment.uuid, LABEL_LIST_NULLS_METADATA_KEY
+        )));
+    }
+
+    Ok(())
+}
+
+/// Merge one caller-defined group of source LabelList segments into a single segment.
+///
+/// A LabelList index is a bitmap over the unnested list values plus a `list_nulls`
+/// row set, so segments are recombined by unioning the underlying bitmap states and
+/// null sets (see [`lance_index::scalar::label_list::merge_label_list_indices`])
+/// rather than rebuilding from source text.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+
+    let dataset_fragments = dataset.fragment_bitmap.as_ref();
+    let mut effective_old_frags = RoaringBitmap::new();
+    let mut deleted_old_frags = RoaringBitmap::new();
+    for segment in &segments {
+        if segment.fragment_bitmap.is_none() {
+            return Err(Error::invalid_input(format!(
+                "CreateIndex: segment {} is missing fragment coverage",
+                segment.uuid
+            )));
+        }
+        if let Some(effective) = segment.effective_fragment_bitmap(dataset_fragments) {
+            effective_old_frags |= effective;
+        }
+        if let Some(deleted) = segment.deleted_fragment_bitmap(dataset_fragments) {
+            deleted_old_frags |= deleted;
+        }
+    }
+
+    let fragment_bitmap = effective_old_frags.clone();
+    let old_data_filter = if deleted_old_frags.is_empty() {
+        None
+    } else {
+        crate::index::append::build_old_data_filter(
+            dataset,
+            &effective_old_frags,
+            &deleted_old_frags,
+        )
+        .await?
+    };
+
+    let mut source_indices = Vec::with_capacity(segments.len());
+    for segment in &segments {
+        validate_nullable_segment_for_merge(dataset, field_id, segment).await?;
+        let scalar_index =
+            super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?;
+        let label_list_index = scalar_index
+            .as_any()
+            .downcast_ref::<LabelListIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "merge_existing_index_segments: expected label list segment {}, got {:?}",
+                    segment.uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push(Arc::new(label_list_index.clone()));
+    }
+
+    let new_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
+    let created_index = lance_index::scalar::label_list::merge_label_list_indices(
+        &source_indices,
+        &new_store,
+        old_data_filter,
+        lance_index::progress::noop_progress(),
+    )
+    .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version: dataset.manifest.version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar/zonemap.rs b/rust/lance/src/index/scalar/zonemap.rs
new file mode 100644
index 00000000000..a3524b4955e
--- /dev/null
+++ b/rust/lance/src/index/scalar/zonemap.rs
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::scalar::index_files_to_table;
+use lance_index::scalar::lance_format::LanceIndexStore;
+use lance_index::scalar::zonemap::ZoneMapIndex;
+use lance_table::format::IndexMetadata;
+use roaring::RoaringBitmap;
+use uuid::Uuid;
+
+use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt};
+
+/// Merge one caller-defined group of source ZoneMap segments into a single segment.
+pub(in crate::index) async fn merge_segments(
+    dataset: &Dataset,
+    segments: Vec<IndexMetadata>,
+) -> Result<IndexMetadata> {
+    if segments.is_empty() {
+        return Err(Error::index("No segment metadata was provided".to_string()));
+    }
+
+    let field_id = *segments[0].fields.first().ok_or_else(|| {
+        Error::invalid_input(format!(
+            "CreateIndex: segment {} is missing field ids",
+            segments[0].uuid
+        ))
+    })?;
+    let field_path = dataset.schema().field_path(field_id)?;
+
+    let mut scalar_indices = Vec::with_capacity(segments.len());
+    let mut fragment_bitmap = RoaringBitmap::new();
+    let dataset_fragments = dataset.fragment_bitmap.as_ref();
+    for segment in &segments {
+        let effective = segment
+            .effective_fragment_bitmap(dataset_fragments)
+            .ok_or_else(|| {
+                Error::invalid_input(format!(
+                    "CreateIndex: segment {} is missing fragment coverage",
+                    segment.uuid
+                ))
+            })?;
+        fragment_bitmap |= effective;
+        let scalar_index =
+            super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?;
+        scalar_indices.push((segment.uuid, scalar_index));
+    }
+
+    let mut source_indices = Vec::with_capacity(scalar_indices.len());
+    for (segment_uuid, scalar_index) in &scalar_indices {
+        let zonemap_index = scalar_index
+            .as_any()
+            .downcast_ref::<ZoneMapIndex>()
+            .ok_or_else(|| {
+                Error::index(format!(
+                    "merge_existing_index_segments: expected zonemap segment {}, got {:?}",
+                    segment_uuid,
+                    scalar_index.index_type()
+                ))
+            })?;
+        source_indices.push(zonemap_index);
+    }
+
+    let new_uuid = Uuid::new_v4();
+    let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?;
+    let created_index = lance_index::scalar::zonemap::merge_zonemap_indices(
+        &source_indices,
+        &new_store,
+        &fragment_bitmap,
+    )
+    .await?;
+
+    Ok(IndexMetadata {
+        uuid: new_uuid,
+        fields: vec![field_id],
+        dataset_version: dataset.manifest.version,
+        fragment_bitmap: Some(fragment_bitmap),
+        index_details: Some(Arc::new(created_index.index_details)),
+        index_version: created_index.index_version as i32,
+        created_at: Some(chrono::Utc::now()),
+        base_id: None,
+        files: Some(index_files_to_table(created_index.files)),
+        ..segments[0].clone()
+    })
+}
diff --git a/rust/lance/src/index/scalar_logical.rs b/rust/lance/src/index/scalar_logical.rs
index 162a36a0c97..d6b30188573 100644
--- a/rust/lance/src/index/scalar_logical.rs
+++ b/rust/lance/src/index/scalar_logical.rs
@@ -3,12 +3,13 @@
 
 //! Query-time logical views over scalar index segments.
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::any::Any;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use deepsize::{Context, DeepSizeOf};
 use futures::future::try_join_all;
+use lance_core::deepsize::{Context, DeepSizeOf};
 use lance_core::{Error, Result};
 use lance_index::metrics::MetricsCollector;
 use lance_index::scalar::{AnyQuery, CreatedIndex, ScalarIndex, SearchResult, UpdateCriteria};
@@ -22,6 +23,21 @@ use crate::dataset::Dataset;
 use crate::index::scalar::fetch_index_details;
 use crate::index::{DatasetIndexExt, DatasetIndexInternalExt};
 
+/// Query-time view that exposes several physical scalar index segments as a single [`ScalarIndex`].
+///
+/// A named scalar index can be built incrementally, producing multiple physical segments that
+/// each cover a disjoint set of fragments. When such an index is opened, the loader bundles
+/// the segments into a `LogicalScalarIndex` so the scanner can treat them as one index: queries
+/// are fanned out to every segment in parallel and the row-address results are unioned together.
+///
+/// All segments must share the same [`IndexType`]; mixing types is rejected at construction.
+/// Per-segment [`SearchResult`] precision is preserved when combining: a union of `Exact`
+/// results stays `Exact`, a union containing `AtMost` results yields `AtMost`, and a union
+/// containing `AtLeast` results yields `AtLeast`. Combining `AtMost` and `AtLeast` segments in
+/// the same query is not supported.
+///
+/// This is a read-only wrapper. [`ScalarIndex::remap`] and [`ScalarIndex::update`] both return
+/// an error — callers must rebuild the index to consolidate segments before mutating it.
 #[derive(Debug)]
 pub struct LogicalScalarIndex {
     name: String,
@@ -31,7 +47,17 @@ pub struct LogicalScalarIndex {
 }
 
 impl LogicalScalarIndex {
-    fn try_new(name: String, column: String, segments: Vec<Arc<dyn ScalarIndex>>) -> Result<Self> {
+    /// Merge several already-opened segments of one scalar index into a single
+    /// searchable [`ScalarIndex`].
+    ///
+    /// Used internally by `open_named_scalar_index`, and exposed so a
+    /// distributed query engine can open an explicit subset of a scalar
+    /// index's segments and present them as one index.
+    pub fn try_new(
+        name: String,
+        column: String,
+        segments: Vec<Arc<dyn ScalarIndex>>,
+    ) -> Result<Self> {
         let Some(first) = segments.first() else {
             return Err(Error::invalid_input(format!(
                 "LogicalScalarIndex '{}' on column '{}' must contain at least one segment",
@@ -76,13 +102,6 @@ impl Index for LogicalScalarIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn lance_index::vector::VectorIndex>> {
-        Err(Error::invalid_input(format!(
-            "LogicalScalarIndex '{}' is not a vector index",
-            self.name
-        )))
-    }
-
     fn statistics(&self) -> Result<serde_json::Value> {
         Ok(json!({
             "index_name": self.name,
@@ -132,13 +151,19 @@ impl ScalarIndex for LogicalScalarIndex {
         combine_search_results(results)
     }
 
+    fn results_are_row_addresses(&self) -> bool {
+        // All segments of a logical index share the same underlying index type,
+        // so they agree on the result domain.
+        self.segments[0].results_are_row_addresses()
+    }
+
     fn can_remap(&self) -> bool {
         false
     }
 
     async fn remap(
         &self,
-        _mapping: &std::collections::HashMap<u64, Option<u64>>,
+        _mapping: &RowAddrRemap,
         _dest_store: &dyn lance_index::scalar::IndexStore,
     ) -> Result<CreatedIndex> {
         Err(Error::invalid_input(format!(
@@ -210,7 +235,14 @@ fn index_intersects_dataset(index: &IndexMetadata, dataset: &Dataset) -> bool {
         .is_some_and(|index_bitmap| index_bitmap.intersection_len(&dataset.fragment_bitmap) > 0)
 }
 
-async fn load_named_scalar_segments(
+/// List the committed, dataset-intersecting segments of a named scalar index.
+///
+/// Returns one [`IndexMetadata`] per usable segment. The result length is the
+/// segment count: `1` means a single (non-segmented) index, `> 1` means the
+/// index is split across multiple segments that a distributed engine may route
+/// to different executors. All returned segments are validated to share the
+/// same underlying index type.
+pub async fn load_named_scalar_segments(
     dataset: &Dataset,
     column: &str,
     index_name: &str,
@@ -263,6 +295,11 @@ fn union_fragment_bitmaps(indices: &[IndexMetadata], index_name: &str) -> Result
     Ok(combined)
 }
 
+/// Return the union of fragment bitmaps across every usable segment of a named scalar index.
+///
+/// Only segments whose fragment bitmap intersects the dataset's current fragment set are
+/// considered. Returns `Ok(None)` when no such segment exists, `Ok(Some(bitmap))` otherwise.
+/// Errors if the segments disagree on their underlying index type.
 pub async fn scalar_index_fragment_bitmap(
     dataset: &Dataset,
     column: &str,
@@ -279,6 +316,13 @@ pub async fn scalar_index_fragment_bitmap(
     }
 }
 
+/// Open a named scalar index, transparently bundling multiple segments when present.
+///
+/// Loads every segment registered under `index_name` whose fragment bitmap intersects the
+/// dataset. If exactly one usable segment exists it is returned directly; if multiple exist
+/// they are wrapped in a [`LogicalScalarIndex`] so the caller sees a single [`ScalarIndex`].
+/// Errors if no usable segment exists (the scanner planned a query against an index that is
+/// not present) or if the segments mix incompatible types.
 pub async fn open_named_scalar_index(
     dataset: &Dataset,
     column: &str,
@@ -292,13 +336,15 @@ pub async fn open_named_scalar_index(
             index_name, column
         ))),
         1 => {
-            let uuid = indices[0].uuid.to_string();
-            dataset.open_scalar_index(column, &uuid, metrics).await
+            dataset
+                .open_scalar_index(column, &indices[0].uuid, metrics)
+                .await
         }
         _ => {
-            let segments = try_join_all(indices.iter().map(|index| {
-                let uuid = index.uuid.to_string();
-                async move { dataset.open_scalar_index(column, &uuid, metrics).await }
+            let segments = try_join_all(indices.iter().map(|index| async move {
+                dataset
+                    .open_scalar_index(column, &index.uuid, metrics)
+                    .await
             }))
             .await?;
 
@@ -323,8 +369,13 @@ mod tests {
     use lance_datagen::array;
     use lance_index::IndexType;
     use lance_index::metrics::NoOpMetricsCollector;
+    use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME;
     use lance_index::scalar::{BuiltinIndexType, SargableQuery, ScalarIndexParams};
 
+    use crate::Dataset;
+    use crate::dataset::WriteParams;
+    use crate::dataset::optimize::{CompactionOptions, compact_files};
+    use crate::dataset::write::WriteMode;
     use crate::index::create::CreateIndexBuilder;
     use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
 
@@ -498,6 +549,112 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_bitmap_segments_commit_and_query_as_logical_index() {
+        let test_dir = TempStrDir::default();
+        let dataset = lance_datagen::gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .into_dataset(
+                test_dir.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(16),
+            )
+            .await
+            .unwrap();
+        let mut dataset = dataset;
+        let fragments = dataset.get_fragments();
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap);
+        let mut staged = Vec::new();
+
+        for fragment_group in fragments.chunks(2) {
+            let fragment_ids = fragment_group
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<Vec<_>>();
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::Bitmap, &params)
+                    .name("value_bitmap".to_string())
+                    .fragments(fragment_ids.clone())
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            assert_eq!(
+                segment
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>(),
+                fragment_ids
+            );
+            let files = segment.files.as_ref().unwrap();
+            assert!(files.iter().any(|file| file.path == BITMAP_LOOKUP_NAME));
+            assert!(files.iter().all(|file| !file.path.starts_with("part_")));
+            staged.push(segment);
+        }
+
+        let staged_uuids = staged
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+        let merged = dataset.merge_existing_index_segments(staged).await.unwrap();
+        assert!(!staged_uuids.contains(&merged.uuid));
+        assert_eq!(
+            merged
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .iter()
+                .collect::<Vec<_>>(),
+            fragments
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<Vec<_>>()
+        );
+        let files = merged.files.as_ref().unwrap();
+        assert!(files.iter().any(|file| file.path == BITMAP_LOOKUP_NAME));
+        assert!(files.iter().all(|file| !file.path.starts_with("part_")));
+
+        dataset
+            .commit_existing_index_segments("value_bitmap", "value", vec![merged])
+            .await
+            .unwrap();
+
+        let committed = dataset.load_indices_by_name("value_bitmap").await.unwrap();
+        assert_eq!(committed.len(), 1);
+        assert_eq!(
+            scalar_index_fragment_bitmap(&dataset, "value", "value_bitmap")
+                .await
+                .unwrap()
+                .unwrap(),
+            dataset.fragment_bitmap.as_ref().clone()
+        );
+
+        let logical =
+            open_named_scalar_index(&dataset, "value", "value_bitmap", &NoOpMetricsCollector)
+                .await
+                .unwrap();
+        assert_eq!(logical.index_type(), IndexType::Bitmap);
+
+        let query = SargableQuery::Equals(ScalarValue::Int32(Some(20)));
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!(
+                "expected exact result from segmented bitmap, got {:?}",
+                other
+            ),
+        };
+
+        let searched_fragments = row_addrs
+            .true_rows()
+            .row_addrs()
+            .unwrap()
+            .map(|row_addr| RowAddress::from(u64::from(row_addr)).fragment_id())
+            .collect::<Vec<_>>();
+        assert_eq!(searched_fragments, vec![1]);
+    }
+
     #[tokio::test]
     async fn test_zonemap_segment_search_keeps_fragment_ids() {
         let dataset = lance_datagen::gen_batch()
@@ -561,6 +718,348 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn test_merge_existing_index_segments_supports_zonemap_segments() {
+        let dataset = lance_datagen::gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(4), FragmentRowCount::from(16))
+            .await
+            .unwrap();
+        let mut dataset = dataset;
+        let fragments = dataset.get_fragments();
+        let zonemap_params = lance_index::scalar::zonemap::ZoneMapIndexBuilderParams::new(8);
+        let params_json = serde_json::to_value(&zonemap_params).unwrap();
+        let params =
+            ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(&params_json);
+        let mut staged = Vec::new();
+
+        for fragment in &fragments {
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::ZoneMap, &params)
+                    .name("value_zonemap_merged".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            staged.push(segment);
+        }
+
+        let staged_uuids = staged
+            .iter()
+            .map(|segment| segment.uuid)
+            .collect::<Vec<_>>();
+        let merged = dataset.merge_existing_index_segments(staged).await.unwrap();
+        assert!(!staged_uuids.contains(&merged.uuid));
+        assert_eq!(
+            merged
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .iter()
+                .collect::<Vec<_>>(),
+            fragments
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<Vec<_>>()
+        );
+        assert!(
+            merged
+                .files
+                .as_ref()
+                .unwrap()
+                .iter()
+                .any(|file| file.path == "zonemap.lance")
+        );
+
+        dataset
+            .commit_existing_index_segments("value_zonemap_merged", "value", vec![merged])
+            .await
+            .unwrap();
+
+        let committed = dataset
+            .load_indices_by_name("value_zonemap_merged")
+            .await
+            .unwrap();
+        assert_eq!(committed.len(), 1);
+
+        let logical = open_named_scalar_index(
+            &dataset,
+            "value",
+            "value_zonemap_merged",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        assert_eq!(logical.index_type(), IndexType::ZoneMap);
+        assert_eq!(
+            logical.statistics().unwrap()["rows_per_zone"],
+            serde_json::json!(8)
+        );
+        assert_eq!(
+            logical.calculate_included_frags().await.unwrap(),
+            dataset.fragment_bitmap.as_ref().clone()
+        );
+
+        let query = SargableQuery::Range(
+            Bound::Included(ScalarValue::Int32(Some(0))),
+            Bound::Included(ScalarValue::Int32(Some(10_000))),
+        );
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let searched_fragments = result
+            .row_addrs()
+            .true_rows()
+            .row_addrs()
+            .unwrap()
+            .map(|row_addr| RowAddress::from(u64::from(row_addr)).fragment_id())
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            searched_fragments,
+            fragments
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<BTreeSet<_>>()
+        );
+
+        let selective_query = SargableQuery::Range(
+            Bound::Included(ScalarValue::Int32(Some(20))),
+            Bound::Included(ScalarValue::Int32(Some(43))),
+        );
+        let selective_result = logical
+            .search(&selective_query, &NoOpMetricsCollector)
+            .await
+            .unwrap();
+        let selective_fragments = selective_result
+            .row_addrs()
+            .true_rows()
+            .row_addrs()
+            .unwrap()
+            .map(|row_addr| RowAddress::from(u64::from(row_addr)).fragment_id())
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            selective_fragments,
+            fragments[1..=2]
+                .iter()
+                .map(|fragment| fragment.id() as u32)
+                .collect::<BTreeSet<_>>()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_existing_zonemap_segments_drops_retired_fragments() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let reader = lance_datagen::gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(64),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        let mut staged = Vec::new();
+        for fragment in dataset.get_fragments() {
+            staged.push(
+                CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::ZoneMap, &params)
+                    .name("value_zonemap_retired".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap(),
+            );
+        }
+        dataset
+            .commit_existing_index_segments("value_zonemap_retired", "value", staged)
+            .await
+            .unwrap();
+
+        dataset.delete("value < 16").await.unwrap();
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 64,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        let live_frags = dataset.fragment_bitmap.as_ref().clone();
+        assert!(!live_frags.contains(0), "compaction should retire frag 0");
+
+        let merged = dataset
+            .merge_existing_index_segments(
+                dataset
+                    .load_indices_by_name("value_zonemap_retired")
+                    .await
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        let coverage = merged.fragment_bitmap.as_ref().unwrap();
+        assert!(!coverage.contains(0), "must drop retired frag 0");
+        assert!(coverage.contains(1), "must keep live indexed frag 1");
+
+        let field_path = dataset.schema().field_path(merged.fields[0]).unwrap();
+        let index = crate::index::scalar::open_scalar_index(
+            &dataset,
+            &field_path,
+            &merged,
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        let query = SargableQuery::Range(
+            Bound::Included(ScalarValue::Int32(Some(0))),
+            Bound::Excluded(ScalarValue::Int32(Some(16))),
+        );
+        let searched_fragments = index
+            .search(&query, &NoOpMetricsCollector)
+            .await
+            .unwrap()
+            .row_addrs()
+            .true_rows()
+            .row_addrs()
+            .unwrap()
+            .map(|row_addr| RowAddress::from(u64::from(row_addr)).fragment_id())
+            .collect::<BTreeSet<_>>();
+        assert!(
+            searched_fragments.is_empty(),
+            "must filter retired-fragment zones"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_merge_then_commit_zonemap_segment_ignores_retired_fragment_coverage() {
+        let tmpdir = TempStrDir::default();
+        let dataset_uri = format!("file://{}", tmpdir.as_str());
+        let reader = lance_datagen::gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(64),
+                lance_datagen::BatchCount::from(2),
+            );
+        let mut dataset = Dataset::write(
+            reader,
+            &dataset_uri,
+            Some(WriteParams {
+                max_rows_per_file: 64,
+                mode: WriteMode::Overwrite,
+                enable_stable_row_ids: true,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap);
+        let segment =
+            CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::ZoneMap, &params)
+                .name("value_zonemap_replace_retired".to_string())
+                .execute_uncommitted()
+                .await
+                .unwrap();
+        let original_coverage = segment.fragment_bitmap.as_ref().unwrap().clone();
+        assert!(original_coverage.contains(0));
+        assert!(original_coverage.contains(1));
+
+        dataset
+            .commit_existing_index_segments("value_zonemap_replace_retired", "value", vec![segment])
+            .await
+            .unwrap();
+
+        dataset.delete("value < 16").await.unwrap();
+        compact_files(
+            &mut dataset,
+            CompactionOptions {
+                target_rows_per_fragment: 64,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+        let live_frags = dataset.fragment_bitmap.as_ref().clone();
+        assert!(!live_frags.contains(0), "compaction should retire frag 0");
+
+        let merged = dataset
+            .merge_existing_index_segments(
+                dataset
+                    .load_indices_by_name("value_zonemap_replace_retired")
+                    .await
+                    .unwrap(),
+            )
+            .await
+            .unwrap();
+        let merged_coverage = merged.fragment_bitmap.as_ref().unwrap().clone();
+        let merged_uuid = merged.uuid;
+
+        dataset
+            .commit_existing_index_segments("value_zonemap_replace_retired", "value", vec![merged])
+            .await
+            .unwrap();
+
+        let committed = dataset
+            .load_indices_by_name("value_zonemap_replace_retired")
+            .await
+            .unwrap();
+        assert_eq!(committed.len(), 1);
+        assert_eq!(committed[0].uuid, merged_uuid);
+
+        let combined_bitmap =
+            scalar_index_fragment_bitmap(&dataset, "value", "value_zonemap_replace_retired")
+                .await
+                .unwrap()
+                .unwrap();
+        assert_eq!(combined_bitmap, merged_coverage);
+    }
+
+    #[tokio::test]
+    async fn test_merge_existing_index_segments_rejects_mismatched_zonemap_params() {
+        let dataset = lance_datagen::gen_batch()
+            .col("value", array::step::<Int32Type>())
+            .into_ram_dataset(FragmentCount::from(2), FragmentRowCount::from(16))
+            .await
+            .unwrap();
+        let mut dataset = dataset;
+        let fragments = dataset.get_fragments();
+        let mut staged = Vec::new();
+
+        for (fragment, rows_per_zone) in fragments.iter().zip([8, 16]) {
+            let zonemap_params =
+                lance_index::scalar::zonemap::ZoneMapIndexBuilderParams::new(rows_per_zone);
+            let params_json = serde_json::to_value(&zonemap_params).unwrap();
+            let params =
+                ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(&params_json);
+            let segment =
+                CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::ZoneMap, &params)
+                    .name("value_zonemap_mismatched".to_string())
+                    .fragments(vec![fragment.id() as u32])
+                    .execute_uncommitted()
+                    .await
+                    .unwrap();
+            staged.push(segment);
+        }
+
+        let err = dataset
+            .merge_existing_index_segments(staged)
+            .await
+            .unwrap_err();
+        assert!(
+            err.to_string().contains("different rows_per_zone values"),
+            "unexpected error: {err}"
+        );
+    }
+
     #[tokio::test]
     async fn test_commit_existing_zonemap_segments_replaces_overlapping_segments() {
         let dataset = lance_datagen::gen_batch()
@@ -631,4 +1130,552 @@ mod tests {
             dataset.fragment_bitmap.as_ref().clone()
         );
     }
+
+    #[tokio::test]
+    async fn test_fmindex_segments_commit_and_query_as_logical_index() {
+        let test_dir = TempStrDir::default();
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            arrow_schema::DataType::Utf8,
+            false,
+        )]));
+        let write_params = crate::dataset::write::WriteParams {
+            max_rows_per_file: 4,
+            ..Default::default()
+        };
+        let batches = vec![
+            arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::StringArray::from(vec![
+                    "the quick brown fox",
+                    "jumps over the lazy dog",
+                    "hello world from rust",
+                    "pack my box with five dozen liquor jugs",
+                    "how vexingly quick daft zebras jump",
+                    "the five boxing wizards jump quickly",
+                    "sphinx of black quartz judge my vow",
+                    "two driven jocks help fax my big quiz",
+                    "waltz bad nymph for quick jigs vex",
+                    "glib jocks quiz nymph to vex dwarf",
+                    "quick brown fox jumps again here",
+                    "lazy dog sleeps under the tree",
+                ]))],
+            )
+            .unwrap(),
+        ];
+        let reader =
+            arrow_array::RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 3);
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        let mut segments = Vec::new();
+        for fragment in &fragments {
+            let segment = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, &params)
+                .name("text_fmindex".to_string())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+
+            assert_eq!(
+                segment
+                    .fragment_bitmap
+                    .as_ref()
+                    .unwrap()
+                    .iter()
+                    .collect::<Vec<_>>(),
+                vec![fragment.id() as u32]
+            );
+            segments.push(segment);
+        }
+
+        dataset
+            .commit_existing_index_segments("text_fmindex", "text", segments)
+            .await
+            .unwrap();
+
+        let committed = dataset.load_indices_by_name("text_fmindex").await.unwrap();
+        assert_eq!(committed.len(), fragments.len());
+
+        let logical =
+            open_named_scalar_index(&dataset, "text", "text_fmindex", &NoOpMetricsCollector)
+                .await
+                .unwrap();
+        assert_eq!(logical.index_type(), IndexType::Fm);
+
+        let query = lance_index::scalar::TextQuery::StringContains("quick".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!(
+                "expected exact result from segmented fmindex, got {:?}",
+                other
+            ),
+        };
+        let match_count = row_addrs.true_rows().row_addrs().unwrap().count();
+        assert_eq!(
+            match_count, 5,
+            "expected exactly 5 matches for 'quick', got {match_count}"
+        );
+
+        // Verify fragment coverage via manifest metadata (not calculate_included_frags,
+        // which derives from row addresses and may not encode fragment IDs for all layouts)
+        assert_eq!(
+            scalar_index_fragment_bitmap(&dataset, "text", "text_fmindex")
+                .await
+                .unwrap()
+                .unwrap(),
+            dataset.fragment_bitmap.as_ref().clone()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fmindex_segments_merge_and_query() {
+        let test_dir = TempStrDir::default();
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            arrow_schema::DataType::Utf8,
+            false,
+        )]));
+        let write_params = crate::dataset::write::WriteParams {
+            max_rows_per_file: 4,
+            ..Default::default()
+        };
+        let batches = vec![
+            arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::StringArray::from(vec![
+                    "alpha beta gamma delta",
+                    "beta gamma delta epsilon",
+                    "gamma delta epsilon zeta",
+                    "delta epsilon zeta eta",
+                    "epsilon zeta eta theta",
+                    "zeta eta theta iota",
+                    "eta theta iota kappa",
+                    "theta iota kappa lambda",
+                ]))],
+            )
+            .unwrap(),
+        ];
+        let reader =
+            arrow_array::RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        let mut staged = Vec::new();
+        for fragment in &fragments {
+            let segment = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, &params)
+                .name("text_fmindex_merge".to_string())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+            staged.push(segment);
+        }
+        assert_eq!(staged.len(), 2);
+
+        let staged_uuids = staged.iter().map(|s| s.uuid).collect::<Vec<_>>();
+        let merged = dataset.merge_existing_index_segments(staged).await.unwrap();
+
+        assert!(!staged_uuids.contains(&merged.uuid));
+        assert_eq!(
+            merged
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .iter()
+                .collect::<Vec<_>>(),
+            fragments.iter().map(|f| f.id() as u32).collect::<Vec<_>>()
+        );
+
+        dataset
+            .commit_existing_index_segments("text_fmindex_merge", "text", vec![merged])
+            .await
+            .unwrap();
+
+        let committed = dataset
+            .load_indices_by_name("text_fmindex_merge")
+            .await
+            .unwrap();
+        assert_eq!(committed.len(), 1);
+
+        let logical = open_named_scalar_index(
+            &dataset,
+            "text",
+            "text_fmindex_merge",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        assert_eq!(logical.index_type(), IndexType::Fm);
+
+        let query = lance_index::scalar::TextQuery::StringContains("delta".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!("expected exact result from merged fmindex, got {:?}", other),
+        };
+        assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 4);
+
+        let query = lance_index::scalar::TextQuery::StringContains("nonexistent".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!("expected exact result from merged fmindex, got {:?}", other),
+        };
+        assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 0);
+    }
+
+    #[tokio::test]
+    async fn test_fmindex_merge_after_compaction_drops_retired_fragments() {
+        use crate::dataset::write::WriteParams;
+
+        let test_dir = TempStrDir::default();
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            arrow_schema::DataType::Utf8,
+            false,
+        )]));
+        // Create two fragments with 4 rows each so compaction can retire one
+        let write_params = WriteParams {
+            max_rows_per_file: 4,
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        let batches = vec![
+            arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::StringArray::from(vec![
+                    "alpha beta gamma",
+                    "beta gamma delta",
+                    "gamma delta epsilon",
+                    "delta epsilon zeta",
+                    "epsilon zeta eta",
+                    "zeta eta theta",
+                    "eta theta iota",
+                    "theta iota kappa",
+                ]))],
+            )
+            .unwrap(),
+        ];
+        let reader =
+            arrow_array::RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+
+        // Build per-fragment FM-Index segments and commit
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        let mut staged = Vec::new();
+        for fragment in &fragments {
+            let segment = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, &params)
+                .name("text_fmindex_compact".to_string())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+            staged.push(segment);
+        }
+        dataset
+            .commit_existing_index_segments("text_fmindex_compact", "text", staged)
+            .await
+            .unwrap();
+
+        // Verify initial state: 2 segments, both fragments live
+        let committed = dataset
+            .load_indices_by_name("text_fmindex_compact")
+            .await
+            .unwrap();
+        assert_eq!(committed.len(), 2);
+
+        // Delete rows from fragment 0 to trigger compaction retirement
+        dataset.delete("text = 'alpha beta gamma'").await.unwrap();
+        dataset.delete("text = 'beta gamma delta'").await.unwrap();
+        crate::dataset::optimize::compact_files(
+            &mut dataset,
+            crate::dataset::optimize::CompactionOptions {
+                target_rows_per_fragment: 4,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        let live_frags: RoaringBitmap = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect();
+        assert!(
+            !live_frags.contains(0),
+            "compaction should retire fragment 0"
+        );
+
+        // Merge: the retired fragment should be dropped from coverage
+        let segments = dataset
+            .load_indices_by_name("text_fmindex_compact")
+            .await
+            .unwrap();
+        let merged = dataset
+            .merge_existing_index_segments(segments)
+            .await
+            .unwrap();
+
+        let coverage = merged.fragment_bitmap.as_ref().unwrap();
+        assert!(
+            !coverage.contains(0),
+            "merged coverage must drop retired fragment 0"
+        );
+        assert!(
+            coverage.contains(1),
+            "merged coverage must keep live fragment 1"
+        );
+
+        // Commit the merged segment and verify search works
+        dataset
+            .commit_existing_index_segments("text_fmindex_compact", "text", vec![merged])
+            .await
+            .unwrap();
+
+        let committed = dataset
+            .load_indices_by_name("text_fmindex_compact")
+            .await
+            .unwrap();
+        assert_eq!(committed.len(), 1);
+
+        let logical = open_named_scalar_index(
+            &dataset,
+            "text",
+            "text_fmindex_compact",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+
+        // "alpha" only existed in the deleted/retired rows
+        let query = lance_index::scalar::TextQuery::StringContains("alpha".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!("expected exact result from merged fmindex, got {:?}", other),
+        };
+        assert_eq!(
+            row_addrs.true_rows().row_addrs().unwrap().count(),
+            0,
+            "deleted rows from retired fragment should not appear in merged index"
+        );
+
+        // "theta" exists in fragment 1 rows only
+        let query = lance_index::scalar::TextQuery::StringContains("theta".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!("expected exact result from merged fmindex, got {:?}", other),
+        };
+        assert!(
+            row_addrs.true_rows().row_addrs().unwrap().count() > 0,
+            "rows from live fragment should still be searchable"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fmindex_merge_single_segment_passthrough() {
+        let test_dir = TempStrDir::default();
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            arrow_schema::DataType::Utf8,
+            false,
+        )]));
+        let write_params = crate::dataset::write::WriteParams {
+            max_rows_per_file: 4,
+            ..Default::default()
+        };
+        let batches = vec![
+            arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::StringArray::from(vec![
+                    "alpha beta gamma delta",
+                    "beta gamma delta epsilon",
+                    "gamma delta epsilon zeta",
+                    "delta epsilon zeta eta",
+                    "epsilon zeta eta theta",
+                    "zeta eta theta iota",
+                    "eta theta iota kappa",
+                    "theta iota kappa lambda",
+                ]))],
+            )
+            .unwrap(),
+        ];
+        let reader =
+            arrow_array::RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+        let fragment_ids: Vec<u32> = fragments.iter().map(|f| f.id() as u32).collect();
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        let segment = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, &params)
+            .name("text_fmindex_single".to_string())
+            .fragments(fragment_ids.clone())
+            .execute_uncommitted()
+            .await
+            .unwrap();
+        let source_uuid = segment.uuid;
+
+        // A single segment whose coverage is fully live is reused, not rebuilt.
+        let merged = dataset
+            .merge_existing_index_segments(vec![segment])
+            .await
+            .unwrap();
+        assert_eq!(
+            merged.uuid, source_uuid,
+            "single-segment merge with unchanged coverage should reuse the segment"
+        );
+        assert_eq!(
+            merged
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .iter()
+                .collect::<Vec<_>>(),
+            fragment_ids
+        );
+
+        dataset
+            .commit_existing_index_segments("text_fmindex_single", "text", vec![merged])
+            .await
+            .unwrap();
+
+        let logical = open_named_scalar_index(
+            &dataset,
+            "text",
+            "text_fmindex_single",
+            &NoOpMetricsCollector,
+        )
+        .await
+        .unwrap();
+        assert_eq!(logical.index_type(), IndexType::Fm);
+
+        let query = lance_index::scalar::TextQuery::StringContains("delta".to_string());
+        let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap();
+        let row_addrs = match result {
+            SearchResult::Exact(row_addrs) => row_addrs,
+            other => panic!("expected exact result from fmindex, got {:?}", other),
+        };
+        assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 4);
+    }
+
+    #[tokio::test]
+    async fn test_fmindex_merge_single_segment_rebuilds_when_coverage_shrinks() {
+        let test_dir = TempStrDir::default();
+
+        let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new(
+            "text",
+            arrow_schema::DataType::Utf8,
+            false,
+        )]));
+        let write_params = crate::dataset::write::WriteParams {
+            max_rows_per_file: 4,
+            enable_stable_row_ids: true,
+            ..Default::default()
+        };
+        let batches = vec![
+            arrow_array::RecordBatch::try_new(
+                schema.clone(),
+                vec![Arc::new(arrow_array::StringArray::from(vec![
+                    "alpha beta gamma",
+                    "beta gamma delta",
+                    "gamma delta epsilon",
+                    "delta epsilon zeta",
+                    "epsilon zeta eta",
+                    "zeta eta theta",
+                    "eta theta iota",
+                    "theta iota kappa",
+                ]))],
+            )
+            .unwrap(),
+        ];
+        let reader =
+            arrow_array::RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone());
+        let mut dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 2);
+
+        let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm);
+        let segment = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, &params)
+            .name("text_fmindex_shrink".to_string())
+            .fragments(fragments.iter().map(|f| f.id() as u32).collect())
+            .execute_uncommitted()
+            .await
+            .unwrap();
+        let source_uuid = segment.uuid;
+
+        // Retire fragment 0: delete its rows and compact it away.
+        dataset.delete("text = 'alpha beta gamma'").await.unwrap();
+        dataset.delete("text = 'beta gamma delta'").await.unwrap();
+        crate::dataset::optimize::compact_files(
+            &mut dataset,
+            crate::dataset::optimize::CompactionOptions {
+                target_rows_per_fragment: 4,
+                ..Default::default()
+            },
+            None,
+        )
+        .await
+        .unwrap();
+
+        let live_frags: RoaringBitmap = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect();
+        assert!(
+            !live_frags.contains(0),
+            "compaction should retire fragment 0"
+        );
+        assert!(live_frags.contains(1), "fragment 1 should stay live");
+
+        // Coverage shrank, so even a single segment must be rebuilt.
+        let merged = dataset
+            .merge_existing_index_segments(vec![segment])
+            .await
+            .unwrap();
+        assert_ne!(
+            merged.uuid, source_uuid,
+            "shrunk coverage must trigger a rebuild"
+        );
+        assert_eq!(
+            merged
+                .fragment_bitmap
+                .as_ref()
+                .unwrap()
+                .iter()
+                .collect::<Vec<_>>(),
+            vec![1]
+        );
+    }
 }
diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs
index ff7d2383c67..bfc4a4f3474 100644
--- a/rust/lance/src/index/vector.rs
+++ b/rust/lance/src/index/vector.rs
@@ -4,11 +4,13 @@
 //! Vector Index for Fast Approximate Nearest Neighbor (ANN) Search
 //!
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::sync::Arc;
 use std::{any::Any, collections::HashMap};
 
 pub mod builder;
 pub(crate) mod details;
+pub mod hamming;
 pub mod ivf;
 pub mod pq;
 pub mod utils;
@@ -17,8 +19,9 @@ pub mod utils;
 mod fixture_test;
 
 use self::{ivf::*, pq::PQIndex};
+use arrow_array::Array;
 use arrow_schema::{DataType, Schema};
-use builder::IvfIndexBuilder;
+use builder::{IvfIndexBuilder, VectorIndexBuildSummary};
 use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use futures::stream;
@@ -29,7 +32,7 @@ use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::optimize::OptimizeOptions;
 use lance_index::progress::{IndexBuildProgress, noop_progress};
 use lance_index::vector::bq::builder::RabitQuantizer;
-use lance_index::vector::bq::{RQBuildParams, RQRotationType};
+use lance_index::vector::bq::{RQBuildParams, RQRotationType, validate_supported_rq_num_bits};
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::hnsw::HNSW;
 use lance_index::vector::ivf::builder::recommended_num_partitions;
@@ -52,9 +55,10 @@ use lance_index::vector::{
     sq::{ScalarQuantizer, builder::SQBuildParams},
 };
 use lance_index::{INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, IndexType};
+use lance_io::object_store::ObjectStore;
 use lance_io::traits::Reader;
 use lance_linalg::distance::*;
-use lance_table::format::{IndexMetadata, list_index_files_with_sizes};
+use lance_table::format::{IndexFile, IndexMetadata};
 use serde::Serialize;
 use tracing::instrument;
 use utils::get_vector_type;
@@ -513,6 +517,7 @@ async fn prepare_vector_segment_build(
     progress: Arc<dyn IndexBuildProgress>,
     mode: &str,
     require_precomputed_ivf: bool,
+    fragment_ids: Option<&[u32]>,
 ) -> Result<(DataType, IndexType, IvfBuildParams, Box<dyn Shuffler>)> {
     let stages = &params.stages;
 
@@ -543,16 +548,40 @@ async fn prepare_vector_segment_build(
         )));
     }
 
-    let num_rows = dataset.count_rows(None).await?;
     let index_type = params.index_type();
-    let num_partitions = ivf_params0.num_partitions.unwrap_or_else(|| {
-        recommended_num_partitions(
-            num_rows,
-            ivf_params0
-                .target_partition_size
-                .unwrap_or(index_type.target_partition_size()),
-        )
-    });
+    if index_type == IndexType::IvfRq {
+        let Some(StageParams::RQ(rq_params)) = stages.last() else {
+            return Err(Error::index(format!(
+                "{mode}: invalid stages: {:?}",
+                stages
+            )));
+        };
+        validate_supported_rq_num_bits(rq_params.num_bits)?;
+    }
+
+    let num_partitions = match (ivf_params0.num_partitions, ivf_params0.centroids.as_ref()) {
+        (Some(num_partitions), Some(centroids)) if num_partitions != centroids.len() => {
+            return Err(Error::index(format!(
+                "{mode}: num_partitions {} does not match precomputed IVF centroids length {}",
+                num_partitions,
+                centroids.len()
+            )));
+        }
+        (Some(num_partitions), _) => num_partitions,
+        (None, Some(centroids)) => centroids.len(),
+        (None, None) => {
+            let num_rows = match fragment_ids {
+                Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await?,
+                None => dataset.count_rows(None).await?,
+            };
+            recommended_num_partitions(
+                num_rows,
+                ivf_params0
+                    .target_partition_size
+                    .unwrap_or(index_type.target_partition_size()),
+            )
+        }
+    };
     let mut ivf_params = ivf_params0.clone();
     ivf_params.num_partitions = Some(num_partitions);
 
@@ -576,12 +605,12 @@ pub(crate) async fn build_distributed_vector_index(
     dataset: &Dataset,
     column: &str,
     _name: &str,
-    uuid: &str,
+    uuid: Uuid,
     params: &VectorIndexParams,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
     fragment_ids: &[u32],
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<Uuid> {
+) -> Result<(Uuid, Vec<IndexFile>)> {
     let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build(
         dataset,
         column,
@@ -589,6 +618,7 @@ pub(crate) async fn build_distributed_vector_index(
         progress.clone(),
         "Build Distributed Vector Index",
         true,
+        Some(fragment_ids),
     )
     .await?;
     let stages = &params.stages;
@@ -602,8 +632,7 @@ pub(crate) async fn build_distributed_vector_index(
 
     let filtered_dataset = dataset.clone();
 
-    let segment_uuid = Uuid::parse_str(uuid)
-        .map_err(|err| Error::invalid_input(format!("Invalid index UUID '{uuid}': {err}")))?;
+    let segment_uuid = uuid;
     let index_dir = dataset.indices_dir().join(segment_uuid.to_string());
 
     let fragment_filter = fragment_ids.to_vec();
@@ -647,7 +676,7 @@ pub(crate) async fn build_distributed_vector_index(
             DataType::Float16 | DataType::Float32 | DataType::Float64 => {
                 let ivf_model = make_ivf_model();
 
-                IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new(
+                let summary = IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new(
                     filtered_dataset,
                     column.to_owned(),
                     index_dir.clone(),
@@ -663,11 +692,12 @@ pub(crate) async fn build_distributed_vector_index(
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+                return Ok((segment_uuid, summary.files));
             }
             DataType::UInt8 => {
                 let ivf_model = make_ivf_model();
 
-                IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
+                let summary = IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
                     filtered_dataset,
                     column.to_owned(),
                     index_dir.clone(),
@@ -683,6 +713,7 @@ pub(crate) async fn build_distributed_vector_index(
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+                return Ok((segment_uuid, summary.files));
             }
             _ => {
                 return Err(Error::index(format!(
@@ -711,7 +742,7 @@ pub(crate) async fn build_distributed_vector_index(
                     let ivf_model = make_ivf_model();
                     let global_pq = make_global_pq(pq_params)?;
 
-                    IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new(
+                    let summary = IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new(
                         filtered_dataset,
                         column.to_owned(),
                         index_dir.clone(),
@@ -731,6 +762,7 @@ pub(crate) async fn build_distributed_vector_index(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok((segment_uuid, summary.files));
                 }
             }
         }
@@ -742,7 +774,7 @@ pub(crate) async fn build_distributed_vector_index(
                     stages
                 )));
             };
-            IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new(
+            let summary = IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
                 index_dir.clone(),
@@ -757,6 +789,7 @@ pub(crate) async fn build_distributed_vector_index(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok((segment_uuid, summary.files));
         }
 
         IndexType::IvfHnswFlat => {
@@ -769,7 +802,7 @@ pub(crate) async fn build_distributed_vector_index(
 
             match element_type {
                 DataType::UInt8 => {
-                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
+                    let summary = IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
                         filtered_dataset,
                         column.to_owned(),
                         index_dir.clone(),
@@ -784,9 +817,10 @@ pub(crate) async fn build_distributed_vector_index(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok((segment_uuid, summary.files));
                 }
                 _ => {
-                    IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
+                    let summary = IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
                         filtered_dataset,
                         column.to_owned(),
                         index_dir.clone(),
@@ -801,6 +835,7 @@ pub(crate) async fn build_distributed_vector_index(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok((segment_uuid, summary.files));
                 }
             }
         }
@@ -822,7 +857,7 @@ pub(crate) async fn build_distributed_vector_index(
             let ivf_model = make_ivf_model();
             let global_pq = make_global_pq(pq_params)?;
 
-            IvfIndexBuilder::<HNSW, ProductQuantizer>::new(
+            let summary = IvfIndexBuilder::<HNSW, ProductQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
                 index_dir.clone(),
@@ -842,6 +877,7 @@ pub(crate) async fn build_distributed_vector_index(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok((segment_uuid, summary.files));
         }
 
         IndexType::IvfHnswSq => {
@@ -857,7 +893,7 @@ pub(crate) async fn build_distributed_vector_index(
                     stages
                 )));
             };
-            IvfIndexBuilder::<HNSW, ScalarQuantizer>::new(
+            let summary = IvfIndexBuilder::<HNSW, ScalarQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
                 index_dir.clone(),
@@ -872,6 +908,7 @@ pub(crate) async fn build_distributed_vector_index(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok((segment_uuid, summary.files));
         }
 
         IndexType::IvfRq => {
@@ -884,7 +921,7 @@ pub(crate) async fn build_distributed_vector_index(
 
             let ivf_model = make_ivf_model();
 
-            IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new(
+            let summary = IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new(
                 filtered_dataset,
                 column.to_owned(),
                 index_dir.clone(),
@@ -903,6 +940,7 @@ pub(crate) async fn build_distributed_vector_index(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok((segment_uuid, summary.files));
         }
 
         _ => {
@@ -911,9 +949,7 @@ pub(crate) async fn build_distributed_vector_index(
                 index_type
             )));
         }
-    };
-
-    Ok(segment_uuid)
+    }
 }
 
 /// Build a Vector Index
@@ -922,11 +958,60 @@ pub(crate) async fn build_vector_index(
     dataset: &Dataset,
     column: &str,
     name: &str,
-    uuid: &str,
+    uuid: Uuid,
     params: &VectorIndexParams,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
+    build_vector_index_impl(
+        dataset,
+        column,
+        name,
+        uuid,
+        params,
+        frag_reuse_index,
+        progress,
+        None,
+    )
+    .await
+}
+
+/// Build a standalone vector index segment over a subset of fragments.
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn build_filtered_vector_index(
+    dataset: &Dataset,
+    column: &str,
+    name: &str,
+    uuid: Uuid,
+    params: &VectorIndexParams,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    fragment_ids: &[u32],
+    progress: Arc<dyn IndexBuildProgress>,
+) -> Result<Vec<IndexFile>> {
+    build_vector_index_impl(
+        dataset,
+        column,
+        name,
+        uuid,
+        params,
+        frag_reuse_index,
+        progress,
+        Some(fragment_ids),
+    )
+    .await
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn build_vector_index_impl(
+    dataset: &Dataset,
+    column: &str,
+    name: &str,
+    uuid: Uuid,
+    params: &VectorIndexParams,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
+    progress: Arc<dyn IndexBuildProgress>,
+    fragment_ids: Option<&[u32]>,
+) -> Result<Vec<IndexFile>> {
     let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build(
         dataset,
         column,
@@ -934,6 +1019,7 @@ pub(crate) async fn build_vector_index(
         progress.clone(),
         "Build Vector Index",
         false,
+        fragment_ids,
     )
     .await?;
     let stages = &params.stages;
@@ -941,10 +1027,10 @@ pub(crate) async fn build_vector_index(
     match index_type {
         IndexType::IvfFlat => match element_type {
             DataType::Float16 | DataType::Float32 | DataType::Float64 => {
-                IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new(
+                let summary = IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new(
                     dataset.clone(),
                     column.to_owned(),
-                    dataset.indices_dir().clone().join(uuid),
+                    dataset.indices_dir().clone().join(uuid.to_string()),
                     params.metric_type,
                     shuffler,
                     Some(ivf_params),
@@ -952,15 +1038,17 @@ pub(crate) async fn build_vector_index(
                     (),
                     frag_reuse_index,
                 )?
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+                Ok(summary.files)
             }
             DataType::UInt8 => {
-                IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
+                let summary = IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new(
                     dataset.clone(),
                     column.to_owned(),
-                    dataset.indices_dir().clone().join(uuid),
+                    dataset.indices_dir().clone().join(uuid.to_string()),
                     params.metric_type,
                     shuffler,
                     Some(ivf_params),
@@ -968,16 +1056,16 @@ pub(crate) async fn build_vector_index(
                     (),
                     frag_reuse_index,
                 )?
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+                Ok(summary.files)
             }
-            _ => {
-                return Err(Error::index(format!(
-                    "Build Vector Index: invalid data type: {:?}",
-                    element_type
-                )));
-            }
+            _ => Err(Error::index(format!(
+                "Build Vector Index: invalid data type: {:?}",
+                element_type
+            ))),
         },
         IndexType::IvfPq => {
             let len = stages.len();
@@ -990,7 +1078,13 @@ pub(crate) async fn build_vector_index(
 
             match params.version {
                 IndexFileVersion::Legacy => {
-                    build_ivf_pq_index(
+                    if fragment_ids.is_some() {
+                        return Err(Error::index(
+                            "Build Vector Index: filtered IVF_PQ builds do not support legacy format"
+                                .to_string(),
+                        ));
+                    }
+                    let files = build_ivf_pq_index(
                         dataset,
                         column,
                         name,
@@ -1001,12 +1095,13 @@ pub(crate) async fn build_vector_index(
                         progress.clone(),
                     )
                     .await?;
+                    Ok(files)
                 }
                 IndexFileVersion::V3 => {
                     let mut builder = IvfIndexBuilder::<FlatIndex, ProductQuantizer>::new(
                         dataset.clone(),
                         column.to_owned(),
-                        dataset.indices_dir().join(uuid),
+                        dataset.indices_dir().join(uuid.to_string()),
                         params.metric_type,
                         shuffler,
                         Some(ivf_params),
@@ -1015,11 +1110,13 @@ pub(crate) async fn build_vector_index(
                         frag_reuse_index,
                     )?;
 
-                    builder
+                    let summary = builder
                         .with_transpose(!params.skip_transpose)
+                        .with_optional_fragment_filter(fragment_ids)
                         .with_progress(progress.clone())
                         .build()
                         .await?;
+                    Ok(summary.files)
                 }
             }
         }
@@ -1031,10 +1128,10 @@ pub(crate) async fn build_vector_index(
                 )));
             };
 
-            IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new(
+            let summary = IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new(
                 dataset.clone(),
                 column.to_owned(),
-                dataset.indices_dir().clone().join(uuid),
+                dataset.indices_dir().clone().join(uuid.to_string()),
                 params.metric_type,
                 shuffler,
                 Some(ivf_params),
@@ -1042,9 +1139,11 @@ pub(crate) async fn build_vector_index(
                 (),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
+            Ok(summary.files)
         }
         IndexType::IvfRq => {
             let StageParams::RQ(rq_params) = &stages[1] else {
@@ -1057,7 +1156,7 @@ pub(crate) async fn build_vector_index(
             let mut builder = IvfIndexBuilder::<FlatIndex, RabitQuantizer>::new(
                 dataset.clone(),
                 column.to_owned(),
-                dataset.indices_dir().join(uuid),
+                dataset.indices_dir().join(uuid.to_string()),
                 params.metric_type,
                 shuffler,
                 Some(ivf_params),
@@ -1066,11 +1165,13 @@ pub(crate) async fn build_vector_index(
                 frag_reuse_index,
             )?;
 
-            builder
+            let summary = builder
                 .with_transpose(!params.skip_transpose)
+                .with_optional_fragment_filter(fragment_ids)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+            Ok(summary.files)
         }
         IndexType::IvfHnswFlat => {
             let StageParams::Hnsw(hnsw_params) = &stages[1] else {
@@ -1081,10 +1182,10 @@ pub(crate) async fn build_vector_index(
             };
             match element_type {
                 DataType::UInt8 => {
-                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
+                    let summary = IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new(
                         dataset.clone(),
                         column.to_owned(),
-                        dataset.indices_dir().clone().join(uuid),
+                        dataset.indices_dir().clone().join(uuid.to_string()),
                         params.metric_type,
                         shuffler,
                         Some(ivf_params),
@@ -1092,15 +1193,17 @@ pub(crate) async fn build_vector_index(
                         hnsw_params.clone(),
                         frag_reuse_index,
                     )?
+                    .with_optional_fragment_filter(fragment_ids)
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    Ok(summary.files)
                 }
                 _ => {
-                    IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
+                    let summary = IvfIndexBuilder::<HNSW, FlatQuantizer>::new(
                         dataset.clone(),
                         column.to_owned(),
-                        dataset.indices_dir().clone().join(uuid),
+                        dataset.indices_dir().clone().join(uuid.to_string()),
                         params.metric_type,
                         shuffler,
                         Some(ivf_params),
@@ -1108,9 +1211,11 @@ pub(crate) async fn build_vector_index(
                         hnsw_params.clone(),
                         frag_reuse_index,
                     )?
+                    .with_optional_fragment_filter(fragment_ids)
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    Ok(summary.files)
                 }
             }
         }
@@ -1127,10 +1232,10 @@ pub(crate) async fn build_vector_index(
                     stages
                 )));
             };
-            IvfIndexBuilder::<HNSW, ProductQuantizer>::new(
+            let summary = IvfIndexBuilder::<HNSW, ProductQuantizer>::new(
                 dataset.clone(),
                 column.to_owned(),
-                dataset.indices_dir().clone().join(uuid),
+                dataset.indices_dir().clone().join(uuid.to_string()),
                 params.metric_type,
                 shuffler,
                 Some(ivf_params),
@@ -1138,9 +1243,11 @@ pub(crate) async fn build_vector_index(
                 hnsw_params.clone(),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
+            Ok(summary.files)
         }
         IndexType::IvfHnswSq => {
             let StageParams::Hnsw(hnsw_params) = &stages[1] else {
@@ -1155,10 +1262,10 @@ pub(crate) async fn build_vector_index(
                     stages
                 )));
             };
-            IvfIndexBuilder::<HNSW, ScalarQuantizer>::new(
+            let summary = IvfIndexBuilder::<HNSW, ScalarQuantizer>::new(
                 dataset.clone(),
                 column.to_owned(),
-                dataset.indices_dir().clone().join(uuid),
+                dataset.indices_dir().clone().join(uuid.to_string()),
                 params.metric_type,
                 shuffler,
                 Some(ivf_params),
@@ -1166,18 +1273,17 @@ pub(crate) async fn build_vector_index(
                 hnsw_params.clone(),
                 frag_reuse_index,
             )?
+            .with_optional_fragment_filter(fragment_ids)
             .with_progress(progress.clone())
             .build()
             .await?;
+            Ok(summary.files)
         }
-        _ => {
-            return Err(Error::index(format!(
-                "Build Vector Index: invalid index type: {:?}",
-                index_type
-            )));
-        }
-    };
-    Ok(())
+        _ => Err(Error::index(format!(
+            "Build Vector Index: invalid index type: {:?}",
+            index_type
+        ))),
+    }
 }
 
 /// Build a Vector Index incrementally using an existing index's IVF model and quantizer
@@ -1186,12 +1292,12 @@ pub(crate) async fn build_vector_index(
 pub(crate) async fn build_vector_index_incremental(
     dataset: &Dataset,
     column: &str,
-    uuid: &str,
+    uuid: Uuid,
     params: &VectorIndexParams,
     existing_index: Arc<dyn VectorIndex>,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
     progress: Arc<dyn IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<VectorIndexBuildSummary> {
     let stages = &params.stages;
 
     if stages.is_empty() {
@@ -1243,7 +1349,7 @@ pub(crate) async fn build_vector_index_incremental(
         Some(progress.clone()),
     );
 
-    let index_dir = dataset.indices_dir().join(uuid);
+    let index_dir = dataset.indices_dir().join(uuid.to_string());
 
     // Determine the index type and build incrementally
     let (sub_index_type, quantization_type) = existing_index.sub_index_type();
@@ -1251,7 +1357,7 @@ pub(crate) async fn build_vector_index_incremental(
     match (sub_index_type, quantization_type) {
         // IVF_FLAT
         (SubIndexType::Flat, QuantizationType::Flat) => {
-            IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new_incremental(
+            let summary = IvfIndexBuilder::<FlatIndex, FlatQuantizer>::new_incremental(
                 dataset.clone(),
                 column.to_owned(),
                 index_dir,
@@ -1266,9 +1372,10 @@ pub(crate) async fn build_vector_index_incremental(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok(summary);
         }
         (SubIndexType::Flat, QuantizationType::FlatBin) => {
-            IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_incremental(
+            let summary = IvfIndexBuilder::<FlatIndex, FlatBinQuantizer>::new_incremental(
                 dataset.clone(),
                 column.to_owned(),
                 index_dir,
@@ -1283,6 +1390,7 @@ pub(crate) async fn build_vector_index_incremental(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok(summary);
         }
         // IVF_PQ
         (SubIndexType::Flat, QuantizationType::Product) => {
@@ -1296,17 +1404,18 @@ pub(crate) async fn build_vector_index_incremental(
                 frag_reuse_index,
                 OptimizeOptions::append(),
             )?;
-            builder
+            let summary = builder
                 .with_ivf(ivf_model)
                 .with_quantizer(quantizer.try_into()?)
                 .with_transpose(!params.skip_transpose)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+            return Ok(summary);
         }
         // IVF_SQ
         (SubIndexType::Flat, QuantizationType::Scalar) => {
-            IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new_incremental(
+            let summary = IvfIndexBuilder::<FlatIndex, ScalarQuantizer>::new_incremental(
                 dataset.clone(),
                 column.to_owned(),
                 index_dir,
@@ -1321,6 +1430,7 @@ pub(crate) async fn build_vector_index_incremental(
             .with_progress(progress.clone())
             .build()
             .await?;
+            return Ok(summary);
         }
         // IVF_RQ
         (SubIndexType::Flat, QuantizationType::Rabit) => {
@@ -1334,13 +1444,14 @@ pub(crate) async fn build_vector_index_incremental(
                 frag_reuse_index,
                 OptimizeOptions::append(),
             )?;
-            builder
+            let summary = builder
                 .with_ivf(ivf_model)
                 .with_quantizer(quantizer.try_into()?)
                 .with_transpose(!params.skip_transpose)
                 .with_progress(progress.clone())
                 .build()
                 .await?;
+            return Ok(summary);
         }
         // IVF_HNSW variants
         (SubIndexType::Hnsw, quantization_type) => {
@@ -1353,7 +1464,7 @@ pub(crate) async fn build_vector_index_incremental(
 
             match quantization_type {
                 QuantizationType::Flat => {
-                    IvfIndexBuilder::<HNSW, FlatQuantizer>::new_incremental(
+                    let summary = IvfIndexBuilder::<HNSW, FlatQuantizer>::new_incremental(
                         dataset.clone(),
                         column.to_owned(),
                         index_dir,
@@ -1368,9 +1479,10 @@ pub(crate) async fn build_vector_index_incremental(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok(summary);
                 }
                 QuantizationType::FlatBin => {
-                    IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new_incremental(
+                    let summary = IvfIndexBuilder::<HNSW, FlatBinQuantizer>::new_incremental(
                         dataset.clone(),
                         column.to_owned(),
                         index_dir,
@@ -1385,9 +1497,10 @@ pub(crate) async fn build_vector_index_incremental(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok(summary);
                 }
                 QuantizationType::Product => {
-                    IvfIndexBuilder::<HNSW, ProductQuantizer>::new_incremental(
+                    let summary = IvfIndexBuilder::<HNSW, ProductQuantizer>::new_incremental(
                         dataset.clone(),
                         column.to_owned(),
                         index_dir,
@@ -1402,9 +1515,10 @@ pub(crate) async fn build_vector_index_incremental(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok(summary);
                 }
                 QuantizationType::Scalar => {
-                    IvfIndexBuilder::<HNSW, ScalarQuantizer>::new_incremental(
+                    let summary = IvfIndexBuilder::<HNSW, ScalarQuantizer>::new_incremental(
                         dataset.clone(),
                         column.to_owned(),
                         index_dir,
@@ -1419,6 +1533,7 @@ pub(crate) async fn build_vector_index_incremental(
                     .with_progress(progress.clone())
                     .build()
                     .await?;
+                    return Ok(summary);
                 }
                 QuantizationType::Rabit => {
                     return Err(Error::index(
@@ -1428,8 +1543,6 @@ pub(crate) async fn build_vector_index_incremental(
             }
         }
     }
-
-    Ok(())
 }
 
 /// Build an empty vector index without training on data
@@ -1438,9 +1551,9 @@ pub(crate) async fn build_empty_vector_index(
     _dataset: &Dataset,
     column: &str,
     name: &str,
-    _uuid: &str,
+    _uuid: Uuid,
     _params: &VectorIndexParams,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     // For now, return a NotImplementedError to indicate this functionality
     // is still being developed
     Err(Error::not_supported_source(
@@ -1453,24 +1566,24 @@ pub(crate) async fn build_empty_vector_index(
     ))
 }
 
-#[instrument(level = "debug", skip_all, fields(old_uuid = old_uuid.to_string(), new_uuid = new_uuid.to_string(), num_rows = mapping.len()))]
+#[instrument(level = "debug", skip_all, fields(old_uuid = old_uuid.to_string(), new_uuid = new_uuid.to_string()))]
 pub(crate) async fn remap_vector_index(
     dataset: Arc<Dataset>,
     column: &str,
     old_uuid: &Uuid,
     new_uuid: &Uuid,
     old_metadata: &IndexMetadata,
-    mapping: &HashMap<u64, Option<u64>>,
-) -> Result<()> {
+    mapping: &RowAddrRemap,
+) -> Result<Vec<IndexFile>> {
     let old_index = dataset
-        .open_vector_index(column, &old_uuid.to_string(), &NoOpMetricsCollector)
+        .open_vector_index(column, old_uuid, &NoOpMetricsCollector)
         .await?;
 
     if let Some(ivf_index) = old_index.as_any().downcast_ref::<IVFIndex>() {
-        remap_index_file(
+        let file = remap_index_file(
             dataset.as_ref(),
-            &old_uuid.to_string(),
-            &new_uuid.to_string(),
+            old_uuid,
+            new_uuid,
             old_metadata.dataset_version,
             ivf_index,
             mapping,
@@ -1482,26 +1595,26 @@ pub(crate) async fn remap_vector_index(
             vec![],
         )
         .await?;
+        Ok(vec![file])
     } else {
         // it's v3 index
-        remap_index_file_v3(
+        let files = remap_index_file_v3(
             dataset.as_ref(),
-            &new_uuid.to_string(),
+            new_uuid,
             old_index,
             mapping,
             column.to_string(),
         )
         .await?;
+        Ok(files)
     }
-
-    Ok(())
 }
 
 /// Open the Vector index on dataset, specified by the `uuid`.
 #[instrument(level = "debug", skip(dataset, vec_idx, reader))]
 pub(crate) async fn open_vector_index(
     dataset: Arc<Dataset>,
-    uuid: &str,
+    uuid: &Uuid,
     vec_idx: &lance_index::pb::VectorIndex,
     reader: Arc<dyn Reader>,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
@@ -1532,7 +1645,7 @@ pub(crate) async fn open_vector_index(
                 }
                 let ivf = IvfModel::try_from(ivf_pb.to_owned())?;
                 last_stage = Some(Arc::new(IVFIndex::try_new(
-                    uuid,
+                    *uuid,
                     ivf,
                     reader.clone(),
                     last_stage.unwrap(),
@@ -1575,11 +1688,29 @@ pub(crate) async fn open_vector_index(
     Ok(idx)
 }
 
+/// Open an index file without a HEAD request when the size is already known.
+///
+/// `file_sizes` maps a file name to its size in bytes (see
+/// `IndexMetadata::file_size_map`). If `file_name` is missing, which is the case
+/// for older indices that did not record sizes, this falls back to `open`, which
+/// issues a HEAD to learn the size.
+pub(crate) async fn open_index_file(
+    object_store: &ObjectStore,
+    path: &Path,
+    file_name: &str,
+    file_sizes: &HashMap<String, u64>,
+) -> Result<Box<dyn Reader>> {
+    match file_sizes.get(file_name) {
+        Some(&size) => object_store.open_with_size(path, size as usize).await,
+        None => object_store.open(path).await,
+    }
+}
+
 #[instrument(level = "debug", skip(dataset, reader))]
 pub(crate) async fn open_vector_index_v2(
     dataset: Arc<Dataset>,
     column: &str,
-    uuid: &str,
+    uuid: &Uuid,
     reader: PreviousFileReader,
     frag_reuse_index: Option<Arc<FragReuseIndex>>,
 ) -> Result<Arc<dyn VectorIndex>> {
@@ -1599,11 +1730,21 @@ pub(crate) async fn open_vector_index_v2(
         .ok_or_else(|| Error::index(format!("Index with id {} does not exist", uuid)))?;
     let index_dir = dataset.indice_files_dir(&index_meta)?;
     let object_store = dataset.object_store_for_index(&index_meta).await?;
+    let file_sizes = index_meta.file_size_map();
 
     let index: Arc<dyn VectorIndex> = match index_metadata.index_type.as_str() {
         "IVF_HNSW_PQ" => {
-            let aux_path = index_dir.clone().join(uuid).join(INDEX_AUXILIARY_FILE_NAME);
-            let aux_reader = object_store.open(&aux_path).await?;
+            let aux_path = index_dir
+                .clone()
+                .join(uuid.to_string())
+                .join(INDEX_AUXILIARY_FILE_NAME);
+            let aux_reader = open_index_file(
+                object_store.as_ref(),
+                &aux_path,
+                INDEX_AUXILIARY_FILE_NAME,
+                &file_sizes,
+            )
+            .await?;
 
             let ivf_data = IvfModel::load(&reader).await?;
             let options = HNSWIndexOptions { use_residual: true };
@@ -1617,7 +1758,7 @@ pub(crate) async fn open_vector_index_v2(
             let ivf = IvfModel::try_from(pb_ivf)?;
 
             Arc::new(IVFIndex::try_new(
-                uuid,
+                *uuid,
                 ivf,
                 reader.object_reader.clone(),
                 Arc::new(hnsw),
@@ -1629,8 +1770,17 @@ pub(crate) async fn open_vector_index_v2(
         }
 
         "IVF_HNSW_SQ" => {
-            let aux_path = index_dir.clone().join(uuid).join(INDEX_AUXILIARY_FILE_NAME);
-            let aux_reader = object_store.open(&aux_path).await?;
+            let aux_path = index_dir
+                .clone()
+                .join(uuid.to_string())
+                .join(INDEX_AUXILIARY_FILE_NAME);
+            let aux_reader = open_index_file(
+                object_store.as_ref(),
+                &aux_path,
+                INDEX_AUXILIARY_FILE_NAME,
+                &file_sizes,
+            )
+            .await?;
 
             let ivf_data = IvfModel::load(&reader).await?;
             let options = HNSWIndexOptions {
@@ -1647,7 +1797,7 @@ pub(crate) async fn open_vector_index_v2(
             let ivf = IvfModel::try_from(pb_ivf)?;
 
             Arc::new(IVFIndex::try_new(
-                uuid,
+                *uuid,
                 ivf,
                 reader.object_reader.clone(),
                 Arc::new(hnsw),
@@ -1704,11 +1854,7 @@ pub async fn initialize_vector_index(
     let column_name = field_names[0];
 
     let source_vector_index = source_dataset
-        .open_vector_index(
-            column_name,
-            &source_index.uuid.to_string(),
-            &NoOpMetricsCollector,
-        )
+        .open_vector_index(column_name, &source_index.uuid, &NoOpMetricsCollector)
         .await?;
 
     let metric_type = source_vector_index.metric_type();
@@ -1777,10 +1923,10 @@ pub async fn initialize_vector_index(
         .open_frag_reuse_index(&NoOpMetricsCollector)
         .await?;
 
-    build_vector_index_incremental(
+    let summary = build_vector_index_incremental(
         target_dataset,
         column_name,
-        &new_uuid.to_string(),
+        new_uuid,
         &params,
         source_vector_index,
         frag_reuse_index,
@@ -1788,10 +1934,6 @@ pub async fn initialize_vector_index(
     )
     .await?;
 
-    // Capture file sizes for the new vector index
-    let index_dir = target_dataset.indices_dir().join(new_uuid.to_string());
-    let files = list_index_files_with_sizes(&target_dataset.object_store, &index_dir).await?;
-
     let field = target_dataset.schema().field(column_name).ok_or_else(|| {
         Error::index(format!(
             "Column '{}' not found in target dataset",
@@ -1811,7 +1953,7 @@ pub async fn initialize_vector_index(
         index_version: source_index.index_version,
         created_at: Some(chrono::Utc::now()),
         base_id: None,
-        files: Some(files),
+        files: Some(summary.files),
     };
 
     let transaction = Transaction::new(
@@ -1880,6 +2022,7 @@ fn derive_rabit_params(rabit_quantizer: &RabitQuantizer) -> RQBuildParams {
     RQBuildParams {
         num_bits: rabit_quantizer.num_bits(),
         rotation_type: rabit_quantizer.rotation_type(),
+        rotation: None,
     }
 }
 
@@ -1946,6 +2089,125 @@ mod tests {
     use lance_index::metrics::NoOpMetricsCollector;
     use lance_linalg::distance::MetricType;
 
+    /// `open_index_file` skips the HEAD when the size is known and still falls
+    /// back to a HEAD for older indices that did not record sizes. A HEAD is
+    /// issued as a `get_opts` call with `head = true`, so a proxy store counts
+    /// those against the index file.
+    ///
+    /// Regression test for <https://github.com/lance-format/lance/issues/6944>.
+    #[tokio::test]
+    async fn test_open_index_file_skips_head_when_size_known() {
+        use lance_index::INDEX_FILE_NAME;
+        use lance_io::assert_io_eq;
+        use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};
+
+        let (store, base) = ObjectStore::from_uri_and_params(
+            Arc::new(ObjectStoreRegistry::default()),
+            "memory:///",
+            &ObjectStoreParams::default(),
+        )
+        .await
+        .unwrap();
+
+        let path = base.join(INDEX_FILE_NAME);
+        // Larger than the block size so size discovery needs a separate HEAD.
+        let data = vec![7u8; 2 * store.block_size()];
+        store.put(&path, &data).await.unwrap();
+
+        let file_sizes = HashMap::from([(INDEX_FILE_NAME.to_string(), data.len() as u64)]);
+
+        // Size recorded in the manifest, so reading the size issues no HEAD.
+        let _ = store.io_stats_incremental(); // reset
+        let reader = open_index_file(store.as_ref(), &path, INDEX_FILE_NAME, &file_sizes)
+            .await
+            .unwrap();
+        assert_eq!(reader.size().await.unwrap(), data.len());
+        let stats = store.io_stats_incremental();
+        assert_io_eq!(
+            stats,
+            read_iops,
+            0,
+            "a known file size must not trigger a HEAD request"
+        );
+
+        // Size unknown, as in an older index, so it falls back to a HEAD.
+        let _ = store.io_stats_incremental(); // reset
+        let reader = open_index_file(store.as_ref(), &path, INDEX_FILE_NAME, &HashMap::new())
+            .await
+            .unwrap();
+        assert_eq!(reader.size().await.unwrap(), data.len());
+        let stats = store.io_stats_incremental();
+        assert_io_eq!(
+            stats,
+            read_iops,
+            1,
+            "an unknown file size must fall back to exactly one HEAD request"
+        );
+    }
+
+    /// `open_index_file` looks up sizes in `IndexMetadata::file_size_map()` by
+    /// bare file name. This pins that a freshly created HNSW index records both
+    /// the main and auxiliary files under those exact names with nonzero sizes,
+    /// which is what lets the open path skip the HEAD.
+    #[tokio::test]
+    async fn test_hnsw_index_records_file_sizes() {
+        use lance_index::{INDEX_AUXILIARY_FILE_NAME, INDEX_FILE_NAME};
+
+        let test_dir = TempStrDir::default();
+        let uri = format!("{}/ds", test_dir.as_str());
+
+        let reader = lance_datagen::gen_batch()
+            .col("vector", array::rand_vec::<Float32Type>(32.into()))
+            .into_reader_rows(RowCount::from(400), BatchCount::from(1));
+        let mut dataset = Dataset::write(reader, &uri, None).await.unwrap();
+
+        let params = VectorIndexParams::with_ivf_hnsw_pq_params(
+            MetricType::L2,
+            IvfBuildParams {
+                num_partitions: Some(8),
+                ..Default::default()
+            },
+            HnswBuildParams {
+                max_level: 6,
+                m: 24,
+                ef_construction: 120,
+                prefetch_distance: None,
+            },
+            PQBuildParams {
+                num_sub_vectors: 8,
+                num_bits: 8,
+                ..Default::default()
+            },
+        );
+        dataset
+            .create_index(
+                &["vector"],
+                IndexType::Vector,
+                Some("hnsw".to_string()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        let index = indices.iter().find(|idx| idx.name == "hnsw").unwrap();
+        let file_sizes = index.file_size_map();
+
+        assert!(
+            file_sizes.get(INDEX_FILE_NAME).copied().unwrap_or(0) > 0,
+            "manifest should record a nonzero {INDEX_FILE_NAME} size, got {file_sizes:?}"
+        );
+        assert!(
+            file_sizes
+                .get(INDEX_AUXILIARY_FILE_NAME)
+                .copied()
+                .unwrap_or(0)
+                > 0,
+            "manifest should record a nonzero {INDEX_AUXILIARY_FILE_NAME} size, got {file_sizes:?}"
+        );
+    }
+
     #[tokio::test]
     async fn test_initialize_vector_index_ivf_pq() {
         let test_dir = TempStrDir::default();
@@ -2016,11 +2278,7 @@ mod tests {
 
         // Verify the index type and parameters match
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let stats = target_vector_index.statistics().unwrap();
@@ -2048,11 +2306,7 @@ mod tests {
 
         // Verify centroids are shared between source and target indices
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -2231,11 +2485,7 @@ mod tests {
 
         // Verify the index type and parameters match
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let stats = target_vector_index.statistics().unwrap();
@@ -2267,11 +2517,7 @@ mod tests {
 
         // Verify centroids are shared between source and target indices
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -2377,7 +2623,7 @@ mod tests {
         let invalid_id = max_id + 1000;
 
         // let params = VectorIndexParams::ivf_flat(4, MetricType::L2);
-        let uuid = Uuid::new_v4().to_string();
+        let uuid = Uuid::new_v4();
 
         let mut ivf_params = IvfBuildParams {
             num_partitions: Some(4),
@@ -2405,7 +2651,7 @@ mod tests {
             &dataset,
             "vector",
             "vector_ivf_flat_dist",
-            &uuid,
+            uuid,
             &params,
             None,
             &[invalid_id],
@@ -2431,7 +2677,7 @@ mod tests {
             .into_reader_rows(RowCount::from(128), BatchCount::from(1));
         let dataset = Dataset::write(reader, &uri, None).await.unwrap();
 
-        let uuid = Uuid::new_v4().to_string();
+        let uuid = Uuid::new_v4();
         let mut ivf_params = IvfBuildParams {
             num_partitions: Some(4),
             ..Default::default()
@@ -2458,7 +2704,7 @@ mod tests {
             &dataset,
             "vector",
             "vector_ivf_flat_dist",
-            &uuid,
+            uuid,
             &params,
             None,
             &[],
@@ -2520,7 +2766,7 @@ mod tests {
         let dataset = Dataset::write(reader, &uri, None).await.unwrap();
 
         let params = VectorIndexParams::ivf_flat(4, MetricType::L2);
-        let uuid = Uuid::new_v4().to_string();
+        let uuid = Uuid::new_v4();
         let progress = Arc::new(RecordingProgress {
             train_ivf_complete: AtomicBool::new(false),
             saw_train_ivf_progress_before_complete: AtomicBool::new(false),
@@ -2531,7 +2777,7 @@ mod tests {
             &dataset,
             "vector",
             "vector_ivf_flat_progress",
-            &uuid,
+            uuid,
             &params,
             None,
             progress.clone(),
@@ -2565,11 +2811,11 @@ mod tests {
         let dataset = Dataset::write(reader, &uri, None).await.unwrap();
 
         let params = VectorIndexParams::ivf_flat(4, MetricType::L2);
-        let uuid = Uuid::new_v4().to_string();
+        let uuid = Uuid::new_v4();
 
         // Pre-create a malformed global training file that is missing the
         // `lance:global_ivf_centroids` metadata key.
-        let out_base = dataset.indices_dir().join(&*uuid);
+        let out_base = dataset.indices_dir().join(uuid.to_string());
         let training_path = out_base.clone().join("global_training.idx");
 
         let writer = dataset
@@ -2600,7 +2846,7 @@ mod tests {
             &dataset,
             "vector",
             "vector_ivf_flat_dist",
-            &uuid,
+            uuid,
             &params,
             None,
             &[valid_id],
@@ -2689,20 +2935,12 @@ mod tests {
 
         // Open both indices to compare centroids
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -2816,11 +3054,7 @@ mod tests {
         // Verify that the optimized index still shares centroids with the source
         let target_indices = target_dataset.load_indices().await.unwrap();
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -2926,11 +3160,7 @@ mod tests {
 
         // Verify the index type and parameters match
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let stats = target_vector_index.statistics().unwrap();
@@ -2962,11 +3192,7 @@ mod tests {
 
         // Verify centroids are shared between source and target indices
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -3152,11 +3378,7 @@ mod tests {
 
         // Verify the index type and parameters match
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let stats = target_vector_index.statistics().unwrap();
@@ -3184,11 +3406,7 @@ mod tests {
 
         // Verify centroids are shared between source and target indices
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -3410,11 +3628,7 @@ mod tests {
 
         // Verify the index type and parameters match
         let target_vector_index = target_dataset
-            .open_vector_index(
-                "vector",
-                &target_indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &target_indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let stats = target_vector_index.statistics().unwrap();
@@ -3442,11 +3656,7 @@ mod tests {
 
         // Verify centroids are shared between source and target indices
         let source_vector_index = source_dataset
-            .open_vector_index(
-                "vector",
-                &source_index.uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &source_index.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs
index 08d8c32a001..f881d28e745 100644
--- a/rust/lance/src/index/vector/builder.rs
+++ b/rust/lance/src/index/vector/builder.rs
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 use std::{collections::HashMap, pin::Pin};
@@ -69,6 +70,7 @@ use lance_io::stream::RecordBatchStream;
 use lance_io::{object_store::ObjectStore, stream::RecordBatchStreamAdapter};
 use lance_linalg::distance::{DistanceType, Dot, L2, Normalize};
 use lance_linalg::kernels::normalize_fsl;
+use lance_table::format::IndexFile;
 use log::info;
 use object_store::path::Path;
 use prost::Message;
@@ -171,6 +173,11 @@ type BuildStream<S, Q> =
 
 type UnindexedStream = Box<dyn Stream<Item = Result<RecordBatch>> + Send + Unpin + 'static>;
 
+pub struct VectorIndexBuildSummary {
+    pub indices_merged: usize,
+    pub files: Vec<IndexFile>,
+}
+
 impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q> {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -282,9 +289,8 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
         })
     }
 
-    // build the index with the all data in the dataset,
-    // return the number of indices merged
-    pub async fn build(&mut self) -> Result<usize> {
+    // build the index and return the files created by the writer.
+    pub async fn build(&mut self) -> Result<VectorIndexBuildSummary> {
         let progress = self.progress.clone();
 
         // step 1. train IVF & quantizer
@@ -318,13 +324,16 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             .stage_start("merge_partitions", num_partitions, "partitions")
             .await?;
         let build_idx_stream = self.build_partitions().boxed().await?;
-        self.merge_partitions(build_idx_stream).await?;
+        let files = self.merge_partitions(build_idx_stream).await?;
         progress.stage_complete("merge_partitions").await?;
 
-        Ok(self.merged_num)
+        Ok(VectorIndexBuildSummary {
+            indices_merged: self.merged_num,
+            files,
+        })
     }
 
-    pub async fn remap(&mut self, mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    pub async fn remap(&mut self, mapping: &RowAddrRemap) -> Result<Vec<IndexFile>> {
         if self.existing_indices.is_empty() {
             return Err(Error::invalid_input(
                 "No existing indices available for remapping",
@@ -359,13 +368,14 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                 }
             });
 
-        self.merge_partitions(
-            stream::iter(build_iter)
-                .buffered(get_num_compute_intensive_cpus())
-                .boxed(),
-        )
-        .await?;
-        Ok(())
+        let files = self
+            .merge_partitions(
+                stream::iter(build_iter)
+                    .buffered(get_num_compute_intensive_cpus())
+                    .boxed(),
+            )
+            .await?;
+        Ok(files)
     }
 
     pub fn with_ivf(&mut self, ivf: IvfModel) -> &mut Self {
@@ -385,7 +395,14 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
 
     /// Set fragment filter for distributed indexing
     pub fn with_fragment_filter(&mut self, fragment_ids: Vec<u32>) -> &mut Self {
-        self.fragment_filter = Some(fragment_ids);
+        self.fragment_filter = Some(Dataset::normalize_fragment_ids(&fragment_ids));
+        self
+    }
+
+    pub fn with_optional_fragment_filter(&mut self, fragment_ids: Option<&[u32]>) -> &mut Self {
+        if let Some(fragment_ids) = fragment_ids {
+            self.fragment_filter = Some(Dataset::normalize_fragment_ids(fragment_ids));
+        }
         self
     }
 
@@ -543,19 +560,11 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             return Ok(None);
         };
         match &self.fragment_filter {
-            Some(fragment_ids) => {
-                let fragments: Vec<_> = dataset
-                    .get_fragments()
-                    .into_iter()
-                    .filter(|f| fragment_ids.contains(&(f.id() as u32)))
-                    .collect();
-                let counts = futures::stream::iter(fragments)
-                    .map(|f| async move { f.count_rows(None).await })
-                    .buffer_unordered(16) // ref: Dataset::count_all_rows()
-                    .try_collect::<Vec<_>>()
-                    .await?;
-                Ok(Some(counts.iter().sum::<usize>() as u64))
-            }
+            Some(fragment_ids) => Ok(Some(
+                dataset
+                    .count_rows_in_existing_fragments(fragment_ids)
+                    .await? as u64,
+            )),
             None => Ok(Some(dataset.count_rows(None).await? as u64)),
         }
     }
@@ -593,14 +602,9 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                         "applying fragment filter for distributed indexing: {:?}",
                         fragment_ids
                     );
-                    // Filter fragments by converting fragment_ids to Fragment objects
-                    let all_fragments = dataset.fragments();
-                    let filtered_fragments: Vec<_> = all_fragments
-                        .iter()
-                        .filter(|fragment| fragment_ids.contains(&(fragment.id as u32)))
-                        .cloned()
-                        .collect();
-                    builder.with_fragments(filtered_fragments);
+                    builder.with_fragments(
+                        dataset.get_existing_fragment_metadata_from_ids(fragment_ids),
+                    );
                 }
 
                 let (vector_type, _) = get_vector_type(dataset.schema(), &self.column)?;
@@ -1036,7 +1040,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
                 continue;
             }
 
-            let part_storage = existing_index.load_partition_storage(part_id).await?;
+            let part_storage = existing_index.load_partition_storage(part_id, None).await?;
             let mut part_batches = part_storage.to_batches()?.collect::<Vec<_>>();
             // for PQ, the PQ codes are transposed, so we need to transpose them back
             match Q::quantization_type() {
@@ -1108,7 +1112,10 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
     }
 
     #[instrument(name = "merge_partitions", level = "debug", skip_all)]
-    async fn merge_partitions(&mut self, mut build_stream: BuildStream<S, Q>) -> Result<()> {
+    async fn merge_partitions(
+        &mut self,
+        mut build_stream: BuildStream<S, Q>,
+    ) -> Result<Vec<IndexFile>> {
         let Some(ivf) = self.ivf.as_ref() else {
             return Err(Error::invalid_input("IVF not set before merge partitions"));
         };
@@ -1347,12 +1354,21 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> IvfIndexBuilder<S, Q>
             serde_json::to_string(&partition_index_metadata)?,
         );
 
-        storage_writer.finish().await?;
-        index_writer.finish().await?;
+        let storage_summary = storage_writer.finish().await?;
+        let index_summary = index_writer.finish().await?;
 
         log::info!("merging {} partitions done", ivf.num_partitions());
 
-        Ok(())
+        Ok(vec![
+            IndexFile {
+                path: INDEX_AUXILIARY_FILE_NAME.to_string(),
+                size_bytes: storage_summary.size_bytes,
+            },
+            IndexFile {
+                path: INDEX_FILE_NAME.to_string(),
+                size_bytes: index_summary.size_bytes,
+            },
+        ])
     }
 
     // take raw vectors from the dataset
@@ -2371,7 +2387,7 @@ mod tests {
 
         let indices = dataset.load_indices_by_name("idx").await.unwrap();
         let initial_index = dataset
-            .open_vector_index("vec", &indices[0].uuid.to_string(), &NoOpMetricsCollector)
+            .open_vector_index("vec", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let initial_ivf = initial_index.ivf_model();
@@ -2401,7 +2417,7 @@ mod tests {
         let indices = dataset.load_indices_by_name("idx").await.unwrap();
         assert_eq!(indices.len(), 1, "expected merge-all on split");
         let optimized = dataset
-            .open_vector_index("vec", &indices[0].uuid.to_string(), &NoOpMetricsCollector)
+            .open_vector_index("vec", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let ivf = optimized.ivf_model();
diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs
index 83e9b92c209..63f9375792e 100644
--- a/rust/lance/src/index/vector/details.rs
+++ b/rust/lance/src/index/vector/details.rs
@@ -503,7 +503,15 @@ pub async fn infer_vector_index_details(
     let index_dir = dataset.indice_files_dir(index)?;
     let file_dir = index_dir.clone().join(uuid.as_str());
     let index_file = file_dir.clone().join(INDEX_FILE_NAME);
-    let reader: Arc<dyn Reader> = dataset.object_store.open(&index_file).await?.into();
+    let file_sizes = index.file_size_map();
+    let reader: Arc<dyn Reader> = super::open_index_file(
+        dataset.object_store.as_ref(),
+        &index_file,
+        INDEX_FILE_NAME,
+        &file_sizes,
+    )
+    .await?
+    .into();
 
     let tailing_bytes = read_last_block(reader.as_ref()).await?;
     let (major_version, minor_version) = read_version(&tailing_bytes)?;
diff --git a/rust/lance/src/index/vector/fixture_test.rs b/rust/lance/src/index/vector/fixture_test.rs
index 6d577c73ff8..a03cc313466 100644
--- a/rust/lance/src/index/vector/fixture_test.rs
+++ b/rust/lance/src/index/vector/fixture_test.rs
@@ -5,10 +5,10 @@
 
 #[cfg(test)]
 mod test {
+    use lance_core::utils::row_addr_remap::RowAddrRemap;
     use std::{
         any::Any,
         cell::OnceCell,
-        collections::HashMap,
         sync::{Arc, Mutex},
     };
 
@@ -18,8 +18,8 @@ mod test {
     use arrow_schema::{DataType, Field, Schema};
     use async_trait::async_trait;
     use datafusion::execution::SendableRecordBatchStream;
-    use deepsize::{Context, DeepSizeOf};
     use lance_arrow::FixedSizeListArrayExt;
+    use lance_core::deepsize::{Context, DeepSizeOf};
     use lance_core::{cache::LanceCache, utils::tempfile::TempStdFile};
     use lance_index::vector::v3::subindex::SubIndexType;
     use lance_index::{Index, IndexType, vector::Query};
@@ -54,7 +54,8 @@ mod test {
 
     impl DeepSizeOf for ResidualCheckMockIndex {
         fn deep_size_of_children(&self, cx: &mut Context) -> usize {
-            self.assert_query_value.deep_size_of_children(cx) + self.ret_val.get_array_memory_size()
+            self.assert_query_value.deep_size_of_children(cx)
+                + self.ret_val.deep_size_of_children(cx)
         }
     }
 
@@ -70,10 +71,6 @@ mod test {
             self
         }
 
-        fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-            Ok(self)
-        }
-
         async fn prewarm(&self) -> Result<()> {
             Ok(())
         }
@@ -152,7 +149,7 @@ mod test {
             todo!("this method is for only IVF_HNSW_* index");
         }
 
-        async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+        async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
             Ok(())
         }
 
@@ -211,7 +208,7 @@ mod test {
                 ]))),
             });
             IVFIndex::try_new(
-                &Uuid::new_v4().to_string(),
+                Uuid::new_v4(),
                 ivf,
                 reader.into(),
                 mock_sub_index,
@@ -268,6 +265,7 @@ mod test {
                 use_index: true,
                 query_parallelism: lance_index::vector::DEFAULT_QUERY_PARALLELISM,
                 dist_q_c: 0.0,
+                approx_mode: Default::default(),
             };
             let idx = make_idx.clone()(expected_query_at_subindex, metric).await;
             let (partition_ids, _) = idx.find_partitions(&q).unwrap();
diff --git a/rust/lance/src/index/vector/hamming.rs b/rust/lance/src/index/vector/hamming.rs
new file mode 100644
index 00000000000..3240ab3021b
--- /dev/null
+++ b/rust/lance/src/index/vector/hamming.rs
@@ -0,0 +1,1304 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Hamming distance clustering for IVF_FLAT indices.
+//!
+//! This module provides functionality to perform pairwise hamming distance
+//! computation and clustering on specific partitions of IVF_FLAT indices.
+//!
+//! A logical IVF_FLAT index may consist of multiple physical segments (e.g.
+//! delta segments created by `optimize_indices` in append mode, or segments
+//! committed by distributed index builds). All segments of one logical index
+//! are assumed to share the same global IVF centroids, so one partition id
+//! refers to the same centroid region in every segment; this is validated and
+//! an error is returned if the centroids differ.
+
+use std::sync::Arc;
+use std::time::Instant;
+
+use arrow_array::cast::AsArray;
+use arrow_array::types::UInt64Type;
+use arrow_array::{Array, FixedSizeListArray, RecordBatchReader};
+use arrow_schema::DataType;
+use lance_core::{Error, Result};
+use lance_index::metrics::NoOpMetricsCollector;
+use lance_index::vector::VectorIndex;
+use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex};
+use lance_index::vector::flat::storage::FLAT_COLUMN;
+use lance_index::vector::ivf::storage::IvfModel;
+use lance_index::vector::storage::VectorStore;
+use lance_linalg::distance::{
+    BinaryHashValues, ClusteringResult, cluster_pairwise_result,
+    extract_binary_hashes_from_fixed_list, pairwise_hamming_distance_binary_parallel,
+    pairwise_hamming_distance_parallel,
+};
+use lance_table::format::IndexMetadata;
+use rand::rng;
+use rand::seq::index::sample;
+use uuid::Uuid;
+
+use crate::dataset::Dataset;
+use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, filter_index_segments_by_ids};
+
+use super::ivf::v2::IVFIndex;
+
+/// One opened physical segment of a logical IVF_FLAT binary index.
+struct HashIndexSegment {
+    metadata: IndexMetadata,
+    index: Arc<dyn VectorIndex>,
+}
+
+impl HashIndexSegment {
+    fn ivf_flat_bin(&self) -> &IVFIndex<FlatIndex, FlatBinQuantizer> {
+        self.index
+            .as_any()
+            .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
+            .expect("segment type validated in open_hash_index_segments")
+    }
+}
+
+/// Validate that a column stores fixed-width binary hashes as
+/// `FixedSizeList<UInt8, N>`, where `N` is a positive multiple of 8 bytes.
+fn validate_hash_column(column: &str, data_type: &DataType) -> Result<usize> {
+    match data_type {
+        DataType::FixedSizeList(inner, size) if *inner.data_type() == DataType::UInt8 => {
+            if *size <= 0 || !(*size as usize).is_multiple_of(8) {
+                return Err(Error::invalid_input(format!(
+                    "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+                     multiple of 8 bytes, got FixedSizeList<UInt8, {}>",
+                    column, size
+                )));
+            }
+            Ok(*size as usize)
+        }
+        DataType::FixedSizeList(inner, size) => Err(Error::invalid_input(format!(
+            "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+             multiple of 8 bytes, got FixedSizeList<{:?}, {}>",
+            column,
+            inner.data_type(),
+            size
+        ))),
+        _ => Err(Error::invalid_input(format!(
+            "Column '{}' must be FixedSizeList<UInt8, N> where N is a positive \
+             multiple of 8 bytes, got {:?}",
+            column, data_type
+        ))),
+    }
+}
+
+/// Validate that every segment of a logical IVF index shares the same global
+/// centroids, so one partition id refers to the same centroid region in each
+/// segment. Fails if any segment has no centroids or diverging centroids.
+fn validate_shared_centroids<'a>(
+    index_name: &str,
+    models: impl IntoIterator<Item = (Uuid, &'a IvfModel)>,
+) -> Result<()> {
+    struct Reference<'a> {
+        uuid: Uuid,
+        num_partitions: usize,
+        centroids: &'a FixedSizeListArray,
+    }
+
+    let mut reference: Option<Reference<'a>> = None;
+    for (uuid, model) in models {
+        let centroids = model.centroids_array().ok_or_else(|| {
+            Error::invalid_input(format!(
+                "Index '{}' segment {} has no IVF centroids; hamming clustering requires \
+                 segments built from a shared global IVF model",
+                index_name, uuid
+            ))
+        })?;
+        match &reference {
+            None => {
+                reference = Some(Reference {
+                    uuid,
+                    num_partitions: model.num_partitions(),
+                    centroids,
+                });
+            }
+            Some(reference) => {
+                if centroids.to_data() != reference.centroids.to_data() {
+                    return Err(Error::invalid_input(format!(
+                        "Index '{}' segments do not share the same global IVF centroids: \
+                         segment {} ({} partitions) differs from segment {} ({} partitions); \
+                         retrain the index to merge segments before hamming clustering",
+                        index_name,
+                        uuid,
+                        model.num_partitions(),
+                        reference.uuid,
+                        reference.num_partitions
+                    )));
+                }
+            }
+        }
+    }
+    Ok(())
+}
+
+/// Open the physical segments of a logical IVF_FLAT binary index.
+///
+/// When `segment_ids` is `None` all segments are opened; otherwise only the
+/// requested segments are opened and every requested id must exist. Validates
+/// that all selected segments index the same fixed-width binary hash column,
+/// are IVF_FLAT indices for binary data, and share the same global centroids.
+async fn open_hash_index_segments(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: Option<&[Uuid]>,
+) -> Result<Vec<HashIndexSegment>> {
+    let metadatas = dataset.load_indices_by_name(index_name).await?;
+    if metadatas.is_empty() {
+        return Err(Error::invalid_input(format!(
+            "Index '{}' not found on dataset",
+            index_name
+        )));
+    }
+
+    let metadatas = match segment_ids {
+        None => metadatas,
+        Some(segment_ids) => {
+            if segment_ids.is_empty() {
+                return Err(Error::invalid_input(format!(
+                    "Segment selection for index '{}' must not be empty; \
+                     omit index_segments to use all segments",
+                    index_name
+                )));
+            }
+            filter_index_segments_by_ids(index_name, metadatas, segment_ids)?
+        }
+    };
+
+    let fields = metadatas[0].fields.clone();
+    if let Some(mismatched) = metadatas.iter().find(|meta| meta.fields != fields) {
+        return Err(Error::invalid_input(format!(
+            "Index '{}' segments cover different fields: segment {} covers {:?} \
+             while segment {} covers {:?}",
+            index_name, metadatas[0].uuid, fields, mismatched.uuid, mismatched.fields
+        )));
+    }
+
+    let field_id = fields
+        .first()
+        .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?;
+    let schema = dataset.schema();
+    let field = schema.field_by_id(*field_id).ok_or_else(|| {
+        Error::invalid_input(format!(
+            "Field with id {} not found in schema for index '{}'",
+            field_id, index_name
+        ))
+    })?;
+    validate_hash_column(&field.name, &field.data_type())?;
+
+    let mut segments = Vec::with_capacity(metadatas.len());
+    for metadata in metadatas {
+        let index = dataset
+            .open_vector_index(&field.name, &metadata.uuid, &NoOpMetricsCollector)
+            .await?;
+        if index
+            .as_any()
+            .downcast_ref::<IVFIndex<FlatIndex, FlatBinQuantizer>>()
+            .is_none()
+        {
+            return Err(Error::invalid_input(format!(
+                "Index '{}' segment {} is not an IVF_FLAT index for binary data",
+                index_name, metadata.uuid
+            )));
+        }
+        segments.push(HashIndexSegment { metadata, index });
+    }
+
+    validate_shared_centroids(
+        index_name,
+        segments
+            .iter()
+            .map(|segment| (segment.metadata.uuid, segment.ivf_flat_bin().ivf_model())),
+    )?;
+
+    Ok(segments)
+}
+
+async fn hamming_clustering_for_ivf_partition_impl(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: Option<&[Uuid]>,
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?;
+
+    // All segments share centroids, so the partition count is uniform.
+    let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions();
+    if partition_id >= num_partitions {
+        return Err(Error::invalid_input(format!(
+            "Partition ID {} is out of range (0..{})",
+            partition_id, num_partitions
+        )));
+    }
+
+    // Concatenate the partition's row ids and hashes across segments; identical
+    // hashes land in the same partition of every segment, so one pairwise pass
+    // over the union finds cross-segment duplicates.
+    let mut all_row_ids: Vec<u64> = Vec::new();
+    let mut hash_chunks = Vec::new();
+    let mut num_hashes = 0;
+    for segment in &segments {
+        let storage = segment
+            .ivf_flat_bin()
+            .load_partition_storage(partition_id, None)
+            .await?;
+        all_row_ids.extend(storage.row_ids().copied());
+        for batch in storage.to_batches()? {
+            let vectors = batch
+                .column_by_name(FLAT_COLUMN)
+                .ok_or_else(|| {
+                    Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN))
+                })?
+                .as_fixed_size_list();
+            let hashes = extract_binary_hashes_from_fixed_list(vectors)?;
+            num_hashes += hashes.len();
+            hash_chunks.push(hashes);
+        }
+        if all_row_ids.len() != num_hashes {
+            return Err(Error::internal(format!(
+                "Index '{}' segment {} partition {}: row id count {} does not match hash count {}",
+                index_name,
+                segment.metadata.uuid,
+                partition_id,
+                all_row_ids.len(),
+                num_hashes
+            )));
+        }
+    }
+
+    if all_row_ids.is_empty() {
+        let empty = ClusteringResult {
+            clusters: Vec::new(),
+        };
+        return Ok(empty.into_reader(None));
+    }
+    let all_hashes = BinaryHashValues::concat(&hash_chunks)?;
+
+    // Compute pairwise hamming distances with threshold filtering
+    let pairwise_result = pairwise_hamming_distance_binary_parallel(
+        &all_hashes,
+        Some(&all_row_ids),
+        Some(hamming_threshold),
+    );
+
+    // Cluster the results
+    let clustering = cluster_pairwise_result(&pairwise_result);
+
+    Ok(clustering.into_reader(None))
+}
+
+/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index.
+///
+/// This function loads a specific partition from every segment of an IVF_FLAT
+/// index on a hash column, computes pairwise hamming distances between all
+/// hashes in the combined partition, filters by threshold, and clusters the
+/// results using union-find. See [`hamming_clustering_for_ivf_partition_segments`]
+/// to restrict the computation to selected segments.
+///
+/// # Arguments
+///
+/// * `dataset` - The Lance dataset
+/// * `index_name` - Name of the IVF_FLAT index on the hash column
+/// * `partition_id` - The partition ID within the IVF_FLAT index
+/// * `hamming_threshold` - Maximum hamming distance to consider as similar
+///
+/// # Returns
+///
+/// A `RecordBatchReader` yielding batches with columns:
+/// - `representative`: UInt64 - The representative row ID for each cluster
+/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The index doesn't exist or is not an IVF_FLAT index
+/// - The indexed column has wrong type (must be `FixedSizeList<UInt8, N>` where
+///   `N` is a positive multiple of 8 bytes)
+/// - The index segments do not share the same global IVF centroids
+/// - The partition ID is out of range
+pub async fn hamming_clustering_for_ivf_partition(
+    dataset: &Dataset,
+    index_name: &str,
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    hamming_clustering_for_ivf_partition_impl(
+        dataset,
+        index_name,
+        None,
+        partition_id,
+        hamming_threshold,
+    )
+    .await
+}
+
+/// Perform pairwise hamming distance clustering on a partition of selected
+/// segments of an IVF_FLAT index.
+///
+/// Same as [`hamming_clustering_for_ivf_partition`] but only the requested
+/// physical segments contribute rows. Segment ids are the index UUIDs reported
+/// by index descriptions; every requested id must belong to the named index and
+/// the selection must not be empty.
+pub async fn hamming_clustering_for_ivf_partition_segments(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: &[Uuid],
+    partition_id: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    hamming_clustering_for_ivf_partition_impl(
+        dataset,
+        index_name,
+        Some(segment_ids),
+        partition_id,
+        hamming_threshold,
+    )
+    .await
+}
+
+async fn get_ivf_partition_info_impl(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: Option<&[Uuid]>,
+) -> Result<Vec<PartitionInfo>> {
+    let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?;
+
+    let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions();
+    let mut partition_infos: Vec<PartitionInfo> = (0..num_partitions)
+        .map(|partition_id| PartitionInfo {
+            partition_id,
+            size: 0,
+        })
+        .collect();
+    for segment in &segments {
+        // Sizes come from the partition storage; the IVF model of a v3 index
+        // file does not carry partition lengths.
+        let index = segment.ivf_flat_bin();
+        for info in partition_infos.iter_mut() {
+            info.size += index.partition_size(info.partition_id);
+        }
+    }
+
+    Ok(partition_infos)
+}
+
+/// Get partition statistics for an IVF_FLAT index.
+///
+/// Partition sizes are aggregated across all segments of the logical index.
+/// See [`get_ivf_partition_info_segments`] to restrict the statistics to
+/// selected segments.
+pub async fn get_ivf_partition_info(
+    dataset: &Dataset,
+    index_name: &str,
+) -> Result<Vec<PartitionInfo>> {
+    get_ivf_partition_info_impl(dataset, index_name, None).await
+}
+
+/// Get partition statistics for selected segments of an IVF_FLAT index.
+///
+/// Same as [`get_ivf_partition_info`] but only the requested physical segments
+/// contribute to the partition sizes.
+pub async fn get_ivf_partition_info_segments(
+    dataset: &Dataset,
+    index_name: &str,
+    segment_ids: &[Uuid],
+) -> Result<Vec<PartitionInfo>> {
+    get_ivf_partition_info_impl(dataset, index_name, Some(segment_ids)).await
+}
+
+/// Information about an IVF partition.
+#[derive(Debug, Clone)]
+pub struct PartitionInfo {
+    pub partition_id: usize,
+    pub size: usize,
+}
+
+/// Perform pairwise hamming distance clustering on sampled rows from a dataset.
+///
+/// This function samples N rows randomly from the dataset, extracts hashes,
+/// computes pairwise hamming distances, and clusters the results.
+/// It's useful for benchmarking and testing without requiring an IVF index.
+///
+/// # Arguments
+///
+/// * `dataset` - The Lance dataset
+/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, N>`
+///   where `N` is a positive multiple of 8 bytes)
+/// * `sample_size` - Number of rows to sample (if None or >= total rows, uses all rows)
+/// * `hamming_threshold` - Maximum hamming distance to consider as similar
+///
+/// # Returns
+///
+/// A `RecordBatchReader` yielding batches with columns:
+/// - `representative`: UInt64 - The representative row ID for each cluster
+/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
+pub async fn hamming_clustering_for_sample(
+    dataset: &Dataset,
+    column: &str,
+    sample_size: Option<usize>,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    // Validate column exists and has correct type
+    let schema = dataset.schema();
+    let field = schema.field(column).ok_or_else(|| {
+        Error::invalid_input(format!("Column '{}' not found in dataset schema", column))
+    })?;
+    validate_hash_column(column, &field.data_type())?;
+
+    // Get total row count
+    let total_rows: usize = dataset
+        .get_fragments()
+        .iter()
+        .filter_map(|f| f.metadata().physical_rows)
+        .sum();
+
+    let use_sampling = sample_size.is_some_and(|s| s < total_rows);
+    let effective_sample = sample_size.unwrap_or(total_rows).min(total_rows);
+
+    // Read data
+    let (hashes, row_ids) = if use_sampling {
+        // Random sample using take() with _rowid (take uses positional indices)
+        let indices: Vec<u64> = sample(&mut rng(), total_rows, effective_sample)
+            .iter()
+            .map(|i| i as u64)
+            .collect();
+
+        let batch = dataset
+            .take(
+                &indices,
+                crate::dataset::ProjectionRequest::from_columns(
+                    [column, "_rowid"],
+                    dataset.schema(),
+                ),
+            )
+            .await?;
+
+        let rowid_col = batch.column_by_name("_rowid").ok_or_else(|| {
+            Error::invalid_input("_rowid column not found in take result".to_string())
+        })?;
+        let row_ids = rowid_col.as_primitive::<UInt64Type>();
+        let row_id_vec: Vec<u64> = row_ids.values().to_vec();
+
+        let hash_col = batch.column_by_name(column).ok_or_else(|| {
+            Error::invalid_input(format!("Column '{}' not found in result", column))
+        })?;
+        let hashes_arr = hash_col.as_fixed_size_list();
+        let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
+
+        (hashes, row_id_vec)
+    } else {
+        // Full scan
+        let batch = dataset
+            .scan()
+            .project(&[column])?
+            .with_row_id()
+            .try_into_batch()
+            .await?;
+
+        let rowid_col = batch.column_by_name("_rowid").ok_or_else(|| {
+            Error::invalid_input("_rowid column not found in scan result".to_string())
+        })?;
+        let row_ids = rowid_col.as_primitive::<UInt64Type>();
+        let row_id_vec: Vec<u64> = row_ids.values().to_vec();
+
+        let hash_col = batch.column_by_name(column).ok_or_else(|| {
+            Error::invalid_input(format!("Column '{}' not found in result", column))
+        })?;
+        let hashes_arr = hash_col.as_fixed_size_list();
+        let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
+
+        (hashes, row_id_vec)
+    };
+
+    if hashes.len() < 2 {
+        let empty = ClusteringResult {
+            clusters: Vec::new(),
+        };
+        return Ok(empty.into_reader(None));
+    }
+
+    // Compute pairwise hamming distances
+    let pairwise =
+        pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(hamming_threshold));
+
+    // Cluster edges
+    let clustering = cluster_pairwise_result(&pairwise);
+
+    Ok(clustering.into_reader(None))
+}
+
+/// Perform pairwise hamming distance clustering on a contiguous range of rows from a fragment.
+///
+/// This function reads a contiguous range of rows from a specific fragment,
+/// extracts hashes, computes pairwise hamming distances, and clusters the results.
+/// Unlike sampling, this reads sequential rows which is useful for distributed
+/// processing where each worker handles a specific range of a fragment.
+///
+/// # Arguments
+///
+/// * `dataset` - The Lance dataset
+/// * `column` - Name of the hash column (must be `FixedSizeList<UInt8, N>`
+///   where `N` is a positive multiple of 8 bytes)
+/// * `fragment_id` - The fragment ID to read from
+/// * `start_row` - The starting row offset within the fragment
+/// * `num_rows` - Number of rows to read from the start position
+/// * `hamming_threshold` - Maximum hamming distance to consider as similar
+///
+/// # Returns
+///
+/// A `RecordBatchReader` yielding batches with columns:
+/// - `representative`: UInt64 - The representative row ID for each cluster
+/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The fragment doesn't exist
+/// - The column has wrong type (must be `FixedSizeList<UInt8, N>` where `N`
+///   is a positive multiple of 8 bytes)
+/// - The row range is out of bounds
+pub async fn hamming_clustering_for_range(
+    dataset: &Dataset,
+    column: &str,
+    fragment_id: usize,
+    start_row: usize,
+    num_rows: usize,
+    hamming_threshold: u32,
+) -> Result<Box<dyn RecordBatchReader + Send>> {
+    // Validate column exists and has correct type
+    let schema = dataset.schema();
+    let field = schema.field(column).ok_or_else(|| {
+        Error::invalid_input(format!("Column '{}' not found in dataset schema", column))
+    })?;
+    validate_hash_column(column, &field.data_type())?;
+
+    // Get the fragment
+    let fragment = dataset.get_fragment(fragment_id).ok_or_else(|| {
+        Error::invalid_input(format!("Fragment with ID {} not found", fragment_id))
+    })?;
+
+    // Get fragment metadata for physical row count
+    let fragment_meta = fragment.metadata().clone();
+    let physical_rows = fragment_meta
+        .physical_rows
+        .ok_or_else(|| Error::invalid_input("Fragment has no physical_rows metadata"))?;
+
+    // Validate the range
+    if start_row >= physical_rows {
+        return Err(Error::invalid_input(format!(
+            "start_row {} is out of range for fragment with {} physical rows",
+            start_row, physical_rows
+        )));
+    }
+
+    // Adjust num_rows if it exceeds available rows
+    let effective_num_rows = num_rows.min(physical_rows - start_row);
+
+    if effective_num_rows == 0 {
+        let empty = ClusteringResult {
+            clusters: Vec::new(),
+        };
+        return Ok(empty.into_reader(None));
+    }
+
+    // Use scanner with the specific fragment and limit/offset
+    let batch = dataset
+        .scan()
+        .with_fragments(vec![fragment_meta])
+        .project(&[column])?
+        .with_row_id()
+        .limit(Some(effective_num_rows as i64), Some(start_row as i64))?
+        .try_into_batch()
+        .await?;
+
+    // Extract row IDs
+    let rowid_col = batch.column_by_name("_rowid").ok_or_else(|| {
+        Error::invalid_input("_rowid column not found in scan result".to_string())
+    })?;
+    let row_ids = rowid_col.as_primitive::<UInt64Type>();
+    let row_id_vec: Vec<u64> = row_ids.values().to_vec();
+
+    // Extract hashes
+    let hash_col = batch
+        .column_by_name(column)
+        .ok_or_else(|| Error::invalid_input(format!("Column '{}' not found in result", column)))?;
+    let hashes_arr = hash_col.as_fixed_size_list();
+    let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?;
+
+    if hashes.len() < 2 {
+        let empty = ClusteringResult {
+            clusters: Vec::new(),
+        };
+        return Ok(empty.into_reader(None));
+    }
+
+    // Compute pairwise hamming distances
+    let pairwise = pairwise_hamming_distance_binary_parallel(
+        &hashes,
+        Some(&row_id_vec),
+        Some(hamming_threshold),
+    );
+
+    // Cluster edges
+    let clustering = cluster_pairwise_result(&pairwise);
+
+    Ok(clustering.into_reader(None))
+}
+
+/// Perform pairwise hamming distance clustering on provided hashes (no I/O).
+///
+/// This is useful for benchmarking the pure compute performance without I/O.
+/// Logs timing information via tracing.
+///
+/// # Arguments
+///
+/// * `hashes` - Vector of 64-bit hash values
+/// * `row_ids` - Optional row IDs (defaults to indices if None)
+/// * `hamming_threshold` - Maximum hamming distance to consider as similar
+///
+/// # Returns
+///
+/// A `RecordBatchReader` yielding batches with columns:
+/// - `representative`: UInt64 - The representative row ID for each cluster
+/// - `duplicates`: `List<UInt64>` - List of duplicate row IDs in each cluster
+pub fn hamming_clustering_from_hashes(
+    hashes: &[u64],
+    row_ids: Option<&[u64]>,
+    hamming_threshold: u32,
+) -> Box<dyn RecordBatchReader + Send> {
+    let num_rows = hashes.len();
+    if num_rows < 2 {
+        let empty = ClusteringResult {
+            clusters: Vec::new(),
+        };
+        return empty.into_reader(None);
+    }
+
+    let total_pairs = (num_rows as u64) * (num_rows as u64 - 1) / 2;
+
+    // Compute pairwise hamming distances
+    let t_compute_start = Instant::now();
+    let pairwise = pairwise_hamming_distance_parallel(hashes, row_ids, Some(hamming_threshold));
+    let compute_time = t_compute_start.elapsed();
+
+    // Cluster edges
+    let t_cluster_start = Instant::now();
+    let clustering = cluster_pairwise_result(&pairwise);
+    let cluster_time = t_cluster_start.elapsed();
+
+    // Log timing info
+    let pairs_per_sec = if compute_time.as_secs_f64() > 0.0 {
+        total_pairs as f64 / compute_time.as_secs_f64()
+    } else {
+        0.0
+    };
+    tracing::info!(
+        num_rows,
+        total_pairs,
+        edges = pairwise.len(),
+        compute_time_ms = compute_time.as_millis(),
+        cluster_time_ms = cluster_time.as_millis(),
+        pairs_per_sec_millions = pairs_per_sec / 1_000_000.0,
+        num_clusters = clustering.num_clusters(),
+        num_duplicates = clustering.num_duplicates(),
+        "Hamming clustering completed"
+    );
+
+    clustering.into_reader(None)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use arrow_array::Array;
+
+    /// Helper to collect all clusters from a reader.
+    fn collect_clusters(reader: Box<dyn RecordBatchReader + Send>) -> Vec<(u64, Vec<u64>)> {
+        let mut clusters = Vec::new();
+        for batch in reader {
+            let batch = batch.unwrap();
+            let reps = batch
+                .column(0)
+                .as_any()
+                .downcast_ref::<arrow_array::UInt64Array>()
+                .unwrap();
+            let dups = batch
+                .column(1)
+                .as_any()
+                .downcast_ref::<arrow_array::ListArray>()
+                .unwrap();
+
+            for i in 0..batch.num_rows() {
+                let rep = reps.value(i);
+                let dup_arr = dups.value(i);
+                let dup_values = dup_arr
+                    .as_any()
+                    .downcast_ref::<arrow_array::UInt64Array>()
+                    .unwrap();
+                let duplicates: Vec<u64> = dup_values.values().to_vec();
+                clusters.push((rep, duplicates));
+            }
+        }
+        clusters
+    }
+
+    #[test]
+    fn test_validate_hash_column_generic_widths() {
+        use arrow_schema::{DataType, Field};
+        use std::sync::Arc;
+
+        fn hash_type(size: i32) -> DataType {
+            DataType::FixedSizeList(Arc::new(Field::new("item", DataType::UInt8, true)), size)
+        }
+
+        assert_eq!(validate_hash_column("hash", &hash_type(8)).unwrap(), 8);
+        assert_eq!(validate_hash_column("hash", &hash_type(16)).unwrap(), 16);
+        assert_eq!(validate_hash_column("hash", &hash_type(32)).unwrap(), 32);
+
+        let err = validate_hash_column("hash", &hash_type(12)).unwrap_err();
+        assert!(err.to_string().contains("12"), "{}", err);
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+
+        let err = validate_hash_column("hash", &hash_type(4)).unwrap_err();
+        assert!(err.to_string().contains("4"), "{}", err);
+        assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err);
+
+        let err = validate_hash_column("hash", &hash_type(-8)).unwrap_err();
+        assert!(err.to_string().contains("-8"), "{}", err);
+    }
+
+    #[test]
+    fn test_hamming_clustering_from_hashes_basic() {
+        // Create some test hashes with known distances
+        let hashes = vec![
+            0b0000u64, // hash 0
+            0b0001u64, // hash 1 - distance 1 from hash 0
+            0b0011u64, // hash 2 - distance 1 from hash 1, distance 2 from hash 0
+            0b1111u64, // hash 3 - distance 2 from hash 2, distance 4 from hash 0
+        ];
+
+        let reader = hamming_clustering_from_hashes(&hashes, None, 1);
+        let clusters = collect_clusters(reader);
+
+        // With threshold 1, pairs (0,1) and (1,2) should be connected
+        // This forms one cluster: {0, 1, 2}
+        assert_eq!(clusters.len(), 1);
+        assert_eq!(clusters[0].1.len(), 2); // 2 duplicates in the cluster
+    }
+
+    #[test]
+    fn test_hamming_clustering_from_hashes_no_clusters() {
+        // All hashes are far apart
+        let hashes = vec![
+            0x0000000000000000u64,
+            0xFFFFFFFFFFFFFFFFu64,
+            0xAAAAAAAAAAAAAAAAu64,
+        ];
+
+        let reader = hamming_clustering_from_hashes(&hashes, None, 5);
+        let clusters = collect_clusters(reader);
+
+        // With threshold 5, no pairs should be connected (min distance is 32)
+        assert_eq!(clusters.len(), 0);
+    }
+
+    #[test]
+    fn test_hamming_clustering_from_hashes_with_row_ids() {
+        let hashes = vec![0b0000u64, 0b0001u64];
+        let row_ids = vec![100u64, 200u64];
+
+        let reader = hamming_clustering_from_hashes(&hashes, Some(&row_ids), 1);
+        let clusters = collect_clusters(reader);
+
+        assert_eq!(clusters.len(), 1);
+        assert_eq!(clusters[0].0, 100); // representative
+        assert_eq!(clusters[0].1, vec![200]); // duplicates
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_ivf_partition() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use lance_index::vector::ivf::IvfBuildParams;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        // Create test data with hash column (FixedSizeList<UInt8, 8>)
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                8,
+            ),
+            false,
+        )]));
+
+        // Generate hashes with some duplicates (similar hashes)
+        let num_rows = 100;
+        let mut hash_bytes = Vec::with_capacity(num_rows * 8);
+        for i in 0..num_rows {
+            // Create groups of similar hashes
+            let base = (i / 10) as u64; // 10 groups
+            let variation = (i % 10) as u64;
+            let hash = base.wrapping_mul(0x123456789) ^ variation;
+            hash_bytes.extend_from_slice(&hash.to_le_bytes());
+        }
+        let values = UInt8Array::from(hash_bytes);
+        let hash_array =
+            FixedSizeListArray::try_new_from_values(values, 8).expect("create hash array");
+
+        let batch =
+            arrow_array::RecordBatch::try_new(schema.clone(), vec![Arc::new(hash_array)]).unwrap();
+
+        // Write dataset
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let mut dataset = crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        // Create IVF_FLAT index with 4 partitions
+        let ivf_params = IvfBuildParams::new(4);
+        let params = crate::index::vector::VectorIndexParams::with_ivf_flat_params(
+            lance_linalg::distance::MetricType::Hamming,
+            ivf_params,
+        );
+
+        dataset
+            .create_index(
+                &["hash"],
+                crate::index::IndexType::Vector,
+                None,
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        // Load and test
+        let dataset = crate::Dataset::open(uri).await.unwrap();
+        let indices = dataset.load_indices().await.unwrap();
+        let index_name = &indices[0].name;
+
+        // Test clustering on partition 0
+        let reader = hamming_clustering_for_ivf_partition(&dataset, index_name, 0, 10)
+            .await
+            .unwrap();
+        let clusters = collect_clusters(reader);
+
+        // Verify we get valid results (may or may not have clusters depending on data distribution)
+        // At minimum, verify no panics and valid schema
+        for (rep, dups) in &clusters {
+            assert!(*rep < num_rows as u64 * 10); // row IDs should be reasonable
+            for dup in dups {
+                assert!(*dup < num_rows as u64 * 10);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_ivf_partition_invalid_index() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                8,
+            ),
+            false,
+        )]));
+
+        let values = UInt8Array::from(vec![0u8; 80]); // 10 rows * 8 bytes
+        let hash_array = FixedSizeListArray::try_new_from_values(values, 8).unwrap();
+        let batch =
+            arrow_array::RecordBatch::try_new(schema.clone(), vec![Arc::new(hash_array)]).unwrap();
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        let dataset = crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        // Test with non-existent index
+        let result = hamming_clustering_for_ivf_partition(&dataset, "nonexistent", 0, 10).await;
+        assert!(result.is_err());
+        let err = result.err().unwrap();
+        assert!(err.to_string().contains("not found"), "Error: {}", err);
+    }
+
+    #[test]
+    fn test_validate_shared_centroids() {
+        use arrow_array::UInt8Array;
+        use lance_arrow::FixedSizeListArrayExt;
+
+        fn model_from_bytes(bytes: Vec<u8>) -> IvfModel {
+            let centroids =
+                FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 8).unwrap();
+            IvfModel::new(centroids, None)
+        }
+
+        let uuid_a = Uuid::new_v4();
+        let uuid_b = Uuid::new_v4();
+
+        let model_a = model_from_bytes(vec![0u8; 16]);
+        let model_b = model_from_bytes(vec![0u8; 16]);
+        validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_b)]).unwrap();
+
+        let mut diverged = vec![0u8; 16];
+        diverged[0] = 1;
+        let model_c = model_from_bytes(diverged);
+        let err =
+            validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_c)]).unwrap_err();
+        assert!(
+            err.to_string()
+                .contains("do not share the same global IVF centroids"),
+            "{}",
+            err
+        );
+        assert!(err.to_string().contains(&uuid_b.to_string()), "{}", err);
+
+        let model_d = model_from_bytes(vec![0u8; 24]);
+        let err =
+            validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_d)]).unwrap_err();
+        assert!(err.to_string().contains("2 partitions"), "{}", err);
+        assert!(err.to_string().contains("3 partitions"), "{}", err);
+
+        let err = validate_shared_centroids("idx", [(uuid_a, &IvfModel::empty())]).unwrap_err();
+        assert!(err.to_string().contains("has no IVF centroids"), "{}", err);
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_ivf_partition_multi_segment_128_bit() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use lance_index::optimize::OptimizeOptions;
+        use lance_index::vector::ivf::IvfBuildParams;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        const HASH_BYTES: i32 = 16;
+
+        fn hash_batch(schema: Arc<Schema>, values: &[u64]) -> arrow_array::RecordBatch {
+            let mut bytes = Vec::with_capacity(values.len() * HASH_BYTES as usize);
+            for value in values {
+                bytes.extend_from_slice(&value.to_le_bytes());
+                let high = value.rotate_left(17) ^ 0xA5A5_A5A5_A5A5_A5A5;
+                bytes.extend_from_slice(&high.to_le_bytes());
+            }
+            let array =
+                FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), HASH_BYTES)
+                    .unwrap();
+            arrow_array::RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap()
+        }
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                HASH_BYTES,
+            ),
+            false,
+        )]));
+
+        // 25 distinct hash values, two copies each; the same batch is written to
+        // fragment 0 and appended as fragment 1, so every value has duplicates
+        // in both fragments.
+        let values: Vec<u64> = (0..50)
+            .map(|i| ((i / 2) as u64).wrapping_mul(0x9E3779B97F4A7C15))
+            .collect();
+        let num_values = 25;
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+        let reader = RecordBatchIterator::new(
+            vec![Ok(hash_batch(schema.clone(), &values))],
+            schema.clone(),
+        );
+        let mut dataset = crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        let params = crate::index::vector::VectorIndexParams::with_ivf_flat_params(
+            lance_linalg::distance::MetricType::Hamming,
+            IvfBuildParams::new(4),
+        );
+        dataset
+            .create_index(
+                &["hash"],
+                crate::index::IndexType::Vector,
+                Some("hash_idx".into()),
+                &params,
+                false,
+            )
+            .await
+            .unwrap();
+
+        let reader = RecordBatchIterator::new(
+            vec![Ok(hash_batch(schema.clone(), &values))],
+            schema.clone(),
+        );
+        dataset.append(reader, None).await.unwrap();
+        dataset
+            .optimize_indices(&OptimizeOptions::append())
+            .await
+            .unwrap();
+
+        let segments = dataset.load_indices_by_name("hash_idx").await.unwrap();
+        assert_eq!(
+            segments.len(),
+            2,
+            "expected a delta segment after optimize append"
+        );
+
+        // Partition sizes aggregate across both segments.
+        let infos = get_ivf_partition_info(&dataset, "hash_idx").await.unwrap();
+        assert_eq!(infos.len(), 4);
+        assert_eq!(infos.iter().map(|info| info.size).sum::<usize>(), 100);
+
+        // Clustering each partition with threshold 0 must group all four copies
+        // of every value, including the copies in the appended fragment.
+        const FRAG1_START: u64 = 1 << 32;
+        let mut clusters = Vec::new();
+        for partition_id in 0..4 {
+            let reader =
+                hamming_clustering_for_ivf_partition(&dataset, "hash_idx", partition_id, 0)
+                    .await
+                    .unwrap();
+            clusters.extend(collect_clusters(reader));
+        }
+        assert_eq!(clusters.len(), num_values);
+        for (representative, duplicates) in &clusters {
+            assert_eq!(duplicates.len(), 3);
+            assert!(*representative < FRAG1_START);
+            assert!(
+                duplicates.iter().any(|row_id| *row_id >= FRAG1_START),
+                "cluster {} should contain rows from the appended fragment",
+                representative
+            );
+        }
+
+        // Selecting only the original segment reproduces the single-segment scope.
+        let first_segment = segments
+            .iter()
+            .find(|meta| meta.fragment_bitmap.as_ref().unwrap().contains(0))
+            .unwrap();
+        let mut old_clusters = Vec::new();
+        for partition_id in 0..4 {
+            let reader = hamming_clustering_for_ivf_partition_segments(
+                &dataset,
+                "hash_idx",
+                &[first_segment.uuid],
+                partition_id,
+                0,
+            )
+            .await
+            .unwrap();
+            old_clusters.extend(collect_clusters(reader));
+        }
+        assert_eq!(old_clusters.len(), num_values);
+        for (representative, duplicates) in &old_clusters {
+            assert_eq!(duplicates.len(), 1);
+            assert!(*representative < FRAG1_START);
+            assert!(duplicates.iter().all(|row_id| *row_id < FRAG1_START));
+        }
+        let infos = get_ivf_partition_info_segments(&dataset, "hash_idx", &[first_segment.uuid])
+            .await
+            .unwrap();
+        assert_eq!(infos.iter().map(|info| info.size).sum::<usize>(), 50);
+
+        // Invalid segment selections are rejected.
+        let err = hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[], 0, 0)
+            .await
+            .err()
+            .unwrap();
+        assert!(err.to_string().contains("must not be empty"), "{}", err);
+        let missing = Uuid::new_v4();
+        let err =
+            hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[missing], 0, 0)
+                .await
+                .err()
+                .unwrap();
+        assert!(err.to_string().contains(&missing.to_string()), "{}", err);
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_sample_integration() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                8,
+            ),
+            false,
+        )]));
+
+        // Create 50 rows with some duplicate hashes
+        let num_rows = 50;
+        let mut hash_bytes = Vec::with_capacity(num_rows * 8);
+        for i in 0..num_rows {
+            // Create some identical hashes (groups of 5)
+            let hash = (i / 5) as u64;
+            hash_bytes.extend_from_slice(&hash.to_le_bytes());
+        }
+        let values = UInt8Array::from(hash_bytes);
+        let hash_array = FixedSizeListArray::try_new_from_values(values, 8).unwrap();
+        let batch =
+            arrow_array::RecordBatch::try_new(schema.clone(), vec![Arc::new(hash_array)]).unwrap();
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        let dataset = crate::Dataset::open(uri).await.unwrap();
+
+        // Test full scan (no sampling)
+        let reader = hamming_clustering_for_sample(&dataset, "hash", None, 0)
+            .await
+            .unwrap();
+        let clusters = collect_clusters(reader);
+
+        // With threshold 0 (exact match) and groups of 5 identical hashes,
+        // we should have 10 clusters with 4 duplicates each
+        assert_eq!(clusters.len(), 10);
+        for (_, dups) in &clusters {
+            assert_eq!(dups.len(), 4);
+        }
+
+        // Test with sampling
+        let reader = hamming_clustering_for_sample(&dataset, "hash", Some(20), 0)
+            .await
+            .unwrap();
+        let clusters = collect_clusters(reader);
+        // With sampling, we may get fewer clusters
+        assert!(clusters.len() <= 10);
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_range_integration() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                8,
+            ),
+            false,
+        )]));
+
+        // Create 50 rows with some duplicate hashes (groups of 5 identical hashes)
+        let num_rows = 50;
+        let mut hash_bytes = Vec::with_capacity(num_rows * 8);
+        for i in 0..num_rows {
+            let hash = (i / 5) as u64;
+            hash_bytes.extend_from_slice(&hash.to_le_bytes());
+        }
+        let values = UInt8Array::from(hash_bytes);
+        let hash_array = FixedSizeListArray::try_new_from_values(values, 8).unwrap();
+        let batch =
+            arrow_array::RecordBatch::try_new(schema.clone(), vec![Arc::new(hash_array)]).unwrap();
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        let dataset = crate::Dataset::open(uri).await.unwrap();
+
+        // Get fragment info
+        let fragments = dataset.get_fragments();
+        assert_eq!(fragments.len(), 1);
+        let fragment_id = fragments[0].id() as usize;
+
+        // Test reading range from the fragment
+        // Reading rows 0-25 should cover groups 0-4 (5 groups, each with 5 rows)
+        let reader = hamming_clustering_for_range(&dataset, "hash", fragment_id, 0, 25, 0)
+            .await
+            .unwrap();
+        let clusters = collect_clusters(reader);
+
+        // With threshold 0 and 25 rows (groups 0-4), we should have 5 clusters
+        // Each cluster has 4 duplicates (5 identical hashes - 1 representative = 4 duplicates)
+        assert_eq!(clusters.len(), 5);
+        for (_, dups) in &clusters {
+            assert_eq!(dups.len(), 4);
+        }
+
+        // Test reading a different range (rows 25-50)
+        let reader = hamming_clustering_for_range(&dataset, "hash", fragment_id, 25, 25, 0)
+            .await
+            .unwrap();
+        let clusters = collect_clusters(reader);
+
+        // Should have 5 clusters (groups 5-9)
+        assert_eq!(clusters.len(), 5);
+        for (_, dups) in &clusters {
+            assert_eq!(dups.len(), 4);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_hamming_clustering_for_range_invalid_fragment() {
+        use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array};
+        use arrow_schema::{Field, Schema};
+        use lance_arrow::FixedSizeListArrayExt;
+        use std::sync::Arc;
+        use tempfile::tempdir;
+
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "hash",
+            arrow_schema::DataType::FixedSizeList(
+                Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)),
+                8,
+            ),
+            false,
+        )]));
+
+        let values = UInt8Array::from(vec![0u8; 80]); // 10 rows * 8 bytes
+        let hash_array = FixedSizeListArray::try_new_from_values(values, 8).unwrap();
+        let batch =
+            arrow_array::RecordBatch::try_new(schema.clone(), vec![Arc::new(hash_array)]).unwrap();
+
+        let temp_dir = tempdir().unwrap();
+        let uri = temp_dir.path().to_str().unwrap();
+
+        let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        crate::Dataset::write(reader, uri, None).await.unwrap();
+
+        let dataset = crate::Dataset::open(uri).await.unwrap();
+
+        // Test with non-existent fragment
+        let result = hamming_clustering_for_range(&dataset, "hash", 999, 0, 10, 0).await;
+        assert!(result.is_err());
+        let err = result.err().unwrap();
+        assert!(err.to_string().contains("not found"), "Error: {}", err);
+
+        // Test with out-of-range start_row
+        let result = hamming_clustering_for_range(&dataset, "hash", 0, 1000, 10, 0).await;
+        assert!(result.is_err());
+        let err = result.err().unwrap();
+        assert!(err.to_string().contains("out of range"), "Error: {}", err);
+    }
+}
diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs
index b7e6567b025..1d56300fc4b 100644
--- a/rust/lance/src/index/vector/ivf.rs
+++ b/rust/lance/src/index/vector/ivf.rs
@@ -13,7 +13,9 @@ use super::{
     utils::PartitionLoadLock,
 };
 use crate::dataset::index::dataset_format_version;
+use crate::index::DatasetIndexExt;
 use crate::index::DatasetIndexInternalExt;
+use crate::index::vector::open_index_file;
 use crate::index::vector::utils::{get_vector_dim, get_vector_type};
 use crate::{
     dataset::Dataset,
@@ -21,6 +23,7 @@ use crate::{
 };
 use crate::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion};
 use arrow::array::ArrayData;
+use arrow::compute::concat_batches;
 use arrow::datatypes::UInt8Type;
 use arrow_arith::numeric::sub;
 use arrow_array::Float32Array;
@@ -33,7 +36,6 @@ use arrow_buffer::MutableBuffer;
 use arrow_schema::{DataType, Schema};
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
-use deepsize::DeepSizeOf;
 use futures::TryFutureExt;
 use futures::{
     Stream, TryStreamExt,
@@ -41,6 +43,8 @@ use futures::{
 };
 use io::write_hnsw_quantization_index_partitions;
 use lance_arrow::*;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use lance_core::{
     Error, ROW_ID_FIELD, Result,
     cache::{LanceCache, UnsizedCacheKey, WeakLanceCache},
@@ -48,6 +52,7 @@ use lance_core::{
     utils::parse::parse_env_as_bool,
     utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS},
 };
+use lance_encoding::decoder::FilterExpression;
 use lance_file::{
     format::MAGIC,
     previous::writer::{
@@ -60,14 +65,18 @@ use lance_index::metrics::MetricsCollector;
 use lance_index::metrics::NoOpMetricsCollector;
 use lance_index::vector::DISTANCE_TYPE_KEY;
 use lance_index::vector::bq::builder::RabitQuantizer;
-use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
+use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatMetadata, FlatQuantizer};
+use lance_index::vector::flat::storage::{FLAT_COLUMN, FlatBinStorage, FlatFloatStorage};
 use lance_index::vector::hnsw::HnswMetadata;
 use lance_index::vector::hnsw::builder::HNSW_METADATA_KEY;
 use lance_index::vector::ivf::storage::IVF_METADATA_KEY;
 use lance_index::vector::ivf::storage::IvfModel;
 use lance_index::vector::kmeans::{KMeans, KMeansParams};
-use lance_index::vector::pq::storage::transpose;
+use lance_index::vector::pq::storage::{
+    PQ_METADATA_KEY, ProductQuantizationMetadata, ProductQuantizationStorage, transpose,
+};
 use lance_index::vector::quantizer::QuantizationType;
+use lance_index::vector::storage::STORAGE_METADATA_KEY;
 use lance_index::vector::v3::shuffler::create_ivf_shuffler;
 use lance_index::vector::v3::subindex::{IvfSubIndex, SubIndexType};
 use lance_index::{
@@ -82,13 +91,17 @@ use lance_index::{
             storage::IVF_PARTITION_KEY,
         },
         pq::{PQBuildParams, ProductQuantizer},
-        quantizer::{Quantization, QuantizationMetadata, Quantizer},
-        sq::ScalarQuantizer,
+        quantizer::{Quantization, QuantizationMetadata, Quantizer, QuantizerStorage},
+        sq::{
+            ScalarQuantizer,
+            storage::{SQ_METADATA_KEY, ScalarQuantizationMetadata, ScalarQuantizationStorage},
+        },
     },
 };
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
 use lance_io::{
+    ReadBatchParams,
     encodings::plain::PlainEncoder,
     local::to_local_path,
     object_store::ObjectStore,
@@ -97,7 +110,7 @@ use lance_io::{
 };
 use lance_linalg::distance::{DistanceType, Dot, L2, MetricType};
 use lance_linalg::{distance::Normalize, kernels::normalize_fsl_owned};
-use lance_table::format::{IndexMetadata as TableIndexMetadata, list_index_files_with_sizes};
+use lance_table::format::{IndexFile, IndexMetadata as TableIndexMetadata};
 use log::{info, warn};
 use object_store::path::Path;
 use prost::Message;
@@ -149,7 +162,7 @@ impl UnsizedCacheKey for LegacyIVFPartitionKey {
 /// IVF Index.
 /// WARNING: Internal API with no stability guarantees.
 pub struct IVFIndex {
-    uuid: String,
+    uuid: Uuid,
 
     /// Ivf model
     pub ivf: IvfModel,
@@ -167,17 +180,16 @@ pub struct IVFIndex {
 }
 
 impl DeepSizeOf for IVFIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
-        self.uuid.deep_size_of_children(context)
-            + self.reader.deep_size_of_children(context)
-            + self.sub_index.deep_size_of_children(context)
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // `Uuid` is a fixed 16-byte struct with no heap children, so contributes 0.
+        self.reader.deep_size_of_children(context) + self.sub_index.deep_size_of_children(context)
     }
 }
 
 impl IVFIndex {
     /// Create a new IVF index.
     pub(crate) fn try_new(
-        uuid: &str,
+        uuid: Uuid,
         ivf: IvfModel,
         reader: Arc<dyn Reader>,
         sub_index: Arc<dyn VectorIndex>,
@@ -193,7 +205,7 @@ impl IVFIndex {
 
         let num_partitions = ivf.num_partitions();
         Ok(Self {
-            uuid: uuid.to_owned(),
+            uuid,
             ivf,
             reader,
             sub_index,
@@ -374,16 +386,116 @@ pub(crate) fn select_segment_for_single_rebalance(
     Ok(selected.map(|candidate| candidate.segment_id))
 }
 
+fn validate_shared_vector_model(indices: &[Arc<dyn VectorIndex>], operation: &str) -> Result<()> {
+    let Some(first) = indices.first() else {
+        return Ok(());
+    };
+    if indices.len() == 1 {
+        return Ok(());
+    }
+
+    let first_metric = first.metric_type();
+    let first_index_type = first.sub_index_type();
+    let first_centroids = first.ivf_model().centroids_array();
+    let first_quantizer = first.quantizer();
+    let first_quantizer_type = first_quantizer.quantization_type();
+
+    for (idx, index) in indices.iter().enumerate().skip(1) {
+        if index.metric_type() != first_metric {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has metric {:?}, expected {:?}",
+                index.metric_type(),
+                first_metric
+            )));
+        }
+        let index_type = index.sub_index_type();
+        if std::mem::discriminant(&index_type.0) != std::mem::discriminant(&first_index_type.0)
+            || index_type.1 != first_index_type.1
+        {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has type {:?}, expected {:?}",
+                index_type, first_index_type
+            )));
+        }
+        match (first_centroids, index.ivf_model().centroids_array()) {
+            (Some(expected), Some(actual)) if expected.to_data() != actual.to_data() => {
+                return Err(Error::index(format!(
+                    "{operation}: vector index segments do not share IVF centroids"
+                )));
+            }
+            (Some(_), None) | (None, Some(_)) => {
+                return Err(Error::index(format!(
+                    "{operation}: vector index segments do not share IVF centroids"
+                )));
+            }
+            _ => {}
+        }
+
+        let quantizer = index.quantizer();
+        if quantizer.quantization_type() != first_quantizer_type {
+            return Err(Error::index(format!(
+                "{operation}: vector index segment {idx} has quantizer {:?}, expected {:?}",
+                quantizer.quantization_type(),
+                first_quantizer_type
+            )));
+        }
+        if !shared_quantizer_model(&first_quantizer, &quantizer) {
+            return Err(Error::index(format!(
+                "{operation}: vector index segments do not share quantizer metadata"
+            )));
+        }
+    }
+
+    Ok(())
+}
+
+fn shared_quantizer_model(left: &Quantizer, right: &Quantizer) -> bool {
+    match (left, right) {
+        (Quantizer::Flat(left), Quantizer::Flat(right)) => {
+            left.metadata(None).dim == right.metadata(None).dim
+        }
+        (Quantizer::FlatBin(left), Quantizer::FlatBin(right)) => {
+            left.metadata(None).dim == right.metadata(None).dim
+        }
+        (Quantizer::Product(left), Quantizer::Product(right)) => {
+            left.num_sub_vectors == right.num_sub_vectors
+                && left.num_bits == right.num_bits
+                && left.dimension == right.dimension
+                && left.distance_type == right.distance_type
+                && left.codebook.to_data() == right.codebook.to_data()
+        }
+        (Quantizer::Scalar(left), Quantizer::Scalar(right)) => {
+            left.metadata(None) == right.metadata(None)
+        }
+        (Quantizer::Rabit(left), Quantizer::Rabit(right)) => {
+            let left = left.metadata(None);
+            let right = right.metadata(None);
+            left.rotation_type == right.rotation_type
+                && left.code_dim == right.code_dim
+                && left.num_bits == right.num_bits
+                && left.packed == right.packed
+                && left.query_estimator == right.query_estimator
+                && left.fast_rotation_signs == right.fast_rotation_signs
+                && match (&left.rotate_mat, &right.rotate_mat) {
+                    (Some(left), Some(right)) => left.to_data() == right.to_data(),
+                    (None, None) => true,
+                    _ => false,
+                }
+        }
+        _ => false,
+    }
+}
+
 // TODO: move to `lance-index` crate.
 ///
-/// Returns (new_uuid, num_indices_merged)
+/// Returns (new_uuid, num_indices_merged, files)
 pub(crate) async fn optimize_vector_indices(
     dataset: Dataset,
     unindexed: Option<impl RecordBatchStream + Unpin + 'static>,
     vector_column: &str,
     logical_index: &LogicalIvfView<'_>,
     options: &OptimizeOptions,
-) -> Result<(Uuid, usize)> {
+) -> Result<(Uuid, usize, Vec<IndexFile>)> {
     let existing_indices = logical_index.indices().cloned().collect::<Vec<_>>();
     // Sanity check the indices
     if existing_indices.is_empty() {
@@ -395,6 +507,7 @@ pub(crate) async fn optimize_vector_indices(
     // try cast to v1 IVFIndex,
     // fallback to v2 IVFIndex if it's not v1 IVFIndex
     if !existing_indices[0].as_any().is::<IVFIndex>() {
+        validate_shared_vector_model(&existing_indices, "optimizing vector index")?;
         return optimize_vector_indices_v2(
             &dataset,
             unindexed,
@@ -420,49 +533,51 @@ pub(crate) async fn optimize_vector_indices(
             "optimizing vector index: the first index isn't IVF".to_string(),
         ))?;
 
-    let merged = if let Some(pq_index) = first_idx.sub_index.as_any().downcast_ref::<PQIndex>() {
-        optimize_ivf_pq_indices(
-            first_idx,
-            pq_index,
-            vector_column,
-            unindexed,
-            &existing_indices,
-            options,
-            writer,
-            dataset.version().version,
-        )
-        .await?
-    } else if let Some(hnsw_sq) = first_idx
-        .sub_index
-        .as_any()
-        .downcast_ref::<HNSWIndex<ScalarQuantizer>>()
-    {
-        let aux_file = dataset
-            .indices_dir()
-            .join(new_uuid.to_string())
-            .join(INDEX_AUXILIARY_FILE_NAME);
-        let aux_writer = object_store.create(&aux_file).await?;
-        optimize_ivf_hnsw_indices(
-            Arc::new(dataset),
-            first_idx,
-            hnsw_sq,
-            vector_column,
-            unindexed,
-            &existing_indices,
-            options,
-            writer,
-            aux_writer,
-        )
-        .await?
-    } else {
-        return Err(Error::index(
-            "optimizing vector index: the sub index isn't PQ or HNSW".to_string(),
-        ));
-    };
+    let (merged, files) =
+        if let Some(pq_index) = first_idx.sub_index.as_any().downcast_ref::<PQIndex>() {
+            let (merged, file) = optimize_ivf_pq_indices(
+                first_idx,
+                pq_index,
+                vector_column,
+                unindexed,
+                &existing_indices,
+                options,
+                writer,
+                dataset.version().version,
+            )
+            .await?;
+            (merged, vec![file])
+        } else if let Some(hnsw_sq) = first_idx
+            .sub_index
+            .as_any()
+            .downcast_ref::<HNSWIndex<ScalarQuantizer>>()
+        {
+            let aux_file = dataset
+                .indices_dir()
+                .join(new_uuid.to_string())
+                .join(INDEX_AUXILIARY_FILE_NAME);
+            let aux_writer = object_store.create(&aux_file).await?;
+            optimize_ivf_hnsw_indices(
+                Arc::new(dataset),
+                first_idx,
+                hnsw_sq,
+                vector_column,
+                unindexed,
+                &existing_indices,
+                options,
+                writer,
+                aux_writer,
+            )
+            .await?
+        } else {
+            return Err(Error::index(
+                "optimizing vector index: the sub index isn't PQ or HNSW".to_string(),
+            ));
+        };
 
     // never change the index version,
     // because we won't update the legacy vector index format
-    Ok((new_uuid, merged))
+    Ok((new_uuid, merged, files))
 }
 
 pub(crate) async fn optimize_vector_indices_v2(
@@ -471,7 +586,7 @@ pub(crate) async fn optimize_vector_indices_v2(
     vector_column: &str,
     existing_indices: &[Arc<dyn VectorIndex>],
     options: &OptimizeOptions,
-) -> Result<(Uuid, usize)> {
+) -> Result<(Uuid, usize, Vec<IndexFile>)> {
     // Sanity check the indices
     if existing_indices.is_empty() {
         return Err(Error::index(
@@ -496,7 +611,7 @@ pub(crate) async fn optimize_vector_indices_v2(
     let shuffler = create_ivf_shuffler(temp_dir_path, num_partitions, format_version, None);
 
     let (_, element_type) = get_vector_type(dataset.schema(), vector_column)?;
-    let merged_num = match index_type {
+    let summary = match index_type {
         // IVF_FLAT
         (SubIndexType::Flat, QuantizationType::Flat) => {
             if element_type == DataType::UInt8 {
@@ -705,7 +820,7 @@ pub(crate) async fn optimize_vector_indices_v2(
         }
     };
 
-    Ok((new_uuid, merged_num))
+    Ok((new_uuid, summary.indices_merged, summary.files))
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -718,7 +833,7 @@ async fn optimize_ivf_pq_indices(
     options: &OptimizeOptions,
     mut writer: Box<dyn Writer>,
     dataset_version: u64,
-) -> Result<usize> {
+) -> Result<(usize, IndexFile)> {
     let metric_type = first_idx.metric_type;
     let dim = first_idx.ivf.dimension();
 
@@ -785,9 +900,16 @@ async fn optimize_ivf_pq_indices(
     // TODO: for now the IVF_PQ index file format hasn't been updated, so keep the old version,
     // change it to latest version value after refactoring the IVF_PQ
     writer.write_magics(pos, 0, 1, MAGIC).await?;
+    let size_bytes = writer.tell().await? as u64;
     Writer::shutdown(writer.as_mut()).await?;
 
-    Ok(existing_indices.len() - start_pos)
+    Ok((
+        existing_indices.len() - start_pos,
+        IndexFile {
+            path: INDEX_FILE_NAME.to_string(),
+            size_bytes,
+        },
+    ))
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -801,7 +923,7 @@ async fn optimize_ivf_hnsw_indices<Q: Quantization>(
     options: &OptimizeOptions,
     writer: Box<dyn Writer>,
     aux_writer: Box<dyn Writer>,
-) -> Result<usize> {
+) -> Result<(usize, Vec<IndexFile>)> {
     let distance_type = first_idx.metric_type;
     let quantizer = hnsw_index.quantizer().clone();
     let ivf = lance_index::vector::ivf::new_ivf_transformer_with_quantizer(
@@ -938,13 +1060,26 @@ async fn optimize_ivf_hnsw_indices<Q: Quantization>(
     writer.add_metadata(IVF_PARTITION_KEY, &hnsw_metadata_json.to_string());
 
     ivf_mut.write(&mut writer).await?;
-    writer.finish().await?;
+    // `finish` writes the footer and returns the authoritative on-disk size.
+    let index_size = writer.finish().await?.size_bytes;
 
     // Write the aux file
     aux_ivf.write(&mut aux_writer).await?;
-    aux_writer.finish().await?;
+    let aux_size = aux_writer.finish().await?.size_bytes;
 
-    Ok(existing_indices.len() - start_pos)
+    Ok((
+        existing_indices.len() - start_pos,
+        vec![
+            IndexFile {
+                path: INDEX_FILE_NAME.to_string(),
+                size_bytes: index_size,
+            },
+            IndexFile {
+                path: INDEX_AUXILIARY_FILE_NAME.to_string(),
+                size_bytes: aux_size,
+            },
+        ],
+    ))
 }
 
 #[derive(Serialize)]
@@ -1054,10 +1189,6 @@ impl Index for IVFIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Ok(self)
-    }
-
     fn index_type(&self) -> IndexType {
         if self.sub_index.as_any().downcast_ref::<PQIndex>().is_some() {
             IndexType::IvfPq
@@ -1101,7 +1232,7 @@ impl Index for IVFIndex {
 
         Ok(serde_json::to_value(IvfIndexStatistics {
             index_type: self.index_type().to_string(),
-            uuid: self.uuid.clone(),
+            uuid: self.uuid.to_string(),
             uri: to_local_path(self.reader.path()),
             metric_type: self.metric_type.to_string(),
             num_partitions: self.ivf.num_partitions(),
@@ -1214,7 +1345,7 @@ impl VectorIndex for IVFIndex {
         todo!("this method is for only IVF_HNSW_* index");
     }
 
-    async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
         // This will be needed if we want to clean up IVF to allow more than just
         // one layer (e.g. IVF -> IVF -> PQ).  We need to pass on the call to
         // remap to the lower layers.
@@ -1589,12 +1720,12 @@ pub async fn build_ivf_pq_index(
     dataset: &Dataset,
     column: &str,
     index_name: &str,
-    uuid: &str,
+    uuid: Uuid,
     metric_type: MetricType,
     ivf_params: &IvfBuildParams,
     pq_params: &PQBuildParams,
     progress: std::sync::Arc<dyn lance_index::progress::IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     let (ivf_model, pq) = build_ivf_model_and_pq(
         dataset,
         column,
@@ -1607,7 +1738,7 @@ pub async fn build_ivf_pq_index(
     let stream = scan_index_field_stream(dataset, column).await?;
     let precomputed_partitions = load_precomputed_partitions_if_available(ivf_params).await?;
 
-    write_ivf_pq_file(
+    let file = write_ivf_pq_file(
         dataset.object_store.as_ref(),
         dataset.indices_dir(),
         column,
@@ -1623,7 +1754,8 @@ pub async fn build_ivf_pq_index(
         ivf_params.shuffle_partition_concurrency,
         ivf_params.precomputed_shuffle_buffers.clone(),
     )
-    .await
+    .await?;
+    Ok(vec![file])
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -1631,7 +1763,7 @@ pub async fn build_ivf_hnsw_pq_index(
     dataset: &Dataset,
     column: &str,
     index_name: &str,
-    uuid: &str,
+    uuid: Uuid,
     metric_type: MetricType,
     ivf_params: &IvfBuildParams,
     hnsw_params: &HnswBuildParams,
@@ -1688,7 +1820,7 @@ impl RemapPageTask {
         mut self,
         reader: Arc<dyn Reader>,
         index: &IVFIndex,
-        mapping: &HashMap<u64, Option<u64>>,
+        mapping: &RowAddrRemap,
     ) -> Result<Self> {
         let mut page = index
             .sub_index
@@ -1732,13 +1864,13 @@ fn generate_remap_tasks(offsets: &[usize], lengths: &[u32]) -> Result<Vec<RemapP
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn remap_index_file_v3(
     dataset: &Dataset,
-    new_uuid: &str,
+    new_uuid: &Uuid,
     index: Arc<dyn VectorIndex>,
-    mapping: &HashMap<u64, Option<u64>>,
+    mapping: &RowAddrRemap,
     column: String,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     let dataset = dataset.clone();
-    let index_dir = dataset.indices_dir().join(new_uuid);
+    let index_dir = dataset.indices_dir().join(new_uuid.to_string());
     let (_, element_type) = get_vector_type(dataset.schema(), &column)?;
     match index.sub_index_type() {
         (SubIndexType::Flat, QuantizationType::Flat) => match element_type {
@@ -1829,25 +1961,39 @@ pub(crate) async fn remap_index_file_v3(
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn remap_index_file(
     dataset: &Dataset,
-    old_uuid: &str,
-    new_uuid: &str,
+    old_uuid: &Uuid,
+    new_uuid: &Uuid,
     old_version: u64,
     index: &IVFIndex,
-    mapping: &HashMap<u64, Option<u64>>,
+    mapping: &RowAddrRemap,
     name: String,
     column: String,
     transforms: Vec<pb::Transform>,
-) -> Result<()> {
+) -> Result<IndexFile> {
     let object_store = dataset.object_store.as_ref();
-    let old_path = dataset.indices_dir().join(old_uuid).join(INDEX_FILE_NAME);
-    let new_path = dataset.indices_dir().join(new_uuid).join(INDEX_FILE_NAME);
+    let old_path = dataset
+        .indices_dir()
+        .join(old_uuid.to_string())
+        .join(INDEX_FILE_NAME);
+    let new_path = dataset
+        .indices_dir()
+        .join(new_uuid.to_string())
+        .join(INDEX_FILE_NAME);
 
-    let reader: Arc<dyn Reader> = object_store.open(&old_path).await?.into();
+    let file_sizes = dataset
+        .load_index(old_uuid)
+        .await?
+        .map(|index| index.file_size_map())
+        .unwrap_or_default();
+    let reader: Arc<dyn Reader> =
+        open_index_file(object_store, &old_path, INDEX_FILE_NAME, &file_sizes)
+            .await?
+            .into();
     let mut writer = object_store.create(&new_path).await?;
 
     let tasks = generate_remap_tasks(&index.ivf.offsets, &index.ivf.lengths)?;
 
-    let mut task_stream = stream::iter(tasks.into_iter())
+    let mut task_stream = stream::iter(tasks)
         .map(|task| task.load_and_remap(reader.clone(), index, mapping))
         .buffered(object_store.io_parallelism());
 
@@ -1883,9 +2029,13 @@ pub(crate) async fn remap_index_file(
     // TODO: for now the IVF_PQ index file format hasn't been updated, so keep the old version,
     // change it to latest version value after refactoring the IVF_PQ
     writer.write_magics(pos, 0, 1, MAGIC).await?;
+    let size_bytes = writer.tell().await? as u64;
     Writer::shutdown(writer.as_mut()).await?;
 
-    Ok(())
+    Ok(IndexFile {
+        path: INDEX_FILE_NAME.to_string(),
+        size_bytes,
+    })
 }
 
 /// Write the index to the index file.
@@ -1896,7 +2046,7 @@ async fn write_ivf_pq_file(
     index_dir: Path,
     column: &str,
     index_name: &str,
-    uuid: &str,
+    uuid: Uuid,
     dataset_version: u64,
     mut ivf: IvfModel,
     pq: ProductQuantizer,
@@ -1906,8 +2056,11 @@ async fn write_ivf_pq_file(
     shuffle_partition_batches: usize,
     shuffle_partition_concurrency: usize,
     precomputed_shuffle_buffers: Option<(Path, Vec<String>)>,
-) -> Result<()> {
-    let path = index_dir.clone().join(uuid).join(INDEX_FILE_NAME);
+) -> Result<IndexFile> {
+    let path = index_dir
+        .clone()
+        .join(uuid.to_string())
+        .join(INDEX_FILE_NAME);
     let mut writer = object_store.create(&path).await?;
 
     let start = std::time::Instant::now();
@@ -1944,9 +2097,13 @@ async fn write_ivf_pq_file(
     // TODO: for now the IVF_PQ index file format hasn't been updated, so keep the old version,
     // change it to latest version value after refactoring the IVF_PQ
     writer.write_magics(pos, 0, 1, MAGIC).await?;
+    let size_bytes = writer.tell().await? as u64;
     Writer::shutdown(writer.as_mut()).await?;
 
-    Ok(())
+    Ok(IndexFile {
+        path: INDEX_FILE_NAME.to_string(),
+        size_bytes,
+    })
 }
 
 pub async fn write_ivf_pq_file_from_existing_index(
@@ -1989,7 +2146,7 @@ async fn write_ivf_hnsw_file(
     dataset: &Dataset,
     column: &str,
     _index_name: &str,
-    uuid: &str,
+    uuid: Uuid,
     mut ivf: IvfModel,
     quantizer: Quantizer,
     distance_type: DistanceType,
@@ -2001,7 +2158,10 @@ async fn write_ivf_hnsw_file(
     precomputed_shuffle_buffers: Option<(Path, Vec<String>)>,
 ) -> Result<()> {
     let object_store = dataset.object_store.as_ref();
-    let path = dataset.indices_dir().join(uuid).join(INDEX_FILE_NAME);
+    let path = dataset
+        .indices_dir()
+        .join(uuid.to_string())
+        .join(INDEX_FILE_NAME);
     let writer = object_store.create(&path).await?;
 
     let schema = lance_core::datatypes::Schema::try_from(HNSW::schema().as_ref())?;
@@ -2022,7 +2182,7 @@ async fn write_ivf_hnsw_file(
 
     let aux_path = dataset
         .indices_dir()
-        .join(uuid)
+        .join(uuid.to_string())
         .join(INDEX_AUXILIARY_FILE_NAME);
     let aux_writer = object_store.create(&aux_path).await?;
     let schema = Schema::new(vec![
@@ -2158,7 +2318,7 @@ pub(crate) async fn merge_segments_with_progress(
     let index_version = infer_source_index_version(&segments)?;
     let segment_uuid = Uuid::new_v4();
     let final_dir = indices_dir.clone().join(segment_uuid.to_string());
-    merge_segments_to_dir(
+    let files = merge_segments_to_dir(
         object_store,
         indices_dir,
         &final_dir,
@@ -2167,7 +2327,6 @@ pub(crate) async fn merge_segments_with_progress(
         progress,
     )
     .await?;
-    let files = list_index_files_with_sizes(object_store, &final_dir).await?;
 
     merged_segment = TableIndexMetadata {
         uuid: segment_uuid,
@@ -2194,7 +2353,7 @@ async fn merge_segments_to_dir(
     segments: &[TableIndexMetadata],
     _requested_index_type: Option<IndexType>,
     progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<Vec<IndexFile>> {
     reset_final_segment_dir(object_store, final_dir).await?;
 
     debug_assert!(
@@ -2221,14 +2380,15 @@ async fn merge_segments_to_dir(
         })
         .collect::<Vec<_>>();
 
-    lance_index::vector::distributed::index_merger::merge_partial_vector_auxiliary_files(
-        object_store,
-        &aux_paths,
-        final_dir,
-        progress.clone(),
-    )
-    .await?;
-    write_root_vector_index_from_auxiliary(
+    let auxiliary_file =
+        lance_index::vector::distributed::index_merger::merge_partial_vector_auxiliary_files(
+            object_store,
+            &aux_paths,
+            final_dir,
+            progress.clone(),
+        )
+        .await?;
+    let index_file = write_root_vector_index_from_auxiliary(
         object_store,
         final_dir,
         None,
@@ -2237,7 +2397,7 @@ async fn merge_segments_to_dir(
     )
     .await?;
 
-    Ok(())
+    Ok(vec![auxiliary_file, index_file])
 }
 
 fn infer_source_index_version(group: &[TableIndexMetadata]) -> Result<i32> {
@@ -2267,7 +2427,7 @@ async fn write_root_vector_index_from_auxiliary(
     requested_index_type: Option<IndexType>,
     centroid_source_index_paths: &[Path],
     progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
-) -> Result<()> {
+) -> Result<IndexFile> {
     let aux_path = index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME);
     let scheduler = ScanScheduler::new(
         Arc::new(object_store.clone()),
@@ -2336,9 +2496,9 @@ async fn write_root_vector_index_from_auxiliary(
     let ivf_bytes = pb_ivf.encode_to_vec().into();
 
     // Determine index metadata JSON from auxiliary or requested index type.
-    let index_meta_json =
+    let mut idx_meta: IndexMetadata =
         if let Some(idx_json) = meta.file_schema.metadata.get(INDEX_METADATA_SCHEMA_KEY) {
-            idx_json.clone()
+            serde_json::from_str(idx_json)?
         } else {
             let dt = meta
                 .file_schema
@@ -2352,11 +2512,25 @@ async fn write_root_vector_index_from_auxiliary(
                         .to_string(),
                 )
             })?;
-            serde_json::to_string(&IndexMetadata {
+            IndexMetadata {
                 index_type: index_type.to_string(),
                 distance_type: dt,
-            })?
+            }
         };
+    if let Some(source_hnsw_index_metadata) =
+        read_hnsw_index_metadata_from_sources(object_store, &scheduler, centroid_source_index_paths)
+            .await?
+    {
+        if idx_meta.index_type.starts_with("IVF_HNSW")
+            && !index_metadata_eq(&idx_meta, &source_hnsw_index_metadata)
+        {
+            return Err(Error::invalid_input(format!(
+                "HNSW index metadata mismatch while merging index segments: expected {:?}, got {:?}",
+                idx_meta, source_hnsw_index_metadata
+            )));
+        }
+        idx_meta = source_hnsw_index_metadata;
+    }
 
     // Write root index.idx via V2 writer so downstream opens through v2 path.
     let index_path = index_dir.clone().join(INDEX_FILE_NAME);
@@ -2377,16 +2551,8 @@ async fn write_root_vector_index_from_auxiliary(
         },
     )?;
 
-    // Attach precise index metadata (type + distance).
-    v2_writer.add_schema_metadata(INDEX_METADATA_SCHEMA_KEY, &index_meta_json);
-
-    // Add IVF protobuf as a global buffer and reference via IVF_METADATA_KEY.
-    let pos = v2_writer.add_global_buffer(ivf_bytes).await?;
-    v2_writer.add_schema_metadata(IVF_METADATA_KEY, pos.to_string());
-
     // For HNSW variants, attach per-partition metadata list; for FLAT-based
     // variants, attach minimal placeholder metadata.
-    let idx_meta: IndexMetadata = serde_json::from_str(&index_meta_json)?;
     let is_hnsw = idx_meta.index_type.starts_with("IVF_HNSW");
     let is_flat_based = matches!(
         idx_meta.index_type.as_str(),
@@ -2394,27 +2560,361 @@ async fn write_root_vector_index_from_auxiliary(
     );
 
     if is_hnsw {
-        let default_meta = HnswMetadata::default();
-        let meta_vec: Vec<String> = (0..nlist)
-            .map(|_| serde_json::to_string(&default_meta).unwrap())
-            .collect();
-        let meta_vec_json = serde_json::to_string(&meta_vec)?;
-        v2_writer.add_schema_metadata(HNSW_METADATA_KEY, meta_vec_json);
-    } else if is_flat_based {
-        let meta_vec: Vec<String> = (0..nlist).map(|_| "{}".to_string()).collect();
-        let meta_vec_json = serde_json::to_string(&meta_vec)?;
-        v2_writer.add_schema_metadata("lance:flat", meta_vec_json);
+        let hnsw_params = read_hnsw_build_params_from_sources(
+            object_store,
+            &scheduler,
+            centroid_source_index_paths,
+        )
+        .await?;
+        write_hnsw_root_index_from_auxiliary(
+            &mut v2_writer,
+            &aux_reader,
+            &ivf_model,
+            &hnsw_params,
+            &idx_meta,
+            progress.clone(),
+        )
+        .await?;
+    } else {
+        // Attach precise index metadata (type + distance).
+        let index_meta_json = serde_json::to_string(&idx_meta)?;
+        v2_writer.add_schema_metadata(INDEX_METADATA_SCHEMA_KEY, &index_meta_json);
+
+        // Add IVF protobuf as a global buffer and reference via IVF_METADATA_KEY.
+        let pos = v2_writer.add_global_buffer(ivf_bytes).await?;
+        v2_writer.add_schema_metadata(IVF_METADATA_KEY, pos.to_string());
+
+        if is_flat_based {
+            let meta_vec: Vec<String> = (0..nlist).map(|_| "{}".to_string()).collect();
+            let meta_vec_json = serde_json::to_string(&meta_vec)?;
+            v2_writer.add_schema_metadata("lance:flat", meta_vec_json);
+        }
+
+        let empty_batch = RecordBatch::new_empty(arrow_schema);
+        v2_writer.write_batch(&empty_batch).await?;
     }
 
-    let empty_batch = RecordBatch::new_empty(arrow_schema);
-    v2_writer.write_batch(&empty_batch).await?;
-    v2_writer.finish().await?;
+    let summary = v2_writer.finish().await?;
     progress.stage_progress("write_root_index", 1).await?;
     progress.stage_complete("write_root_index").await?;
 
+    Ok(IndexFile {
+        path: INDEX_FILE_NAME.to_string(),
+        size_bytes: summary.size_bytes,
+    })
+}
+
+async fn read_hnsw_index_metadata_from_sources(
+    object_store: &ObjectStore,
+    scheduler: &Arc<ScanScheduler>,
+    source_index_paths: &[Path],
+) -> Result<Option<IndexMetadata>> {
+    let mut index_metadata: Option<IndexMetadata> = None;
+
+    for source_index_path in source_index_paths {
+        if !object_store.exists(source_index_path).await? {
+            continue;
+        }
+
+        let fh = scheduler
+            .open_file(source_index_path, &CachedFileSize::unknown())
+            .await?;
+        let reader = V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await?;
+        let Some(metadata_json) = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .get(INDEX_METADATA_SCHEMA_KEY)
+        else {
+            continue;
+        };
+        let metadata: IndexMetadata = serde_json::from_str(metadata_json)?;
+        if !metadata.index_type.starts_with("IVF_HNSW") {
+            continue;
+        }
+
+        if let Some(index_metadata) = index_metadata.as_ref() {
+            if !index_metadata_eq(index_metadata, &metadata) {
+                return Err(Error::invalid_input(format!(
+                    "HNSW index metadata mismatch while merging index segments: \
+                     expected {:?}, got {:?} in {}",
+                    index_metadata, metadata, source_index_path
+                )));
+            }
+        } else {
+            index_metadata = Some(metadata);
+        }
+    }
+
+    Ok(index_metadata)
+}
+
+fn index_metadata_eq(left: &IndexMetadata, right: &IndexMetadata) -> bool {
+    left.index_type == right.index_type && left.distance_type == right.distance_type
+}
+
+async fn read_hnsw_build_params_from_sources(
+    object_store: &ObjectStore,
+    scheduler: &Arc<ScanScheduler>,
+    source_index_paths: &[Path],
+) -> Result<HnswBuildParams> {
+    let mut build_params: Option<HnswBuildParams> = None;
+
+    for source_index_path in source_index_paths {
+        if !object_store.exists(source_index_path).await? {
+            continue;
+        }
+
+        let fh = scheduler
+            .open_file(source_index_path, &CachedFileSize::unknown())
+            .await?;
+        let reader = V2Reader::try_open(
+            fh,
+            None,
+            Arc::default(),
+            &LanceCache::no_cache(),
+            V2ReaderOptions::default(),
+        )
+        .await?;
+        let Some(metadata_json) = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .get(HNSW_METADATA_KEY)
+        else {
+            continue;
+        };
+        let partition_metadata: Vec<String> = serde_json::from_str(metadata_json)?;
+        for metadata in partition_metadata {
+            if metadata.is_empty() {
+                continue;
+            }
+            let metadata: HnswMetadata = serde_json::from_str(&metadata)?;
+            if let Some(build_params) = build_params.as_ref() {
+                if !hnsw_build_params_eq(build_params, &metadata.params) {
+                    return Err(Error::invalid_input(format!(
+                        "HNSW build parameters mismatch while merging index segments: \
+                         expected {:?}, got {:?} in {}",
+                        build_params, metadata.params, source_index_path
+                    )));
+                }
+            } else {
+                build_params = Some(metadata.params);
+            }
+        }
+    }
+
+    Ok(build_params.unwrap_or_default())
+}
+
+fn hnsw_build_params_eq(left: &HnswBuildParams, right: &HnswBuildParams) -> bool {
+    left.max_level == right.max_level
+        && left.m == right.m
+        && left.ef_construction == right.ef_construction
+        && left.prefetch_distance == right.prefetch_distance
+}
+
+async fn write_hnsw_root_index_from_auxiliary(
+    writer: &mut V2Writer,
+    aux_reader: &V2Reader,
+    aux_ivf: &IvfModel,
+    hnsw_params: &HnswBuildParams,
+    index_metadata: &IndexMetadata,
+    progress: Arc<dyn lance_index::progress::IndexBuildProgress>,
+) -> Result<()> {
+    let mut index_ivf = if let Some(centroids) = aux_ivf.centroids.clone() {
+        IvfModel::new(centroids, aux_ivf.loss)
+    } else {
+        IvfModel::empty()
+    };
+    let distance_type = DistanceType::try_from(index_metadata.distance_type.as_str())?;
+    let mut partition_index_metadata = Vec::with_capacity(aux_ivf.num_partitions());
+
+    progress
+        .stage_start(
+            "rebuild_hnsw_graph",
+            Some(aux_ivf.num_partitions() as u64),
+            "partitions",
+        )
+        .await?;
+
+    for partition_id in 0..aux_ivf.num_partitions() {
+        let row_range = aux_ivf.row_range(partition_id);
+        if row_range.is_empty() {
+            index_ivf.add_partition(0);
+            partition_index_metadata.push(String::new());
+            progress
+                .stage_progress("rebuild_hnsw_graph", partition_id as u64 + 1)
+                .await?;
+            continue;
+        }
+
+        let batch = read_v2_partition_batch(aux_reader, row_range).await?;
+        let hnsw = build_hnsw_from_storage_batch(
+            &index_metadata.index_type,
+            batch,
+            aux_reader,
+            distance_type,
+            hnsw_params,
+        )
+        .await?;
+        let index_batch = hnsw.to_batch()?;
+
+        writer.write_batch(&index_batch).await?;
+        index_ivf.add_partition(index_batch.num_rows() as u32);
+        partition_index_metadata.push(serde_json::to_string(&hnsw.metadata())?);
+        progress
+            .stage_progress("rebuild_hnsw_graph", partition_id as u64 + 1)
+            .await?;
+    }
+
+    progress.stage_complete("rebuild_hnsw_graph").await?;
+
+    write_hnsw_index_metadata(writer, &index_ivf, distance_type, index_metadata).await?;
+    writer.add_schema_metadata(
+        HNSW_METADATA_KEY,
+        serde_json::to_string(&partition_index_metadata)?,
+    );
+
     Ok(())
 }
 
+async fn read_v2_partition_batch(reader: &V2Reader, range: Range<usize>) -> Result<RecordBatch> {
+    let schema = Arc::new(reader.schema().as_ref().into());
+    let stream = reader
+        .read_stream(
+            ReadBatchParams::Range(range),
+            u32::MAX,
+            4,
+            FilterExpression::no_filter(),
+        )
+        .await?;
+    let batches = stream.try_collect::<Vec<_>>().await?;
+    if batches.is_empty() {
+        Ok(RecordBatch::new_empty(schema))
+    } else {
+        Ok(concat_batches(&schema, batches.iter())?)
+    }
+}
+
+async fn build_hnsw_from_storage_batch(
+    index_type: &str,
+    batch: RecordBatch,
+    aux_reader: &V2Reader,
+    distance_type: DistanceType,
+    hnsw_params: &HnswBuildParams,
+) -> Result<HNSW> {
+    match index_type {
+        "IVF_HNSW_FLAT" => {
+            let metadata = read_storage_metadata::<FlatMetadata>(aux_reader, "")?;
+            let vector_type = batch
+                .column_by_name(FLAT_COLUMN)
+                .ok_or_else(|| {
+                    Error::index(format!(
+                        "{FLAT_COLUMN} column missing from HNSW_FLAT storage"
+                    ))
+                })?
+                .as_fixed_size_list()
+                .value_type();
+            if vector_type == DataType::UInt8 && distance_type == DistanceType::Hamming {
+                let storage =
+                    FlatBinStorage::try_from_batch(batch, &metadata, distance_type, None)?;
+                HNSW::index_vectors(&storage, hnsw_params.clone())
+            } else {
+                let storage =
+                    FlatFloatStorage::try_from_batch(batch, &metadata, distance_type, None)?;
+                HNSW::index_vectors(&storage, hnsw_params.clone())
+            }
+        }
+        "IVF_HNSW_PQ" => {
+            let metadata = read_pq_storage_metadata(aux_reader).await?;
+            let storage =
+                ProductQuantizationStorage::try_from_batch(batch, &metadata, distance_type, None)?;
+            HNSW::index_vectors(&storage, hnsw_params.clone())
+        }
+        "IVF_HNSW_SQ" => {
+            let metadata =
+                read_storage_metadata::<ScalarQuantizationMetadata>(aux_reader, SQ_METADATA_KEY)?;
+            let storage =
+                ScalarQuantizationStorage::try_from_batch(batch, &metadata, distance_type, None)?;
+            HNSW::index_vectors(&storage, hnsw_params.clone())
+        }
+        other => Err(Error::index(format!(
+            "Cannot rebuild HNSW graph for unsupported index type {other}"
+        ))),
+    }
+}
+
+async fn write_hnsw_index_metadata(
+    writer: &mut V2Writer,
+    ivf: &IvfModel,
+    distance_type: DistanceType,
+    index_metadata: &IndexMetadata,
+) -> Result<()> {
+    let pb_ivf: lance_index::pb::Ivf = ivf.try_into()?;
+    let pos = writer
+        .add_global_buffer(pb_ivf.encode_to_vec().into())
+        .await?;
+    writer.add_schema_metadata(IVF_METADATA_KEY, pos.to_string());
+    writer.add_schema_metadata(
+        INDEX_METADATA_SCHEMA_KEY,
+        serde_json::to_string(&IndexMetadata {
+            index_type: index_metadata.index_type.clone(),
+            distance_type: distance_type.to_string(),
+        })?,
+    );
+    Ok(())
+}
+
+async fn read_pq_storage_metadata(reader: &V2Reader) -> Result<ProductQuantizationMetadata> {
+    let mut metadata =
+        read_storage_metadata::<ProductQuantizationMetadata>(reader, PQ_METADATA_KEY)?;
+    if metadata.codebook.is_none() {
+        let tensor_bytes = reader
+            .read_global_buffer(metadata.codebook_position as u32)
+            .await?;
+        let codebook_tensor: lance_index::pb::Tensor = Message::decode(tensor_bytes)?;
+        metadata.codebook = Some(FixedSizeListArray::try_from(&codebook_tensor)?);
+    }
+    Ok(metadata)
+}
+
+fn read_storage_metadata<T>(reader: &V2Reader, storage_metadata_key: &str) -> Result<T>
+where
+    T: serde::de::DeserializeOwned,
+{
+    if !storage_metadata_key.is_empty()
+        && let Some(metadata) = reader
+            .metadata()
+            .file_schema
+            .metadata
+            .get(storage_metadata_key)
+    {
+        return Ok(serde_json::from_str(metadata)?);
+    }
+
+    let storage_metadata = reader
+        .metadata()
+        .file_schema
+        .metadata
+        .get(STORAGE_METADATA_KEY)
+        .ok_or_else(|| Error::index(format!("{STORAGE_METADATA_KEY} missing from storage file")))?;
+    let metadata_entries: Vec<String> = serde_json::from_str(storage_metadata)?;
+    let metadata = metadata_entries.first().ok_or_else(|| {
+        Error::index(format!(
+            "{STORAGE_METADATA_KEY} did not contain any storage metadata entries"
+        ))
+    })?;
+    Ok(serde_json::from_str(metadata)?)
+}
+
 async fn do_train_ivf_model<T: ArrowPrimitiveType>(
     centroids: Option<Arc<FixedSizeListArray>>,
     data: &PrimitiveArray<T>,
@@ -3868,19 +4368,26 @@ async fn train_streaming_coreset_ivf_model(
 
     let coreset_len = coreset.len();
     let (coreset_data, coreset_weights, coreset_losses) = coreset.into_fsl_parts(dimension)?;
-    let weighted_hierarchical_params = WeightedHierarchicalKMeansParams {
-        dimension,
-        target_k: num_partitions,
-        metric_type: DistanceType::L2,
-        max_iters: params.max_iters,
-        on_progress: on_progress.clone(),
+    // Scope `weighted_hierarchical_params` so the `on_progress` clone it holds
+    // (which owns a clone of `progress_tx`) is dropped as soon as training
+    // returns. Otherwise it would outlive the `progress_worker.await` below,
+    // keeping a channel sender alive so `progress_rx.recv()` never returns
+    // `None` and the progress worker — and thus this function — hangs forever.
+    let mut centroids = {
+        let weighted_hierarchical_params = WeightedHierarchicalKMeansParams {
+            dimension,
+            target_k: num_partitions,
+            metric_type: DistanceType::L2,
+            max_iters: params.max_iters,
+            on_progress: on_progress.clone(),
+        };
+        train_weighted_hierarchical_f32_kmeans(
+            &coreset_data,
+            &coreset_weights,
+            &coreset_losses,
+            &weighted_hierarchical_params,
+        )?
     };
-    let mut centroids = train_weighted_hierarchical_f32_kmeans(
-        &coreset_data,
-        &coreset_weights,
-        &coreset_losses,
-        &weighted_hierarchical_params,
-    )?;
     let refine_iters = 3;
     if refine_iters > 0 {
         let refined = refine_weighted_f32_kmeans(
@@ -4178,6 +4685,41 @@ mod tests {
 
     const DIM: usize = 32;
 
+    #[test]
+    fn test_shared_quantizer_model_compares_skipped_payloads() {
+        let codebook = |offset| {
+            let values = Float32Array::from_iter_values((0..32).map(|v| v as f32 + offset));
+            FixedSizeListArray::try_new_from_values(values, 2).unwrap()
+        };
+        let pq1 = Quantizer::Product(ProductQuantizer::new(
+            1,
+            4,
+            2,
+            codebook(0.0),
+            DistanceType::L2,
+        ));
+        let pq2 = Quantizer::Product(ProductQuantizer::new(
+            1,
+            4,
+            2,
+            codebook(100.0),
+            DistanceType::L2,
+        ));
+        assert!(!shared_quantizer_model(&pq1, &pq2));
+
+        let rq1 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::<Float32Type>(
+            1,
+            8,
+            lance_index::vector::bq::RQRotationType::Matrix,
+        ));
+        let rq2 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::<Float32Type>(
+            1,
+            8,
+            lance_index::vector::bq::RQRotationType::Matrix,
+        ));
+        assert!(!shared_quantizer_model(&rq1, &rq2));
+    }
+
     async fn compute_test_ivf_loss(dataset: &Dataset, column: &str, ivf: &IvfModel) -> f64 {
         let centroids = ivf
             .centroids_array()
@@ -4467,6 +5009,7 @@ mod tests {
                     use_index: true,
                     query_parallelism: lance_index::vector::DEFAULT_QUERY_PARALLELISM,
                     dist_q_c: 0.0,
+                    approx_mode: Default::default(),
                 };
                 let (partitions, _) = index.find_partitions(&query).unwrap();
                 let nearest_partition_id = partitions.value(0) as usize;
@@ -4642,13 +5185,12 @@ mod tests {
         let pq_params = PQBuildParams::new(NUM_SUBVECTORS as usize, NUM_BITS as usize);
 
         let uuid = Uuid::new_v4();
-        let uuid_str = uuid.to_string();
 
         build_ivf_pq_index(
             &dataset,
             WellKnownIvfPqData::COLUMN,
             INDEX_NAME,
-            &uuid_str,
+            uuid,
             MetricType::L2,
             &ivf_params,
             &pq_params,
@@ -4693,7 +5235,7 @@ mod tests {
             .unwrap();
 
         let index = dataset_mut
-            .open_vector_index(WellKnownIvfPqData::COLUMN, &uuid_str, &NoOpMetricsCollector)
+            .open_vector_index(WellKnownIvfPqData::COLUMN, &uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -4737,15 +5279,14 @@ mod tests {
         let mapping = build_mapping(row_ids_to_modify, row_ids_to_remove, max_id);
 
         let new_uuid = Uuid::new_v4();
-        let new_uuid_str = new_uuid.to_string();
 
         remap_index_file(
             &dataset_mut,
-            &uuid_str,
-            &new_uuid_str,
+            &uuid,
+            &new_uuid,
             dataset_mut.version().version,
             ivf_index,
-            &mapping,
+            &RowAddrRemap::direct(mapping),
             INDEX_NAME.to_string(),
             WellKnownIvfPqData::COLUMN.to_string(),
             vec![],
@@ -4789,11 +5330,7 @@ mod tests {
             .unwrap();
 
         let remapped = dataset_mut
-            .open_vector_index(
-                WellKnownIvfPqData::COLUMN,
-                &new_uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index(WellKnownIvfPqData::COLUMN, &new_uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let ivf_remapped = remapped.as_any().downcast_ref::<IVFIndex>().unwrap();
@@ -5267,6 +5804,94 @@ mod tests {
         );
     }
 
+    /// Regression test for a hang in the streaming *coreset* trainer
+    /// (`train_streaming_coreset_ivf_model`, taken when `num_partitions > 256`).
+    ///
+    /// That function spawns a `progress_worker` task that loops on
+    /// `progress_rx.recv()` and only terminates once every clone of the mpsc
+    /// sender is dropped. The `on_progress` closure owns a sender clone, and
+    /// `WeightedHierarchicalKMeansParams` used to retain an `on_progress` clone
+    /// that outlived the `progress_worker.await` at the end of the function.
+    /// With a live sender remaining, `recv()` never returned `None`, the worker
+    /// never finished, and the trainer hung forever after all compute was done.
+    ///
+    /// The build is wrapped in a timeout so the regression fails fast rather
+    /// than hanging the test process indefinitely.
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_streaming_coreset_ivf_training_terminates() {
+        use lance_index::progress::IndexBuildProgress;
+        use std::sync::atomic::{AtomicU64, Ordering};
+        use std::time::Duration;
+
+        #[derive(Debug, Default)]
+        struct CountingProgress {
+            progress_calls: AtomicU64,
+        }
+
+        #[async_trait::async_trait]
+        impl IndexBuildProgress for CountingProgress {
+            async fn stage_start(&self, _: &str, _: Option<u64>, _: &str) -> Result<()> {
+                Ok(())
+            }
+            async fn stage_progress(&self, _: &str, _: u64) -> Result<()> {
+                self.progress_calls.fetch_add(1, Ordering::Relaxed);
+                Ok(())
+            }
+            async fn stage_complete(&self, _: &str) -> Result<()> {
+                Ok(())
+            }
+        }
+
+        const SMALL_DIM: usize = 8;
+
+        let test_dir = TempStrDir::default();
+        let uri = format!("{}/ds", test_dir.as_str());
+        let reader = gen_batch()
+            .col("id", array::step::<UInt64Type>())
+            .col(
+                "vector",
+                array::rand_vec::<Float32Type>((SMALL_DIM as u32).into()),
+            )
+            .into_reader_rows(RowCount::from(2048), BatchCount::from(4));
+        let dataset = Dataset::write(reader, &uri, None).await.unwrap();
+
+        // > 256 partitions routes through `train_streaming_coreset_ivf_model`.
+        let mut params = IvfBuildParams::new(257);
+        params.sample_rate = 8;
+        params.streaming_sample_rate = Some(4);
+        params.streaming_refine_passes = 1;
+        params.max_iters = 2;
+
+        let progress = Arc::new(CountingProgress::default());
+
+        let ivf_model = tokio::time::timeout(
+            Duration::from_secs(120),
+            build_ivf_model(
+                &dataset,
+                "vector",
+                SMALL_DIM,
+                MetricType::L2,
+                &params,
+                None,
+                progress.clone(),
+            ),
+        )
+        .await
+        .expect(
+            "streaming coreset IVF training hung: progress worker never terminated after training",
+        )
+        .unwrap();
+
+        assert_eq!(ivf_model.num_partitions(), 257);
+        assert_eq!(ivf_model.dimension(), SMALL_DIM);
+        // The progress worker must have processed reports and then joined
+        // cleanly (proven by `build_ivf_model` returning at all).
+        assert!(
+            progress.progress_calls.load(Ordering::Relaxed) > 0,
+            "expected the progress worker to receive at least one report"
+        );
+    }
+
     #[test]
     fn test_fixed_training_ranges_are_sorted_and_bounded() {
         let ranges = generate_fixed_training_ranges(10_000, 1_234, 1_024, 16);
@@ -5849,11 +6474,7 @@ mod tests {
             .unwrap();
         let indices = dataset.load_indices().await.unwrap();
         let idx = dataset
-            .open_generic_index(
-                "vector",
-                indices[0].uuid.to_string().as_str(),
-                &NoOpMetricsCollector,
-            )
+            .open_generic_index("vector", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let ivf_idx = idx.as_any().downcast_ref::<v2::IvfPq>().unwrap();
@@ -5872,7 +6493,7 @@ mod tests {
         );
 
         // PQ code is on residual space
-        let pq_store = ivf_idx.load_partition_storage(0).await.unwrap();
+        let pq_store = ivf_idx.load_partition_storage(0, None).await.unwrap();
         pq_store
             .codebook()
             .values()
@@ -6174,4 +6795,80 @@ mod tests {
         let indices = dataset.load_indices().await.unwrap();
         assert!(!indices.is_empty(), "should have at least one index");
     }
+
+    #[tokio::test]
+    async fn test_optimize_ivf_hnsw_records_actual_file_sizes() {
+        // Regression test: `optimize_ivf_hnsw_indices` must record the on-disk
+        // file sizes, which are only known after `finish()` writes the footer.
+        const DIM: usize = 16;
+        let data = gen_batch()
+            .col(
+                "vec",
+                array::rand_vec::<Float32Type>(Dimension::from(DIM as u32)),
+            )
+            .into_batch_rows(RowCount::from(1_000))
+            .unwrap();
+        let schema = data.schema();
+
+        let mut dataset = InsertBuilder::new("memory://")
+            .execute(vec![data])
+            .await
+            .unwrap();
+
+        // Legacy file version keeps the index on the v1 path that goes through
+        // `optimize_ivf_hnsw_indices`.
+        let mut params = VectorIndexParams::with_ivf_hnsw_sq_params(
+            MetricType::L2,
+            IvfBuildParams::new(2),
+            HnswBuildParams::default().num_edges(50),
+            SQBuildParams::default(),
+        );
+        params.version(IndexFileVersion::Legacy);
+        dataset
+            .create_index(&["vec"], IndexType::Vector, None, &params, false)
+            .await
+            .unwrap();
+
+        // Append unindexed data so `optimize_indices` has something to merge.
+        let more = gen_batch()
+            .col(
+                "vec",
+                array::rand_vec::<Float32Type>(Dimension::from(DIM as u32)),
+            )
+            .into_batch_rows(RowCount::from(500))
+            .unwrap();
+        let more = RecordBatch::try_new(schema.clone(), more.columns().to_vec()).unwrap();
+        let mut dataset = InsertBuilder::new(Arc::new(dataset))
+            .with_params(&WriteParams {
+                mode: WriteMode::Append,
+                ..Default::default()
+            })
+            .execute(vec![more])
+            .await
+            .unwrap();
+
+        dataset.optimize_indices(&Default::default()).await.unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        let index = indices.first().expect("should have an index");
+        let files = index
+            .files
+            .as_ref()
+            .expect("optimized index should record file sizes");
+        assert!(!files.is_empty(), "index should record at least one file");
+
+        let indices_dir = dataset.indices_dir().join(index.uuid.to_string());
+        for file in files {
+            let actual = dataset
+                .object_store
+                .size(&indices_dir.clone().join(file.path.as_str()))
+                .await
+                .unwrap();
+            assert_eq!(
+                file.size_bytes, actual,
+                "recorded size for {} must match the on-disk size",
+                file.path
+            );
+        }
+    }
 }
diff --git a/rust/lance/src/index/vector/ivf/io.rs b/rust/lance/src/index/vector/ivf/io.rs
index bb52dd4b484..a49398a0750 100644
--- a/rust/lance/src/index/vector/ivf/io.rs
+++ b/rust/lance/src/index/vector/ivf/io.rs
@@ -25,7 +25,6 @@ use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_file::previous::writer::FileWriter as PreviousFileWriter;
 use lance_index::metrics::NoOpMetricsCollector;
-use lance_index::scalar::IndexWriter;
 use lance_index::vector::hnsw::HNSW;
 use lance_index::vector::hnsw::{HnswMetadata, builder::HnswBuildParams};
 use lance_index::vector::ivf::storage::IvfModel;
@@ -44,6 +43,7 @@ use lance_table::format::SelfDescribingFileReader;
 use lance_table::io::manifest::ManifestDescribing;
 use object_store::path::Path;
 use tokio::sync::Semaphore;
+use tokio::task::JoinHandle;
 
 use crate::Result;
 
@@ -278,164 +278,234 @@ pub(super) async fn write_hnsw_quantization_index_partitions(
     }
 
     let object_store = ObjectStore::local();
-    let mut part_files = Vec::with_capacity(ivf.num_partitions());
-    let mut aux_part_files = Vec::with_capacity(ivf.num_partitions());
-    let tmp_part_dir = Path::from_filesystem_path(TempStdDir::default())?;
-    let mut tasks = Vec::with_capacity(ivf.num_partitions());
-    let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL));
-    for part_id in 0..ivf.num_partitions() {
-        part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id)));
-        aux_part_files.push(
-            tmp_part_dir
-                .clone()
-                .join(format!("hnsw_part_aux_{}", part_id)),
-        );
+    // Partitions are staged in this scratch dir, then merged into the final index.
+    // Share the guard with every task via `Arc` so its `Drop` removes the dir only
+    // after the last task finishes -- never while one is still writing.
+    let tmp_part_dir_guard = Arc::new(TempStdDir::default());
+    let tmp_part_dir = Path::from_filesystem_path(&**tmp_part_dir_guard)?;
+
+    // `Option` per handle so the consume loop can `take()` each one, leaving the
+    // not-yet-consumed handles for the error-path drain.
+    let mut tasks: Vec<Option<JoinHandle<Result<usize>>>> =
+        Vec::with_capacity(ivf.num_partitions());
+
+    let build_result: Result<(Vec<HnswMetadata>, IvfModel)> = async {
+        let mut part_files = Vec::with_capacity(ivf.num_partitions());
+        let mut aux_part_files = Vec::with_capacity(ivf.num_partitions());
+        let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL));
+        for part_id in 0..ivf.num_partitions() {
+            part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id)));
+            aux_part_files.push(
+                tmp_part_dir
+                    .clone()
+                    .join(format!("hnsw_part_aux_{}", part_id)),
+            );
 
-        let mut code_array: Vec<Arc<dyn Array>> = vec![];
-        let mut row_id_array: Vec<Arc<dyn Array>> = vec![];
+            let mut code_array: Vec<Arc<dyn Array>> = vec![];
+            let mut row_id_array: Vec<Arc<dyn Array>> = vec![];
 
-        // We don't transform vectors to SQ codes while shuffling,
-        // so we won't merge SQ codes from the stream.
+            // We don't transform vectors to SQ codes while shuffling,
+            // so we won't merge SQ codes from the stream.
 
-        if let Some(&previous_indices) = existing_indices.as_ref() {
-            for &idx in previous_indices.iter() {
-                let sub_index = idx
-                    .load_partition(part_id, true, &NoOpMetricsCollector)
-                    .await?;
-                let row_ids = Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned()));
-                row_id_array.push(row_ids);
+            if let Some(&previous_indices) = existing_indices.as_ref() {
+                for &idx in previous_indices.iter() {
+                    let sub_index = idx
+                        .load_partition(part_id, true, &NoOpMetricsCollector)
+                        .await?;
+                    let row_ids =
+                        Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned()));
+                    row_id_array.push(row_ids);
+                }
             }
-        }
 
-        let code_column = match &quantizer {
-            Quantizer::Product(pq) => Some(pq.column()),
-            _ => None,
-        };
-        merge_streams(
-            &mut streams_heap,
-            &mut new_streams,
-            part_id as u32,
-            code_column,
-            &mut code_array,
-            &mut row_id_array,
-        )
-        .await?;
-
-        if row_id_array.is_empty() {
-            tasks.push(tokio::spawn(async { Ok(0) }));
-            continue;
-        }
-
-        let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
-        let part_writer = PreviousFileWriter::<ManifestDescribing>::try_new(
-            &object_store,
-            part_file,
-            Schema::try_from(writer.schema())?,
-            &Default::default(),
-        )
-        .await?;
+            let code_column = match &quantizer {
+                Quantizer::Product(pq) => Some(pq.column()),
+                _ => None,
+            };
+            merge_streams(
+                &mut streams_heap,
+                &mut new_streams,
+                part_id as u32,
+                code_column,
+                &mut code_array,
+                &mut row_id_array,
+            )
+            .await?;
 
-        let aux_part_writer = match auxiliary_writer.as_ref() {
-            Some(writer) => Some(
-                PreviousFileWriter::<ManifestDescribing>::try_new(
-                    &object_store,
-                    aux_part_file,
-                    Schema::try_from(writer.schema())?,
-                    &Default::default(),
-                )
-                .await?,
-            ),
-            None => None,
-        };
+            if row_id_array.is_empty() {
+                tasks.push(Some(tokio::spawn(async { Ok(0) })));
+                continue;
+            }
 
-        let dataset = dataset.clone();
-        let column = column.to_owned();
-        let hnsw_params = hnsw_params.clone();
-        let quantizer = quantizer.clone();
-        let sem = sem.clone();
-        tasks.push(tokio::spawn(async move {
-            let _permit = sem.acquire().await.expect("semaphore error");
-
-            log::debug!("Building HNSW partition {}", part_id);
-            let result = build_hnsw_quantization_partition(
-                dataset,
-                &column,
-                distance_type,
-                hnsw_params,
-                part_writer,
-                aux_part_writer,
-                quantizer,
-                row_id_array,
-                code_array,
+            let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
+            let part_writer = PreviousFileWriter::<ManifestDescribing>::try_new(
+                &object_store,
+                part_file,
+                Schema::try_from(writer.schema())?,
+                &Default::default(),
             )
-            .await;
-            log::debug!("Finished building HNSW partition {}", part_id);
-            result
-        }));
-    }
-
-    let mut aux_ivf = IvfModel::empty();
-    let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions());
-    for (part_id, task) in tasks.into_iter().enumerate() {
-        let offset = writer.len();
-        let num_rows = task.await??;
+            .await?;
 
-        if num_rows == 0 {
-            ivf.add_partition(0);
-            aux_ivf.add_partition(0);
-            hnsw_metadata.push(HnswMetadata::default());
-            continue;
+            let aux_part_writer = match auxiliary_writer.as_ref() {
+                Some(writer) => Some(
+                    PreviousFileWriter::<ManifestDescribing>::try_new(
+                        &object_store,
+                        aux_part_file,
+                        Schema::try_from(writer.schema())?,
+                        &Default::default(),
+                    )
+                    .await?,
+                ),
+                None => None,
+            };
+
+            let dataset = dataset.clone();
+            let column = column.to_owned();
+            let hnsw_params = hnsw_params.clone();
+            let quantizer = quantizer.clone();
+            let sem = sem.clone();
+            let tmp_part_dir_guard = tmp_part_dir_guard.clone();
+            tasks.push(Some(tokio::spawn(async move {
+                // Hold a guard clone so the scratch dir stays alive while this task writes.
+                let _tmp_part_dir_guard = tmp_part_dir_guard;
+                let _permit = sem.acquire().await.map_err(|err| {
+                    Error::io(format!(
+                        "failed to acquire HNSW partition build permit: {err}"
+                    ))
+                })?;
+
+                log::debug!("Building HNSW partition {}", part_id);
+                let result = build_hnsw_quantization_partition(
+                    dataset,
+                    &column,
+                    distance_type,
+                    hnsw_params,
+                    part_writer,
+                    aux_part_writer,
+                    quantizer,
+                    row_id_array,
+                    code_array,
+                )
+                .await;
+                log::debug!("Finished building HNSW partition {}", part_id);
+                result
+            })));
         }
 
-        let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
-        let part_reader =
-            PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?;
+        let mut aux_ivf = IvfModel::empty();
+        let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions());
+        for (part_id, task) in tasks.iter_mut().enumerate() {
+            let task = task
+                .take()
+                .expect("each partition task is consumed exactly once");
+            let offset = writer.len();
+            let num_rows = task.await??;
+
+            if num_rows == 0 {
+                ivf.add_partition(0);
+                aux_ivf.add_partition(0);
+                hnsw_metadata.push(HnswMetadata::default());
+                continue;
+            }
 
-        let batches = futures::stream::iter(0..part_reader.num_batches())
-            .map(|batch_id| {
-                part_reader.read_batch(
-                    batch_id as i32,
-                    ReadBatchParams::RangeFull,
-                    part_reader.schema(),
-                )
-            })
-            .buffered(object_store.io_parallelism())
-            .try_collect::<Vec<_>>()
-            .await?;
-        writer.write(&batches).await?;
-
-        ivf.add_partition((writer.len() - offset) as u32);
-        hnsw_metadata.push(serde_json::from_str(
-            part_reader.schema().metadata[HNSW::metadata_key()].as_str(),
-        )?);
-        std::mem::drop(part_reader);
-        object_store.delete(part_file).await?;
-
-        if let Some(aux_writer) = auxiliary_writer.as_mut() {
-            let aux_part_reader =
-                PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None)
-                    .await?;
+            let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]);
+            let part_reader =
+                PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?;
 
-            let batches = futures::stream::iter(0..aux_part_reader.num_batches())
+            let batches = futures::stream::iter(0..part_reader.num_batches())
                 .map(|batch_id| {
-                    aux_part_reader.read_batch(
+                    part_reader.read_batch(
                         batch_id as i32,
                         ReadBatchParams::RangeFull,
-                        aux_part_reader.schema(),
+                        part_reader.schema(),
                     )
                 })
                 .buffered(object_store.io_parallelism())
                 .try_collect::<Vec<_>>()
                 .await?;
-            std::mem::drop(aux_part_reader);
-            object_store.delete(aux_part_file).await?;
+            writer.write(&batches).await?;
+
+            ivf.add_partition((writer.len() - offset) as u32);
+            hnsw_metadata.push(serde_json::from_str(
+                part_reader.schema().metadata[HNSW::metadata_key()].as_str(),
+            )?);
+            std::mem::drop(part_reader);
+            object_store.delete(part_file).await?;
+
+            if let Some(aux_writer) = auxiliary_writer.as_mut() {
+                let aux_part_reader =
+                    PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None)
+                        .await?;
+
+                let batches = futures::stream::iter(0..aux_part_reader.num_batches())
+                    .map(|batch_id| {
+                        aux_part_reader.read_batch(
+                            batch_id as i32,
+                            ReadBatchParams::RangeFull,
+                            aux_part_reader.schema(),
+                        )
+                    })
+                    .buffered(object_store.io_parallelism())
+                    .try_collect::<Vec<_>>()
+                    .await?;
+                std::mem::drop(aux_part_reader);
+                object_store.delete(aux_part_file).await?;
 
-            aux_writer.write(&batches).await?;
-            aux_ivf.add_partition(num_rows as u32);
+                aux_writer.write(&batches).await?;
+                aux_ivf.add_partition(num_rows as u32);
+            }
+        }
+
+        Ok((hnsw_metadata, aux_ivf))
+    }
+    .await;
+
+    // On error, abort and await the partition builds we never consumed so none
+    // keep running in the background; see `drain_partition_tasks`.
+    if build_result.is_err() {
+        for err in drain_partition_tasks(&mut tasks).await {
+            log::warn!(
+                "HNSW partition build task failed while draining after an earlier error: {err}"
+            );
+        }
+    }
+
+    build_result
+}
+
+/// Abort and await every still-outstanding partition-build task, returning the
+/// non-cancellation errors they surfaced.
+///
+/// A dropped [`JoinHandle`] detaches its task, so every handle is aborted up front
+/// before any is awaited: otherwise a task slow to observe its own cancellation
+/// would keep running -- and keep writing into the scratch dir -- while an earlier
+/// handle is still being awaited. Awaiting then resolves only once each task has
+/// actually stopped. Cancellation errors are the expected result of the abort and
+/// dropped; failures and panics from tasks that had already finished before the
+/// abort are returned so the caller can surface them (a task still in flight is
+/// cancelled, so this best-effort drain only reports errors already produced).
+async fn drain_partition_tasks(tasks: &mut [Option<JoinHandle<Result<usize>>>]) -> Vec<Error> {
+    for task in tasks.iter() {
+        if let Some(handle) = task.as_ref() {
+            handle.abort();
         }
     }
 
-    Ok((hnsw_metadata, aux_ivf))
+    let mut errors = Vec::with_capacity(tasks.len());
+    for task in tasks.iter_mut() {
+        let Some(handle) = task.take() else {
+            continue;
+        };
+        match handle.await {
+            Ok(Ok(_)) => {}
+            Ok(Err(e)) => errors.push(e),
+            Err(join_err) if join_err.is_cancelled() => {}
+            Err(join_err) => errors.push(Error::io(format!(
+                "HNSW partition build task panicked: {join_err}"
+            ))),
+        }
+    }
+    errors
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -473,24 +543,29 @@ async fn build_hnsw_quantization_partition(
     let build_hnsw =
         build_and_write_hnsw(vectors.clone(), (*hnsw_params).clone(), metric_type, writer);
 
+    // Build PQ storage as a child future, joined below: it writes `aux_writer`'s
+    // file into the scratch dir, so it is cancelled together with this task when the
+    // error-path drain aborts it, and the join surfaces its errors.
     let build_store = match quantizer {
         Quantizer::Flat(_) => {
             return Err(Error::index(
                 "Flat quantizer is not supported for IVF_HNSW".to_string(),
             ));
         }
-        Quantizer::Product(pq) => tokio::spawn(build_and_write_pq_storage(
-            metric_type,
-            row_ids,
-            code_array,
-            pq,
-            aux_writer.unwrap(),
-        )),
-
-        _ => unreachable!("IVF_HNSW_SQ has been moved to v2 index builder"),
+        Quantizer::Product(pq) => {
+            let aux_writer = aux_writer.ok_or_else(|| {
+                Error::index("IVF_HNSW_PQ requires an auxiliary writer for PQ storage".to_string())
+            })?;
+            build_and_write_pq_storage(metric_type, row_ids, code_array, pq, aux_writer)
+        }
+        _ => {
+            return Err(Error::index(
+                "IVF_HNSW_SQ is not supported in the legacy HNSW partition writer".to_string(),
+            ));
+        }
     };
 
-    let index_rows = futures::join!(build_hnsw, build_store).0?;
+    let (index_rows, ()) = futures::try_join!(build_hnsw, build_store)?;
     assert!(
         index_rows >= num_rows,
         "index rows {} must be greater than or equal to num rows {}",
@@ -508,8 +583,8 @@ async fn build_and_write_hnsw(
 ) -> Result<usize> {
     let batch = params.build(vectors, distance_type).await?.to_batch()?;
     let metadata = batch.schema_ref().metadata().clone();
-    writer.write_record_batch(batch).await?;
-    writer.finish_with_metadata(&metadata).await
+    writer.write(&[batch]).await?;
+    Ok(writer.finish_with_metadata(&metadata).await?.num_rows as usize)
 }
 
 async fn build_and_write_pq_storage(
@@ -521,7 +596,7 @@ async fn build_and_write_pq_storage(
 ) -> Result<()> {
     let storage = spawn_cpu(move || build_pq_storage(metric_type, row_ids, code_array, pq)).await?;
 
-    writer.write_record_batch(storage.batch().clone()).await?;
+    writer.write(&[storage.batch().clone()]).await?;
     writer.finish().await?;
     Ok(())
 }
@@ -530,6 +605,9 @@ async fn build_and_write_pq_storage(
 mod tests {
     use super::*;
 
+    use std::path::PathBuf;
+    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
     use crate::Dataset;
     use crate::index::vector::ivf::v2;
     use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams};
@@ -538,6 +616,8 @@ mod tests {
     use lance_core::utils::tempfile::TempStrDir;
     use lance_index::IndexType;
     use lance_index::metrics::NoOpMetricsCollector;
+    use lance_index::vector::ivf::IvfBuildParams;
+    use lance_index::vector::pq::PQBuildParams;
     use lance_testing::datagen::generate_random_array;
 
     #[tokio::test]
@@ -579,11 +659,7 @@ mod tests {
         assert_eq!(ds.get_fragments().len(), 2);
 
         let idx = ds
-            .open_vector_index(
-                "vector",
-                &indices[0].uuid.to_string(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let _ivf_idx = idx
@@ -593,4 +669,248 @@ mod tests {
 
         //let indices = /ds.
     }
+
+    /// The scratch dir must outlive every partition task: dropping the caller-side
+    /// guard while tasks are still in flight must not remove it, because each task
+    /// still holds an `Arc` clone of the guard.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_scratch_dir_outlives_partition_tasks() {
+        let tmp_part_dir_guard = Arc::new(TempStdDir::default());
+        let scratch_path = tmp_part_dir_guard.to_path_buf();
+
+        // Park each task until the caller-side guard is dropped, so the dir's
+        // survival is attributable solely to the clones the tasks still hold.
+        let running = Arc::new(AtomicUsize::new(0));
+        let released = Arc::new(AtomicBool::new(false));
+
+        const NUM_TASKS: usize = 3;
+        let mut tasks = Vec::with_capacity(NUM_TASKS);
+        for _ in 0..NUM_TASKS {
+            let task_guard = tmp_part_dir_guard.clone();
+            let running = running.clone();
+            let released = released.clone();
+            tasks.push(tokio::spawn(async move {
+                running.fetch_add(1, Ordering::SeqCst);
+                while !released.load(Ordering::SeqCst) {
+                    tokio::task::yield_now().await;
+                }
+                // Still holding `task_guard`, so the directory must be live.
+                task_guard.exists()
+            }));
+        }
+
+        // Drop the caller-side guard only once every task is parked holding its clone.
+        while running.load(Ordering::SeqCst) < NUM_TASKS {
+            tokio::task::yield_now().await;
+        }
+        drop(tmp_part_dir_guard);
+        assert!(
+            scratch_path.exists(),
+            "scratch dir removed while partition tasks still held the guard"
+        );
+
+        released.store(true, Ordering::SeqCst);
+        for task in tasks {
+            assert!(
+                task.await.unwrap(),
+                "a partition task observed its scratch dir already removed"
+            );
+        }
+        assert!(
+            !scratch_path.exists(),
+            "scratch dir was not removed after the last task's guard clone dropped"
+        );
+    }
+
+    /// The drain step must outlive every spawned task: it aborts and awaits each
+    /// one so that, once it returns, no task is still running against the scratch
+    /// directory. It must also surface late failures rather than swallow them.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_drain_partition_tasks_waits_and_reports_errors() {
+        // Scratch dir guard analogous to the one held by
+        // `write_hnsw_quantization_index_partitions`; tasks read from it, and it
+        // is dropped only after the drain completes.
+        let scratch_guard = TempStdDir::default();
+        let scratch_path = scratch_guard.to_path_buf();
+
+        // Count of live task futures. `LiveGuard` decrements on both completion and
+        // cancellation, so a zero count after the drain proves every task terminated
+        // rather than being detached.
+        let live = Arc::new(AtomicUsize::new(0));
+        let saw_missing_dir = Arc::new(AtomicBool::new(false));
+
+        struct LiveGuard(Arc<AtomicUsize>);
+        impl Drop for LiveGuard {
+            fn drop(&mut self) {
+                self.0.fetch_sub(1, Ordering::SeqCst);
+            }
+        }
+
+        const NUM_SLOW: usize = 3;
+        let mut tasks: Vec<Option<JoinHandle<Result<usize>>>> = Vec::with_capacity(NUM_SLOW + 1);
+
+        // Tasks that never resolve on their own: the drain's abort is the only thing
+        // that can stop them, which is exactly what this test exercises.
+        for _ in 0..NUM_SLOW {
+            let live = live.clone();
+            let saw_missing_dir = saw_missing_dir.clone();
+            let scratch_path = scratch_path.clone();
+            tasks.push(Some(tokio::spawn(async move {
+                live.fetch_add(1, Ordering::SeqCst);
+                let _guard = LiveGuard(live.clone());
+                if !scratch_path.exists() {
+                    saw_missing_dir.store(true, Ordering::SeqCst);
+                }
+                futures::future::pending::<()>().await;
+                Ok(0)
+            })));
+        }
+
+        // A task that fails; its error must be returned, not silently dropped.
+        // The drain aborts every handle up front, and an abort only preserves a
+        // task's output if the task has already finished -- an in-flight task is
+        // cancelled and its error lost. So wait until this task has actually
+        // completed before handing it to the drain; otherwise whether its error
+        // surfaces would depend on the scheduler and the test would be flaky.
+        let failing =
+            tokio::spawn(async move { Err(Error::io("late partition failure".to_string())) });
+        while !failing.is_finished() {
+            tokio::task::yield_now().await;
+        }
+        tasks.push(Some(failing));
+
+        // Ensure all slow tasks are actually running before draining, so the
+        // drain has to await their cancellation rather than aborting them before
+        // they ever start.
+        while live.load(Ordering::SeqCst) < NUM_SLOW {
+            tokio::task::yield_now().await;
+        }
+
+        let errors = drain_partition_tasks(&mut tasks).await;
+
+        assert!(tasks.iter().all(Option::is_none), "handles left undrained");
+        assert_eq!(
+            live.load(Ordering::SeqCst),
+            0,
+            "a task was still running after the drain returned"
+        );
+        assert!(
+            !saw_missing_dir.load(Ordering::SeqCst),
+            "scratch dir was removed while a task was still running"
+        );
+        assert_eq!(errors.len(), 1, "expected exactly the one late failure");
+        assert!(
+            errors[0].to_string().contains("late partition failure"),
+            "late failure was not surfaced: {}",
+            errors[0]
+        );
+
+        // The guard, not the drain, removes the scratch dir.
+        assert!(scratch_path.exists());
+        drop(scratch_guard);
+        assert!(!scratch_path.exists());
+    }
+
+    /// `write_hnsw_quantization_index_partitions` stages each partition in a scratch
+    /// directory owned by a [`TempStdDir`] guard, which must remove it once the build
+    /// finishes so the OS temp dir does not grow without bound across legacy
+    /// IVF_HNSW_* builds.
+    ///
+    /// The OS temp dir is process-global, so an in-process check can't attribute a
+    /// leak to our own build. Instead we run the build in a child process with
+    /// `TMPDIR` pointed at an isolated dir we own, then assert nothing survives.
+    #[test]
+    fn test_hnsw_pq_scratch_dir_is_not_leaked() {
+        // Isolated temp root for the child. Owned here so it -- and anything the
+        // child leaks into it -- is removed when this guard drops at test end.
+        let isolated_root = TempStdDir::default();
+
+        let child_test = "index::vector::ivf::io::tests::build_legacy_hnsw_pq_in_child_process";
+        let output = std::process::Command::new(std::env::current_exe().unwrap())
+            .args([child_test, "--exact", "--ignored", "--nocapture"])
+            .env("TMPDIR", isolated_root.as_ref())
+            .env("LANCE_HNSW_LEAK_TEST_ROOT", isolated_root.as_ref())
+            .output()
+            .expect("failed to spawn child test process");
+        assert!(
+            output.status.success(),
+            "child build process failed:\n--- stdout ---\n{}\n--- stderr ---\n{}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        );
+
+        // Each build stages its partitions in one `.tmp*` dir directly under TMPDIR.
+        // Every guard removes its dir when it drops, so none should survive.
+        let leaked: Vec<PathBuf> = std::fs::read_dir(&isolated_root)
+            .expect("read isolated temp root")
+            .flatten()
+            .map(|entry| entry.path())
+            .filter(|path| {
+                path.is_dir()
+                    && path
+                        .file_name()
+                        .and_then(|name| name.to_str())
+                        .is_some_and(|name| name.starts_with(".tmp"))
+            })
+            .collect();
+
+        assert!(
+            leaked.is_empty(),
+            "legacy IVF_HNSW_PQ build leaked {} scratch director{} under the temp dir; \
+             the TempStdDir guard should remove each one when it drops: {:?}",
+            leaked.len(),
+            if leaked.len() == 1 { "y" } else { "ies" },
+            leaked,
+        );
+    }
+
+    /// Child half of [`test_hnsw_pq_scratch_dir_is_not_leaked`]. Ignored so it only
+    /// runs when the parent spawns it with `TMPDIR` and `LANCE_HNSW_LEAK_TEST_ROOT`
+    /// pointed at an isolated dir. Builds a few legacy IVF_HNSW_PQ indices; the
+    /// parent does the leak detection.
+    #[tokio::test]
+    #[ignore = "spawned as a child process by test_hnsw_pq_scratch_dir_is_not_leaked"]
+    async fn build_legacy_hnsw_pq_in_child_process() {
+        // Only do work when spawned by the parent; a bare `--ignored` run leaves
+        // the variable unset, so no-op rather than fail.
+        let Ok(root) = std::env::var("LANCE_HNSW_LEAK_TEST_ROOT") else {
+            return;
+        };
+
+        const DIM: usize = 32;
+        const ROWS: usize = 1024;
+        const NLIST: usize = 4;
+        const NUM_BUILDS: usize = 3;
+
+        // Keep the dataset out of the temp dir's `.tmp*` namespace so the parent
+        // never confuses it with a leaked scratch directory.
+        let dataset_uri = format!("{root}/dataset");
+        let values = generate_random_array(ROWS * DIM);
+        let fsl = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap());
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "vector",
+            fsl.data_type().clone(),
+            false,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![fsl]).unwrap();
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone());
+        let ds = Dataset::write(batches, &dataset_uri, Default::default())
+            .await
+            .unwrap();
+
+        for _ in 0..NUM_BUILDS {
+            crate::index::vector::ivf::build_ivf_hnsw_pq_index(
+                &ds,
+                "vector",
+                "idx",
+                uuid::Uuid::new_v4(),
+                MetricType::L2,
+                &IvfBuildParams::new(NLIST),
+                &HnswBuildParams::default(),
+                &PQBuildParams::new(4, 8),
+            )
+            .await
+            .unwrap();
+        }
+    }
 }
diff --git a/rust/lance/src/index/vector/ivf/partition_serde.rs b/rust/lance/src/index/vector/ivf/partition_serde.rs
index f8d13a2f0b5..ad737620a94 100644
--- a/rust/lance/src/index/vector/ivf/partition_serde.rs
+++ b/rust/lance/src/index/vector/ivf/partition_serde.rs
@@ -3,46 +3,35 @@
 
 //! Serialization and zero-copy deserialization for IVF partition cache entries.
 //!
-//! The format is:
-//!
-//! ```text
-//! [header_len: u64 LE]
-//! [header: JSON bytes]
-//! [sub_index Arrow IPC stream]
-//! [... quantizer-specific IPC streams ...]
-//! [storage Arrow IPC stream]
-//! ```
-//!
-//! Each IPC section is a self-delimiting Arrow IPC stream (schema + batches + EOS
-//! marker), written directly to the underlying writer without buffering. On
-//! deserialization, each message is read into a per-message buffer and zero-copy
-//! decoded via [`lance_arrow::ipc`].
+//! Each entry is a protobuf header (see `lance-index/protos-cache/cache.proto`, with the
+//! distance and rotation types as proto enums) followed by 64-byte-aligned
+//! Arrow IPC sections in a fixed, version-keyed order: the sub-index, then any
+//! quantizer-specific arrays (PQ codebook, RabitQ Matrix rotation), then the
+//! quantizer storage batches. Sections decode zero-copy via [`lance_arrow::ipc`].
 
-use std::io::Write;
 use std::sync::Arc;
 
 use arrow_array::{FixedSizeListArray, RecordBatch};
 use arrow_schema::{DataType, Field, Schema};
-use bytes::Bytes;
-use lance_arrow::ipc::{
-    read_ipc_stream_at, read_ipc_stream_single_at, read_len_prefixed_bytes_at, write_ipc_stream,
-    write_ipc_stream_batches, write_len_prefixed_bytes,
-};
-use lance_core::cache::CacheCodecImpl;
+use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter};
 use lance_core::{Error, Result};
 use lance_index::vector::bq::RQRotationType;
 use lance_index::vector::bq::builder::RabitQuantizer;
-use lance_index::vector::bq::storage::RabitQuantizationMetadata;
+use lance_index::vector::bq::storage::{RabitQuantizationMetadata, RabitQueryEstimator};
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatMetadata, FlatQuantizer};
 use lance_index::vector::pq::ProductQuantizer;
 use lance_index::vector::pq::storage::ProductQuantizationMetadata;
 use lance_index::vector::quantizer::{Quantization, QuantizerStorage};
 use lance_index::vector::sq::ScalarQuantizer;
-use lance_index::vector::sq::storage::ScalarQuantizationMetadata;
 use lance_index::vector::storage::VectorStore;
 use lance_index::vector::v3::subindex::IvfSubIndex;
 use lance_linalg::distance::DistanceType;
-use serde::{Deserialize, Serialize};
+
+use lance_index::cache_pb::{
+    DistanceType as PbDistanceType, FlatPartitionHeader, PqPartitionHeader, RabitPartitionHeader,
+    RabitQueryEstimator as PbRabitQueryEstimator, RotationType as PbRotationType,
+    SqPartitionHeader,
+};
 
 use super::v2::PartitionEntry;
 
@@ -68,7 +57,7 @@ type ArcAny = Arc<dyn std::any::Any + Send + Sync>;
 
 fn serialize_partition_entry<S, Concrete>(
     any: &ArcAny,
-    writer: &mut dyn Write,
+    writer: &mut CacheEntryWriter<'_>,
 ) -> lance_core::Result<()>
 where
     S: IvfSubIndex + 'static,
@@ -81,14 +70,16 @@ where
     concrete.serialize(writer)
 }
 
-fn deserialize_partition_entry<S, Q, Concrete>(data: &Bytes) -> lance_core::Result<ArcAny>
+fn deserialize_partition_entry<S, Q, Concrete>(
+    reader: &mut CacheEntryReader<'_>,
+) -> lance_core::Result<ArcAny>
 where
     S: IvfSubIndex + 'static,
     Q: Quantization + 'static,
     Concrete: Quantization + 'static,
     PartitionEntry<S, Concrete>: CacheCodecImpl,
 {
-    let concrete = PartitionEntry::<S, Concrete>::deserialize(data)?;
+    let concrete = PartitionEntry::<S, Concrete>::deserialize(reader)?;
     let any: ArcAny = Arc::new(concrete);
     Ok(any
         .downcast::<PartitionEntry<S, Q>>()
@@ -109,6 +100,8 @@ where
     PartitionEntry<S, Concrete>: CacheCodecImpl,
 {
     lance_core::cache::CacheCodec::new(
+        <PartitionEntry<S, Concrete> as CacheCodecImpl>::TYPE_ID,
+        <PartitionEntry<S, Concrete> as CacheCodecImpl>::CURRENT_VERSION,
         serialize_partition_entry::<S, Concrete>,
         deserialize_partition_entry::<S, Q, Concrete>,
     )
@@ -118,51 +111,64 @@ where
 // Common helpers
 // ---------------------------------------------------------------------------
 
-fn distance_type_to_u8(dt: DistanceType) -> u8 {
+// Distance and rotation discriminants travel as proto enums in the header;
+// these map to/from the in-memory Rust enums.
+
+fn distance_type_to_proto(dt: DistanceType) -> PbDistanceType {
     match dt {
-        DistanceType::L2 => 0,
-        DistanceType::Cosine => 1,
-        DistanceType::Dot => 2,
-        DistanceType::Hamming => 3,
+        DistanceType::L2 => PbDistanceType::L2,
+        DistanceType::Cosine => PbDistanceType::Cosine,
+        DistanceType::Dot => PbDistanceType::Dot,
+        DistanceType::Hamming => PbDistanceType::Hamming,
     }
 }
 
-fn u8_to_distance_type(v: u8) -> Result<DistanceType> {
-    match v {
-        0 => Ok(DistanceType::L2),
-        1 => Ok(DistanceType::Cosine),
-        2 => Ok(DistanceType::Dot),
-        3 => Ok(DistanceType::Hamming),
-        _ => Err(Error::io(format!("unknown distance type: {v}"))),
+fn proto_to_distance_type(dt: PbDistanceType) -> DistanceType {
+    match dt {
+        PbDistanceType::L2 => DistanceType::L2,
+        PbDistanceType::Cosine => DistanceType::Cosine,
+        PbDistanceType::Dot => DistanceType::Dot,
+        PbDistanceType::Hamming => DistanceType::Hamming,
     }
 }
 
-fn rotation_type_to_u8(rt: RQRotationType) -> u8 {
+fn rotation_type_to_proto(rt: RQRotationType) -> PbRotationType {
     match rt {
-        RQRotationType::Matrix => 0,
-        RQRotationType::Fast => 1,
+        RQRotationType::Matrix => PbRotationType::Matrix,
+        RQRotationType::Fast => PbRotationType::Fast,
     }
 }
 
-fn u8_to_rotation_type(v: u8) -> Result<RQRotationType> {
-    match v {
-        0 => Ok(RQRotationType::Matrix),
-        1 => Ok(RQRotationType::Fast),
-        _ => Err(Error::io(format!("unknown rotation type: {v}"))),
+fn proto_to_rotation_type(rt: PbRotationType) -> RQRotationType {
+    match rt {
+        PbRotationType::Matrix => RQRotationType::Matrix,
+        PbRotationType::Fast => RQRotationType::Fast,
     }
 }
 
-/// Write a JSON-serializable header using [`write_len_prefixed_bytes`].
-fn write_json_header(writer: &mut dyn Write, header: &impl Serialize) -> Result<()> {
-    let header_json = serde_json::to_vec(header)?;
-    write_len_prefixed_bytes(writer, &header_json)?;
-    Ok(())
+fn query_estimator_to_proto(qe: RabitQueryEstimator) -> PbRabitQueryEstimator {
+    match qe {
+        RabitQueryEstimator::ResidualQuery => PbRabitQueryEstimator::ResidualQuery,
+        RabitQueryEstimator::RawQuery => PbRabitQueryEstimator::RawQuery,
+    }
 }
 
-/// Read a JSON header written by [`write_json_header`].
-fn read_json_header<T: serde::de::DeserializeOwned>(data: &Bytes, offset: &mut usize) -> Result<T> {
-    let bytes = read_len_prefixed_bytes_at(data, offset).map_err(|e| Error::io(e.to_string()))?;
-    serde_json::from_slice(&bytes).map_err(|e| Error::io(e.to_string()))
+fn proto_to_query_estimator(qe: PbRabitQueryEstimator) -> RabitQueryEstimator {
+    match qe {
+        PbRabitQueryEstimator::ResidualQuery => RabitQueryEstimator::ResidualQuery,
+        PbRabitQueryEstimator::RawQuery => RabitQueryEstimator::RawQuery,
+    }
+}
+
+/// Read a storage section expected to hold exactly one batch.
+fn read_single_storage_batch(r: &mut CacheEntryReader<'_>) -> Result<RecordBatch> {
+    let mut batches = r.read_ipc_batches()?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(Error::io(format!(
+            "expected exactly 1 storage batch, got {n}"
+        ))),
+    }
 }
 
 /// Wrap a `FixedSizeListArray` in a single-column `RecordBatch` with the given
@@ -202,17 +208,11 @@ fn batch_to_codebook(batch: &RecordBatch) -> Result<FixedSizeListArray> {
 // PQ
 // ---------------------------------------------------------------------------
 
-#[derive(Serialize, Deserialize)]
-struct PqPartitionHeader {
-    distance_type: u8,
-    nbits: u32,
-    num_sub_vectors: usize,
-    dimension: usize,
-    transposed: bool,
-}
-
 impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ProductQuantizer> {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
+    const TYPE_ID: &'static str = "lance.vector.ivf.PartitionEntry.PQ";
+    const CURRENT_VERSION: u32 = 1;
+
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         let metadata = self.storage.metadata();
         let distance_type = self.storage.distance_type();
 
@@ -221,32 +221,28 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ProductQuantizer> {
         })?;
 
         let header = PqPartitionHeader {
-            distance_type: distance_type_to_u8(distance_type),
+            distance_type: distance_type_to_proto(distance_type) as i32,
             nbits: metadata.nbits,
-            num_sub_vectors: metadata.num_sub_vectors,
-            dimension: metadata.dimension,
+            num_sub_vectors: metadata.num_sub_vectors as u64,
+            dimension: metadata.dimension as u64,
             transposed: metadata.transposed,
         };
 
-        write_json_header(writer, &header)?;
-        write_ipc_stream(&self.index.to_batch()?, writer)?;
-        write_ipc_stream(&codebook_to_batch(codebook)?, writer)?;
-        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+        w.write_header(&header)?;
+        w.write_ipc(&self.index.to_batch()?)?;
+        w.write_ipc(&codebook_to_batch(codebook)?)?;
+        w.write_ipc_batches(self.storage.to_batches()?)?;
 
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let header: PqPartitionHeader = read_json_header(data, &mut offset)?;
-        let distance_type = u8_to_distance_type(header.distance_type)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: PqPartitionHeader = r.read_header()?;
+        let distance_type = proto_to_distance_type(header.distance_type());
 
-        let sub_index_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
-        let codebook_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
-        let storage_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let sub_index_batch = r.read_ipc()?;
+        let codebook_batch = r.read_ipc()?;
+        let storage_batch = read_single_storage_batch(r)?;
 
         let index = S::load(sub_index_batch)?;
         let codebook = batch_to_codebook(&codebook_batch)?;
@@ -254,8 +250,8 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ProductQuantizer> {
         let metadata = ProductQuantizationMetadata {
             codebook_position: 0,
             nbits: header.nbits,
-            num_sub_vectors: header.num_sub_vectors,
-            dimension: header.dimension,
+            num_sub_vectors: header.num_sub_vectors as usize,
+            dimension: header.dimension as usize,
             codebook: Some(codebook),
             codebook_tensor: Vec::new(),
             transposed: header.transposed,
@@ -276,41 +272,35 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ProductQuantizer> {
 // Flat (Float32)
 // ---------------------------------------------------------------------------
 
-#[derive(Serialize, Deserialize)]
-struct FlatPartitionHeader {
-    distance_type: u8,
-    dim: usize,
-}
-
 impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatQuantizer> {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        let metadata = self.storage.metadata();
-        let distance_type = self.storage.distance_type();
+    const TYPE_ID: &'static str = "lance.vector.ivf.PartitionEntry.Flat";
+    const CURRENT_VERSION: u32 = 1;
 
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let metadata = self.storage.metadata();
         let header = FlatPartitionHeader {
-            distance_type: distance_type_to_u8(distance_type),
-            dim: metadata.dim,
+            distance_type: distance_type_to_proto(self.storage.distance_type()) as i32,
+            dim: metadata.dim as u64,
         };
 
-        write_json_header(writer, &header)?;
-        write_ipc_stream(&self.index.to_batch()?, writer)?;
-        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+        w.write_header(&header)?;
+        w.write_ipc(&self.index.to_batch()?)?;
+        w.write_ipc_batches(self.storage.to_batches()?)?;
 
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let header: FlatPartitionHeader = read_json_header(data, &mut offset)?;
-        let distance_type = u8_to_distance_type(header.distance_type)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: FlatPartitionHeader = r.read_header()?;
+        let distance_type = proto_to_distance_type(header.distance_type());
 
-        let sub_index_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
-        let storage_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let sub_index_batch = r.read_ipc()?;
+        let storage_batch = read_single_storage_batch(r)?;
 
         let index = S::load(sub_index_batch)?;
-        let metadata = FlatMetadata { dim: header.dim };
+        let metadata = FlatMetadata {
+            dim: header.dim as usize,
+        };
         let storage = <FlatQuantizer as Quantization>::Storage::try_from_batch(
             storage_batch,
             &metadata,
@@ -327,34 +317,34 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatQuantizer> {
 // ---------------------------------------------------------------------------
 
 impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatBinQuantizer> {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        let metadata = self.storage.metadata();
-        let distance_type = self.storage.distance_type();
+    const TYPE_ID: &'static str = "lance.vector.ivf.PartitionEntry.FlatBin";
+    const CURRENT_VERSION: u32 = 1;
 
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let metadata = self.storage.metadata();
         let header = FlatPartitionHeader {
-            distance_type: distance_type_to_u8(distance_type),
-            dim: metadata.dim,
+            distance_type: distance_type_to_proto(self.storage.distance_type()) as i32,
+            dim: metadata.dim as u64,
         };
 
-        write_json_header(writer, &header)?;
-        write_ipc_stream(&self.index.to_batch()?, writer)?;
-        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+        w.write_header(&header)?;
+        w.write_ipc(&self.index.to_batch()?)?;
+        w.write_ipc_batches(self.storage.to_batches()?)?;
 
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let header: FlatPartitionHeader = read_json_header(data, &mut offset)?;
-        let distance_type = u8_to_distance_type(header.distance_type)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: FlatPartitionHeader = r.read_header()?;
+        let distance_type = proto_to_distance_type(header.distance_type());
 
-        let sub_index_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
-        let storage_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let sub_index_batch = r.read_ipc()?;
+        let storage_batch = read_single_storage_batch(r)?;
 
         let index = S::load(sub_index_batch)?;
-        let metadata = FlatMetadata { dim: header.dim };
+        let metadata = FlatMetadata {
+            dim: header.dim as usize,
+        };
         let storage = <FlatBinQuantizer as Quantization>::Storage::try_from_batch(
             storage_batch,
             &metadata,
@@ -370,56 +360,41 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, FlatBinQuantizer> {
 // SQ
 // ---------------------------------------------------------------------------
 
-#[derive(Serialize, Deserialize)]
-struct SqPartitionHeader {
-    distance_type: u8,
-    num_bits: u16,
-    dim: usize,
-    bounds_start: f64,
-    bounds_end: f64,
-}
-
 impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ScalarQuantizer> {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        let metadata = self.storage.metadata();
-        let distance_type = self.storage.distance_type();
+    const TYPE_ID: &'static str = "lance.vector.ivf.PartitionEntry.SQ";
+    const CURRENT_VERSION: u32 = 1;
 
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let metadata = self.storage.metadata();
         let header = SqPartitionHeader {
-            distance_type: distance_type_to_u8(distance_type),
-            num_bits: metadata.num_bits,
-            dim: metadata.dim,
+            distance_type: distance_type_to_proto(self.storage.distance_type()) as i32,
+            num_bits: metadata.num_bits as u32,
+            dim: metadata.dim as u64,
             bounds_start: metadata.bounds.start,
             bounds_end: metadata.bounds.end,
         };
 
-        write_json_header(writer, &header)?;
-        write_ipc_stream(&self.index.to_batch()?, writer)?;
-        // SQ storage may contain multiple batches; stream them all in one IPC stream.
-        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+        w.write_header(&header)?;
+        w.write_ipc(&self.index.to_batch()?)?;
+        // SQ storage may contain multiple batches; write them all in one section.
+        w.write_ipc_batches(self.storage.to_batches()?)?;
 
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let header: SqPartitionHeader = read_json_header(data, &mut offset)?;
-        let distance_type = u8_to_distance_type(header.distance_type)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: SqPartitionHeader = r.read_header()?;
+        let distance_type = proto_to_distance_type(header.distance_type());
 
-        let sub_index_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
-        let storage_batches =
-            read_ipc_stream_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let sub_index_batch = r.read_ipc()?;
+        let storage_batches = r.read_ipc_batches()?;
 
         let index = S::load(sub_index_batch)?;
-        let metadata = ScalarQuantizationMetadata {
-            dim: header.dim,
-            num_bits: header.num_bits,
-            bounds: header.bounds_start..header.bounds_end,
-        };
+        let num_bits = header.num_bits as u16;
         let storage = <ScalarQuantizer as Quantization>::Storage::try_new(
-            metadata.num_bits,
+            num_bits,
             distance_type,
-            metadata.bounds,
+            header.bounds_start..header.bounds_end,
             storage_batches,
             None,
         )?;
@@ -432,80 +407,69 @@ impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, ScalarQuantizer> {
 // RabitQ
 // ---------------------------------------------------------------------------
 
-#[derive(Serialize, Deserialize)]
-struct RabitPartitionHeader {
-    distance_type: u8,
-    num_bits: u8,
-    code_dim: u32,
-    /// 0 = Matrix, 1 = Fast
-    rotation_type: u8,
-    /// Fast rotation signs (only set when rotation_type == Fast).
-    fast_rotation_signs: Option<Vec<u8>>,
-}
-
 impl<S: IvfSubIndex> CacheCodecImpl for PartitionEntry<S, RabitQuantizer> {
-    fn serialize(&self, writer: &mut dyn Write) -> Result<()> {
-        let metadata = self.storage.metadata();
-        let distance_type = self.storage.distance_type();
+    const TYPE_ID: &'static str = "lance.vector.ivf.PartitionEntry.Rabit";
+    const CURRENT_VERSION: u32 = 1;
 
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        let metadata = self.storage.metadata();
         let header = RabitPartitionHeader {
-            distance_type: distance_type_to_u8(distance_type),
-            num_bits: metadata.num_bits,
+            distance_type: distance_type_to_proto(self.storage.distance_type()) as i32,
+            num_bits: metadata.num_bits as u32,
             code_dim: metadata.code_dim,
-            rotation_type: rotation_type_to_u8(metadata.rotation_type),
+            rotation_type: rotation_type_to_proto(metadata.rotation_type) as i32,
+            query_estimator: query_estimator_to_proto(metadata.query_estimator) as i32,
             fast_rotation_signs: metadata.fast_rotation_signs.clone(),
         };
 
-        write_json_header(writer, &header)?;
-
-        write_ipc_stream(&self.index.to_batch()?, writer)?;
+        w.write_header(&header)?;
+        w.write_ipc(&self.index.to_batch()?)?;
 
-        // Write the rotation matrix IPC stream only for Matrix rotation; the
-        // Fast rotation case stores its signs compactly in the JSON header.
+        // Write the rotation matrix IPC section only for Matrix rotation; the
+        // Fast rotation case stores its signs compactly in the proto header.
         if metadata.rotation_type == RQRotationType::Matrix {
             let mat = metadata.rotate_mat.as_ref().ok_or_else(|| {
                 Error::io(
                     "RabitQ Matrix metadata missing rotate_mat during serialization".to_string(),
                 )
             })?;
-            write_ipc_stream(&fsl_to_batch(mat, "rotate_mat")?, writer)?;
+            w.write_ipc(&fsl_to_batch(mat, "rotate_mat")?)?;
         }
 
-        write_ipc_stream_batches(self.storage.to_batches()?, writer)?;
+        w.write_ipc_batches(self.storage.to_batches()?)?;
 
         Ok(())
     }
 
-    fn deserialize(data: &Bytes) -> Result<Self> {
-        let mut offset = 0;
-        let header: RabitPartitionHeader = read_json_header(data, &mut offset)?;
-        let distance_type = u8_to_distance_type(header.distance_type)?;
-        let rotation_type = u8_to_rotation_type(header.rotation_type)?;
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        let header: RabitPartitionHeader = r.read_header()?;
+        let distance_type = proto_to_distance_type(header.distance_type());
+        let rotation_type = proto_to_rotation_type(header.rotation_type());
 
-        let sub_index_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let sub_index_batch = r.read_ipc()?;
 
         let rotate_mat = if rotation_type == RQRotationType::Matrix {
-            let mat_batch = read_ipc_stream_single_at(data, &mut offset)
-                .map_err(|e| Error::io(e.to_string()))?;
+            let mat_batch = r.read_ipc()?;
             Some(batch_to_fsl(&mat_batch)?)
         } else {
             None
         };
 
-        let storage_batch =
-            read_ipc_stream_single_at(data, &mut offset).map_err(|e| Error::io(e.to_string()))?;
+        let storage_batch = read_single_storage_batch(r)?;
 
         let index = S::load(sub_index_batch)?;
+        // Read the proto enum accessor before moving fields out of `header`.
+        let query_estimator = proto_to_query_estimator(header.query_estimator());
         let metadata = RabitQuantizationMetadata {
             rotate_mat,
             rotate_mat_position: None,
             fast_rotation_signs: header.fast_rotation_signs,
             rotation_type,
             code_dim: header.code_dim,
-            num_bits: header.num_bits,
+            num_bits: header.num_bits as u8,
             // The storage batch already has packed codes; skip re-packing.
             packed: true,
+            query_estimator,
         };
         let storage = <RabitQuantizer as Quantization>::Storage::try_from_batch(
             storage_batch,
@@ -543,6 +507,21 @@ mod tests {
     use lance_index::vector::flat::storage::FlatFloatStorage;
     use lance_index::vector::sq::storage::ScalarQuantizationStorage;
 
+    /// Serialize a codec body (no envelope) for tests.
+    fn ser_body<T: CacheCodecImpl>(entry: &T) -> Vec<u8> {
+        let mut buf = Vec::new();
+        entry
+            .serialize(&mut CacheEntryWriter::new(&mut buf))
+            .unwrap();
+        buf
+    }
+
+    /// Deserialize a codec body (no envelope) at the current build's version.
+    fn de_body<T: CacheCodecImpl>(bytes: Vec<u8>) -> Result<T> {
+        let data = bytes::Bytes::from(bytes);
+        T::deserialize(&mut CacheEntryReader::new(&data, 0, T::CURRENT_VERSION))
+    }
+
     // ----- PQ helpers -------------------------------------------------------
 
     fn make_test_codebook(dim: usize, num_sub_vectors: usize) -> FixedSizeListArray {
@@ -610,12 +589,9 @@ mod tests {
             storage,
         };
 
-        let mut serialized = Vec::new();
-        entry.serialize(&mut serialized).unwrap();
-        let deserialized = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
-            &bytes::Bytes::from(serialized),
-        )
-        .unwrap();
+        let serialized = ser_body(&entry);
+        let deserialized =
+            de_body::<PartitionEntry<FlatIndex, ProductQuantizer>>(serialized).unwrap();
 
         assert_eq!(entry.storage, deserialized.storage);
     }
@@ -663,12 +639,8 @@ mod tests {
                 storage,
             };
 
-            let mut bytes = Vec::new();
-            entry.serialize(&mut bytes).unwrap();
-            let restored = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
-                &bytes::Bytes::from(bytes),
-            )
-            .unwrap();
+            let bytes = ser_body(&entry);
+            let restored = de_body::<PartitionEntry<FlatIndex, ProductQuantizer>>(bytes).unwrap();
             assert_eq!(
                 restored.storage.distance_type(),
                 entry.storage.distance_type()
@@ -686,12 +658,9 @@ mod tests {
             storage,
         };
 
-        let mut serialized = Vec::new();
-        entry.serialize(&mut serialized).unwrap();
-        let deserialized = PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(
-            &bytes::Bytes::from(serialized),
-        )
-        .unwrap();
+        let serialized = ser_body(&entry);
+        let deserialized =
+            de_body::<PartitionEntry<FlatIndex, ProductQuantizer>>(serialized).unwrap();
         assert_eq!(entry.storage, deserialized.storage);
     }
 
@@ -704,13 +673,9 @@ mod tests {
             index: FlatIndex::default(),
             storage,
         };
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
+        let mut bytes = ser_body(&entry);
         bytes.truncate(3);
-        assert!(
-            PartitionEntry::<FlatIndex, ProductQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .is_err()
-        );
+        assert!(de_body::<PartitionEntry<FlatIndex, ProductQuantizer>>(bytes).is_err());
     }
 
     // ----- Flat helpers -----------------------------------------------------
@@ -748,11 +713,8 @@ mod tests {
             storage,
         };
 
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, FlatQuantizer>>(bytes).unwrap();
 
         assert_eq!(
             restored.storage.metadata().dim,
@@ -778,11 +740,8 @@ mod tests {
                 index: FlatIndex::default(),
                 storage,
             };
-            let mut bytes = Vec::new();
-            entry.serialize(&mut bytes).unwrap();
-            let restored =
-                PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                    .unwrap();
+            let bytes = ser_body(&entry);
+            let restored = de_body::<PartitionEntry<FlatIndex, FlatQuantizer>>(bytes).unwrap();
             assert_eq!(restored.storage.distance_type(), dt);
         }
     }
@@ -795,11 +754,8 @@ mod tests {
             storage,
         };
 
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, FlatQuantizer>>(bytes).unwrap();
 
         let restored_batch = restored.storage.to_batches().unwrap().next().unwrap();
         let schema = restored_batch.schema();
@@ -820,11 +776,8 @@ mod tests {
             storage,
         };
 
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, FlatQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, FlatQuantizer>>(bytes).unwrap();
 
         let restored_batch = restored.storage.to_batches().unwrap().next().unwrap();
         let schema = restored_batch.schema();
@@ -876,11 +829,8 @@ mod tests {
             storage,
         };
 
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, ScalarQuantizer>>(bytes).unwrap();
 
         let m = entry.storage.metadata();
         let rm = restored.storage.metadata();
@@ -906,12 +856,8 @@ mod tests {
                 index: FlatIndex::default(),
                 storage,
             };
-            let mut bytes = Vec::new();
-            entry.serialize(&mut bytes).unwrap();
-            let restored = PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(
-                &bytes::Bytes::from(bytes),
-            )
-            .unwrap();
+            let bytes = ser_body(&entry);
+            let restored = de_body::<PartitionEntry<FlatIndex, ScalarQuantizer>>(bytes).unwrap();
             assert_eq!(restored.storage.distance_type(), dt);
         }
     }
@@ -952,11 +898,8 @@ mod tests {
             index: FlatIndex::default(),
             storage,
         };
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, ScalarQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, ScalarQuantizer>>(bytes).unwrap();
 
         assert_eq!(restored.storage.len(), 30);
         let orig_ids: Vec<u64> = entry.storage.row_ids().copied().collect();
@@ -970,14 +913,27 @@ mod tests {
         num_rows: usize,
         code_dim: usize,
         distance_type: DistanceType,
+    ) -> <RabitQuantizer as Quantization>::Storage {
+        make_rabit_storage(
+            num_rows,
+            code_dim,
+            distance_type,
+            RQRotationType::Fast,
+            RabitQueryEstimator::ResidualQuery,
+        )
+    }
+
+    fn make_rabit_storage(
+        num_rows: usize,
+        code_dim: usize,
+        distance_type: DistanceType,
+        rotation_type: RQRotationType,
+        query_estimator: RabitQueryEstimator,
     ) -> <RabitQuantizer as Quantization>::Storage {
         use lance_arrow::FixedSizeListArrayExt;
 
-        let quantizer = RabitQuantizer::new_with_rotation::<Float32Type>(
-            1,
-            code_dim as i32,
-            RQRotationType::Fast,
-        );
+        let quantizer =
+            RabitQuantizer::new_with_rotation::<Float32Type>(1, code_dim as i32, rotation_type);
         let values: Vec<f32> = (0..num_rows * code_dim)
             .map(|i| (i % 100) as f32 / 100.0 - 0.5)
             .collect();
@@ -989,7 +945,8 @@ mod tests {
             .as_fixed_size_list()
             .clone();
 
-        let metadata = quantizer.metadata(None);
+        let mut metadata = quantizer.metadata(None);
+        metadata.query_estimator = query_estimator;
         let batch = RecordBatch::try_from_iter(vec![
             (
                 lance_core::ROW_ID,
@@ -1036,17 +993,15 @@ mod tests {
             storage,
         };
 
-        let mut bytes = Vec::new();
-        entry.serialize(&mut bytes).unwrap();
-        let restored =
-            PartitionEntry::<FlatIndex, RabitQuantizer>::deserialize(&bytes::Bytes::from(bytes))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, RabitQuantizer>>(bytes).unwrap();
 
         let m = entry.storage.metadata();
         let rm = restored.storage.metadata();
         assert_eq!(rm.num_bits, m.num_bits);
         assert_eq!(rm.code_dim, m.code_dim);
         assert_eq!(rm.rotation_type, m.rotation_type);
+        assert_eq!(rm.query_estimator, m.query_estimator);
         assert_eq!(rm.fast_rotation_signs, m.fast_rotation_signs);
         assert!(rm.packed);
         assert_eq!(
@@ -1077,19 +1032,129 @@ mod tests {
                 index: FlatIndex::default(),
                 storage,
             };
-            let mut bytes = Vec::new();
-            entry.serialize(&mut bytes).unwrap();
-            let restored = PartitionEntry::<FlatIndex, RabitQuantizer>::deserialize(
-                &bytes::Bytes::from(bytes),
-            )
+            let bytes = ser_body(&entry);
+            let restored = de_body::<PartitionEntry<FlatIndex, RabitQuantizer>>(bytes).unwrap();
+            // The codec round-trips the distance type faithfully.
+            assert_eq!(
+                restored.storage.distance_type(),
+                entry.storage.distance_type()
+            );
+        }
+    }
+
+    #[test]
+    fn test_roundtrip_rabitq_raw_query_estimator() {
+        // The query estimator is a non-default value here; it must survive the
+        // round trip so raw-query search keeps working after a cache reload.
+        let storage = make_rabit_storage(
+            40,
+            32,
+            DistanceType::L2,
+            RQRotationType::Fast,
+            RabitQueryEstimator::RawQuery,
+        );
+        assert_eq!(
+            storage.metadata().query_estimator,
+            RabitQueryEstimator::RawQuery
+        );
+        let entry = PartitionEntry::<FlatIndex, RabitQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, RabitQuantizer>>(bytes).unwrap();
+        assert_eq!(
+            restored.storage.metadata().query_estimator,
+            RabitQueryEstimator::RawQuery
+        );
+    }
+
+    /// Matrix rotation writes an extra `rotate_mat` IPC section between the
+    /// sub-index and storage sections; exercise that the codec preserves it.
+    #[test]
+    fn test_roundtrip_flat_rabitq_matrix() {
+        let storage = make_rabit_storage(
+            40,
+            32,
+            DistanceType::L2,
+            RQRotationType::Matrix,
+            RabitQueryEstimator::ResidualQuery,
+        );
+        let entry = PartitionEntry::<FlatIndex, RabitQuantizer> {
+            index: FlatIndex::default(),
+            storage,
+        };
+
+        let bytes = ser_body(&entry);
+        let restored = de_body::<PartitionEntry<FlatIndex, RabitQuantizer>>(bytes).unwrap();
+
+        let m = entry.storage.metadata();
+        let rm = restored.storage.metadata();
+        assert_eq!(rm.rotation_type, RQRotationType::Matrix);
+        assert_eq!(rm.code_dim, m.code_dim);
+        assert_eq!(rm.num_bits, m.num_bits);
+        // The rotation matrix itself must survive the round trip.
+        let orig_mat = m
+            .rotate_mat
+            .as_ref()
+            .expect("matrix rotation has rotate_mat");
+        let rest_mat = rm
+            .rotate_mat
+            .as_ref()
+            .expect("restored matrix rotation has rotate_mat");
+        assert_eq!(
+            orig_mat.values().as_primitive::<Float32Type>().values(),
+            rest_mat.values().as_primitive::<Float32Type>().values(),
+        );
+    }
+
+    /// SQ storage (a multi-batch IPC section) must decode zero-copy through the
+    /// full envelope even though the proto header and sub-index section push it
+    /// to a non-aligned starting offset.
+    #[test]
+    fn test_partition_storage_is_zero_copy_through_envelope() {
+        use lance_core::cache::CacheCodec;
+        const ALIGN: usize = 64;
+
+        let entry = PartitionEntry::<FlatIndex, ScalarQuantizer> {
+            index: FlatIndex::default(),
+            storage: make_sq_storage(64, 32, DistanceType::L2),
+        };
+        let codec = CacheCodec::from_impl::<PartitionEntry<FlatIndex, ScalarQuantizer>>();
+        let any: Arc<dyn std::any::Any + Send + Sync> = Arc::new(entry);
+        let mut buf = Vec::new();
+        codec.serialize(&any, &mut buf).unwrap();
+
+        let mut v = vec![0u8; buf.len() + ALIGN];
+        let pad = (ALIGN - (v.as_ptr() as usize % ALIGN)) % ALIGN;
+        v[pad..pad + buf.len()].copy_from_slice(&buf);
+        let data = bytes::Bytes::from(v).slice(pad..pad + buf.len());
+
+        let restored = codec.deserialize(&data).hit().unwrap();
+        let restored = restored
+            .downcast::<PartitionEntry<FlatIndex, ScalarQuantizer>>()
             .unwrap();
-            assert_eq!(restored.storage.distance_type(), dt);
+
+        let base = data.as_ptr() as usize;
+        let end = base + data.len();
+        let first = restored.storage.to_batches().unwrap().next().unwrap();
+        for col in first.columns() {
+            for buffer in col.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= base && ptr < end,
+                    "storage buffer was realigned out of the input — misaligned IPC section",
+                );
+            }
         }
     }
 
     #[test]
     fn test_ivf_index_state_roundtrip() {
-        use crate::index::vector::ivf::v2::{IvfIndexState, IvfStateEntryBox};
+        use crate::index::vector::ivf::v2::{
+            IvfIndexState, IvfStateEntryBox, empty_rabit_search_cache_cell,
+        };
         use lance_index::vector::flat::index::FlatQuantizer;
         use lance_index::vector::ivf::storage::IvfModel;
         use lance_index::vector::quantizer::QuantizationType;
@@ -1114,21 +1179,17 @@ mod tests {
             cache_key_prefix: "prefix/".to_string(),
             index_file_size: 1024,
             aux_file_size: 512,
+            rq_search_cache: empty_rabit_search_cache_cell(),
         };
 
         let entry = IvfStateEntryBox(Arc::new(state));
 
-        let mut bytes = Vec::new();
-        CacheCodecImpl::serialize(&entry, &mut bytes).unwrap();
-
-        let restored =
-            <IvfStateEntryBox as CacheCodecImpl>::deserialize(&bytes::Bytes::from(bytes.clone()))
-                .unwrap();
+        let bytes = ser_body(&entry);
+        let restored = de_body::<IvfStateEntryBox>(bytes.clone()).unwrap();
 
         // Re-serialize the restored entry and compare bytes — a stronger check
         // than field-by-field comparison and avoids needing to downcast.
-        let mut restored_bytes = Vec::new();
-        CacheCodecImpl::serialize(&restored, &mut restored_bytes).unwrap();
+        let restored_bytes = ser_body(&restored);
         assert_eq!(bytes, restored_bytes);
     }
 }
diff --git a/rust/lance/src/index/vector/ivf/v2.rs b/rust/lance/src/index/vector/ivf/v2.rs
index 672264f0dac..732e6befd90 100644
--- a/rust/lance/src/index/vector/ivf/v2.rs
+++ b/rust/lance/src/index/vector/ivf/v2.rs
@@ -3,12 +3,13 @@
 
 //! IVF - Inverted File index.
 
-use std::io::Write as IoWrite;
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::marker::PhantomData;
 use std::{
     any::Any,
+    borrow::Cow,
     collections::{BinaryHeap, HashMap},
-    sync::Arc,
+    sync::{Arc, LazyLock, Mutex},
 };
 
 use crate::index::vector::{IndexFileVersion, builder::index_type_string};
@@ -21,23 +22,29 @@ use async_trait::async_trait;
 use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use deepsize::DeepSizeOf;
 use futures::future::BoxFuture;
 use futures::prelude::stream::{self, TryStreamExt};
 use futures::{StreamExt, TryFutureExt};
 use lance_arrow::RecordBatchExt;
-use lance_arrow::ipc::write_len_prefixed_bytes;
-use lance_core::cache::{CacheCodec, CacheCodecImpl, CacheKey, LanceCache, WeakLanceCache};
+use lance_core::cache::{
+    CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache,
+    WeakLanceCache,
+};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu};
 use lance_core::utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS};
 use lance_core::{Error, ROW_ID, Result};
 use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
 use lance_file::LanceEncodingsIo;
 use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions};
+use lance_index::cache_pb::IvfStateHeader;
 use lance_index::frag_reuse::FragReuseIndex;
 use lance_index::metrics::{LocalMetricsCollector, MetricsCollector, NoOpMetricsCollector};
 use lance_index::vector::VectorIndexCacheEntry;
 use lance_index::vector::bq::builder::RabitQuantizer;
+use lance_index::vector::bq::ex_dot::{blocked_ex_code_bytes, padded_query_len};
+use lance_index::vector::bq::rabit_ex_bits;
+use lance_index::vector::bq::storage::{RabitQueryEstimator, SEGMENT_NUM_CODES};
 use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer};
 use lance_index::vector::graph::OrderedNode;
 use lance_index::vector::hnsw::HNSW;
@@ -48,7 +55,8 @@ use lance_index::vector::quantizer::{
 };
 use lance_index::vector::sq::ScalarQuantizer;
 use lance_index::vector::storage::{
-    QueryResidual, QueryScratch, QueryScratchCapacity, QueryScratchPool, VectorStore,
+    QueryResidual, QueryScratch, QueryScratchCapacity, QueryScratchPool, RabitRawQueryContext,
+    VectorStore,
 };
 use lance_index::vector::v3::subindex::SubIndexType;
 use lance_index::{
@@ -61,7 +69,7 @@ use lance_index::{
 };
 use lance_index::{INDEX_METADATA_SCHEMA_KEY, IndexMetadata};
 use lance_io::local::to_local_path;
-use lance_io::scheduler::SchedulerConfig;
+use lance_io::scheduler::{IoStats, ScanStats, SchedulerConfig};
 use lance_io::utils::CachedFileSize;
 use lance_io::{
     ReadBatchParams, object_store::ObjectStore, scheduler::ScanScheduler, traits::Reader,
@@ -73,9 +81,12 @@ use roaring::RoaringBitmap;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 use tracing::{info, instrument};
+use uuid::Uuid;
 
 use super::{IvfIndexPartitionStatistics, IvfIndexStatistics, maybe_centroids_for_stats};
 
+pub(crate) type RabitSearchCacheCell = Arc<Mutex<Option<Option<Arc<RabitSearchCache>>>>>;
+
 /// Serializable state of an IVF index, sufficient to reconstruct the index
 /// without re-reading global buffers from object storage.
 ///
@@ -107,19 +118,126 @@ pub(crate) struct IvfIndexState<Q: Quantization> {
     /// when reconstructing from cache.
     pub(crate) index_file_size: u64,
     pub(crate) aux_file_size: u64,
+    /// Runtime-only cache, intentionally excluded from the CacheCodec wire format.
+    pub(crate) rq_search_cache: RabitSearchCacheCell,
 }
 
+/// Number of prepared partitions handed to a single `spawn_cpu` dispatch on the
+/// streaming search path.
+///
+/// The streaming path deliberately avoids per-partition CPU-task fan-out (a measured
+/// 14-30% latency win, see #6475). Searching a batch of partitions per `spawn_cpu`
+/// keeps most of that benefit — the per-dispatch overhead is paid once per
+/// `STREAMING_SEARCH_BATCH_SIZE` partitions instead of once per partition — while
+/// keeping the channel `recv`/`send` in async code so no CPU-pool thread ever parks on
+/// a channel (which can deadlock the pool on small hosts, see #7642). `should_stop` is
+/// still checked per partition, so early-stop granularity is unchanged.
+///
+/// This is a tunable knob: larger batches amortize dispatch overhead further and keep
+/// more work on a single CPU thread, at the cost of more prepared partitions held in
+/// memory at once. The batch is an upper bound: the search loop greedily drains
+/// whatever is already prepared rather than waiting for a full batch, so a slow
+/// producer yields small batches (matching the old search-as-it-arrives latency) and
+/// only a fast producer fills whole ones. Override with the
+/// `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE` environment variable.
+pub(crate) const DEFAULT_STREAMING_SEARCH_BATCH_SIZE: usize = 16;
+
+pub(crate) static STREAMING_SEARCH_BATCH_SIZE: LazyLock<usize> = LazyLock::new(|| {
+    let batch_size = std::env::var("LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE")
+        .map(|value| {
+            value
+                .parse()
+                .expect("failed to parse LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE")
+        })
+        .unwrap_or(DEFAULT_STREAMING_SEARCH_BATCH_SIZE);
+    assert!(
+        batch_size > 0,
+        "LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE must be greater than 0, got {batch_size}"
+    );
+    batch_size
+});
+
 struct PreparedPartitionSearch<S: IvfSubIndex, Q: Quantization> {
     query: Query,
     pre_filter: Arc<dyn PreFilter>,
     partition_id: usize,
     partition_centroid: Option<ArrayRef>,
+    rq_search_cache: Option<Arc<RabitSearchCache>>,
+    raw_query_context: Option<Arc<RabitRawQueryContext>>,
     part_entry: Arc<dyn VectorIndexCacheEntry>,
     _marker: PhantomData<(S, Q)>,
 }
 
+#[derive(Debug)]
+pub(crate) struct RabitSearchCache {
+    rotated_centroids: Vec<f32>,
+    code_dim: usize,
+}
+
+pub(crate) fn empty_rabit_search_cache_cell() -> RabitSearchCacheCell {
+    Arc::new(Mutex::new(None))
+}
+
+fn rabit_search_cache_cell(cache: Option<Arc<RabitSearchCache>>) -> RabitSearchCacheCell {
+    Arc::new(Mutex::new(Some(cache)))
+}
+
+fn rotated_partition_centroid_slice(
+    cache: Option<&RabitSearchCache>,
+    partition_id: usize,
+) -> Option<&[f32]> {
+    let cache = cache?;
+    let start = partition_id.checked_mul(cache.code_dim)?;
+    let end = start.checked_add(cache.code_dim)?;
+    cache.rotated_centroids.get(start..end)
+}
+
+/// `f32` scratch needed for the ex-bit query state: a zero-padded query copy
+/// when the rotated dim is not a multiple of the 64-dim kernel block (the
+/// FastScan ex LUT is built directly from the query, with no f32 table).
+fn rabit_ex_scratch_len(dim: usize, num_bits: u8) -> usize {
+    let multi_bit = rabit_ex_bits(num_bits)
+        .map(|ex_bits| ex_bits > 0)
+        .unwrap_or(true);
+    if !multi_bit || dim.is_multiple_of(64) {
+        0
+    } else {
+        padded_query_len(dim)
+    }
+}
+
+fn rabit_u8_scratch_len(dim: usize, num_bits: u8) -> usize {
+    let binary_dist_table_len = dim * 4;
+    let ex_dist_table_len = rabit_ex_bits(num_bits)
+        .ok()
+        .and_then(|ex_bits| match ex_bits {
+            2 | 4 | 8 => Some(blocked_ex_code_bytes(dim, ex_bits)),
+            _ => None,
+        })
+        .map(|ex_code_len| ex_code_len * 2 * SEGMENT_NUM_CODES)
+        .unwrap_or_default();
+    binary_dist_table_len.max(ex_dist_table_len)
+}
+
+fn rabit_query_scratch_capacity(
+    dim: usize,
+    max_partition_len: usize,
+    num_bits: u8,
+) -> QueryScratchCapacity {
+    let dist_table_len = dim * 4;
+    let ex_scratch_len = rabit_ex_scratch_len(dim, num_bits);
+    let u8_scratch_len = rabit_u8_scratch_len(dim, num_bits);
+
+    QueryScratchCapacity::new(
+        max_partition_len,
+        dim + dist_table_len + ex_scratch_len,
+        max_partition_len.max(dist_table_len),
+        u8_scratch_len,
+    )
+}
+
 impl<Q: Quantization> DeepSizeOf for IvfIndexState<Q> {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.index_file_path.deep_size_of_children(context)
             + self.uuid.deep_size_of_children(context)
             + self.ivf.deep_size_of_children(context)
@@ -127,31 +245,16 @@ impl<Q: Quantization> DeepSizeOf for IvfIndexState<Q> {
             + self.sub_index_metadata.deep_size_of_children(context)
             + self.metadata.deep_size_of_children(context)
             + self.cache_key_prefix.deep_size_of_children(context)
+            + self
+                .rq_search_cache
+                .lock()
+                .ok()
+                .and_then(|cache| cache.as_ref().and_then(|cache| cache.as_ref().cloned()))
+                .map(|cache| cache.rotated_centroids.len() * std::mem::size_of::<f32>())
+                .unwrap_or_default()
     }
 }
 
-/// Serialization header for the `IvfIndexState` wire format.
-///
-/// Kept as a flat, non-generic struct so the JSON header format is stable
-/// regardless of `Q`. `quantizer_metadata_json` holds the serialized
-/// `Q::Metadata`; large blobs (PQ codebook, RQ matrix) follow as raw bytes.
-#[derive(serde::Serialize, serde::Deserialize)]
-struct IvfIndexStateHeader {
-    index_file_path: String,
-    uuid: String,
-    distance_type: String,
-    sub_index_metadata: Vec<String>,
-    sub_index_type: String,
-    quantization_type: String,
-    quantizer_metadata_json: String,
-    #[serde(default)]
-    cache_key_prefix: String,
-    #[serde(default)]
-    index_file_size: u64,
-    #[serde(default)]
-    aux_file_size: u64,
-}
-
 /// Object-safe interface for a type-erased `IvfIndexState<Q>`.
 ///
 /// Stored as `Arc<dyn IvfStateEntry>` inside [`IvfStateEntryBox`], which is
@@ -159,13 +262,14 @@ struct IvfIndexStateHeader {
 /// wrapper lets the cache infrastructure work with a sized type while the
 /// hot paths call `reconstruct` without knowing `Q`.
 pub(crate) trait IvfStateEntry: DeepSizeOf + Send + Sync + 'static {
-    fn serialize_state(&self, writer: &mut dyn IoWrite) -> Result<()>;
+    fn serialize_state(&self, w: &mut CacheEntryWriter<'_>) -> Result<()>;
 
     fn reconstruct<'a>(
         &'a self,
         object_store: Arc<ObjectStore>,
         file_metadata_cache: &'a LanceCache,
         index_cache: LanceCache,
+        frag_reuse_index: Option<Arc<FragReuseIndex>>,
     ) -> BoxFuture<'a, Result<Arc<dyn VectorIndex>>>;
 }
 
@@ -178,47 +282,44 @@ pub(crate) trait IvfStateEntry: DeepSizeOf + Send + Sync + 'static {
 pub(crate) struct IvfStateEntryBox(pub(crate) Arc<dyn IvfStateEntry>);
 
 impl DeepSizeOf for IvfStateEntryBox {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.0.deep_size_of_children(context)
     }
 }
 
-/// Wire format (unchanged from the non-generic `IvfIndexState`):
-/// `[header_json_len: u64 LE][header JSON][ivf_pb_len: u64 LE][ivf protobuf]
-///  [extra_len: u64 LE][extra bytes][aux_ivf_pb_len: u64 LE][aux_ivf protobuf]`
+/// Wire format:
+/// ```text
+/// HEADER   : IvfStateHeader proto (paths, types, quantizer metadata JSON)
+/// RAW_BLOB : IVF model protobuf
+/// RAW_BLOB : quantizer extra-metadata buffer (may be empty)
+/// RAW_BLOB : auxiliary IVF model protobuf
+/// ```
 impl CacheCodecImpl for IvfStateEntryBox {
-    fn serialize(&self, writer: &mut dyn IoWrite) -> Result<()> {
-        self.0.serialize_state(writer)
-    }
+    const TYPE_ID: &'static str = "lance.vector.ivf.IvfState";
+    const CURRENT_VERSION: u32 = 1;
 
-    fn deserialize(data: &bytes::Bytes) -> Result<Self> {
-        use lance_arrow::ipc::read_len_prefixed_bytes_at;
+    fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
+        self.0.serialize_state(w)
+    }
 
-        // Parse the common wire format, then dispatch on quantization_type to
+    fn deserialize(r: &mut CacheEntryReader<'_>) -> Result<Self> {
+        // Parse the common header, then dispatch on quantization_type to
         // construct the right IvfIndexState<Q>.
-        let mut offset = 0;
-        let header_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
-        let header: IvfIndexStateHeader = serde_json::from_slice(&header_bytes)
-            .map_err(|e| lance_core::Error::io(format!("IvfIndexState header: {e}")))?;
+        let header: IvfStateHeader = r.read_header()?;
 
-        let ivf_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let ivf_bytes = r.read_raw()?;
         let ivf = IvfModel::try_from(
             pb::Ivf::decode(ivf_bytes.as_ref())
                 .map_err(|e| lance_core::Error::io(format!("IvfIndexState IVF decode: {e}")))?,
         )?;
 
-        let extra_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let extra_bytes = r.read_raw()?;
 
-        // aux_ivf was added after initial deployment; fall back to ivf on
-        // clean EOF (legacy format without the field).
-        let aux_ivf = if offset + 8 <= data.len() {
-            let aux_ivf_bytes = read_len_prefixed_bytes_at(data, &mut offset)?;
+        let aux_ivf_bytes = r.read_raw()?;
+        let aux_ivf =
             IvfModel::try_from(pb::Ivf::decode(aux_ivf_bytes.as_ref()).map_err(|e| {
                 lance_core::Error::io(format!("IvfIndexState aux IVF decode: {e}"))
-            })?)?
-        } else {
-            ivf.clone()
-        };
+            })?)?;
 
         let distance_type = DistanceType::try_from(header.distance_type.as_str())?;
         let sub_index_type = SubIndexType::try_from(header.sub_index_type.as_str())?;
@@ -227,7 +328,7 @@ impl CacheCodecImpl for IvfStateEntryBox {
         // Helper: parse Q::Metadata from the JSON+extra_bytes in the header,
         // then build an IvfStateEntryBox wrapping IvfIndexState<Q>.
         fn make_entry<Q: Quantization + 'static>(
-            header: IvfIndexStateHeader,
+            header: IvfStateHeader,
             ivf: IvfModel,
             aux_ivf: IvfModel,
             extra_bytes: bytes::Bytes,
@@ -258,6 +359,7 @@ impl CacheCodecImpl for IvfStateEntryBox {
                 cache_key_prefix: header.cache_key_prefix,
                 index_file_size: header.index_file_size,
                 aux_file_size: header.aux_file_size,
+                rq_search_cache: empty_rabit_search_cache_cell(),
             })))
         }
 
@@ -312,15 +414,15 @@ impl CacheCodecImpl for IvfStateEntryBox {
 }
 
 impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
-    fn serialize_state(&self, writer: &mut dyn IoWrite) -> Result<()> {
+    fn serialize_state(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> {
         let quantizer_metadata_json = serde_json::to_string(&self.metadata)
             .map_err(|e| lance_core::Error::io(format!("IvfIndexState metadata: {e}")))?;
         let extra = self.metadata.extra_metadata()?;
         let extra = extra.as_deref().unwrap_or(&[]);
 
-        let header = IvfIndexStateHeader {
+        let header = IvfStateHeader {
             index_file_path: self.index_file_path.clone(),
-            uuid: self.uuid.clone(),
+            uuid: self.uuid.to_string(),
             distance_type: self.distance_type.to_string(),
             sub_index_metadata: self.sub_index_metadata.clone(),
             sub_index_type: self.sub_index_type.to_string(),
@@ -330,15 +432,13 @@ impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
             index_file_size: self.index_file_size,
             aux_file_size: self.aux_file_size,
         };
-        let header_json = serde_json::to_vec(&header)
-            .map_err(|e| lance_core::Error::io(format!("IvfIndexState header: {e}")))?;
         let ivf_bytes = pb::Ivf::try_from(&self.ivf)?.encode_to_vec();
         let aux_ivf_bytes = pb::Ivf::try_from(&self.aux_ivf)?.encode_to_vec();
 
-        write_len_prefixed_bytes(writer, &header_json)?;
-        write_len_prefixed_bytes(writer, &ivf_bytes)?;
-        write_len_prefixed_bytes(writer, extra)?;
-        write_len_prefixed_bytes(writer, &aux_ivf_bytes)?;
+        w.write_header(&header)?;
+        w.write_raw(&ivf_bytes)?;
+        w.write_raw(extra)?;
+        w.write_raw(&aux_ivf_bytes)?;
         Ok(())
     }
 
@@ -347,6 +447,7 @@ impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
         object_store: Arc<ObjectStore>,
         file_metadata_cache: &'a LanceCache,
         index_cache: LanceCache,
+        frag_reuse_index: Option<Arc<FragReuseIndex>>,
     ) -> BoxFuture<'a, Result<Arc<dyn VectorIndex>>> {
         Box::pin(async move {
             match self.sub_index_type {
@@ -356,6 +457,7 @@ impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
                         object_store,
                         file_metadata_cache,
                         index_cache,
+                        frag_reuse_index,
                     )
                     .await
                 }
@@ -365,6 +467,7 @@ impl<Q: Quantization + 'static> IvfStateEntry for IvfIndexState<Q> {
                         object_store,
                         file_metadata_cache,
                         index_cache,
+                        frag_reuse_index,
                     )
                     .await
                 }
@@ -394,8 +497,8 @@ struct CachedIndexReaders {
     aux_reader: Arc<FileReader>,
 }
 
-impl deepsize::DeepSizeOf for CachedIndexReaders {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+impl lance_core::deepsize::DeepSizeOf for CachedIndexReaders {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         // FileReader doesn't impl DeepSizeOf. We approximate by counting the
         // fixed struct size for each reader plus the Arc<CachedFileMetadata>
         // heap contents. The metadata Arcs are also held by FileMetadataCacheKey
@@ -516,7 +619,7 @@ pub struct IVFIndex<S: IvfSubIndex + 'static, Q: Quantization + 'static> {
     /// Object-store path to the index file (forward-slash separated).
     /// Used by `cacheable_state()` for cross-platform reconstruction.
     index_path: String,
-    uuid: String,
+    uuid: Uuid,
 
     /// Ivf model
     ivf: IvfModel,
@@ -530,32 +633,115 @@ pub struct IVFIndex<S: IvfSubIndex + 'static, Q: Quantization + 'static> {
     index_cache: WeakLanceCache,
 
     io_parallelism: usize,
+    /// Cumulative I/O performed while opening this index (file footers, IVF
+    /// centroids, quantization metadata).  Captured once in `try_new`; exposed
+    /// via [`VectorIndex::open_io_stats`] so the opening query can attribute the
+    /// one-time open cost to its plan metrics.
+    open_io_stats: ScanStats,
     scratch_pool: Arc<QueryScratchPool>,
+    use_query_residual: bool,
     use_residual_scratch: bool,
+    rq_search_cache: Option<Arc<RabitSearchCache>>,
 
     _marker: PhantomData<(S, Q)>,
 }
 
 impl<S: IvfSubIndex, Q: Quantization> DeepSizeOf for IVFIndex<S, Q> {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // `Uuid` is a fixed 16-byte struct with no heap children, so contributes 0.
         self.uri.deep_size_of_children(context)
             + self.index_path.deep_size_of_children(context)
             + self.ivf.deep_size_of_children(context)
             + self.sub_index_metadata.deep_size_of_children(context)
-            + self.uuid.deep_size_of_children(context)
             + self.storage.deep_size_of_children(context)
             + self.scratch_pool.deep_size_of_children(context)
+            + self
+                .rq_search_cache
+                .as_ref()
+                .map(|cache| cache.rotated_centroids.len() * std::mem::size_of::<f32>())
+                .unwrap_or_default()
         // Skipping session since it is a weak ref
     }
 }
 
 impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
+    fn use_query_residual(
+        storage: &IvfQuantizationStorage<Q>,
+        distance_type: DistanceType,
+    ) -> bool {
+        if Q::quantization_type() == QuantizationType::Rabit
+            && let Ok(Quantizer::Rabit(rq)) = storage.quantizer()
+        {
+            return rq.metadata_ref().query_estimator == RabitQueryEstimator::ResidualQuery;
+        }
+        Q::use_residual(distance_type)
+    }
+
+    fn build_rq_search_cache(
+        ivf: &IvfModel,
+        storage: &IvfQuantizationStorage<Q>,
+    ) -> Result<Option<Arc<RabitSearchCache>>> {
+        if Q::quantization_type() != QuantizationType::Rabit {
+            return Ok(None);
+        }
+        let Quantizer::Rabit(rq) = storage.quantizer()? else {
+            return Ok(None);
+        };
+        if rq.metadata_ref().query_estimator != RabitQueryEstimator::RawQuery {
+            return Ok(None);
+        }
+        let centroids = ivf
+            .centroids_array()
+            .ok_or_else(|| Error::index("IVF_RQ raw-query search requires centroids"))?;
+        let rotated_centroids = rq.rotate_fsl_to_f32(centroids)?;
+        Ok(Some(Arc::new(RabitSearchCache {
+            rotated_centroids,
+            code_dim: rq.code_dim(),
+        })))
+    }
+
+    fn rq_search_cache_from_state(
+        state: &IvfIndexState<Q>,
+        storage: &IvfQuantizationStorage<Q>,
+    ) -> Result<Option<Arc<RabitSearchCache>>> {
+        let mut cache = state
+            .rq_search_cache
+            .lock()
+            .map_err(|_| Error::internal("RQ search cache lock was poisoned".to_string()))?;
+        if let Some(cache) = cache.as_ref() {
+            return Ok(cache.clone());
+        }
+        let built = Self::build_rq_search_cache(&state.ivf, storage)?;
+        *cache = Some(built.clone());
+        Ok(built)
+    }
+
+    fn prepare_rq_raw_query_context(
+        &self,
+        query: &ArrayRef,
+    ) -> Result<Option<Arc<RabitRawQueryContext>>> {
+        if Q::quantization_type() != QuantizationType::Rabit || self.use_query_residual {
+            return Ok(None);
+        }
+        let Quantizer::Rabit(rq) = self.storage.quantizer()? else {
+            return Ok(None);
+        };
+        if rq.metadata_ref().query_estimator != RabitQueryEstimator::RawQuery {
+            return Ok(None);
+        }
+        Ok(Some(Arc::new(
+            rq.metadata_ref()
+                .prepare_raw_query_context(query.as_ref())?,
+        )))
+    }
+
     async fn prepare_partition(
         &self,
         partition_id: usize,
         query: &Query,
         pre_filter: Arc<dyn PreFilter>,
         metrics: &dyn MetricsCollector,
+        raw_query_context: Option<Arc<RabitRawQueryContext>>,
     ) -> Result<PreparedPartitionSearch<S, Q>> {
         let (part_entry, ()) = tokio::try_join!(
             self.load_partition(partition_id, true, metrics),
@@ -566,6 +752,8 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             pre_filter,
             partition_id,
             partition_centroid: self.ivf.centroid(partition_id),
+            rq_search_cache: self.rq_search_cache.clone(),
+            raw_query_context,
             part_entry,
             _marker: PhantomData,
         })
@@ -577,6 +765,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         query: &Query,
         pre_filter: Arc<dyn PreFilter>,
         metrics: &dyn MetricsCollector,
+        raw_query_context: Option<Arc<RabitRawQueryContext>>,
     ) -> Result<PreparedPartitionSearch<S, Q>> {
         let part_entry = self.load_partition(partition_id, true, metrics).await?;
         Ok(PreparedPartitionSearch {
@@ -584,13 +773,15 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             pre_filter,
             partition_id,
             partition_centroid: self.ivf.centroid(partition_id),
+            rq_search_cache: self.rq_search_cache.clone(),
+            raw_query_context,
             part_entry,
             _marker: PhantomData,
         })
     }
 
     fn run_prepared_partition_search(
-        distance_type: DistanceType,
+        use_query_residual: bool,
         use_residual_scratch: bool,
         prepared: PreparedPartitionSearch<S, Q>,
         metrics: &dyn MetricsCollector,
@@ -601,16 +792,23 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             pre_filter,
             partition_id,
             partition_centroid,
+            rq_search_cache,
+            raw_query_context,
             part_entry,
             _marker: _,
         } = prepared;
-        let residual = Self::residual_for_scratch(
+        let rotated_partition_centroid =
+            rotated_partition_centroid_slice(rq_search_cache.as_deref(), partition_id);
+        let residual = Self::query_context_for_scratch(
+            use_query_residual,
             use_residual_scratch,
             partition_id,
             partition_centroid.as_ref(),
+            rotated_partition_centroid,
+            raw_query_context.as_deref(),
         )?;
         let query = Self::preprocess_partition_query_owned(
-            distance_type,
+            use_query_residual,
             use_residual_scratch,
             partition_id,
             partition_centroid.as_ref(),
@@ -640,7 +838,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
 
     #[allow(clippy::too_many_arguments)]
     fn accumulate_prepared_partition_search(
-        distance_type: DistanceType,
+        use_query_residual: bool,
         use_residual_scratch: bool,
         prepared: PreparedPartitionSearch<S, Q>,
         heap: &mut BinaryHeap<OrderedNode<u64>>,
@@ -652,16 +850,23 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             pre_filter,
             partition_id,
             partition_centroid,
+            rq_search_cache,
+            raw_query_context,
             part_entry,
             _marker: _,
         } = prepared;
-        let residual = Self::residual_for_scratch(
+        let rotated_partition_centroid =
+            rotated_partition_centroid_slice(rq_search_cache.as_deref(), partition_id);
+        let residual = Self::query_context_for_scratch(
+            use_query_residual,
             use_residual_scratch,
             partition_id,
             partition_centroid.as_ref(),
+            rotated_partition_centroid,
+            raw_query_context.as_deref(),
         )?;
         let query = Self::preprocess_partition_query_owned(
-            distance_type,
+            use_query_residual,
             use_residual_scratch,
             partition_id,
             partition_centroid.as_ref(),
@@ -689,16 +894,26 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         )
     }
 
-    fn residual_for_scratch<'a>(
+    fn query_context_for_scratch<'a>(
+        use_query_residual: bool,
         use_residual_scratch: bool,
         partition_id: usize,
         partition_centroid: Option<&'a ArrayRef>,
+        rotated_partition_centroid: Option<&'a [f32]>,
+        raw_query_context: Option<&'a RabitRawQueryContext>,
     ) -> Result<Option<QueryResidual<'a>>> {
         if use_residual_scratch {
             let partition_centroid = partition_centroid.ok_or_else(|| {
                 Error::index(format!("partition centroid {partition_id} does not exist"))
             })?;
             Ok(Some(QueryResidual::Centroid(partition_centroid.as_ref())))
+        } else if !use_query_residual
+            && (rotated_partition_centroid.is_some() || raw_query_context.is_some())
+        {
+            Ok(Some(QueryResidual::RabitRawQuery {
+                rotated_centroid: rotated_partition_centroid,
+                query: raw_query_context,
+            }))
         } else {
             Ok(None)
         }
@@ -716,14 +931,14 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
     }
 
     fn preprocess_partition_query(
-        distance_type: DistanceType,
+        use_query_residual: bool,
         use_residual_scratch: bool,
         partition_id: usize,
         partition_centroid: Option<&ArrayRef>,
         query: &Query,
     ) -> Result<Query> {
         Self::preprocess_partition_query_owned(
-            distance_type,
+            use_query_residual,
             use_residual_scratch,
             partition_id,
             partition_centroid,
@@ -732,13 +947,13 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
     }
 
     fn preprocess_partition_query_owned(
-        distance_type: DistanceType,
+        use_query_residual: bool,
         use_residual_scratch: bool,
         partition_id: usize,
         partition_centroid: Option<&ArrayRef>,
         mut query: Query,
     ) -> Result<Query> {
-        if Q::use_residual(distance_type) {
+        if use_query_residual {
             let partition_centroid = partition_centroid.ok_or_else(|| {
                 Error::index(format!("partition centroid {partition_id} does not exist"))
             })?;
@@ -751,36 +966,37 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         Ok(query)
     }
 
-    fn query_scratch_capacity(ivf: &IvfModel) -> QueryScratchCapacity {
+    fn query_scratch_capacity(
+        ivf: &IvfModel,
+        storage: &IvfQuantizationStorage<Q>,
+    ) -> QueryScratchCapacity {
         if Q::quantization_type() != QuantizationType::Rabit {
             return QueryScratchCapacity::default();
         }
 
         let dim = ivf.dimension();
-        let dist_table_len = dim * 4;
         let max_partition_len = ivf.lengths.iter().copied().max().unwrap_or_default() as usize;
+        let num_bits = match storage.quantizer() {
+            Ok(Quantizer::Rabit(rq)) => rq.metadata_ref().num_bits,
+            _ => 9,
+        };
 
-        QueryScratchCapacity::new(
-            max_partition_len,
-            dim + dist_table_len,
-            max_partition_len,
-            dist_table_len,
-        )
+        rabit_query_scratch_capacity(dim, max_partition_len, num_bits)
     }
 
-    fn use_residual_scratch(ivf: &IvfModel, distance_type: DistanceType) -> bool {
+    fn use_residual_scratch(ivf: &IvfModel, use_query_residual: bool) -> bool {
         Q::quantization_type() == QuantizationType::Rabit
-            && Q::use_residual(distance_type)
+            && use_query_residual
             && ivf
                 .centroids_array()
                 .map(|centroids| centroids.value_type() == DataType::Float32)
                 .unwrap_or(false)
     }
 
-    fn query_scratch_pool(ivf: &IvfModel) -> QueryScratchPool {
+    fn query_scratch_pool(ivf: &IvfModel, storage: &IvfQuantizationStorage<Q>) -> QueryScratchPool {
         QueryScratchPool::with_capacity(
             get_num_compute_intensive_cpus(),
-            Self::query_scratch_capacity(ivf),
+            Self::query_scratch_capacity(ivf, storage),
         )
     }
 
@@ -788,7 +1004,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
     pub(crate) async fn try_new(
         object_store: Arc<ObjectStore>,
         index_dir: Path,
-        uuid: String,
+        uuid: Uuid,
         frag_reuse_index: Option<Arc<FragReuseIndex>>,
         file_metadata_cache: &LanceCache,
         index_cache: LanceCache,
@@ -798,7 +1014,11 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         let scheduler_config = SchedulerConfig::max_bandwidth(&object_store);
         let scheduler = ScanScheduler::new(object_store, scheduler_config);
 
-        let uri = index_dir.clone().join(uuid.as_str()).join(INDEX_FILE_NAME);
+        let uuid_str = uuid.to_string();
+        let uri = index_dir
+            .clone()
+            .join(uuid_str.as_str())
+            .join(INDEX_FILE_NAME);
         let cached_size = file_sizes
             .get(INDEX_FILE_NAME)
             .map(|&size| CachedFileSize::new(size))
@@ -847,7 +1067,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
                 .open_file(
                     &index_dir
                         .clone()
-                        .join(uuid.as_str())
+                        .join(uuid_str.as_str())
                         .join(INDEX_AUXILIARY_FILE_NAME),
                     &aux_cached_size,
                 )
@@ -869,7 +1089,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             .await;
         let aux_path = index_dir
             .clone()
-            .join(uuid.as_str())
+            .join(uuid_str.as_str())
             .join(INDEX_AUXILIARY_FILE_NAME);
         file_metadata_cache
             .with_key_prefix(aux_path.as_ref())
@@ -879,7 +1099,9 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         // Cache open readers so the first reconstruction also skips file opens.
         file_metadata_cache
             .insert_with_key(
-                &CachedIndexReadersKey { uuid: uuid.clone() },
+                &CachedIndexReadersKey {
+                    uuid: uuid_str.clone(),
+                },
                 Arc::new(CachedIndexReaders {
                     index_reader: Arc::new(index_reader.clone()),
                     aux_reader: Arc::new(storage.reader().clone()),
@@ -887,15 +1109,25 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             )
             .await;
 
-        let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf));
-        let use_residual_scratch = Self::use_residual_scratch(&ivf, distance_type);
+        let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage));
+        let use_query_residual = Self::use_query_residual(&storage, distance_type);
+        let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual);
+        let rq_search_cache = Self::build_rq_search_cache(&ivf, &storage)?;
+
+        // The scheduler is freshly created above and, at this point, has served
+        // only the open-time reads (file footers, IVF centroids, quantization
+        // metadata) -- partition reads happen later, during queries.  So its
+        // cumulative stats are exactly the one-time index-open I/O.
+        let open_io_stats = scheduler.stats();
 
         Ok(Self {
             uri: to_local_path(&uri),
             index_path: uri.as_ref().to_string(),
             uuid,
             scratch_pool,
+            use_query_residual,
             use_residual_scratch,
+            rq_search_cache,
             ivf,
             reader: index_reader,
             storage,
@@ -903,6 +1135,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             distance_type,
             index_cache: WeakLanceCache::from(&index_cache),
             io_parallelism,
+            open_io_stats,
             _marker: PhantomData,
         })
     }
@@ -912,7 +1145,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
     pub(crate) fn from_cached_state(
         uri: String,
         index_path: String,
-        uuid: String,
+        uuid: Uuid,
         ivf: IvfModel,
         reader: FileReader,
         storage: IvfQuantizationStorage<Q>,
@@ -920,15 +1153,19 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         distance_type: DistanceType,
         index_cache: LanceCache,
         io_parallelism: usize,
+        rq_search_cache: Option<Arc<RabitSearchCache>>,
     ) -> Self {
-        let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf));
-        let use_residual_scratch = Self::use_residual_scratch(&ivf, distance_type);
+        let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage));
+        let use_query_residual = Self::use_query_residual(&storage, distance_type);
+        let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual);
         Self {
             uri,
             index_path,
             uuid,
             scratch_pool,
+            use_query_residual,
             use_residual_scratch,
+            rq_search_cache,
             ivf,
             reader,
             storage,
@@ -936,6 +1173,10 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             distance_type,
             index_cache: WeakLanceCache::from(&index_cache),
             io_parallelism,
+            // Reconstruction from cached state re-opens readers on its own path;
+            // the open-time I/O is not attributed here (it is a one-time cost,
+            // and the first open via `try_new` already accounts for it).
+            open_io_stats: ScanStats::default(),
             _marker: PhantomData,
         }
     }
@@ -963,7 +1204,8 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
                 .get_or_insert_with_key(cache_key, || async {
                     info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id);
                     metrics.record_part_load();
-                    self.load_partition_entry(partition_id).await
+                    self.load_partition_entry(partition_id, metrics.io_stats())
+                        .await
                 })
                 .await?;
             Ok(entry as Arc<dyn VectorIndexCacheEntry>)
@@ -973,11 +1215,18 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             }
             info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id);
             metrics.record_part_load();
-            Ok(Arc::new(self.load_partition_entry(partition_id).await?))
+            Ok(Arc::new(
+                self.load_partition_entry(partition_id, metrics.io_stats())
+                    .await?,
+            ))
         }
     }
 
-    async fn load_partition_entry(&self, partition_id: usize) -> Result<PartitionEntry<S, Q>> {
+    async fn load_partition_entry(
+        &self,
+        partition_id: usize,
+        io_stats: Option<IoStats>,
+    ) -> Result<PartitionEntry<S, Q>> {
         let schema = Arc::new(self.reader.schema().as_ref().into());
         let batch = match self.reader.metadata().num_rows {
             0 => RecordBatch::new_empty(schema),
@@ -986,8 +1235,17 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
                 if row_range.is_empty() {
                     RecordBatch::new_empty(schema)
                 } else {
-                    let batches = self
-                        .reader
+                    // When I/O is being measured, read through a reader whose
+                    // scheduler also records into the per-query sink (a cheap
+                    // clone sharing all cached metadata, no file re-open).
+                    // Otherwise borrow the shared reader as-is, with no clone.
+                    let reader = match &io_stats {
+                        Some(io_stats) => {
+                            Cow::Owned(self.reader.with_io_stats(io_stats.recorder()))
+                        }
+                        None => Cow::Borrowed(&self.reader),
+                    };
+                    let batches = reader
                         .read_stream(
                             ReadBatchParams::Range(row_range),
                             u32::MAX,
@@ -1006,15 +1264,19 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             self.sub_index_metadata[partition_id].clone(),
         )?;
         let idx = S::load(batch)?;
-        let storage = self.load_partition_storage(partition_id).await?;
+        let storage = self.load_partition_storage(partition_id, io_stats).await?;
         Ok(PartitionEntry {
             index: idx,
             storage,
         })
     }
 
-    pub async fn load_partition_storage(&self, partition_id: usize) -> Result<Q::Storage> {
-        self.storage.load_partition(partition_id).await
+    pub async fn load_partition_storage(
+        &self,
+        partition_id: usize,
+        io_stats: Option<IoStats>,
+    ) -> Result<Q::Storage> {
+        self.storage.load_partition(partition_id, io_stats).await
     }
 
     /// preprocess the query vector given the partition id.
@@ -1023,7 +1285,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
     #[instrument(level = "debug", skip(self))]
     pub fn preprocess_query(&self, partition_id: usize, query: &Query) -> Result<Query> {
         Self::preprocess_partition_query(
-            self.distance_type,
+            self.use_query_residual,
             self.use_residual_scratch,
             partition_id,
             self.ivf.centroid(partition_id).as_ref(),
@@ -1036,7 +1298,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
         let (sub_index_type, quantization_type) = self.sub_index_type();
         IvfStateEntryBox(Arc::new(IvfIndexState::<Q> {
             index_file_path: self.index_path.clone(),
-            uuid: self.uuid.clone(),
+            uuid: self.uuid.to_string(),
             ivf: self.ivf.clone(),
             aux_ivf: self.storage.ivf().clone(),
             distance_type: self.distance_type,
@@ -1047,6 +1309,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization> IVFIndex<S, Q> {
             cache_key_prefix: self.index_cache.prefix().to_string(),
             index_file_size: self.reader.metadata().file_size(),
             aux_file_size: self.storage.reader().metadata().file_size(),
+            rq_search_cache: rabit_search_cache_cell(self.rq_search_cache.clone()),
         }))
     }
 }
@@ -1061,10 +1324,6 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> Index for IVFIndex<S,
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Ok(self)
-    }
-
     async fn prewarm(&self) -> Result<()> {
         futures::stream::iter(0..self.ivf.num_partitions())
             .map(Ok)
@@ -1149,7 +1408,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> Index for IVFIndex<S,
 
         Ok(serde_json::to_value(IvfIndexStatistics {
             index_type,
-            uuid: self.uuid.clone(),
+            uuid: self.uuid.to_string(),
             uri: self.uri.clone(),
             metric_type: self.distance_type.to_string(),
             num_partitions: self.ivf.num_partitions(),
@@ -1208,15 +1467,19 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         let part_entry = self.load_partition(partition_id, true, metrics).await?;
         pre_filter.wait_for_ready().await?;
 
-        let residual_centroid = if self.use_residual_scratch {
-            Some(self.ivf.centroid(partition_id).ok_or_else(|| {
-                Error::index(format!("partition centroid {partition_id} does not exist"))
-            })?)
-        } else {
-            None
-        };
-        let query = self.preprocess_query(partition_id, query)?;
+        let partition_centroid = self.ivf.centroid(partition_id);
+        let rq_search_cache = self.rq_search_cache.clone();
+        let raw_query_context = self.prepare_rq_raw_query_context(&query.key)?;
+        let query = Self::preprocess_partition_query(
+            self.use_query_residual,
+            self.use_residual_scratch,
+            partition_id,
+            partition_centroid.as_ref(),
+            query,
+        )?;
         let scratch_pool = self.scratch_pool.clone();
+        let use_query_residual = self.use_query_residual;
+        let use_residual_scratch = self.use_residual_scratch;
         let (batch, local_metrics) = spawn_cpu(move || {
             let param = (&query).into();
             let refine_factor = query.refine_factor.unwrap_or(1) as usize;
@@ -1228,7 +1491,16 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                 .ok_or(Error::internal(
                     "failed to downcast partition entry".to_string(),
                 ))?;
-            let residual = residual_centroid.as_deref().map(QueryResidual::Centroid);
+            let rotated_partition_centroid =
+                rotated_partition_centroid_slice(rq_search_cache.as_deref(), partition_id);
+            let residual = Self::query_context_for_scratch(
+                use_query_residual,
+                use_residual_scratch,
+                partition_id,
+                partition_centroid.as_ref(),
+                rotated_partition_centroid,
+                raw_query_context.as_deref(),
+            )?;
             let batch = scratch_pool.with_scratch(|scratch| {
                 part.index.search_with_scratch(
                     query.key,
@@ -1257,8 +1529,9 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         pre_filter: Arc<dyn PreFilter>,
         metrics: &dyn MetricsCollector,
     ) -> Result<PreparedPartitionSearchHandle> {
+        let raw_query_context = self.prepare_rq_raw_query_context(&query.key)?;
         Ok(Box::new(
-            self.prepare_partition(partition_id, query, pre_filter, metrics)
+            self.prepare_partition(partition_id, query, pre_filter, metrics, raw_query_context)
                 .await?,
         ))
     }
@@ -1273,7 +1546,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
             .map_err(|_| Error::internal("failed to downcast prepared partition search"))?;
         self.scratch_pool.with_scratch(|scratch| {
             Self::run_prepared_partition_search(
-                self.distance_type,
+                self.use_query_residual,
                 self.use_residual_scratch,
                 *prepared,
                 metrics,
@@ -1321,12 +1594,14 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         }
 
         let prepare_parallelism = get_num_compute_intensive_cpus().max(1);
+        let raw_query_context = self.prepare_rq_raw_query_context(&query.key)?;
 
         if control.is_none() && S::supports_global_topk_heap() {
             let heap_capacity = query.k * query.refine_factor.unwrap_or(1) as usize;
             pre_filter.wait_for_ready().await?;
             let prepare_index = self.clone();
             let prepare_metrics = metrics.clone();
+            let prepare_raw_query_context = raw_query_context.clone();
             let prepared = stream::iter(start_idx..end_idx)
                 .map(move |idx| {
                     let part_id = partitions.value(idx);
@@ -1335,6 +1610,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                     let index = prepare_index.clone();
                     let pre_filter = pre_filter.clone();
                     let metrics = prepare_metrics.clone();
+                    let raw_query_context = prepare_raw_query_context.clone();
                     async move {
                         index
                             .prepare_partition_without_prefilter_wait(
@@ -1342,6 +1618,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                                 &query,
                                 pre_filter,
                                 metrics.as_ref(),
+                                raw_query_context,
                             )
                             .await
                     }
@@ -1350,7 +1627,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                 .try_collect::<Vec<_>>()
                 .await?;
 
-            let distance_type = self.distance_type;
+            let use_query_residual = self.use_query_residual;
             let use_residual_scratch = self.use_residual_scratch;
             let search_metrics = metrics.clone();
             let scratch_pool = self.scratch_pool.clone();
@@ -1359,7 +1636,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                 scratch_pool.with_scratch(|scratch| -> DataFusionResult<()> {
                     for prepared in prepared {
                         Self::accumulate_prepared_partition_search(
-                            distance_type,
+                            use_query_residual,
                             use_residual_scratch,
                             prepared,
                             &mut heap,
@@ -1380,12 +1657,16 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
             )));
         }
 
+        // The prepared channel holds a full search batch so that partitions prepared
+        // while the previous batch is being searched are ready for the next greedy
+        // drain, instead of serializing producer and consumer through a single slot.
         let (prepared_tx, mut prepared_rx) =
-            mpsc::channel::<Result<PreparedPartitionSearch<S, Q>>>(1);
+            mpsc::channel::<Result<PreparedPartitionSearch<S, Q>>>(*STREAMING_SEARCH_BATCH_SIZE);
         let (batch_tx, batch_rx) = mpsc::channel::<DataFusionResult<RecordBatch>>(1);
 
         let prepare_index = self.clone();
         let prepare_metrics = metrics.clone();
+        let prepare_raw_query_context = raw_query_context.clone();
         tokio::spawn(async move {
             let prepare_stream = stream::iter(start_idx..end_idx)
                 .map(move |idx| {
@@ -1395,6 +1676,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                     let index = prepare_index.clone();
                     let pre_filter = pre_filter.clone();
                     let metrics = prepare_metrics.clone();
+                    let raw_query_context = prepare_raw_query_context.clone();
                     async move {
                         index
                             .prepare_partition(
@@ -1402,6 +1684,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
                                 &query,
                                 pre_filter,
                                 metrics.as_ref(),
+                                raw_query_context,
                             )
                             .await
                     }
@@ -1417,64 +1700,142 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
             }
         });
 
-        let distance_type = self.distance_type;
+        let use_query_residual = self.use_query_residual;
         let use_residual_scratch = self.use_residual_scratch;
         let search_metrics = metrics.clone();
-        let batch_tx_for_search = batch_tx.clone();
         let search_control = control.clone();
         let scratch_pool = self.scratch_pool.clone();
+        // Search prepared partitions in batches. Each batch is searched in a single
+        // `spawn_cpu` dispatch (amortizing the per-dispatch overhead the single-worker
+        // design in #6475 avoided), but the channel `recv`/`send` stay in async code so
+        // no CPU-pool thread ever parks on a channel — parking one can deadlock the pool
+        // on small hosts (#7642). `should_stop` is checked per partition, so early-stop
+        // granularity is unchanged.
+        //
+        // Batches are formed greedily: wait for one prepared partition, then drain
+        // whatever else is already prepared, up to the batch size. Waiting for a full
+        // batch instead would delay the first search (and the early-stop feedback it
+        // produces) behind up to a whole batch of prepare I/O, which is significant
+        // when prepare parallelism is low.
         tokio::spawn(async move {
-            let search_result = spawn_cpu(move || -> DataFusionResult<()> {
-                scratch_pool.with_scratch(|scratch| {
-                    while let Some(prepared) = prepared_rx.blocking_recv() {
-                        let prepared = match prepared {
-                            Ok(prepared) => prepared,
-                            Err(err) => {
-                                let _ = batch_tx_for_search
-                                    .blocking_send(Err(DataFusionError::from(err)));
-                                return Ok(());
-                            }
-                        };
+            loop {
+                // Stop pulling as soon as the search is done — or the receiver of our
+                // results is gone — so the producer stops preparing partitions we
+                // would never search.
+                if search_control
+                    .as_ref()
+                    .is_some_and(|control| control.should_stop())
+                    || batch_tx.is_closed()
+                {
+                    return;
+                }
 
-                        if search_control
-                            .as_ref()
-                            .is_some_and(|control| control.should_stop())
-                        {
-                            return Ok(());
+                let mut prepared_batch = Vec::with_capacity(*STREAMING_SEARCH_BATCH_SIZE);
+                let mut prepare_error = None;
+                let mut producer_done = false;
+                match prepared_rx.recv().await {
+                    Some(Ok(prepared)) => prepared_batch.push(prepared),
+                    Some(Err(err)) => prepare_error = Some(DataFusionError::from(err)),
+                    None => producer_done = true,
+                }
+                while prepare_error.is_none()
+                    && !producer_done
+                    && prepared_batch.len() < *STREAMING_SEARCH_BATCH_SIZE
+                {
+                    match prepared_rx.try_recv() {
+                        Ok(Ok(prepared)) => prepared_batch.push(prepared),
+                        Ok(Err(err)) => {
+                            prepare_error = Some(DataFusionError::from(err));
                         }
-
-                        let batch = {
-                            Self::run_prepared_partition_search(
-                                distance_type,
-                                use_residual_scratch,
-                                prepared,
-                                search_metrics.as_ref(),
-                                scratch,
-                            )
+                        // Nothing else is prepared yet; search what we have rather
+                        // than waiting for more.
+                        Err(mpsc::error::TryRecvError::Empty) => break,
+                        Err(mpsc::error::TryRecvError::Disconnected) => {
+                            producer_done = true;
                         }
-                        .map_err(DataFusionError::from);
-                        match batch {
-                            Ok(batch) => {
-                                if let Some(control) = search_control.as_ref() {
-                                    control.record_batch(&batch);
+                    }
+                }
+
+                if !prepared_batch.is_empty() {
+                    let scratch_pool = scratch_pool.clone();
+                    let search_metrics = search_metrics.clone();
+                    let search_control = search_control.clone();
+                    // `is_closed` is synchronously callable, so a sender clone lets the
+                    // CPU loop notice a dropped receiver between partitions instead of
+                    // searching out the whole batch for a cancelled query. (A `select!`
+                    // on `closed()` would not help here: `spawn_cpu` closures are not
+                    // cancellable, so abandoning the await leaves the work running.)
+                    let cancel_probe = batch_tx.clone();
+                    let search_output = spawn_cpu(move || {
+                        let mut outputs: Vec<DataFusionResult<RecordBatch>> =
+                            Vec::with_capacity(prepared_batch.len());
+                        // `stopped` means the whole search should end (an error, an
+                        // early-stop signal, or cancellation), not just this batch.
+                        let mut stopped = false;
+                        scratch_pool.with_scratch(|scratch| {
+                            for prepared in prepared_batch {
+                                if search_control
+                                    .as_ref()
+                                    .is_some_and(|control| control.should_stop())
+                                    || cancel_probe.is_closed()
+                                {
+                                    stopped = true;
+                                    break;
                                 }
-                                if batch_tx_for_search.blocking_send(Ok(batch)).is_err() {
-                                    return Ok(());
+                                match Self::run_prepared_partition_search(
+                                    use_query_residual,
+                                    use_residual_scratch,
+                                    prepared,
+                                    search_metrics.as_ref(),
+                                    scratch,
+                                )
+                                .map_err(DataFusionError::from)
+                                {
+                                    Ok(batch) => {
+                                        if let Some(control) = search_control.as_ref() {
+                                            control.record_batch(&batch);
+                                        }
+                                        outputs.push(Ok(batch));
+                                    }
+                                    Err(err) => {
+                                        outputs.push(Err(err));
+                                        stopped = true;
+                                        break;
+                                    }
                                 }
                             }
-                            Err(err) => {
-                                let _ = batch_tx_for_search.blocking_send(Err(err));
-                                return Ok(());
-                            }
+                        });
+                        Ok::<_, DataFusionError>((outputs, stopped))
+                    })
+                    .await;
+
+                    let (outputs, stopped) = match search_output {
+                        Ok(output) => output,
+                        // Defensive: the closure always returns Ok (search errors are
+                        // captured per partition in `outputs`), so this arm should be
+                        // unreachable. Forward and stop rather than drop silently.
+                        Err(err) => {
+                            let _ = batch_tx.send(Err(err)).await;
+                            return;
+                        }
+                    };
+                    for output in outputs {
+                        if batch_tx.send(output).await.is_err() {
+                            return;
                         }
                     }
-                    Ok(())
-                })
-            })
-            .await;
+                    if stopped {
+                        return;
+                    }
+                }
 
-            if let Err(err) = search_result {
-                let _ = batch_tx.send(Err(err)).await;
+                if let Some(err) = prepare_error {
+                    let _ = batch_tx.send(Err(err)).await;
+                    return;
+                }
+                if producer_done {
+                    return;
+                }
             }
         });
 
@@ -1546,7 +1907,7 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
         todo!("this method is for only IVF_HNSW_* index");
     }
 
-    async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
         Err(Error::index(
             "Remapping IVF in this way not supported".to_string(),
         ))
@@ -1572,6 +1933,10 @@ impl<S: IvfSubIndex + 'static, Q: Quantization + 'static> VectorIndex for IVFInd
     fn metric_type(&self) -> DistanceType {
         self.distance_type
     }
+
+    fn open_io_stats(&self) -> Option<ScanStats> {
+        Some(self.open_io_stats)
+    }
 }
 
 pub type IvfFlatIndex = IVFIndex<FlatIndex, FlatQuantizer>;
@@ -1584,6 +1949,7 @@ async fn reconstruct_typed<S: IvfSubIndex + 'static, Q: Quantization + 'static>(
     object_store: Arc<ObjectStore>,
     file_metadata_cache: &LanceCache,
     index_cache: LanceCache,
+    frag_reuse_index: Option<Arc<FragReuseIndex>>,
 ) -> Result<Arc<dyn VectorIndex>> {
     let io_parallelism = object_store.io_parallelism();
 
@@ -1639,13 +2005,16 @@ async fn reconstruct_typed<S: IvfSubIndex + 'static, Q: Quantization + 'static>(
         state.aux_ivf.clone(),
         state.metadata.clone(),
         state.distance_type,
-        None,
+        frag_reuse_index,
     );
+    let rq_search_cache = IVFIndex::<S, Q>::rq_search_cache_from_state(state, &storage)?;
 
+    let parsed_uuid = Uuid::parse_str(&state.uuid)
+        .map_err(|e| Error::index(format!("Invalid UUID in IvfIndexState: {e}")))?;
     let index = IVFIndex::<S, Q>::from_cached_state(
         to_local_path(&index_path),
         index_path.to_string(),
-        state.uuid.clone(),
+        parsed_uuid,
         state.ivf.clone(),
         index_reader,
         storage,
@@ -1653,6 +2022,7 @@ async fn reconstruct_typed<S: IvfSubIndex + 'static, Q: Quantization + 'static>(
         state.distance_type,
         index_cache,
         io_parallelism,
+        rq_search_cache,
     );
     Ok(Arc::new(index))
 }
@@ -1675,7 +2045,10 @@ mod tests {
     use itertools::Itertools;
     use lance_arrow::FixedSizeListArrayExt;
     use lance_index::vector::bq::{
-        RQBuildParams, RQRotationType, storage::RabitQuantizationMetadata,
+        RQBuildParams, RQRotationType,
+        ex_dot::{blocked_ex_code_bytes, padded_query_len},
+        storage::{RABIT_BLOCKED_EX_CODE_COLUMN, RabitQuantizationMetadata, RabitQueryEstimator},
+        transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN},
     };
     use lance_index::vector::storage::VectorStore;
 
@@ -1699,8 +2072,10 @@ mod tests {
     use lance_file::reader::{FileReader, FileReaderOptions};
     use lance_file::writer::FileWriter;
     use lance_index::IndexType;
+    use lance_index::optimize::OptimizeOptions;
     use lance_index::progress::IndexBuildProgress;
     use lance_index::vector::DIST_COL;
+    use lance_index::vector::hnsw::builder::HnswBuildParams;
     use lance_index::vector::ivf::IvfBuildParams;
     use lance_index::vector::kmeans::{KMeansParams, train_kmeans};
     use lance_index::vector::pq::PQBuildParams;
@@ -1712,8 +2087,6 @@ mod tests {
         storage::STORAGE_METADATA_KEY,
     };
     use lance_index::{INDEX_AUXILIARY_FILE_NAME, metrics::NoOpMetricsCollector};
-    use lance_index::{optimize::OptimizeOptions, scalar::IndexReader};
-    use lance_index::{scalar::IndexWriter, vector::hnsw::builder::HnswBuildParams};
     use lance_io::{
         object_store::ObjectStore,
         scheduler::{ScanScheduler, SchedulerConfig},
@@ -1726,12 +2099,67 @@ mod tests {
     use rand::distr::uniform::SampleUniform;
     use rand::{Rng, SeedableRng, rngs::StdRng};
     use rstest::rstest;
+    use uuid::Uuid;
 
     const NUM_ROWS: usize = 512;
     const DIM: usize = 32;
 
     lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>);
 
+    #[test]
+    fn test_rotated_partition_centroid_slice_borrows_cache() {
+        let cache = super::RabitSearchCache {
+            rotated_centroids: vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
+            code_dim: 2,
+        };
+
+        let centroid = super::rotated_partition_centroid_slice(Some(&cache), 1).unwrap();
+
+        assert_eq!(centroid, &[3.0, 4.0]);
+        assert_eq!(centroid.as_ptr(), cache.rotated_centroids[2..].as_ptr());
+        assert!(super::rotated_partition_centroid_slice(Some(&cache), 3).is_none());
+        assert!(super::rotated_partition_centroid_slice(None, 0).is_none());
+    }
+
+    #[test]
+    fn test_rabit_ex_scratch_len_uses_num_bits() {
+        // Block-aligned dims read the rotated query in place.
+        let dim = 960;
+        for num_bits in [1, 3, 5, 7, 9] {
+            assert_eq!(super::rabit_ex_scratch_len(dim, num_bits), 0);
+        }
+
+        // Unaligned multi-bit queries add one padded query copy.
+        let dim = 968;
+        assert_eq!(super::rabit_ex_scratch_len(dim, 1), 0);
+        assert_eq!(super::rabit_ex_scratch_len(dim, 7), padded_query_len(dim));
+    }
+
+    #[test]
+    fn test_rabit_u8_scratch_len_includes_ex_fastscan_tables() {
+        let dim = 960;
+
+        assert_eq!(super::rabit_u8_scratch_len(dim, 1), dim * 4);
+        assert_eq!(super::rabit_u8_scratch_len(dim, 3), dim * 8);
+        assert_eq!(super::rabit_u8_scratch_len(dim, 5), dim * 16);
+        assert_eq!(super::rabit_u8_scratch_len(dim, 7), dim * 4);
+        assert_eq!(super::rabit_u8_scratch_len(dim, 9), dim * 32);
+    }
+
+    #[test]
+    fn test_rabit_query_scratch_capacity_does_not_preallocate_u32() {
+        let dim = 960;
+        let max_partition_len = 4096;
+
+        let capacity = super::rabit_query_scratch_capacity(dim, max_partition_len, 5);
+
+        assert_eq!(capacity.distances, max_partition_len);
+        assert_eq!(capacity.query_f32, dim + dim * 4);
+        assert_eq!(capacity.u16, max_partition_len);
+        assert_eq!(capacity.u8, dim * 16);
+        assert_eq!(capacity.u32, 0);
+    }
+
     async fn generate_test_dataset<T: ArrowPrimitiveType>(
         test_uri: &str,
         range: Range<T::Native>,
@@ -1790,11 +2218,11 @@ mod tests {
         vectors
     }
 
-    async fn get_rq_metadata(
+    async fn open_rq_aux_reader(
         dataset: &Dataset,
         scheduler: Arc<ScanScheduler>,
         index_uuid: &str,
-    ) -> RabitQuantizationMetadata {
+    ) -> FileReader {
         let index_path = dataset
             .indices_dir()
             .join(index_uuid)
@@ -1803,7 +2231,7 @@ mod tests {
             .open_file(&index_path, &CachedFileSize::unknown())
             .await
             .unwrap();
-        let reader = FileReader::try_open(
+        FileReader::try_open(
             file_scheduler,
             None,
             Arc::<DecoderPlugins>::default(),
@@ -1811,7 +2239,15 @@ mod tests {
             FileReaderOptions::default(),
         )
         .await
-        .unwrap();
+        .unwrap()
+    }
+
+    async fn get_rq_metadata(
+        dataset: &Dataset,
+        scheduler: Arc<ScanScheduler>,
+        index_uuid: &str,
+    ) -> RabitQuantizationMetadata {
+        let reader = open_rq_aux_reader(dataset, scheduler, index_uuid).await;
         let metadata = reader.schema().metadata.get(STORAGE_METADATA_KEY).unwrap();
         let metadata_entries: Vec<String> = serde_json::from_str(metadata).unwrap();
         serde_json::from_str(&metadata_entries[0]).unwrap()
@@ -2108,11 +2544,12 @@ mod tests {
     ) -> VectorIndexTestContext {
         let stats_json = dataset.index_statistics(index_name).await.unwrap();
         let stats: serde_json::Value = serde_json::from_str(&stats_json).unwrap();
-        let uuid = stats["indices"][0]["uuid"]
+        let uuid_str = stats["indices"][0]["uuid"]
             .as_str()
             .expect("Index uuid should be present");
+        let uuid = Uuid::parse_str(uuid_str).expect("uuid in stats should be a valid UUID");
         let index = dataset
-            .open_vector_index(column, uuid, &NoOpMetricsCollector)
+            .open_vector_index(column, &uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
 
@@ -2428,7 +2865,7 @@ mod tests {
     async fn load_partition_row_ids(index: &IvfPq, partition_idx: usize) -> Vec<u64> {
         index
             .storage
-            .load_partition(partition_idx)
+            .load_partition(partition_idx, None)
             .await
             .unwrap()
             .row_ids()
@@ -3337,6 +3774,147 @@ mod tests {
         assert!(result.num_rows() > 0);
     }
 
+    #[rstest]
+    #[case::flat("IVF_HNSW_FLAT")]
+    #[case::pq("IVF_HNSW_PQ")]
+    #[case::sq("IVF_HNSW_SQ")]
+    #[tokio::test]
+    async fn test_merge_existing_hnsw_segments_rebuilds_graph(#[case] expected_index_type: &str) {
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+        let (schema, batches) = make_two_fragment_batches();
+        let dataset_uri = format!("{}/merge_hnsw_rebuilds_graph", base_uri);
+        let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+        let params = match expected_index_type {
+            "IVF_HNSW_FLAT" => VectorIndexParams::ivf_hnsw(
+                DistanceType::L2,
+                prepare_global_ivf(&dataset, "vector").await,
+                HnswBuildParams::default(),
+            ),
+            "IVF_HNSW_PQ" => {
+                let (ivf_params, pq_params) = prepare_global_ivf_pq(&dataset, "vector").await;
+                VectorIndexParams::with_ivf_hnsw_pq_params(
+                    DistanceType::L2,
+                    ivf_params,
+                    HnswBuildParams::default(),
+                    pq_params,
+                )
+            }
+            "IVF_HNSW_SQ" => VectorIndexParams::with_ivf_hnsw_sq_params(
+                DistanceType::L2,
+                prepare_global_ivf(&dataset, "vector").await,
+                HnswBuildParams::default(),
+                SQBuildParams::default(),
+            ),
+            other => panic!("unexpected HNSW index type {other}"),
+        };
+        let mut segments = Vec::new();
+
+        for fragment in fragments.iter().take(2) {
+            let segment = dataset
+                .create_index_builder(&["vector"], IndexType::Vector, &params)
+                .name("vector_idx".to_string())
+                .fragments(vec![fragment.id() as u32])
+                .execute_uncommitted()
+                .await
+                .unwrap();
+            segments.push(segment);
+        }
+
+        let merged = dataset
+            .merge_existing_index_segments(segments)
+            .await
+            .unwrap();
+        dataset
+            .commit_existing_index_segments("vector_idx", "vector", vec![merged])
+            .await
+            .unwrap();
+
+        let stats = dataset.index_statistics("vector_idx").await.unwrap();
+        let stats: serde_json::Value = serde_json::from_str(&stats).unwrap();
+        assert_eq!(stats["index_type"].as_str().unwrap(), expected_index_type);
+        assert_eq!(
+            stats["indices"][0]["sub_index"]["index_type"]
+                .as_str()
+                .unwrap(),
+            "HNSW"
+        );
+
+        let query_batch = dataset
+            .scan()
+            .project(&["vector"] as &[&str])
+            .unwrap()
+            .limit(Some(4), None)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        let q = query_batch["vector"].as_fixed_size_list().value(0);
+        let result = dataset
+            .scan()
+            .project(&["_rowid"] as &[&str])
+            .unwrap()
+            .nearest("vector", q.as_ref(), 5)
+            .unwrap()
+            .try_into_batch()
+            .await
+            .unwrap();
+        assert!(result.num_rows() > 0);
+    }
+
+    #[tokio::test]
+    async fn test_merge_existing_hnsw_segments_rejects_mismatched_build_params() {
+        let test_dir = TempStrDir::default();
+        let base_uri = test_dir.as_str();
+        let (schema, batches) = make_two_fragment_batches();
+        let dataset_uri = format!("{}/merge_hnsw_rejects_mismatched_params", base_uri);
+        let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await;
+
+        let fragments = dataset.get_fragments();
+        assert!(fragments.len() >= 2);
+
+        let ivf_params = prepare_global_ivf(&dataset, "vector").await;
+        let default_params = VectorIndexParams::ivf_hnsw(
+            DistanceType::L2,
+            ivf_params.clone(),
+            HnswBuildParams::default(),
+        );
+        let custom_params = VectorIndexParams::ivf_hnsw(
+            DistanceType::L2,
+            ivf_params,
+            HnswBuildParams::default().num_edges(16),
+        );
+
+        let first_segment = dataset
+            .create_index_builder(&["vector"], IndexType::Vector, &default_params)
+            .name("vector_idx".to_string())
+            .fragments(vec![fragments[0].id() as u32])
+            .execute_uncommitted()
+            .await
+            .unwrap();
+        let second_segment = dataset
+            .create_index_builder(&["vector"], IndexType::Vector, &custom_params)
+            .name("vector_idx".to_string())
+            .fragments(vec![fragments[1].id() as u32])
+            .execute_uncommitted()
+            .await
+            .unwrap();
+
+        let error = dataset
+            .merge_existing_index_segments(vec![first_segment, second_segment])
+            .await
+            .unwrap_err();
+        assert!(
+            error
+                .to_string()
+                .contains("HNSW build parameters mismatch while merging index segments"),
+            "{error}"
+        );
+    }
+
     #[tokio::test]
     async fn test_merge_index_metadata_reports_progress() {
         const INDEX_NAME: &str = "vector_idx";
@@ -3938,17 +4516,30 @@ mod tests {
         test_remap(params, nlist, recall_requirement).await;
     }
 
-    // RQ doesn't perform well for random data
-    // need to verify recall with real-world dataset (e.g. sift1m)
+    #[tokio::test]
+    async fn test_build_ivf_sq_dot_with_negative_values() {
+        let nlist = 4;
+        let ivf_params = IvfBuildParams::new(nlist);
+        let sq_params = SQBuildParams::default();
+        let params =
+            VectorIndexParams::with_ivf_sq_params(DistanceType::Dot, ivf_params, sq_params);
+
+        test_index_impl::<Float32Type>(params, nlist, 0.75, -1.0..1.0, None).await;
+    }
+
+    // These queries probe every partition, so recall here measures RaBitQ quantization
+    // error alone. At 1 bit per dimension it averages ~0.67 on this uniformly random,
+    // L2-normalized data, and each build draws a fresh random rotation, so no bar worth
+    // asserting sits clear of the spread. 5 bits lifts recall to ~0.97; its `ex_bits = 4`
+    // also covers a FastScan ex-code kernel that the multi-bit test below never reaches.
     #[rstest]
-    #[case(1, DistanceType::L2, 0.5)]
-    #[case(1, DistanceType::Cosine, 0.5)]
-    #[case(1, DistanceType::Dot, 0.5)]
-    #[case(4, DistanceType::L2, 0.5)]
-    #[case(4, DistanceType::Cosine, 0.5)]
-    #[case(4, DistanceType::Dot, 0.5)]
+    #[case(1, DistanceType::L2, 0.9)]
+    #[case(1, DistanceType::Cosine, 0.9)]
+    #[case(1, DistanceType::Dot, 0.9)]
+    #[case(4, DistanceType::L2, 0.9)]
+    #[case(4, DistanceType::Cosine, 0.9)]
+    #[case(4, DistanceType::Dot, 0.9)]
     #[tokio::test]
-    // #[ignore = "Temporarily skipping flaky 4-bit IVF_RQ tests"]
     async fn test_build_ivf_rq(
         #[case] nlist: usize,
         #[case] distance_type: DistanceType,
@@ -3957,7 +4548,7 @@ mod tests {
     ) {
         let _ = env_logger::try_init();
         let ivf_params = IvfBuildParams::new(nlist);
-        let rq_params = RQBuildParams::with_rotation_type(1, rotation_type);
+        let rq_params = RQBuildParams::with_rotation_type(5, rotation_type);
         let params = VectorIndexParams::with_ivf_rq_params(distance_type, ivf_params, rq_params);
         test_index(params.clone(), nlist, recall_requirement, None).await;
         if distance_type == DistanceType::Cosine {
@@ -3966,6 +4557,55 @@ mod tests {
         test_remap(params.clone(), nlist, recall_requirement).await;
     }
 
+    #[rstest]
+    #[case::l2(DistanceType::L2, 9)]
+    #[case::cosine(DistanceType::Cosine, 9)]
+    // ex_bits=3 and ex_bits=5 have no FastScan support and use the bit-plane
+    // repack, so these searches go through the exact ex-dot rerank kernels
+    // end to end.
+    #[case::l2_plane_repack_3bit(DistanceType::L2, 4)]
+    #[case::l2_plane_repack_5bit(DistanceType::L2, 6)]
+    #[tokio::test]
+    async fn test_build_ivf_rq_multi_bit_persists_split_codes_and_searches(
+        #[case] distance_type: DistanceType,
+        #[case] num_bits: u8,
+    ) {
+        let test_dir = TempStrDir::default();
+        let test_uri = test_dir.as_str();
+        let (mut dataset, vectors) = generate_test_dataset::<Float32Type>(test_uri, 0.0..1.0).await;
+
+        let ivf_params = IvfBuildParams::new(4);
+        let rq_params = RQBuildParams::with_rotation_type(num_bits, RQRotationType::Fast);
+        let params = VectorIndexParams::with_ivf_rq_params(distance_type, ivf_params, rq_params);
+        dataset
+            .create_index(&["vector"], IndexType::Vector, None, &params, true)
+            .await
+            .unwrap();
+
+        let indices = dataset.load_indices().await.unwrap();
+        assert_eq!(indices.len(), 1);
+        let obj_store = Arc::new(ObjectStore::local());
+        let scheduler = ScanScheduler::new(obj_store, SchedulerConfig::default_for_testing());
+        let index_uuid = indices[0].uuid.to_string();
+        let rq_meta = get_rq_metadata(&dataset, scheduler.clone(), &index_uuid).await;
+        assert_eq!(rq_meta.num_bits, num_bits);
+        assert_eq!(rq_meta.query_estimator, RabitQueryEstimator::RawQuery);
+
+        let reader = open_rq_aux_reader(&dataset, scheduler, &index_uuid).await;
+        let schema = reader.schema();
+        let ex_field = schema.field(RABIT_BLOCKED_EX_CODE_COLUMN).unwrap();
+        let DataType::FixedSizeList(_, ex_code_bytes) = ex_field.data_type() else {
+            panic!("RQ ex-code field should be FixedSizeList");
+        };
+        let expected_ex_code_bytes =
+            blocked_ex_code_bytes(rq_meta.rotated_dim(), num_bits - 1) as i32;
+        assert_eq!(ex_code_bytes, expected_ex_code_bytes);
+        assert!(schema.field(EX_ADD_FACTORS_COLUMN).is_some());
+        assert!(schema.field(EX_SCALE_FACTORS_COLUMN).is_some());
+
+        test_recall::<Float32Type>(params, 4, 0.5, "vector", &dataset, vectors).await;
+    }
+
     #[rstest]
     #[case::fast(RQRotationType::Fast)]
     #[case::matrix(RQRotationType::Matrix)]
@@ -4061,6 +4701,22 @@ mod tests {
         test_remap(params, nlist, recall_requirement).await;
     }
 
+    #[tokio::test]
+    async fn test_create_ivf_hnsw_sq_dot_with_negative_values() {
+        let nlist = 4;
+        let ivf_params = IvfBuildParams::new(nlist);
+        let sq_params = SQBuildParams::default();
+        let hnsw_params = HnswBuildParams::default();
+        let params = VectorIndexParams::with_ivf_hnsw_sq_params(
+            DistanceType::Dot,
+            ivf_params,
+            hnsw_params,
+            sq_params,
+        );
+
+        test_index_impl::<Float32Type>(params, nlist, 0.75, -1.0..1.0, None).await;
+    }
+
     #[rstest]
     #[case(4, DistanceType::L2, 0.9)]
     #[case(4, DistanceType::Cosine, 0.9)]
@@ -4179,10 +4835,7 @@ mod tests {
             .as_primitive::<Float32Type>()
             .values()
             .to_vec();
-        let results = dists
-            .into_iter()
-            .zip(row_ids.clone().into_iter())
-            .collect::<Vec<_>>();
+        let results = dists.into_iter().zip(row_ids.clone()).collect::<Vec<_>>();
         let row_ids = row_ids.into_iter().collect::<HashSet<_>>();
 
         let gt = multivec_ground_truth(&vectors, &query, k, params.metric_type);
@@ -4241,11 +4894,7 @@ mod tests {
         let indices = dataset.load_indices_by_name("vector_idx").await.unwrap();
         assert_eq!(indices.len(), 1); // v1 index should be replaced by v3 index
         let index = dataset
-            .open_vector_index(
-                "vector",
-                indices[0].uuid.to_string().as_str(),
-                &NoOpMetricsCollector,
-            )
+            .open_vector_index("vector", &indices[0].uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let v3_index = index.as_any().downcast_ref::<super::IvfPq>();
@@ -4579,10 +5228,7 @@ mod tests {
             .as_primitive::<Float32Type>()
             .values()
             .to_vec();
-        let results = dists
-            .into_iter()
-            .zip(row_ids.into_iter())
-            .collect::<Vec<_>>();
+        let results = dists.into_iter().zip(row_ids).collect::<Vec<_>>();
         let row_ids = results.iter().map(|(_, id)| *id).collect::<HashSet<_>>();
         assert!(row_ids.len() == k);
 
@@ -4638,9 +5284,22 @@ mod tests {
 
         // Rewrite auxiliary file with PQ codebook inlined into schema metadata.
         let mut metadata = reader.schema().metadata.clone();
-        let batch = reader
-            .read_range(0..reader.num_rows() as usize, None)
+        let projection = lance_file::reader::ReaderProjection::from_whole_schema(
+            reader.schema(),
+            reader.metadata().version(),
+        );
+        let batches = reader
+            .read_stream_projected(
+                lance_io::ReadBatchParams::RangeFull,
+                u32::MAX,
+                u32::MAX,
+                projection,
+                lance_encoding::decoder::FilterExpression::no_filter(),
+            )
             .await?;
+        use futures::TryStreamExt as _;
+        let batches = batches.try_collect::<Vec<_>>().await?;
+        let batch = arrow::compute::concat_batches(&batches[0].schema(), &batches)?;
         let new_aux_path = new_dir.clone().join(INDEX_AUXILIARY_FILE_NAME);
         let mut writer = FileWriter::try_new(
             obj_store.create(&new_aux_path).await?,
@@ -4661,7 +5320,10 @@ mod tests {
             STORAGE_METADATA_KEY.to_owned(),
             serde_json::to_string(&vec![pq_metadata])?,
         );
-        writer.finish_with_metadata(metadata).await?;
+        for (key, value) in metadata {
+            writer.add_schema_metadata(key, value);
+        }
+        writer.finish().await?;
 
         // Build new IndexMetadata with the new UUID and file sizes.
         let new_files =
@@ -5702,11 +6364,9 @@ mod tests {
             // Try serialized store first
             let guard = self.serialized.lock().await;
             if let Some((bytes, stored_codec, _)) = guard.get(key) {
-                return Some(
-                    stored_codec
-                        .deserialize(&bytes::Bytes::copy_from_slice(bytes))
-                        .expect("deserialization should succeed"),
-                );
+                return stored_codec
+                    .deserialize(&bytes::Bytes::copy_from_slice(bytes))
+                    .hit();
             }
             drop(guard);
             // Fall through to passthrough
diff --git a/rust/lance/src/index/vector/pq.rs b/rust/lance/src/index/vector/pq.rs
index 1f1a89d80c8..217667afbe9 100644
--- a/rust/lance/src/index/vector/pq.rs
+++ b/rust/lance/src/index/vector/pq.rs
@@ -1,8 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+use lance_core::utils::row_addr_remap::RowAddrRemap;
+use std::any::Any;
 use std::sync::Arc;
-use std::{any::Any, collections::HashMap};
 
 use arrow::compute::concat;
 use arrow_array::types::UInt64Type;
@@ -17,8 +18,8 @@ use arrow_select::take::take;
 use async_trait::async_trait;
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use deepsize::DeepSizeOf;
 use lance_arrow::FixedSizeListArrayExt;
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::utils::address::RowAddress;
 use lance_core::utils::tokio::spawn_cpu;
 use lance_core::{ROW_ID, ROW_ID_FIELD};
@@ -71,17 +72,29 @@ pub struct PQIndex {
 }
 
 impl DeepSizeOf for PQIndex {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         self.pq.deep_size_of_children(context)
             + self
                 .code
                 .as_ref()
-                .map(|code| code.get_array_memory_size())
+                .map(|code| {
+                    if context.mark_seen(Arc::as_ptr(code) as *const () as usize) {
+                        (code.as_ref() as &dyn arrow_array::Array).deep_size_of_children(context)
+                    } else {
+                        0
+                    }
+                })
                 .unwrap_or(0)
             + self
                 .row_ids
                 .as_ref()
-                .map(|row_ids| row_ids.get_array_memory_size())
+                .map(|row_ids| {
+                    if context.mark_seen(Arc::as_ptr(row_ids) as *const () as usize) {
+                        (row_ids.as_ref() as &dyn arrow_array::Array).deep_size_of_children(context)
+                    } else {
+                        0
+                    }
+                })
                 .unwrap_or(0)
     }
 }
@@ -168,10 +181,6 @@ impl Index for PQIndex {
         self
     }
 
-    fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-        Ok(self)
-    }
-
     fn index_type(&self) -> IndexType {
         IndexType::Vector
     }
@@ -430,14 +439,14 @@ impl VectorIndex for PQIndex {
         todo!("this method is for only IVF_HNSW_* index");
     }
 
-    async fn remap(&mut self, mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+    async fn remap(&mut self, mapping: &RowAddrRemap) -> Result<()> {
         let num_vectors = self.row_ids.as_ref().unwrap().len();
         let row_ids = self.row_ids.as_ref().unwrap().values().iter();
         let transposed_codes = self.code.as_ref().unwrap();
         let remapped = row_ids
             .enumerate()
             .filter_map(|(vec_idx, old_row_id)| {
-                let new_row_id = mapping.get(old_row_id).cloned();
+                let new_row_id = mapping.get(*old_row_id);
                 // If the row id is not in the mapping then this row is not remapped and we keep as is
                 let new_row_id = new_row_id.unwrap_or(Some(*old_row_id));
                 new_row_id.map(|new_row_id| {
@@ -636,6 +645,7 @@ pub(crate) fn build_pq_storage(
 mod tests {
     use super::*;
 
+    use std::collections::HashMap;
     use std::{ops::Range, sync::Mutex};
 
     use arrow::datatypes::Float32Type;
@@ -899,6 +909,7 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: Default::default(),
         };
         let is_empty_threads = Arc::new(Mutex::new(Vec::new()));
         let pre_filter = Arc::new(TestPreFilter::with_thread_capture(
diff --git a/rust/lance/src/index/vector/utils.rs b/rust/lance/src/index/vector/utils.rs
index 3046d0f3a83..012d8d3a703 100644
--- a/rust/lance/src/index/vector/utils.rs
+++ b/rust/lance/src/index/vector/utils.rs
@@ -1,16 +1,16 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::collections::HashSet;
 use std::pin::Pin;
 use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
 
 use arrow::array::ArrayData;
 use arrow::datatypes::DataType;
 use arrow_array::new_empty_array;
 use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, cast::AsArray};
 use arrow_buffer::{Buffer, MutableBuffer};
-use futures::{Stream, StreamExt, TryStreamExt, stream};
+use futures::{Stream, StreamExt, stream};
 use lance_arrow::DataTypeExt;
 use lance_core::datatypes::Schema;
 use lance_linalg::distance::DistanceType;
@@ -18,6 +18,7 @@ use log::{info, warn};
 use rand::rngs::SmallRng;
 use rand::seq::{IteratorRandom, SliceRandom};
 use rand::{Rng, SeedableRng};
+use roaring::RoaringTreemap;
 use tokio::sync::Mutex;
 
 use crate::dataset::{Dataset, ProjectionRequest, TakeBuilder, row_offsets_to_row_addresses};
@@ -264,38 +265,7 @@ fn infer_vector_element_type_impl(
 async fn count_rows(dataset: &Dataset, fragment_ids: Option<&[u32]>) -> Result<usize> {
     match fragment_ids {
         None => dataset.count_rows(None).await,
-        Some(fragment_ids) => {
-            let sorted_ids: Vec<u32>;
-            let sorted_fragment_ids = if fragment_ids.windows(2).all(|w| w[0] <= w[1]) {
-                fragment_ids
-            } else {
-                sorted_ids = {
-                    let mut v = fragment_ids.to_vec();
-                    v.sort_unstable();
-                    v
-                };
-                &sorted_ids
-            };
-            let fragments = dataset.get_frags_from_ordered_ids(sorted_fragment_ids);
-            let valid_fragments = fragments
-                .into_iter()
-                .enumerate()
-                .map(|(i, frag)| {
-                    frag.ok_or_else(|| {
-                        Error::index(format!(
-                            "Unexpectedly missing fragment {}",
-                            sorted_fragment_ids[i]
-                        ))
-                    })
-                })
-                .collect::<Result<Vec<_>>>()?;
-            let cnts = stream::iter(valid_fragments)
-                .map(|f| async move { f.count_rows(None).await })
-                .buffer_unordered(16)
-                .try_collect::<Vec<usize>>()
-                .await?;
-            Ok(cnts.iter().sum::<usize>())
-        }
+        Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await,
     }
 }
 
@@ -309,8 +279,6 @@ pub async fn maybe_sample_training_data(
     sample_size_hint: usize,
     fragment_ids: Option<&[u32]>,
 ) -> Result<FixedSizeListArray> {
-    let num_rows = count_rows(dataset, fragment_ids).await?;
-
     let vector_field = dataset.schema().field(column).ok_or(Error::index(format!(
         "Sample training data: column {} does not exist in schema",
         column
@@ -328,6 +296,8 @@ pub async fn maybe_sample_training_data(
         return Ok(new_empty_array(&fsl_type).as_fixed_size_list().clone());
     }
 
+    let num_rows = count_rows(dataset, fragment_ids).await?;
+
     let is_nullable = vector_field.nullable;
 
     let sample_size_hint = match vector_field.data_type() {
@@ -486,18 +456,39 @@ async fn sample_training_data(
             );
             return vector_column_to_fsl(&batch, column);
         }
+        // Rows the consumer still needs. The fragment producer sizes each
+        // prefetch round to this outstanding demand, keeping reads bounded by
+        // the requested sample size.
+        let still_needed = Arc::new(AtomicUsize::new(sample_size_hint));
         let scan = sample_training_data_scan_from_fragments(
             dataset,
             column,
-            sample_size_hint,
             num_rows,
             fragment_ids,
+            still_needed.clone(),
         )?;
         return match vector_field.data_type() {
             DataType::FixedSizeList(_, _) => {
-                sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await
+                sample_nullable_fsl(
+                    column,
+                    sample_size_hint,
+                    byte_width,
+                    vector_field,
+                    scan,
+                    Some(still_needed),
+                )
+                .await
+            }
+            _ => {
+                sample_nullable_fallback(
+                    column,
+                    sample_size_hint,
+                    is_nullable,
+                    scan,
+                    Some(still_needed),
+                )
+                .await
             }
-            _ => sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await,
         };
     }
 
@@ -516,12 +507,20 @@ async fn sample_training_data(
         DataType::FixedSizeList(_, _) => {
             let scan =
                 sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?;
-            sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await
+            sample_nullable_fsl(
+                column,
+                sample_size_hint,
+                byte_width,
+                vector_field,
+                scan,
+                None,
+            )
+            .await
         }
         _ => {
             let scan =
                 sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?;
-            sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await
+            sample_nullable_fallback(column, sample_size_hint, is_nullable, scan, None).await
         }
     }
 }
@@ -550,12 +549,19 @@ fn sample_training_data_scan(
 /// sampling must first map random offsets within the selected fragments to row
 /// addresses and then `take` those rows. Both nullable FSL and multivector
 /// paths reuse this stream to avoid duplicating fragment sampling logic.
+///
+/// Each round is sized to `still_needed` (the consumer's outstanding demand),
+/// so a low-null column reads at most the requested sample. Visited offsets
+/// are tracked in a [`RoaringTreemap`] because sparse or all-null columns
+/// force the stream to visit most selected rows before it can terminate, and
+/// a compressed bitmap keeps that persistent state near `num_rows / 8` bytes
+/// even when fully populated.
 fn sample_training_data_scan_from_fragments(
     dataset: &Dataset,
     column: &str,
-    sample_size_hint: usize,
     num_rows: usize,
     fragment_ids: &[u32],
+    still_needed: Arc<AtomicUsize>,
 ) -> Result<Pin<Box<dyn Stream<Item = Result<RecordBatch>> + Send>>> {
     if fragment_ids.is_empty() {
         return Err(Error::invalid_input(
@@ -563,21 +569,7 @@ fn sample_training_data_scan_from_fragments(
         ));
     }
 
-    let mut ordered_ids = fragment_ids.to_vec();
-    ordered_ids.sort_unstable();
-    ordered_ids.dedup();
-    let selected_fragments = dataset
-        .get_frags_from_ordered_ids(&ordered_ids)
-        .into_iter()
-        .zip(ordered_ids.iter())
-        .map(|(fragment, fragment_id)| {
-            fragment.ok_or_else(|| {
-                Error::invalid_input(format!(
-                    "Unknown fragment id {fragment_id} in training fragment filter"
-                ))
-            })
-        })
-        .collect::<Result<Vec<_>>>()?;
+    let selected_fragments = dataset.get_fragments_from_ids(fragment_ids)?;
     let dataset = Arc::new(dataset.clone());
     let projection = Arc::new(
         ProjectionRequest::from(dataset.schema().project(&[column])?)
@@ -589,19 +581,36 @@ fn sample_training_data_scan_from_fragments(
             dataset,
             projection,
             selected_fragments,
-            HashSet::<u64>::with_capacity(sample_size_hint.min(num_rows)),
+            RoaringTreemap::new(),
             SmallRng::from_os_rng(),
+            still_needed,
         ),
-        move |(dataset, projection, selected_fragments, mut seen_offsets, mut rng)| async move {
-            if seen_offsets.len() >= num_rows {
+        move |(
+            dataset,
+            projection,
+            selected_fragments,
+            mut seen_offsets,
+            mut rng,
+            still_needed,
+        )| async move {
+            if seen_offsets.len() as usize >= num_rows {
+                return Ok(None);
+            }
+            let still = still_needed.load(Ordering::Relaxed);
+            if still == 0 {
                 return Ok(None);
             }
 
-            let remaining = num_rows.saturating_sub(seen_offsets.len());
-            let target = sample_size_hint.saturating_mul(2).min(remaining);
+            let remaining = num_rows.saturating_sub(seen_offsets.len() as usize);
+            // Sizing the round to the outstanding demand keeps a low-null
+            // column's reads bounded by the requested sample, matching the
+            // non-nullable path.
+            let target = still.min(remaining);
             let mut sampled_offsets = if remaining <= target.saturating_mul(4) {
+                // Few offsets remain unseen, so shuffling the unseen set is
+                // cheaper than repeatedly rejecting already-sampled offsets.
                 let mut unseen_indices = (0..num_rows as u64)
-                    .filter(|index| !seen_offsets.contains(index))
+                    .filter(|index| !seen_offsets.contains(*index))
                     .collect::<Vec<_>>();
                 unseen_indices.shuffle(&mut rng);
                 unseen_indices.truncate(target);
@@ -635,7 +644,14 @@ fn sample_training_data_scan_from_fragments(
             .await?;
             Ok(Some((
                 batch,
-                (dataset, projection, selected_fragments, seen_offsets, rng),
+                (
+                    dataset,
+                    projection,
+                    selected_fragments,
+                    seen_offsets,
+                    rng,
+                    still_needed,
+                ),
             )))
         },
     );
@@ -646,22 +662,7 @@ fn resolve_scan_fragments(
     dataset: &Dataset,
     fragment_ids: &[u32],
 ) -> Result<Vec<lance_table::format::Fragment>> {
-    let mut ordered_ids = fragment_ids.to_vec();
-    ordered_ids.sort_unstable();
-    let fragments = dataset.get_frags_from_ordered_ids(&ordered_ids);
-    if let Some(missing_id) = fragments
-        .iter()
-        .zip(ordered_ids.iter())
-        .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id))
-    {
-        return Err(Error::invalid_input(format!(
-            "Unknown fragment id {missing_id} in training fragment filter"
-        )));
-    }
-    Ok(fragments
-        .into_iter()
-        .map(|fragment| fragment.unwrap().metadata().clone())
-        .collect())
+    dataset.get_fragment_metadata_from_ids(fragment_ids)
 }
 
 /// Build a FixedSizeListArray from raw flat value bytes.
@@ -721,6 +722,7 @@ async fn sample_nullable_fsl<S>(
     byte_width: usize,
     vector_field: &lance_core::datatypes::Field,
     mut scan: S,
+    still_needed: Option<Arc<AtomicUsize>>,
 ) -> Result<FixedSizeListArray>
 where
     S: Stream<Item = Result<RecordBatch>> + Unpin,
@@ -731,6 +733,12 @@ where
     let mut rows_scanned: usize = 0;
 
     while num_non_null < sample_size_hint {
+        let remaining_rows = sample_size_hint - num_non_null;
+        // A fragment-limited producer sizes its next prefetch round to this
+        // outstanding demand.
+        if let Some(still_needed) = &still_needed {
+            still_needed.store(remaining_rows, Ordering::Relaxed);
+        }
         let Some(batch) = scan.next().await else {
             break;
         };
@@ -750,7 +758,16 @@ where
             continue;
         }
         let previous_num_non_null = num_non_null;
-        accumulate_fsl_values(&mut values_buf, &mut num_non_null, &array, byte_width, true)?;
+        // `remaining_rows` keeps `values_buf` within its pre-allocated
+        // `sample_size_hint * byte_width` capacity.
+        accumulate_fsl_values(
+            &mut values_buf,
+            &mut num_non_null,
+            &array,
+            byte_width,
+            true,
+            remaining_rows,
+        )?;
         info!(
             "Sample training data: batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled after null filtering)",
             batch_count,
@@ -762,6 +779,11 @@ where
         );
     }
 
+    // Zero the demand so any further poll of the producer terminates instead
+    // of reading another round.
+    if let Some(still_needed) = &still_needed {
+        still_needed.store(0, Ordering::Relaxed);
+    }
     let num_rows_out = num_non_null.min(sample_size_hint);
     values_buf.truncate(num_rows_out * byte_width);
 
@@ -797,7 +819,14 @@ async fn sample_fsl_uniform(
     for (chunk_idx, chunk) in indices.chunks(TAKE_CHUNK_SIZE).enumerate() {
         let batch = dataset.take(chunk, projection.clone()).await?;
         let array = get_column_from_batch(&batch, column)?;
-        accumulate_fsl_values(&mut values_buf, &mut total_rows, &array, byte_width, false)?;
+        accumulate_fsl_values(
+            &mut values_buf,
+            &mut total_rows,
+            &array,
+            byte_width,
+            false,
+            usize::MAX,
+        )?;
         info!(
             "Sample training data: batch {}/{} read {} rows ({}/{} sampled by uniform random sampling)",
             chunk_idx + 1,
@@ -821,13 +850,20 @@ async fn sample_fsl_uniform(
 /// When `filter_nulls` is false and there are no nulls, copies raw bytes
 /// directly from the FSL values buffer (accounting for child array offset).
 /// When `filter_nulls` is true, uses Arrow's `filter` kernel to remove nulls.
+/// At most `max_rows` rows are appended so callers can stop copying once their
+/// sample is full; otherwise one oversized source batch can grow `values_buf`
+/// far beyond the intended cap.
 fn accumulate_fsl_values(
     values_buf: &mut MutableBuffer,
     num_rows: &mut usize,
     array: &ArrayRef,
     byte_width: usize,
     filter_nulls: bool,
+    max_rows: usize,
 ) -> Result<()> {
+    if max_rows == 0 {
+        return Ok(());
+    }
     let needs_filter = filter_nulls && array.null_count() > 0;
 
     if needs_filter {
@@ -835,21 +871,29 @@ fn accumulate_fsl_values(
         let mask = arrow_array::BooleanArray::from(nulls.inner().clone());
         let filtered = arrow::compute::filter(array, &mask)?;
         let fsl = filtered.as_fixed_size_list();
+        let take = fsl.len().min(max_rows);
+        if take == 0 {
+            return Ok(());
+        }
         let values_data = fsl.values().to_data();
-        let value_bytes = &values_data.buffers()[0].as_slice()[..fsl.len() * byte_width];
+        let value_bytes = &values_data.buffers()[0].as_slice()[..take * byte_width];
         values_buf.extend_from_slice(value_bytes);
-        *num_rows += fsl.len();
+        *num_rows += take;
     } else {
         // No nulls: copy raw bytes directly, accounting for child array offset.
         let fsl = array.as_fixed_size_list();
+        let take = fsl.len().min(max_rows);
+        if take == 0 {
+            return Ok(());
+        }
         let values = fsl.values();
         let values_data = values.to_data();
         let elem_size = byte_width / fsl.value_length() as usize;
         let offset_bytes = values_data.offset() * elem_size;
-        let total_bytes = fsl.len() * byte_width;
+        let total_bytes = take * byte_width;
         let buf = &values_data.buffers()[0].as_slice()[offset_bytes..offset_bytes + total_bytes];
         values_buf.extend_from_slice(buf);
-        *num_rows += fsl.len();
+        *num_rows += take;
     }
     Ok(())
 }
@@ -862,6 +906,7 @@ async fn sample_nullable_fallback<S>(
     sample_size_hint: usize,
     is_nullable: bool,
     mut scan: S,
+    still_needed: Option<Arc<AtomicUsize>>,
 ) -> Result<FixedSizeListArray>
 where
     S: Stream<Item = Result<RecordBatch>> + Unpin,
@@ -873,6 +918,12 @@ where
     let mut rows_scanned: usize = 0;
 
     while num_non_null < sample_size_hint {
+        let remaining_rows = sample_size_hint - num_non_null;
+        // A fragment-limited producer sizes its next prefetch round to this
+        // outstanding demand.
+        if let Some(still_needed) = &still_needed {
+            still_needed.store(remaining_rows, Ordering::Relaxed);
+        }
         let Some(batch) = scan.next().await else {
             break;
         };
@@ -898,7 +949,14 @@ where
         } else {
             batch
         };
-        let accepted_rows = batch.num_rows();
+        // Slicing to the outstanding demand keeps the retained batches, and
+        // the post-loop `concat_batches`, bounded by the sample size.
+        let accepted_rows = batch.num_rows().min(remaining_rows);
+        let batch = if accepted_rows < batch.num_rows() {
+            batch.slice(0, accepted_rows)
+        } else {
+            batch
+        };
         num_non_null += accepted_rows;
         info!(
             "Sample training data (fallback): batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled)",
@@ -912,6 +970,12 @@ where
         filtered.push(batch);
     }
 
+    // Zero the demand so any further poll of the producer terminates instead
+    // of reading another round.
+    if let Some(still_needed) = &still_needed {
+        still_needed.store(0, Ordering::Relaxed);
+    }
+
     let Some(schema) = schema else {
         return Err(Error::index("No non-null training data found".to_string()));
     };
@@ -1040,6 +1104,7 @@ mod tests {
 
     use crate::dataset::InsertBuilder;
     use arrow_array::{ArrayRef, Float32Array, types::Float32Type};
+    use arrow_buffer::{BooleanBufferBuilder, NullBuffer};
     use arrow_schema::{DataType, Field};
     use lance_arrow::FixedSizeListArrayExt;
     use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch};
@@ -1201,7 +1266,15 @@ mod tests {
         let mut buf = MutableBuffer::new(0);
         let mut num_rows = 0usize;
         let sliced_ref: ArrayRef = Arc::new(sliced);
-        accumulate_fsl_values(&mut buf, &mut num_rows, &sliced_ref, byte_width, false).unwrap();
+        accumulate_fsl_values(
+            &mut buf,
+            &mut num_rows,
+            &sliced_ref,
+            byte_width,
+            false,
+            usize::MAX,
+        )
+        .unwrap();
 
         assert_eq!(num_rows, 4);
         let result: &[f32] =
@@ -1327,4 +1400,140 @@ mod tests {
         let result = count_rows(&dataset, Some(&[ids[2], ids[0]])).await.unwrap();
         assert_eq!(result, 250);
     }
+
+    /// Nullable FSL with fragment-limited sampling must fill the requested sample
+    /// size when enough non-null rows exist, and terminate cleanly when all
+    /// selected rows are null.
+    #[tokio::test]
+    async fn test_maybe_sample_training_data_fsl_nullable_fragment_limited() {
+        let nrows: usize = 2000;
+        let dims: u32 = 8;
+        let sample_size: usize = 500;
+
+        for (case, null_probability, expected_len) in
+            [("partial_nulls", 0.5, sample_size), ("all_nulls", 1.0, 0)]
+        {
+            let col_gen = array::rand_vec::<Float32Type>(Dimension::from(dims))
+                .with_random_nulls(null_probability);
+            let data = gen_batch()
+                .col("vec", col_gen)
+                .into_batch_rows(RowCount::from(nrows as u64))
+                .unwrap();
+
+            let dataset = InsertBuilder::new("memory://")
+                .execute(vec![data])
+                .await
+                .unwrap();
+
+            let fragment_ids: Vec<u32> = dataset
+                .get_fragments()
+                .iter()
+                .map(|f| f.id() as u32)
+                .collect();
+
+            let training_data =
+                maybe_sample_training_data(&dataset, "vec", sample_size, Some(&fragment_ids))
+                    .await
+                    .unwrap();
+
+            assert_eq!(training_data.len(), expected_len, "{case}");
+            assert_eq!(training_data.null_count(), 0, "{case}");
+            assert_eq!(training_data.value_length(), dims as i32, "{case}");
+        }
+    }
+
+    /// Scan-side regression: each fragment-limited producer round must read at
+    /// most the consumer's outstanding demand. Driving the producer directly
+    /// with a fixed `still_needed` and inspecting the raw batch size catches
+    /// over-reads that a post-truncation output-length check cannot.
+    #[tokio::test]
+    async fn test_sample_fragment_scan_round_caps_at_still_needed() {
+        let nrows: usize = 4000;
+        let dims: u32 = 8;
+        let still: usize = 500;
+
+        let col_gen = array::rand_vec::<Float32Type>(Dimension::from(dims)).with_random_nulls(0.5);
+        let data = gen_batch()
+            .col("vec", col_gen)
+            .into_batch_rows(RowCount::from(nrows as u64))
+            .unwrap();
+
+        let dataset = InsertBuilder::new("memory://fsl_scan_round_cap_test")
+            .execute(vec![data])
+            .await
+            .unwrap();
+
+        let fragment_ids: Vec<u32> = dataset
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect();
+        let num_rows = count_rows(&dataset, Some(&fragment_ids)).await.unwrap();
+
+        // `still_needed` is left large enough that `num_rows` never bounds the
+        // round, so the batch size reflects the demand cap and nothing else.
+        let still_needed = Arc::new(AtomicUsize::new(still));
+        let mut scan = sample_training_data_scan_from_fragments(
+            &dataset,
+            "vec",
+            num_rows,
+            &fragment_ids,
+            still_needed.clone(),
+        )
+        .unwrap();
+
+        let batch = scan.next().await.unwrap().unwrap();
+        assert!(
+            batch.num_rows() <= still,
+            "producer round read {} rows but only {} were outstanding",
+            batch.num_rows(),
+            still
+        );
+    }
+
+    #[test]
+    fn test_accumulate_fsl_values_respects_max_rows() {
+        let dim: usize = 4;
+        let total_rows: usize = 100;
+        let max_rows: usize = 16;
+        let byte_width = dim * std::mem::size_of::<f32>();
+
+        let values: Vec<f32> = (0..total_rows * dim).map(|i| i as f32).collect();
+        let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim as i32)
+            .unwrap();
+        let arr: ArrayRef = Arc::new(fsl);
+
+        let mut buf = MutableBuffer::new(0);
+        let mut num_rows = 0usize;
+        accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap();
+
+        assert_eq!(num_rows, max_rows);
+        assert_eq!(buf.len(), max_rows * byte_width);
+
+        let values: Vec<f32> = (0..total_rows * dim).map(|i| i as f32).collect();
+        let item_field = Arc::new(Field::new("item", DataType::Float32, true));
+
+        // Every other row is null, leaving 50 non-null rows.
+        let mut nulls_builder = BooleanBufferBuilder::new(total_rows);
+        for i in 0..total_rows {
+            nulls_builder.append(i % 2 == 0);
+        }
+        let nulls = NullBuffer::new(nulls_builder.finish());
+
+        let fsl = FixedSizeListArray::try_new(
+            item_field,
+            dim as i32,
+            Arc::new(Float32Array::from(values)),
+            Some(nulls),
+        )
+        .unwrap();
+        let arr: ArrayRef = Arc::new(fsl);
+
+        let mut buf = MutableBuffer::new(0);
+        let mut num_rows = 0usize;
+        accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap();
+
+        assert_eq!(num_rows, max_rows);
+        assert_eq!(buf.len(), max_rows * byte_width);
+    }
 }
diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs
index d0a2934552c..35cd275b8ee 100644
--- a/rust/lance/src/io/commit.rs
+++ b/rust/lance/src/io/commit.rs
@@ -489,13 +489,12 @@ fn fix_schema(manifest: &mut Manifest) -> Result<()> {
     }
 
     // Now, we need to remap the field ids to be unique.
-    let mut field_id_seed = manifest.max_field_id() + 1;
     let mut old_field_id_mapping: HashMap<i32, i32> = HashMap::new();
     let mut fields_with_duplicate_ids = fields_with_duplicate_ids.into_iter().collect::<Vec<_>>();
     fields_with_duplicate_ids.sort_unstable();
-    for field_id in fields_with_duplicate_ids {
+    for (field_id_seed, field_id) in (manifest.max_field_id() + 1..).zip(fields_with_duplicate_ids)
+    {
         old_field_id_mapping.insert(field_id, field_id_seed);
-        field_id_seed += 1;
     }
 
     let mut fragments = manifest.fragments.as_ref().clone();
@@ -592,17 +591,21 @@ pub(crate) async fn migrate_fragments(
 
             let mut data_files = fragment.files.clone();
 
-            // For each of the data files in the fragment, we need to get the file size
-            let object_store = dataset.object_store.as_ref();
+            // For each of the data files in the fragment, we need to get the file size.
+            // Resolve each file against its own storage base: multi-base datasets
+            // keep data files outside the dataset root (DataFile.base_id).
             let get_sizes = data_files
                 .iter()
                 .map(|file| {
                     if let Some(size) = file.file_size_bytes.get() {
                         Either::Left(futures::future::ready(Ok(size)))
                     } else {
-                        Either::Right(async {
+                        let dataset = dataset.clone();
+                        Either::Right(async move {
+                            let object_store = dataset.object_store_for_data_file(file).await?;
+                            let data_dir = dataset.data_file_dir_for_base(file.base_id)?;
                             object_store
-                                .size(&dataset.base.clone().join("data").join(file.path.clone()))
+                                .size(&data_dir.join(file.path.clone()))
                                 .map_ok(|size| {
                                     NonZero::new(size).ok_or_else(|| {
                                         Error::internal(format!("File {} has size 0", file.path))
@@ -688,11 +691,7 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re
             let idx_field = dataset.schema().field_by_id(index.fields[0]).ok_or_else(|| Error::internal(format!("Index with uuid {} referred to field with id {} which did not exist in dataset", index.uuid, index.fields[0])))?;
             // We need to calculate the fragments covered by the index
             let idx = dataset
-                .open_generic_index(
-                    &idx_field.name,
-                    &index.uuid.to_string(),
-                    &NoOpMetricsCollector,
-                )
+                .open_generic_index(&idx_field.name, &index.uuid, &NoOpMetricsCollector)
                 .await?;
             index.fragment_bitmap = Some(idx.calculate_included_frags().await?);
         }
@@ -1691,6 +1690,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path1", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("unused", vec![9], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1703,6 +1703,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("path3", vec![2], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1740,6 +1741,7 @@ mod tests {
                     vec![0, 1, 10],
                     None,
                 )],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1752,6 +1754,7 @@ mod tests {
                     DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None),
                     DataFile::new_legacy_from_fields("path3", vec![10], None),
                 ],
+                overlays: vec![],
                 deletion_file: None,
                 row_id_meta: None,
                 physical_rows: None,
@@ -1842,6 +1845,7 @@ mod tests {
         let fragment = Fragment {
             id: 0,
             files: vec![data_file],
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(100),
diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs
index b242cd5b3dd..fe02b722374 100644
--- a/rust/lance/src/io/commit/conflict_resolver.rs
+++ b/rust/lance/src/io/commit/conflict_resolver.rs
@@ -7,7 +7,7 @@ use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta};
 use crate::io::deletion::read_dataset_deletion_file;
 use crate::{
     Dataset,
-    dataset::transaction::{Operation, Transaction},
+    dataset::transaction::{DataOverlayGroup, Operation, Transaction, UpdateMode},
 };
 use futures::{StreamExt, TryStreamExt};
 use lance_core::{Error, Result, utils::deletion::DeletionVector};
@@ -15,7 +15,9 @@ use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME;
 use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MergedGeneration};
 use lance_select::{RowAddrTreeMap, RowSetOps};
 use lance_table::format::IndexMetadata;
+use lance_table::format::overlay::OverlayCoverage;
 use lance_table::{format::Fragment, io::deletion::write_deletion_file};
+use roaring::RoaringBitmap;
 use std::{
     borrow::Cow,
     collections::{HashMap, HashSet},
@@ -137,6 +139,21 @@ impl<'a> TransactionRebase<'a> {
                     conflicting_mem_wal_merged_gens: Vec::new(),
                 })
             }
+            Operation::DataOverlay { groups } => {
+                let modified_fragment_ids =
+                    groups.iter().map(|g| g.fragment_id).collect::<HashSet<_>>();
+                let initial_fragments =
+                    initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids)
+                        .await;
+                Ok(Self {
+                    transaction,
+                    affected_rows,
+                    initial_fragments,
+                    modified_fragment_ids,
+                    conflicting_frag_reuse_indices: Vec::new(),
+                    conflicting_mem_wal_merged_gens: Vec::new(),
+                })
+            }
             Operation::Merge { fragments, .. } => {
                 let modified_fragment_ids = fragments.iter().map(|f| f.id).collect::<HashSet<_>>();
                 let initial_fragments =
@@ -181,6 +198,22 @@ impl<'a> TransactionRebase<'a> {
         )
     }
 
+    #[track_caller]
+    fn data_replacement_target_removed_err(
+        &self,
+        fragment_id: u64,
+        other_transaction: &Transaction,
+        other_version: u64,
+    ) -> Error {
+        Error::incompatible_transaction_source(
+            format!(
+                "DataReplacement target fragment {} was removed by concurrent {} at version {}.",
+                fragment_id, other_transaction.operation, other_version
+            )
+            .into(),
+        )
+    }
+
     /// Check whether the transaction conflicts with another transaction.
     /// Mutate the current [TransactionRebase] based on `other_transaction` to be used for
     /// eventually finishing the rebase process.
@@ -203,6 +236,9 @@ impl<'a> TransactionRebase<'a> {
             Operation::DataReplacement { .. } => {
                 self.check_data_replacement_txn(other_transaction, other_version)
             }
+            Operation::DataOverlay { .. } => {
+                self.check_data_overlay_txn(other_transaction, other_version)
+            }
             Operation::Merge { .. } => self.check_merge_txn(other_transaction, other_version),
             Operation::Restore { .. } => self.check_restore_txn(other_transaction, other_version),
             Operation::ReserveFragments { .. } => {
@@ -235,6 +271,10 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Project { .. }
                 | Operation::Append { .. }
                 | Operation::UpdateConfig { .. }
+                // A concurrent overlay is inert against the rows we delete
+                // (deletions take precedence over overlays) and otherwise
+                // preserves physical offsets, so it never conflicts.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
                 Operation::Rewrite { groups, .. } => {
                     if groups
@@ -330,6 +370,8 @@ impl<'a> TransactionRebase<'a> {
         if let Operation::Update {
             inserted_rows_filter: self_inserted_rows_filter,
             merged_generations: self_merged_generations,
+            new_fragments: self_new_fragments,
+            update_mode: self_update_mode,
             ..
         } = &self.transaction.operation
         {
@@ -383,6 +425,49 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Clone { .. }
                 | Operation::UpdateConfig { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
+                Operation::DataOverlay { groups } => {
+                    // Our update recomputed rows from the pre-overlay base, so if
+                    // it commits over an overlay it would silently undo the
+                    // overlay's values for any cell it recomputed. A row-moving
+                    // update (RewriteRows) relocates the rows it touches out to
+                    // new fragments; only the rows it actually moved lose their
+                    // overlay, so we conflict only when the moved rows intersect
+                    // the overlay's coverage. An in-place column rewrite
+                    // (RewriteColumns) preserves offsets and just tombstones the
+                    // overlaid fields at build time, so it never conflicts.
+                    let moves_rows = !self_new_fragments.is_empty()
+                        && matches!(self_update_mode, Some(UpdateMode::RewriteRows) | None);
+                    if !moves_rows {
+                        return Ok(());
+                    }
+                    // `affected_rows` holds the physical offsets (per fragment)
+                    // this update moved. The overlay's coverage is in the same
+                    // physical-offset space, so we can intersect the two in
+                    // memory. Without affected rows we cannot be precise, so we
+                    // fall back to a fragment-granular conflict.
+                    for group in groups {
+                        if !self.modified_fragment_ids.contains(&group.fragment_id) {
+                            continue;
+                        }
+                        let Some(affected_rows) = self.affected_rows else {
+                            return Err(
+                                self.retryable_conflict_err(other_transaction, other_version)
+                            );
+                        };
+                        let Some(moved) =
+                            affected_rows.get_fragment_bitmap(group.fragment_id as u32)
+                        else {
+                            continue;
+                        };
+                        let coverage = overlay_group_coverage(group);
+                        if !(moved & &coverage).is_empty() {
+                            return Err(
+                                self.retryable_conflict_err(other_transaction, other_version)
+                            );
+                        }
+                    }
+                    Ok(())
+                }
                 Operation::Append { .. } => {
                     // If current transaction has primary key conflict detection,
                     // we can't safely commit against an Append because we don't
@@ -498,37 +583,33 @@ impl<'a> TransactionRebase<'a> {
             match &other_transaction.operation {
                 Operation::Append { .. }
                 | Operation::Clone { .. }
+                // An overlay committed after this index's version is newer than
+                // the index; the query path excludes its covered cells via the
+                // version gate, so the build does not conflict.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
                 Operation::CreateIndex {
                     new_indices: created_indices,
-                    ..
+                    removed_indices: other_removed_indices,
                 } => {
-                    let self_has_frag_reuse = new_indices
-                        .iter()
-                        .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME);
-                    let other_has_frag_reuse = created_indices
-                        .iter()
-                        .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME);
-                    let self_has_mem_wal =
-                        new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME);
-                    let other_has_mem_wal = created_indices
-                        .iter()
-                        .any(|idx| idx.name == MEM_WAL_INDEX_NAME);
-                    let has_regular_name_conflict = new_indices
-                        .iter()
-                        .filter(|idx| {
-                            idx.name != FRAG_REUSE_INDEX_NAME && idx.name != MEM_WAL_INDEX_NAME
-                        })
-                        .any(|new_index| {
-                            created_indices
-                                .iter()
-                                .any(|created_index| created_index.name == new_index.name)
-                        });
-
-                    if (self_has_frag_reuse && other_has_frag_reuse)
-                        || (self_has_mem_wal && other_has_mem_wal)
-                        || has_regular_name_conflict
-                    {
+                    // Two index transactions conflict when they touch any index
+                    // name in common, in either direction: committing a creation
+                    // over a concurrent same-name removal would resurrect the
+                    // dropped index, and committing a removal over a concurrent
+                    // same-name replacement would silently no-op the drop (its
+                    // stale UUIDs no longer match anything at apply time).
+                    // System singletons need no special casing under this rule.
+                    let names_intersect =
+                        new_indices
+                            .iter()
+                            .chain(removed_indices.iter())
+                            .any(|self_index| {
+                                created_indices
+                                    .iter()
+                                    .chain(other_removed_indices.iter())
+                                    .any(|other_index| other_index.name == self_index.name)
+                            });
+                    if names_intersect {
                         Err(self.retryable_conflict_err(other_transaction, other_version))
                     } else {
                         Ok(())
@@ -537,10 +618,23 @@ impl<'a> TransactionRebase<'a> {
                 // Although some of the rows we indexed may have been deleted / moved,
                 // row ids are still valid, so we allow this optimistically.
                 Operation::Delete { .. } | Operation::Update { .. } => Ok(()),
-                // Merge, reserve, and project don't change row ids, so this should be fine.
+                // Merge and reserve don't change row ids, so this should be fine.
                 Operation::Merge { .. } => Ok(()),
                 Operation::ReserveFragments { .. } => Ok(()),
-                Operation::Project { .. } => Ok(()),
+                // A projection doesn't change row ids either, but it can drop the
+                // indexed column. Committing this index afterwards would publish
+                // metadata referencing a field that no longer exists.
+                Operation::Project { schema } => {
+                    let missing_field = new_indices
+                        .iter()
+                        .flat_map(|idx| idx.fields.iter())
+                        .any(|field_id| schema.field_by_id(*field_id).is_none());
+                    if missing_field {
+                        Err(self.incompatible_conflict_err(other_transaction, other_version))
+                    } else {
+                        Ok(())
+                    }
+                }
                 // Should be compatible with rewrite if it didn't move the rows
                 // we indexed. If it did, we could retry.
                 // TODO: this will change with stable row ids.
@@ -679,6 +773,20 @@ impl<'a> TransactionRebase<'a> {
                         Ok(())
                     }
                 }
+                Operation::DataOverlay { groups } => {
+                    // Rewriting a fragment changes its physical row addresses, so
+                    // an overlay addressed by physical offset on that fragment is
+                    // invalidated and must be re-applied against the new base.
+                    if groups
+                        .iter()
+                        .map(|g| g.fragment_id)
+                        .any(|id| self.modified_fragment_ids.contains(&id))
+                    {
+                        Err(self.retryable_conflict_err(other_transaction, other_version))
+                    } else {
+                        Ok(())
+                    }
+                }
                 Operation::Rewrite {
                     groups,
                     frag_reuse_index: committed_fri,
@@ -858,6 +966,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::Restore { .. }
             | Operation::ReserveFragments { .. }
@@ -891,7 +1000,8 @@ impl<'a> TransactionRebase<'a> {
             | Operation::Merge { .. }
             | Operation::UpdateConfig { .. }
             | Operation::Clone { .. }
-            | Operation::DataReplacement { .. } => Ok(()),
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => Ok(()),
         }
     }
 
@@ -904,13 +1014,75 @@ impl<'a> TransactionRebase<'a> {
             match &other_transaction.operation {
                 Operation::Append { .. }
                 | Operation::Clone { .. }
-                | Operation::Delete { .. }
-                | Operation::Update { .. }
-                | Operation::Merge { .. }
                 | Operation::UpdateConfig { .. }
                 | Operation::ReserveFragments { .. }
                 | Operation::Project { .. }
+                // Both a column replacement and an overlay preserve physical row
+                // addresses; the overlay is newer and wins its covered cells.
+                | Operation::DataOverlay { .. }
                 | Operation::UpdateBases { .. } => Ok(()),
+                Operation::Merge { .. } => {
+                    // Merge rewrites the whole fragment list; always conflict
+                    // (symmetric with check_merge_txn).
+                    Err(self.retryable_conflict_err(other_transaction, other_version))
+                }
+                Operation::Delete {
+                    deleted_fragment_ids,
+                    ..
+                } => {
+                    // A delete only tombstones rows (deletion vector); our positional
+                    // file stays aligned and the rebase preserves the deletion vector.
+                    // Conflict only if our target fragment was removed outright.
+                    for replacement in replacements {
+                        if deleted_fragment_ids.contains(&replacement.0) {
+                            return Err(self.data_replacement_target_removed_err(
+                                replacement.0,
+                                other_transaction,
+                                other_version,
+                            ));
+                        }
+                    }
+                    Ok(())
+                }
+                Operation::Update {
+                    removed_fragment_ids,
+                    updated_fragments,
+                    new_fragments,
+                    fields_modified,
+                    update_mode,
+                    ..
+                } => {
+                    for replacement in replacements {
+                        if removed_fragment_ids.contains(&replacement.0) {
+                            return Err(self.data_replacement_target_removed_err(
+                                replacement.0,
+                                other_transaction,
+                                other_version,
+                            ));
+                        }
+                        if !updated_fragments.iter().any(|f| f.id == replacement.0) {
+                            continue;
+                        }
+                        // A row-rewriting update moves the matched rows out to
+                        // new_fragments our positional file does not cover; a horizontal
+                        // update may rewrite one of our fields in place. Either makes the
+                        // file stale. (RewriteColumns new_fragments are unrelated inserts,
+                        // not moved rows, so they stay aligned.)
+                        let moved_rows = !new_fragments.is_empty()
+                            && matches!(update_mode, Some(UpdateMode::RewriteRows) | None);
+                        let field_rewritten = replacement
+                            .1
+                            .fields
+                            .iter()
+                            .any(|f| *f >= 0 && fields_modified.contains(&(*f as u32)));
+                        if moved_rows || field_rewritten {
+                            return Err(
+                                self.retryable_conflict_err(other_transaction, other_version)
+                            );
+                        }
+                    }
+                    Ok(())
+                }
                 Operation::CreateIndex { new_indices, .. } => {
                     // A data replacement only conflicts if it is updating the field that
                     // is being indexed.
@@ -980,6 +1152,119 @@ impl<'a> TransactionRebase<'a> {
         }
     }
 
+    /// Conflict checks for our DataOverlay transaction against a concurrent one.
+    ///
+    /// Overlays are intentionally permissive (see the Data Overlay Files spec):
+    /// they stack with other overlays and tolerate appends, index builds, data
+    /// replacement, deletes, and in-place column rewrites (Update with
+    /// `RewriteColumns`), because overlay coverage is addressed by physical offset
+    /// and the version gate keeps indexes correct. A concurrent operation
+    /// conflicts when it takes precedence over the overlay for cells the overlay
+    /// covers, dropping the overlay's values: retryably when it rewrites the
+    /// physical layout of one of our fragments (Rewrite, Merge) or re-creates the
+    /// covered rows from the pre-overlay base (a row-moving Update — checked
+    /// row-by-row in `finish_data_overlay`), or removes an overlaid fragment
+    /// outright (a Delete / Update that drops the fragment); and incompatibly for
+    /// whole-dataset replacements (Overwrite / Restore) and MemWAL state updates
+    /// (UpdateMemWalState), which do not rebase against data operations.
+    fn check_data_overlay_txn(
+        &mut self,
+        other_transaction: &Transaction,
+        other_version: u64,
+    ) -> Result<()> {
+        match &other_transaction.operation {
+            Operation::Append { .. }
+            | Operation::CreateIndex { .. }
+            | Operation::ReserveFragments { .. }
+            | Operation::Project { .. }
+            | Operation::UpdateConfig { .. }
+            | Operation::UpdateBases { .. }
+            | Operation::Clone { .. }
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => Ok(()),
+            // A concurrent Delete only tombstones rows via a deletion vector,
+            // which preserves physical offsets; the overlay value for a deleted
+            // offset is simply inert. Conflict only if the whole overlaid
+            // fragment was removed, orphaning the overlay.
+            Operation::Delete {
+                deleted_fragment_ids,
+                ..
+            } => {
+                if deleted_fragment_ids
+                    .iter()
+                    .any(|id| self.modified_fragment_ids.contains(id))
+                {
+                    Err(self.retryable_conflict_err(other_transaction, other_version))
+                } else {
+                    Ok(())
+                }
+            }
+            // A concurrent Update that removed an overlaid fragment orphans the
+            // overlay outright — conflict. A row-moving update (RewriteRows)
+            // deletes the rows it touches and re-creates them in new fragments;
+            // the update took precedence and the re-created rows were computed
+            // from the pre-overlay base, so the overlay's values for those cells
+            // are lost. That is a per-row problem, not an offset one: only the
+            // moved rows are affected. Comparing the moved rows against the
+            // overlay's coverage needs the update's deletion vectors, so we mark
+            // the fragment here and verify row-by-row in `finish_data_overlay`.
+            // An in-place column rewrite (RewriteColumns) preserves rows and just
+            // tombstones the overlaid fields at build time, so it never conflicts.
+            Operation::Update {
+                removed_fragment_ids,
+                updated_fragments,
+                new_fragments,
+                update_mode,
+                ..
+            } => {
+                let removed_ours = removed_fragment_ids
+                    .iter()
+                    .any(|id| self.modified_fragment_ids.contains(id));
+                if removed_ours {
+                    return Err(self.retryable_conflict_err(other_transaction, other_version));
+                }
+                let moves_rows = !new_fragments.is_empty()
+                    && matches!(update_mode, Some(UpdateMode::RewriteRows) | None);
+                if moves_rows {
+                    for updated in updated_fragments {
+                        if let Some((_, needs_row_check)) =
+                            self.initial_fragments.get_mut(&updated.id)
+                        {
+                            *needs_row_check = true;
+                        }
+                    }
+                }
+                Ok(())
+            }
+            Operation::Rewrite { groups, .. } => {
+                // A rewrite (compaction / fold) of a fragment we are overlaying
+                // changes its physical row addresses, so our offsets would be
+                // invalid. Conflict only if it touches one of our fragments.
+                let touches_our_fragment = groups
+                    .iter()
+                    .flat_map(|g| g.old_fragments.iter())
+                    .any(|f| self.modified_fragment_ids.contains(&f.id));
+                if touches_our_fragment {
+                    Err(self.retryable_conflict_err(other_transaction, other_version))
+                } else {
+                    Ok(())
+                }
+            }
+            Operation::Merge { .. } => {
+                // Merge rewrites the whole fragment list; always conflict.
+                Err(self.retryable_conflict_err(other_transaction, other_version))
+            }
+            // Overwrite/Restore replace the dataset; UpdateMemWalState does not
+            // rebase against data operations (mirroring check_update_mem_wal_state_txn,
+            // which likewise treats a concurrent DataOverlay as incompatible).
+            Operation::Overwrite { .. }
+            | Operation::Restore { .. }
+            | Operation::UpdateMemWalState { .. } => {
+                Err(self.incompatible_conflict_err(other_transaction, other_version))
+            }
+        }
+    }
+
     fn check_merge_txn(
         &mut self,
         other_transaction: &Transaction,
@@ -997,7 +1282,8 @@ impl<'a> TransactionRebase<'a> {
             | Operation::Delete { .. }
             | Operation::Rewrite { .. }
             | Operation::Merge { .. }
-            | Operation::DataReplacement { .. } => {
+            | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. } => {
                 Err(self.retryable_conflict_err(other_transaction, other_version))
             }
             Operation::Overwrite { .. }
@@ -1021,6 +1307,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::Restore { .. }
             | Operation::ReserveFragments { .. }
@@ -1049,6 +1336,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::CreateIndex { .. }
             | Operation::Rewrite { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Merge { .. }
             | Operation::ReserveFragments { .. }
             | Operation::Update { .. }
@@ -1073,6 +1361,7 @@ impl<'a> TransactionRebase<'a> {
             | Operation::UpdateConfig { .. }
             | Operation::CreateIndex { .. }
             | Operation::DataReplacement { .. }
+            | Operation::DataOverlay { .. }
             | Operation::Rewrite { .. }
             | Operation::Clone { .. }
             | Operation::ReserveFragments { .. }
@@ -1137,6 +1426,7 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::CreateIndex { .. }
                 | Operation::Rewrite { .. }
                 | Operation::DataReplacement { .. }
+                | Operation::DataOverlay { .. }
                 | Operation::Merge { .. }
                 | Operation::Restore { .. }
                 | Operation::ReserveFragments { .. }
@@ -1209,6 +1499,7 @@ impl<'a> TransactionRebase<'a> {
                 | Operation::Overwrite { .. }
                 | Operation::Delete { .. }
                 | Operation::DataReplacement { .. }
+                | Operation::DataOverlay { .. }
                 | Operation::Merge { .. }
                 | Operation::Restore { .. }
                 | Operation::Clone { .. }
@@ -1299,6 +1590,7 @@ impl<'a> TransactionRebase<'a> {
             }
             Operation::CreateIndex { .. } => self.finish_create_index(dataset).await,
             Operation::Rewrite { .. } => self.finish_rewrite(dataset).await,
+            Operation::DataOverlay { .. } => self.finish_data_overlay(dataset).await,
             Operation::Append { .. }
             | Operation::Overwrite { .. }
             | Operation::DataReplacement { .. }
@@ -1475,6 +1767,90 @@ impl<'a> TransactionRebase<'a> {
         }
     }
 
+    /// Verify no concurrent row-moving Update dropped the values of any cell
+    /// this overlay covers. `check_data_overlay_txn` flags (via the
+    /// `initial_fragments` needs-check bool) each overlaid fragment on which a
+    /// concurrent RewriteRows update relocated rows; here we read the deletion
+    /// vectors and conflict only when the moved rows intersect the overlay's
+    /// coverage.
+    ///
+    /// The moved rows are computed as the current deletion vector minus the
+    /// read-time one. In the rare case where both a concurrent Delete and a
+    /// concurrent Update touched the same flagged fragment, the Delete's rows are
+    /// also counted and may trigger an unnecessary retry — never data loss. Pure
+    /// concurrent deletes leave the fragment unflagged and are not examined here.
+    async fn finish_data_overlay(self, dataset: &Dataset) -> Result<Transaction> {
+        let fragments_to_check: HashSet<u64> = self
+            .initial_fragments
+            .iter()
+            .filter_map(|(id, (_, needs_check))| needs_check.then_some(*id))
+            .collect();
+        if fragments_to_check.is_empty() {
+            return Ok(Transaction {
+                read_version: dataset.manifest.version,
+                ..self.transaction
+            });
+        }
+
+        // Coverage (physical offsets, unioned across fields) per flagged fragment.
+        let Operation::DataOverlay { groups } = &self.transaction.operation else {
+            return Err(wrong_operation_err(&self.transaction.operation));
+        };
+        let mut coverage_by_fragment: HashMap<u64, RoaringBitmap> = HashMap::new();
+        for group in groups {
+            if !fragments_to_check.contains(&group.fragment_id) {
+                continue;
+            }
+            *coverage_by_fragment.entry(group.fragment_id).or_default() |=
+                overlay_group_coverage(group);
+        }
+
+        for (fragment_id, coverage) in coverage_by_fragment {
+            let Some(current_fragment) = dataset
+                .fragments()
+                .as_slice()
+                .iter()
+                .find(|f| f.id == fragment_id)
+            else {
+                // The fragment is gone entirely; the overlay is orphaned.
+                return Err(crate::Error::retryable_commit_conflict_source(
+                    dataset.manifest.version,
+                    format!(
+                        "This {} transaction was preempted: overlaid fragment {} was removed by a concurrent transaction. Please retry.",
+                        self.transaction.uuid, fragment_id
+                    )
+                    .into(),
+                ));
+            };
+            let current_deletions =
+                read_fragment_deletion_bitmap(dataset, current_fragment).await?;
+            let initial_deletions = match self.initial_fragments.get(&fragment_id) {
+                Some((initial_fragment, _)) => {
+                    read_fragment_deletion_bitmap(dataset, initial_fragment).await?
+                }
+                None => RoaringBitmap::new(),
+            };
+            let moved_rows = &current_deletions - &initial_deletions;
+            let conflicting = &moved_rows & &coverage;
+            if !conflicting.is_empty() {
+                let sample: Vec<u32> = conflicting.iter().take(5).collect();
+                return Err(crate::Error::retryable_commit_conflict_source(
+                    dataset.manifest.version,
+                    format!(
+                        "This {} transaction was preempted by a concurrent update that moved overlaid rows on fragment {} (offsets {:?}). Please retry.",
+                        self.transaction.uuid, fragment_id, sample.as_slice()
+                    )
+                    .into(),
+                ));
+            }
+        }
+
+        Ok(Transaction {
+            read_version: dataset.manifest.version,
+            ..self.transaction
+        })
+    }
+
     async fn finish_create_index(mut self, dataset: &Dataset) -> Result<Transaction> {
         if let Operation::CreateIndex {
             new_indices,
@@ -1699,6 +2075,40 @@ async fn initial_fragments_for_rebase(
         .collect::<HashMap<_, _>>()
 }
 
+/// Read a fragment's deletion vector as a bitmap of physical offsets, or an
+/// empty bitmap when the fragment has no deletion file.
+async fn read_fragment_deletion_bitmap(
+    dataset: &Dataset,
+    fragment: &Fragment,
+) -> Result<RoaringBitmap> {
+    match &fragment.deletion_file {
+        Some(deletion_file) => {
+            let dv = read_dataset_deletion_file(dataset, fragment.id, deletion_file).await?;
+            Ok(RoaringBitmap::from(dv.as_ref()))
+        }
+        None => Ok(RoaringBitmap::new()),
+    }
+}
+
+/// The physical offsets a group's overlays cover, unioned across every overlay
+/// and every field. This is the set of cells whose values the overlay supplies,
+/// used to test whether a concurrent row-moving Update actually invalidates the
+/// overlay.
+fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap {
+    let mut union = RoaringBitmap::new();
+    for overlay in &group.overlays {
+        match &overlay.coverage {
+            OverlayCoverage::Shared(bitmap) => union |= bitmap.as_ref(),
+            OverlayCoverage::PerField(bitmaps) => {
+                for bitmap in bitmaps {
+                    union |= bitmap.as_ref();
+                }
+            }
+        }
+    }
+    union
+}
+
 fn wrong_operation_err(op: &Operation) -> Error {
     Error::internal(format!("function called against a wrong operation: {}", op))
 }
@@ -1707,6 +2117,7 @@ fn wrong_operation_err(op: &Operation) -> Error {
 mod tests {
     use std::{num::NonZero, sync::Arc};
 
+    use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows};
     use arrow_array::{Int32Array, RecordBatch};
     use arrow_schema::{DataType, Field, Schema};
     use lance_core::Error;
@@ -2706,49 +3117,485 @@ mod tests {
     }
 
     #[test]
-    fn test_create_index_conflicts_only_on_same_name() {
-        let index0 = IndexMetadata {
-            uuid: uuid::Uuid::new_v4(),
-            name: "test".to_string(),
-            fields: vec![0],
-            dataset_version: 1,
-            fragment_bitmap: None,
-            index_details: None,
-            index_version: 0,
-            created_at: None,
-            base_id: None,
-            files: None,
+    fn test_data_overlay_conflicts() {
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use ConflictResult::*;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        // Our transaction overlays fragment 1.
+        let overlay_op = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
         };
-        let index1 = IndexMetadata {
-            uuid: uuid::Uuid::new_v4(),
-            name: "other".to_string(),
-            ..index0.clone()
+        let update_removing = |removed_fragment_ids: Vec<u64>| Operation::Update {
+            removed_fragment_ids,
+            updated_fragments: vec![],
+            new_fragments: vec![],
+            fields_modified: vec![],
+            merged_generations: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: None,
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
         };
-
-        let txn = Transaction::new(
-            0,
-            Operation::CreateIndex {
-                new_indices: vec![index0.clone()],
-                removed_indices: vec![],
-            },
-            None,
-        );
-        let mut rebase = TransactionRebase {
-            transaction: txn,
-            initial_fragments: HashMap::new(),
-            modified_fragment_ids: HashSet::new(),
-            affected_rows: None,
-            conflicting_frag_reuse_indices: Vec::new(),
-            conflicting_mem_wal_merged_gens: Vec::new(),
+        let delete = |updated: Vec<Fragment>, deleted: Vec<u64>| Operation::Delete {
+            updated_fragments: updated,
+            deleted_fragment_ids: deleted,
+            predicate: "x > 2".to_string(),
+        };
+        // A row-moving update (RewriteRows) relocates the updated rows into
+        // new_fragments; an in-place column rewrite (RewriteColumns) leaves rows
+        // where they are.
+        let update_moving = |updated: Vec<Fragment>, new: Vec<Fragment>| Operation::Update {
+            removed_fragment_ids: vec![],
+            updated_fragments: updated,
+            new_fragments: new,
+            fields_modified: vec![],
+            merged_generations: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteRows),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let update_rewrite_columns = |updated: Vec<Fragment>| Operation::Update {
+            removed_fragment_ids: vec![],
+            updated_fragments: updated,
+            new_fragments: vec![],
+            fields_modified: vec![0],
+            merged_generations: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteColumns),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let rewrite_of = |old: &Fragment| Operation::Rewrite {
+            groups: vec![RewriteGroup {
+                old_fragments: vec![old.clone()],
+                new_fragments: vec![],
+            }],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
         };
 
-        let same_name = Transaction::new(
-            0,
-            Operation::CreateIndex {
-                new_indices: vec![IndexMetadata {
-                    uuid: uuid::Uuid::new_v4(),
-                    ..index0
-                }],
+        let fragment0 = Fragment::new(0);
+        let fragment1 = Fragment::new(1);
+
+        // Each case is checked against our overlay on fragment 1.
+        let cases: Vec<(Operation, ConflictResult)> = vec![
+            // Permissive: preserves physical offsets / leaves fragment 1 in place.
+            (
+                Operation::Append {
+                    fragments: vec![fragment0.clone()],
+                },
+                Compatible,
+            ),
+            (
+                Operation::CreateIndex {
+                    new_indices: vec![],
+                    removed_indices: vec![],
+                },
+                Compatible,
+            ),
+            (
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(
+                        1,
+                        DataFile::new_legacy_from_fields("r.lance", vec![0], None),
+                    )],
+                },
+                Compatible,
+            ),
+            // Another overlay on the same fragment stacks rather than conflicts.
+            (overlay_op(1), Compatible),
+            // A Delete only tombstones rows (deletion vector) on fragment 1, and
+            // an in-place column rewrite preserves offsets, so both are compatible.
+            (delete(vec![fragment1.clone()], vec![]), Compatible),
+            (update_rewrite_columns(vec![fragment1.clone()]), Compatible),
+            (update_removing(vec![2]), Compatible),
+            // ...but removing our overlaid fragment 1 orphans the overlay -> conflict.
+            (delete(vec![], vec![1]), Retryable),
+            (update_removing(vec![1]), Retryable),
+            // A row-moving update re-creates the rows it touches from the
+            // pre-overlay base. Whether that actually drops any overlaid cell is
+            // a per-row question answered in `finish_data_overlay` (see
+            // test_data_overlay_finish_conflicts_with_row_moving_update), so the
+            // check itself defers rather than conflicting; a moving update on any
+            // fragment is compatible at this stage.
+            (
+                update_moving(vec![fragment1.clone()], vec![fragment0.clone()]),
+                Compatible,
+            ),
+            (
+                update_moving(vec![fragment0.clone()], vec![fragment0.clone()]),
+                Compatible,
+            ),
+            // Rewriting fragment 1 invalidates its physical offsets -> conflict;
+            // a rewrite of a different fragment does not.
+            (rewrite_of(&fragment1), Retryable),
+            (rewrite_of(&fragment0), Compatible),
+            // Merge rewrites the whole fragment list; Restore replaces the dataset.
+            (
+                Operation::Merge {
+                    fragments: vec![fragment1.clone()],
+                    schema: lance_core::datatypes::Schema::default(),
+                },
+                Retryable,
+            ),
+            (Operation::Restore { version: 1 }, NotCompatible),
+            // Overwrite/Restore replace the dataset, and UpdateMemWalState does
+            // not rebase against data operations — all hard conflicts.
+            (
+                Operation::Overwrite {
+                    fragments: vec![fragment0.clone()],
+                    schema: lance_core::datatypes::Schema::default(),
+                    config_upsert_values: None,
+                    initial_bases: None,
+                },
+                NotCompatible,
+            ),
+            (
+                Operation::UpdateMemWalState {
+                    merged_generations: vec![],
+                },
+                NotCompatible,
+            ),
+        ];
+
+        for (other, expected) in cases {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, overlay_op(1), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&overlay_op(1))
+                    .collect::<HashSet<_>>(),
+                affected_rows: None,
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_merged_gens: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            match expected {
+                Compatible => assert!(
+                    result.is_ok(),
+                    "overlay should be compatible with {other:?}, got {result:?}"
+                ),
+                Retryable => assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "overlay should retryably conflict with {other:?}, got {result:?}"
+                ),
+                NotCompatible => assert!(
+                    matches!(result, Err(Error::IncompatibleTransaction { .. })),
+                    "overlay should be incompatible with {other:?}, got {result:?}"
+                ),
+            }
+        }
+    }
+
+    #[test]
+    fn test_rewrite_conflicts_with_data_overlay() {
+        // Reverse direction of test_data_overlay_conflicts: our transaction is a
+        // Rewrite and a concurrent DataOverlay has already committed. A rewrite
+        // changes the physical row addresses of the fragments it touches, so an
+        // overlay on one of those fragments is invalidated (retryable); an
+        // overlay on any other fragment is unaffected.
+        use crate::dataset::transaction::DataOverlayGroup;
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let overlay_on = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        // Our transaction rewrites fragment 1.
+        let rewrite_op = Operation::Rewrite {
+            groups: vec![RewriteGroup {
+                old_fragments: vec![Fragment::new(1)],
+                new_fragments: vec![],
+            }],
+            rewritten_indices: vec![],
+            frag_reuse_index: None,
+        };
+
+        for (other, expect_conflict) in [(overlay_on(1), true), (overlay_on(0), false)] {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, rewrite_op.clone(), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&rewrite_op).collect::<HashSet<_>>(),
+                affected_rows: None,
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_merged_gens: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            if expect_conflict {
+                assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "rewrite of fragment 1 should retryably conflict with {other:?}, got {result:?}"
+                );
+            } else {
+                assert!(
+                    result.is_ok(),
+                    "rewrite of fragment 1 should not conflict with {other:?}, got {result:?}"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_update_conflicts_with_data_overlay() {
+        // Reverse direction of test_data_overlay_conflicts: our transaction is an
+        // Update and a concurrent DataOverlay has already committed. A row-moving
+        // update relocates the rows it touches, so an overlay on one of those
+        // fragments can no longer be applied (retryable); an overlay on any other
+        // fragment, or an in-place column rewrite, is compatible.
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let overlay_on = |fragment_id: u64| Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        // Our update always touches fragment 1.
+        let update =
+            |update_mode: Option<UpdateMode>, new_fragments: Vec<Fragment>| Operation::Update {
+                removed_fragment_ids: vec![],
+                updated_fragments: vec![Fragment::new(1)],
+                new_fragments,
+                fields_modified: vec![0],
+                merged_generations: Vec::new(),
+                fields_for_preserving_frag_bitmap: vec![],
+                update_mode,
+                inserted_rows_filter: None,
+                updated_fragment_offsets: None,
+            };
+
+        // The overlay covers physical offset 0 of its fragment. Row addresses
+        // pack the fragment id in the high 32 bits and the offset in the low 32.
+        let rows_on = |fragment_id: u64, offsets: &[u32]| {
+            let mut map = RowAddrTreeMap::new();
+            map.insert_bitmap(
+                fragment_id as u32,
+                RoaringBitmap::from_iter(offsets.iter().copied()),
+            );
+            map
+        };
+
+        // (update, committed overlay, moved rows the update carries, expect conflict)
+        let cases = [
+            // Row-moving update whose moved rows include the overlaid cell -> the
+            // update would undo the overlay, so conflict.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                Some(rows_on(1, &[0])),
+                true,
+            ),
+            // ...but if the moved rows miss the overlaid cell, the overlay survives.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                Some(rows_on(1, &[5])),
+                false,
+            ),
+            // An overlay on a fragment the update did not touch is fine.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(0),
+                Some(rows_on(1, &[0])),
+                false,
+            ),
+            // An in-place column rewrite preserves rows -> compatible.
+            (
+                update(Some(UpdateMode::RewriteColumns), vec![]),
+                overlay_on(1),
+                Some(rows_on(1, &[0])),
+                false,
+            ),
+            // Without affected rows we cannot be precise, so a row-moving update
+            // on the overlaid fragment falls back to a conservative conflict.
+            (
+                update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]),
+                overlay_on(1),
+                None,
+                true,
+            ),
+        ];
+
+        for (update_op, other, affected_rows, expect_conflict) in cases {
+            let mut rebase = TransactionRebase {
+                transaction: Transaction::new(0, update_op.clone(), None),
+                initial_fragments: HashMap::new(),
+                modified_fragment_ids: modified_fragment_ids(&update_op).collect::<HashSet<_>>(),
+                affected_rows: affected_rows.as_ref(),
+                conflicting_frag_reuse_indices: Vec::new(),
+                conflicting_mem_wal_merged_gens: Vec::new(),
+            };
+            let other_txn = Transaction::new(0, other.clone(), None);
+            let result = rebase.check_txn(&other_txn, 1);
+            if expect_conflict {
+                assert!(
+                    matches!(result, Err(Error::RetryableCommitConflict { .. })),
+                    "update should retryably conflict with {other:?}, got {result:?}"
+                );
+            } else {
+                assert!(
+                    result.is_ok(),
+                    "update should be compatible with {other:?}, got {result:?}"
+                );
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[rstest::rstest]
+    #[case::coverage_overlaps_moved_row(vec![0u32], true)]
+    #[case::coverage_disjoint_from_moved_row(vec![3u32], false)]
+    async fn test_data_overlay_finish_conflicts_with_row_moving_update(
+        #[case] coverage_offsets: Vec<u32>,
+        #[case] expect_conflict: bool,
+    ) {
+        // 5 rows in one fragment. A concurrent RewriteRows update moves row 0 out
+        // to a new fragment (deleting it from fragment 0). Our overlay on fragment
+        // 0 conflicts only when its coverage includes the moved row; the decision
+        // is made in finish, which reads the deletion vectors.
+        use crate::dataset::transaction::{DataOverlayGroup, UpdateMode};
+        use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
+        use roaring::RoaringBitmap;
+
+        let dataset = test_dataset(5, 1).await;
+        let mut fragment = dataset.fragments().as_slice()[0].clone();
+
+        let moved_fragment = Fragment::new(0)
+            .with_file(
+                "moved.lance",
+                vec![0],
+                vec![0],
+                &LanceFileVersion::Stable,
+                NonZero::new(10),
+            )
+            .with_physical_rows(1);
+        let update_op = Operation::Update {
+            updated_fragments: vec![apply_deletion(&[0], &mut fragment, &dataset).await],
+            removed_fragment_ids: vec![],
+            new_fragments: vec![moved_fragment],
+            fields_modified: vec![],
+            merged_generations: Vec::new(),
+            fields_for_preserving_frag_bitmap: vec![],
+            update_mode: Some(UpdateMode::RewriteRows),
+            inserted_rows_filter: None,
+            updated_fragment_offsets: None,
+        };
+        let update_txn = Transaction::new_from_version(dataset.manifest.version, update_op);
+
+        let overlay_op = Operation::DataOverlay {
+            groups: vec![DataOverlayGroup {
+                fragment_id: 0,
+                overlays: vec![DataOverlayFile {
+                    data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None),
+                    coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(coverage_offsets)),
+                    committed_version: 0,
+                }],
+            }],
+        };
+        let overlay_txn = Transaction::new_from_version(dataset.manifest.version, overlay_op);
+
+        // Commit the update so the latest dataset reflects the moved (deleted) row.
+        let latest_dataset = CommitBuilder::new(Arc::new(dataset.clone()))
+            .execute(update_txn.clone())
+            .await
+            .unwrap();
+
+        let mut rebase = TransactionRebase::try_new(&dataset, overlay_txn.clone(), None)
+            .await
+            .unwrap();
+        // The check defers the row-level decision to finish, flagging fragment 0.
+        rebase.check_txn(&update_txn, 1).unwrap();
+        assert_eq!(
+            rebase
+                .initial_fragments
+                .iter()
+                .map(|(id, (_, needs_check))| (*id, *needs_check))
+                .collect::<Vec<_>>(),
+            vec![(0, true)],
+        );
+
+        let res = rebase.finish(&latest_dataset).await;
+        if expect_conflict {
+            assert!(
+                matches!(res, Err(crate::Error::RetryableCommitConflict { .. })),
+                "overlay covering the moved row should conflict, got {res:?}"
+            );
+        } else {
+            assert!(
+                res.is_ok(),
+                "overlay disjoint from the moved row should succeed, got {res:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_create_index_conflicts_only_on_same_name() {
+        let index0 = IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            name: "test".to_string(),
+            fields: vec![0],
+            dataset_version: 1,
+            fragment_bitmap: None,
+            index_details: None,
+            index_version: 0,
+            created_at: None,
+            base_id: None,
+            files: None,
+        };
+        let index1 = IndexMetadata {
+            uuid: uuid::Uuid::new_v4(),
+            name: "other".to_string(),
+            ..index0.clone()
+        };
+
+        let txn = Transaction::new(
+            0,
+            Operation::CreateIndex {
+                new_indices: vec![index0.clone()],
+                removed_indices: vec![],
+            },
+            None,
+        );
+        let mut rebase = TransactionRebase {
+            transaction: txn,
+            initial_fragments: HashMap::new(),
+            modified_fragment_ids: HashSet::new(),
+            affected_rows: None,
+            conflicting_frag_reuse_indices: Vec::new(),
+            conflicting_mem_wal_merged_gens: Vec::new(),
+        };
+
+        let same_name = Transaction::new(
+            0,
+            Operation::CreateIndex {
+                new_indices: vec![IndexMetadata {
+                    uuid: uuid::Uuid::new_v4(),
+                    ..index0
+                }],
                 removed_indices: vec![],
             },
             None,
@@ -3179,6 +4026,7 @@ mod tests {
             Operation::DataReplacement { replacements } => {
                 Box::new(replacements.iter().map(|r| r.0))
             }
+            Operation::DataOverlay { groups } => Box::new(groups.iter().map(|g| g.fragment_id)),
         }
     }
 
@@ -3258,7 +4106,7 @@ mod tests {
             (
                 "DataReplacement vs Rewrite on different fragment",
                 Operation::DataReplacement {
-                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01)],
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
                 },
                 Operation::Rewrite {
                     groups: vec![RewriteGroup {
@@ -3270,6 +4118,154 @@ mod tests {
                 },
                 Compatible,
             ),
+            // A concurrent Update/Delete only invalidates our positional file when it
+            // removes our target fragment outright, or (a horizontal update) rewrites
+            // one of our fields. A deletion-vector-only change stays aligned.
+            (
+                "DataReplacement vs Update (RewriteColumns) on a different field",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![Fragment::new(0)],
+                    removed_fragment_ids: vec![],
+                    new_fragments: vec![],
+                    fields_modified: vec![2],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: Some(RewriteColumns),
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                Compatible,
+            ),
+            (
+                // RewriteColumns new_fragments are unrelated inserts, not moved rows.
+                "DataReplacement vs Update (RewriteColumns) with inserts on a different field",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![Fragment::new(0)],
+                    removed_fragment_ids: vec![],
+                    new_fragments: vec![Fragment::new(5)],
+                    fields_modified: vec![2],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: Some(RewriteColumns),
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                Compatible,
+            ),
+            (
+                "DataReplacement vs Update (RewriteColumns) that rewrote one of our fields",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![Fragment::new(0)],
+                    removed_fragment_ids: vec![],
+                    new_fragments: vec![],
+                    fields_modified: vec![1],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: Some(RewriteColumns),
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                Retryable,
+            ),
+            (
+                "DataReplacement vs Update (RewriteRows) that moved our rows",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![Fragment::new(0)],
+                    removed_fragment_ids: vec![],
+                    new_fragments: vec![Fragment::new(5)],
+                    fields_modified: vec![],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: Some(RewriteRows),
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                Retryable,
+            ),
+            (
+                "DataReplacement vs Update that removed our fragment",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![],
+                    removed_fragment_ids: vec![0],
+                    new_fragments: vec![],
+                    fields_modified: vec![],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: None,
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                NotCompatible,
+            ),
+            (
+                "DataReplacement vs Update (RewriteRows) that moved a different fragment's rows",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Update {
+                    updated_fragments: vec![Fragment::new(1)],
+                    removed_fragment_ids: vec![],
+                    new_fragments: vec![Fragment::new(5)],
+                    fields_modified: vec![],
+                    merged_generations: Vec::new(),
+                    fields_for_preserving_frag_bitmap: vec![],
+                    update_mode: Some(RewriteRows),
+                    inserted_rows_filter: None,
+                    updated_fragment_offsets: None,
+                },
+                Compatible,
+            ),
+            (
+                "DataReplacement vs Delete (deletion-vector only) on same fragment",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Delete {
+                    deleted_fragment_ids: vec![],
+                    updated_fragments: vec![Fragment::new(0)],
+                    predicate: "a > 0".to_string(),
+                },
+                Compatible,
+            ),
+            (
+                "DataReplacement vs Delete that removes the fragment",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01.clone())],
+                },
+                Operation::Delete {
+                    deleted_fragment_ids: vec![0],
+                    updated_fragments: vec![],
+                    predicate: "a > 0".to_string(),
+                },
+                NotCompatible,
+            ),
+            // Merge rewrites the whole fragment list -> always conflicts.
+            (
+                "DataReplacement vs Merge",
+                Operation::DataReplacement {
+                    replacements: vec![DataReplacementGroup(0, data_file_frag0_fields01)],
+                },
+                Operation::Merge {
+                    fragments: vec![Fragment::new(0)],
+                    schema: lance_core::datatypes::Schema::default(),
+                },
+                Retryable,
+            ),
         ];
 
         for (description, op1, op2, expected) in cases {
@@ -3296,8 +4292,10 @@ mod tests {
                     );
                 }
                 NotCompatible => {
+                    // Removal returns a non-retryable IncompatibleTransaction so the
+                    // caller can drop the fragment instead of retrying.
                     assert!(
-                        matches!(result, Err(Error::CommitConflict { .. })),
+                        matches!(result, Err(Error::IncompatibleTransaction { .. })),
                         "{}: expected NotCompatible but got {:?}",
                         description,
                         result
@@ -3649,4 +4647,77 @@ mod tests {
 
         assert_eq!(dataset_v2.count_rows(None).await.unwrap(), 5);
     }
+
+    #[tokio::test]
+    async fn test_index_transactions_conflict_on_shared_names() {
+        fn index_meta(name: &str) -> IndexMetadata {
+            IndexMetadata {
+                uuid: Uuid::new_v4(),
+                name: name.to_string(),
+                fields: vec![0],
+                dataset_version: 1,
+                fragment_bitmap: Some([0u32].into_iter().collect()),
+                index_details: None,
+                index_version: 0,
+                created_at: None,
+                base_id: None,
+                files: None,
+            }
+        }
+        fn create_op(name: &str) -> Operation {
+            Operation::CreateIndex {
+                new_indices: vec![index_meta(name)],
+                removed_indices: vec![],
+            }
+        }
+        fn removal_op(name: &str) -> Operation {
+            Operation::CreateIndex {
+                new_indices: vec![],
+                removed_indices: vec![index_meta(name)],
+            }
+        }
+
+        let dataset = test_dataset(10, 1).await;
+        // Index transactions touching the same name conflict in every
+        // direction: a creation over a concurrent removal must not resurrect
+        // the dropped index, and a removal over a concurrent replacement must
+        // not silently no-op the drop. Regular names and both system
+        // singletons behave identically.
+        for name in ["my_idx", FRAG_REUSE_INDEX_NAME, MEM_WAL_INDEX_NAME] {
+            for (self_op, other_op) in [
+                (create_op(name), removal_op(name)),
+                (removal_op(name), create_op(name)),
+                (removal_op(name), removal_op(name)),
+            ] {
+                let staged = Transaction::new(dataset.manifest.version, self_op, None);
+                let committed = Transaction::new(dataset.manifest.version, other_op, None);
+                let mut rebase = TransactionRebase::try_new(&dataset, staged, None)
+                    .await
+                    .unwrap();
+                let err = rebase
+                    .check_txn(&committed, dataset.manifest.version + 1)
+                    .unwrap_err();
+                assert!(
+                    matches!(err, Error::RetryableCommitConflict { .. }),
+                    "{name}: expected a retryable commit conflict, got: {err}"
+                );
+                assert!(
+                    err.to_string()
+                        .contains("preempted by concurrent transaction CreateIndex"),
+                    "{name}: conflict message must identify the concurrent CreateIndex, got: {err}"
+                );
+            }
+
+            // Transactions on disjoint names never conflict, singletons included.
+            let staged = Transaction::new(dataset.manifest.version, create_op(name), None);
+            let unrelated =
+                Transaction::new(dataset.manifest.version, removal_op("other_idx"), None);
+            let mut rebase = TransactionRebase::try_new(&dataset, staged, None)
+                .await
+                .unwrap();
+            rebase
+                .check_txn(&unrelated, dataset.manifest.version + 1)
+                .unwrap_or_else(|err| panic!("{name}: disjoint names must not conflict: {err}"));
+        }
+    }
 }
diff --git a/rust/lance/src/io/commit/external_manifest.rs b/rust/lance/src/io/commit/external_manifest.rs
index df2b84a4878..850d10f9a23 100644
--- a/rust/lance/src/io/commit/external_manifest.rs
+++ b/rust/lance/src/io/commit/external_manifest.rs
@@ -4,10 +4,14 @@
 /// Keep the tests in `lance` crate because it has dependency on [Dataset].
 #[cfg(test)]
 mod test {
+    use std::ops::Range;
     use std::sync::Arc;
+    use std::sync::atomic::{AtomicUsize, Ordering};
     use std::{collections::HashMap, time::Duration};
 
     use async_trait::async_trait;
+    use bytes::Bytes;
+    use futures::stream::BoxStream;
     use futures::{StreamExt, TryStreamExt, future::join_all};
     use lance_core::{Error, Result};
     use lance_table::io::commit::external_manifest::{
@@ -15,7 +19,12 @@ mod test {
     };
     use lance_table::io::commit::{CommitHandler, ManifestNamingScheme};
     use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
-    use object_store::{ObjectStoreExt, local::LocalFileSystem, path::Path};
+    use object_store::memory::InMemory;
+    use object_store::{
+        CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+        ObjectStore as OSObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload,
+        PutResult, RenameOptions, Result as OSResult, local::LocalFileSystem, path::Path,
+    };
     use tokio::sync::Mutex;
 
     use crate::dataset::builder::DatasetBuilder;
@@ -365,6 +374,32 @@ mod test {
         assert_eq!(ds.version().version, 6);
         assert_eq!(ds.count_rows(None).await.unwrap(), 60);
 
+        {
+            inner_store.lock().await.remove(&(ds.base.to_string(), 6));
+        }
+        assert!(
+            handler
+                .version_exists(
+                    &ds.base,
+                    6,
+                    ds.object_store.inner.as_ref(),
+                    ds.manifest_location().naming_scheme,
+                )
+                .await
+                .unwrap()
+        );
+        assert!(
+            !handler
+                .version_exists(
+                    &ds.base,
+                    7,
+                    ds.object_store.inner.as_ref(),
+                    ds.manifest_location().naming_scheme,
+                )
+                .await
+                .unwrap()
+        );
+
         // Open without external store handler again, should see the newly sync'd commit
         let ds = DatasetBuilder::from_uri(ds_uri).load().await.unwrap();
         assert_eq!(ds.version().version, 6);
@@ -394,4 +429,288 @@ mod test {
             .collect::<Vec<_>>();
         assert!(unexpected_entries.is_empty(), "{:?}", unexpected_entries);
     }
+
+    /// S3's `CopyObject` API has a hard 5 GB cap on the source object size.
+    /// Above that, callers must use multipart copy (`UploadPartCopy`) instead.
+    /// `lance-table::io::commit::external_manifest` calls
+    /// `object_store.copy(staging, final)` unconditionally on the manifest
+    /// commit path — which fails for manifests >5 GB.
+    ///
+    /// This wrapper enforces that S3-side cap on top of any inner store, so
+    /// the regression can be reproduced in-process without S3.
+    ///
+    /// It also lets the test override `head().size` for a chosen path, so the
+    /// staging file can *appear* to be 14 GB without actually putting that
+    /// many bytes into the inner store.
+    const S3_COPY_OBJECT_CAP_BYTES: u64 = 5 * 1024 * 1024 * 1024;
+
+    #[derive(Debug)]
+    struct CopyCapStore {
+        inner: Arc<dyn OSObjectStore>,
+        /// path → fake size returned by head(); overrides the inner store.
+        head_size_overrides: Arc<Mutex<HashMap<String, u64>>>,
+        /// Counts calls to `copy_opts` (the fast path). Tests use this to
+        /// assert which branch of `copy_size_aware` was taken — succeeding
+        /// alone is not enough, since the slow path can also succeed for
+        /// small files.
+        copy_calls: AtomicUsize,
+        /// Counts calls to `put_multipart_opts` (the slow read+rewrite path).
+        put_multipart_calls: AtomicUsize,
+    }
+
+    impl CopyCapStore {
+        fn new(inner: Arc<dyn OSObjectStore>) -> Self {
+            Self {
+                inner,
+                head_size_overrides: Arc::new(Mutex::new(HashMap::new())),
+                copy_calls: AtomicUsize::new(0),
+                put_multipart_calls: AtomicUsize::new(0),
+            }
+        }
+
+        async fn override_size(&self, path: &Path, size: u64) {
+            self.head_size_overrides
+                .lock()
+                .await
+                .insert(path.to_string(), size);
+        }
+
+        async fn effective_size(&self, location: &Path, real: u64) -> u64 {
+            self.head_size_overrides
+                .lock()
+                .await
+                .get(&location.to_string())
+                .copied()
+                .unwrap_or(real)
+        }
+
+        fn copy_calls(&self) -> usize {
+            self.copy_calls.load(Ordering::SeqCst)
+        }
+
+        fn put_multipart_calls(&self) -> usize {
+            self.put_multipart_calls.load(Ordering::SeqCst)
+        }
+    }
+
+    impl std::fmt::Display for CopyCapStore {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(f, "CopyCapStore({})", self.inner)
+        }
+    }
+
+    #[async_trait]
+    impl OSObjectStore for CopyCapStore {
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.inner.put_opts(location, bytes, opts).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.put_multipart_calls.fetch_add(1, Ordering::SeqCst);
+            self.inner.put_multipart_opts(location, opts).await
+        }
+
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            // `head()` is a default method on `ObjectStore` that delegates to
+            // `get_opts(location, GetOptions { head: true, .. })`. To make a
+            // staging file *appear* to be 14 GB without holding 14 GB in
+            // memory, we override the size in the returned ObjectMeta here.
+            let mut res = self.inner.get_opts(location, options).await?;
+            let overridden = self.effective_size(location, res.meta.size).await;
+            res.meta.size = overridden;
+            Ok(res)
+        }
+
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            self.inner.get_ranges(location, ranges).await
+        }
+
+        // `head` and `delete` are default methods on `ObjectStore`, derived
+        // from `get_opts`/`delete_stream`. We override `head` indirectly by
+        // overriding `get_opts` below — it returns size based on the
+        // overrides table for the chosen path.
+        fn delete_stream(
+            &self,
+            locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list(prefix)
+        }
+
+        fn list_with_offset(
+            &self,
+            prefix: Option<&Path>,
+            offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.inner.list_with_offset(prefix, offset)
+        }
+
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+            // Mimic S3's CopyObject 5 GB hard cap: read the (possibly-overridden)
+            // size of the source via head() and reject if it crosses the cap.
+            let meta = self.head(from).await?;
+            if meta.size >= S3_COPY_OBJECT_CAP_BYTES {
+                return Err(object_store::Error::Generic {
+                    store: "S3",
+                    source: format!(
+                        "EntityTooLarge: ProposedSize {} exceeds CopyObject 5GB cap",
+                        meta.size
+                    )
+                    .into(),
+                });
+            }
+            self.copy_calls.fetch_add(1, Ordering::SeqCst);
+            self.inner.copy_opts(from, to, opts).await
+        }
+
+        async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+            self.inner.rename_opts(from, to, opts).await
+        }
+    }
+
+    /// Repro for the manifest >5 GB bug.
+    ///
+    /// Drives `ExternalManifestStore::put` (the default impl) against a
+    /// staging file whose `head().size` is reported as 14 GB. That `put`
+    /// calls `object_store.copy(staging, final)` unconditionally — which
+    /// our `CopyCapStore` wrapper rejects with the same `EntityTooLarge`
+    /// error S3 returns in production.
+    ///
+    /// Today this test is RED: the copy step fails on >5 GB.
+    /// After `copy_size_aware` lands, it should turn GREEN by falling back
+    /// to a multipart-equivalent path (option 1: read+rewrite via
+    /// `ObjectWriter`).
+    #[tokio::test]
+    async fn manifest_commit_succeeds_when_staging_exceeds_5gb_copy_cap() {
+        let inner: Arc<dyn OSObjectStore> = Arc::new(InMemory::new());
+        let capped = Arc::new(CopyCapStore::new(inner));
+
+        // Write a small staging file, then lie about its size so the
+        // CopyObject cap fires without holding 14 GB in memory.
+        let base_path = Path::from("repro");
+        let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd");
+        let body = b"fake manifest body";
+        capped
+            .put(&staging_path, PutPayload::from_static(body))
+            .await
+            .expect("seed staging file");
+        capped
+            .override_size(&staging_path, 14_961_429_442) // matches the production failure
+            .await;
+
+        // Spin up an ExternalManifestStore and drive `put` (the same code
+        // path the failing CTAS hits via ExternalManifestCommitHandler).
+        let external = SleepyExternalManifestStore::new();
+        let head_meta = capped.head(&staging_path).await.unwrap();
+
+        let location = external
+            .put(
+                &base_path,
+                1,
+                &staging_path,
+                head_meta.size,
+                head_meta.e_tag,
+                capped.as_ref(),
+                ManifestNamingScheme::V2,
+            )
+            .await
+            .expect(
+                "manifest commit should succeed for a >5 GB staging file via multipart-aware copy",
+            );
+
+        // Branch-taken assertions: the slow read+rewrite path was used.
+        assert_eq!(
+            capped.copy_calls(),
+            0,
+            "CopyObject must not be attempted for >5 GiB sources"
+        );
+        assert!(
+            capped.put_multipart_calls() >= 1,
+            "read+rewrite path must initiate a multipart upload"
+        );
+
+        // End-state assertions: final manifest exists with the original
+        // bytes, and the staging file was deleted.
+        let final_get = capped
+            .inner
+            .get(&location.path)
+            .await
+            .expect("final manifest must exist on the inner store")
+            .bytes()
+            .await
+            .unwrap();
+        assert_eq!(final_get.as_ref(), body);
+        let staging_after = capped.inner.head(&staging_path).await;
+        assert!(
+            matches!(staging_after, Err(object_store::Error::NotFound { .. })),
+            "staging file must be cleaned up after commit, got: {:?}",
+            staging_after
+        );
+    }
+
+    /// Counterpart to manifest_commit_succeeds_when_staging_exceeds_5gb_copy_cap.
+    /// Confirms that for staging files BELOW the 5 GB cap, the fast-path
+    /// server-side `copy()` is still used — i.e. we haven't accidentally
+    /// regressed every commit to read+rewrite.
+    #[tokio::test]
+    async fn manifest_commit_uses_fast_copy_for_small_staging() {
+        let inner: Arc<dyn OSObjectStore> = Arc::new(InMemory::new());
+        let capped = Arc::new(CopyCapStore::new(inner));
+
+        let base_path = Path::from("repro");
+        let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd");
+        capped
+            .put(
+                &staging_path,
+                PutPayload::from_static(b"small manifest body"),
+            )
+            .await
+            .expect("seed staging file");
+        // No size override — the staging file's real size is ~20 bytes,
+        // well below the 5 GB cap, so copy_size_aware must take the fast
+        // path.
+
+        let external = SleepyExternalManifestStore::new();
+        let head_meta = capped.head(&staging_path).await.unwrap();
+
+        external
+            .put(
+                &base_path,
+                1,
+                &staging_path,
+                head_meta.size,
+                head_meta.e_tag,
+                capped.as_ref(),
+                ManifestNamingScheme::V2,
+            )
+            .await
+            .expect("small manifest commit must succeed via fast-path copy");
+
+        // The branch-taken assertion: fast path was used, slow path was not.
+        assert!(
+            capped.copy_calls() >= 1,
+            "small-file commit must use server-side CopyObject"
+        );
+        assert_eq!(
+            capped.put_multipart_calls(),
+            0,
+            "small-file commit must NOT initiate a multipart upload"
+        );
+    }
 }
diff --git a/rust/lance/src/io/commit/namespace_manifest.rs b/rust/lance/src/io/commit/namespace_manifest.rs
index 0587ff96ad4..f4f012adcca 100644
--- a/rust/lance/src/io/commit/namespace_manifest.rs
+++ b/rust/lance/src/io/commit/namespace_manifest.rs
@@ -14,27 +14,77 @@ use lance_table::io::commit::{ManifestLocation, ManifestNamingScheme};
 use object_store::ObjectStore as OSObjectStore;
 use object_store::path::Path;
 
+use lance_namespace::error::NamespaceError;
+
+use crate::dataset::branch_location::BranchLocation;
+
+/// Whether `e` says the requested chain (table or branch) does not exist, as
+/// opposed to a failure talking to the namespace.
+fn is_chain_not_found(e: &lance_core::Error) -> bool {
+    if let lance_core::Error::Namespace { source, .. } = e
+        && let Some(ns_err) = source.downcast_ref::<NamespaceError>()
+    {
+        return matches!(
+            ns_err,
+            NamespaceError::TableNotFound { .. } | NamespaceError::TableBranchNotFound { .. }
+        );
+    }
+    false
+}
+
 #[derive(Debug)]
 pub struct LanceNamespaceExternalManifestStore {
     namespace_client: Arc<dyn LanceNamespace>,
     table_id: Vec<String>,
+    /// Object-store path of the table root (the main branch). The base path the
+    /// trait methods receive is resolved against this to derive which branch a
+    /// request targets, so a single store serves every branch of the table.
+    table_root: Path,
 }
 
 impl LanceNamespaceExternalManifestStore {
-    pub fn new(namespace_client: Arc<dyn LanceNamespace>, table_id: Vec<String>) -> Self {
+    pub fn new(
+        namespace_client: Arc<dyn LanceNamespace>,
+        table_id: Vec<String>,
+        table_root: Path,
+    ) -> Self {
         Self {
             namespace_client,
             table_id,
+            table_root,
         }
     }
+
+    /// Build a store for the table rooted at `table_uri`, resolving the root
+    /// path from the uri without initializing an object store.
+    pub fn for_table_uri(
+        namespace_client: Arc<dyn LanceNamespace>,
+        table_id: Vec<String>,
+        table_uri: &str,
+    ) -> Result<Self> {
+        let table_root = lance_io::object_store::ObjectStore::extract_path_from_uri(
+            Arc::new(lance_io::object_store::ObjectStoreRegistry::default()),
+            table_uri,
+        )?;
+        Ok(Self::new(namespace_client, table_id, table_root))
+    }
+
+    /// Derive the branch targeted by `base` (the table root for main, or a
+    /// branch chain produced by `BranchLocation::find_branch`). The branch
+    /// path layout is owned by [`BranchLocation`]; this store never parses or
+    /// constructs it directly.
+    fn branch_for_base(&self, base: &str) -> Result<Option<String>> {
+        BranchLocation::branch_of(self.table_root.as_ref(), base)
+    }
 }
 
 #[async_trait]
 impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
-    async fn get(&self, _base_uri: &str, version: u64) -> Result<String> {
+    async fn get(&self, base_uri: &str, version: u64) -> Result<String> {
         let request = DescribeTableVersionRequest {
             id: Some(self.table_id.clone()),
             version: Some(version as i64),
+            branch: self.branch_for_base(base_uri)?,
             ..Default::default()
         };
 
@@ -47,15 +97,24 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
         Ok(response.version.manifest_path)
     }
 
-    async fn get_latest_version(&self, _base_uri: &str) -> Result<Option<(u64, String)>> {
+    async fn get_latest_version(&self, base_uri: &str) -> Result<Option<(u64, String)>> {
         let request = ListTableVersionsRequest {
             id: Some(self.table_id.clone()),
             descending: Some(true),
             limit: Some(1),
+            branch: self.branch_for_base(base_uri)?,
             ..Default::default()
         };
 
-        let response = self.namespace_client.list_table_versions(request).await?;
+        let response = match self.namespace_client.list_table_versions(request).await {
+            Ok(response) => response,
+            // A chain that does not exist yet (e.g. probing a branch location
+            // before the branch is created) has no latest version; the
+            // ExternalManifestStore contract reports that as None, not an
+            // error, so existence checks can treat it as a missing dataset.
+            Err(e) if is_chain_not_found(&e) => return Ok(None),
+            Err(e) => return Err(e),
+        };
 
         if response.versions.is_empty() {
             return Ok(None);
@@ -73,7 +132,7 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
     /// Put the manifest to the namespace store.
     async fn put(
         &self,
-        _base_path: &Path,
+        base_path: &Path,
         version: u64,
         staging_path: &Path,
         size: u64,
@@ -94,6 +153,7 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
             manifest_size: Some(size as i64),
             e_tag: e_tag.clone(),
             naming_scheme: Some(naming_scheme_str.to_string()),
+            branch: self.branch_for_base(base_path.as_ref())?,
             ..Default::default()
         };
 
@@ -146,3 +206,93 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore {
         ))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use lance_namespace::models::ListTableVersionsResponse;
+
+    /// A namespace whose list_table_versions always fails with the configured
+    /// error, to pin how get_latest_version classifies failures.
+    #[derive(Debug)]
+    struct FailingNamespace {
+        error: fn() -> lance_core::Error,
+    }
+
+    #[async_trait]
+    impl LanceNamespace for FailingNamespace {
+        fn namespace_id(&self) -> String {
+            "failing".to_string()
+        }
+
+        async fn list_table_versions(
+            &self,
+            _request: ListTableVersionsRequest,
+        ) -> Result<ListTableVersionsResponse> {
+            Err((self.error)())
+        }
+    }
+
+    fn store_with(error: fn() -> lance_core::Error) -> LanceNamespaceExternalManifestStore {
+        LanceNamespaceExternalManifestStore::new(
+            Arc::new(FailingNamespace { error }),
+            vec!["t".to_string()],
+            Path::parse("data/t.lance").unwrap(),
+        )
+    }
+
+    /// A chain that does not exist (missing table or branch) has no latest
+    /// version; everything else is a real failure and must propagate so an
+    /// outage is never mistaken for an absent dataset.
+    #[tokio::test]
+    async fn test_get_latest_version_error_classification() {
+        use lance_namespace::error::NamespaceError;
+
+        let absent = [
+            store_with(|| {
+                NamespaceError::TableNotFound {
+                    message: "missing table".to_string(),
+                }
+                .into()
+            }),
+            store_with(|| {
+                NamespaceError::TableBranchNotFound {
+                    message: "missing branch".to_string(),
+                }
+                .into()
+            }),
+        ];
+        for store in absent {
+            let latest = store.get_latest_version("data/t.lance/tree/dev").await;
+            assert!(
+                matches!(latest, Ok(None)),
+                "a missing chain must read as no latest version, got: {:?}",
+                latest
+            );
+        }
+
+        let failures = [
+            store_with(|| {
+                NamespaceError::Internal {
+                    message: "server error".to_string(),
+                }
+                .into()
+            }),
+            store_with(|| {
+                NamespaceError::Throttling {
+                    message: "slow down".to_string(),
+                }
+                .into()
+            }),
+            store_with(|| lance_core::Error::io("connection reset".to_string())),
+        ];
+        for store in failures {
+            let latest = store.get_latest_version("data/t.lance/tree/dev").await;
+            assert!(
+                latest.is_err(),
+                "a real failure must propagate, got: {:?}",
+                latest
+            );
+        }
+    }
+}
diff --git a/rust/lance/src/io/commit/s3_test.rs b/rust/lance/src/io/commit/s3_test.rs
index b5b1a09c776..4be469ee368 100644
--- a/rust/lance/src/io/commit/s3_test.rs
+++ b/rust/lance/src/io/commit/s3_test.rs
@@ -341,7 +341,10 @@ async fn test_ddb_open_iops() {
     // Checkout original version
     dataset.checkout_version(1).await.unwrap();
     let io_stats = dataset.object_store.as_ref().io_stats_incremental();
-    // Checkout: 1 IOPS: manifest file
-    assert_io_eq!(io_stats, read_iops, 1);
+    // Checkout: 0 read IOPS. Version 1's manifest was already loaded and cached
+    // on this Session when the dataset was opened above, so the checkout serves
+    // the manifest body from the metadata cache. Version resolution is handled
+    // in DynamoDB and issues no S3 read.
+    assert_io_eq!(io_stats, read_iops, 0);
     assert_io_eq!(io_stats, write_iops, 0);
 }
diff --git a/rust/lance/src/io/exec.rs b/rust/lance/src/io/exec.rs
index f06f575de68..a477d60d56d 100644
--- a/rust/lance/src/io/exec.rs
+++ b/rust/lance/src/io/exec.rs
@@ -7,6 +7,8 @@
 
 #[cfg(feature = "substrait")]
 pub mod ann_proto;
+pub mod count_from_mask;
+pub mod count_pushdown;
 mod filter;
 pub mod filtered_read;
 #[cfg(feature = "substrait")]
diff --git a/rust/lance/src/io/exec/ann_proto.rs b/rust/lance/src/io/exec/ann_proto.rs
index 235098cab55..c57ad4ca1b7 100644
--- a/rust/lance/src/io/exec/ann_proto.rs
+++ b/rust/lance/src/io/exec/ann_proto.rs
@@ -16,10 +16,11 @@ use arrow_array::RecordBatch;
 use arrow_schema::{Field, Schema as ArrowSchema};
 use lance_core::{Error, Result};
 use lance_index::pb as index_pb;
-use lance_index::vector::{DEFAULT_QUERY_PARALLELISM, Query};
+use lance_index::vector::{ApproxMode, DEFAULT_QUERY_PARALLELISM, Query};
 use lance_linalg::distance::DistanceType;
 use lance_table::format::IndexMetadata;
 use lance_table::format::pb as table_pb;
+use uuid::Uuid;
 
 use crate::Dataset;
 use crate::pb;
@@ -79,6 +80,22 @@ fn query_vector_from_ipc_bytes(bytes: &[u8]) -> Result<arrow_array::ArrayRef> {
     Ok(batches[0].column(0).clone())
 }
 
+fn approx_mode_to_proto(mode: ApproxMode) -> pb::VectorApproxMode {
+    match mode {
+        ApproxMode::Fast => pb::VectorApproxMode::Fast,
+        ApproxMode::Normal => pb::VectorApproxMode::Normal,
+        ApproxMode::Accurate => pb::VectorApproxMode::Accurate,
+    }
+}
+
+fn approx_mode_from_proto(value: i32) -> ApproxMode {
+    match pb::VectorApproxMode::try_from(value).unwrap_or(pb::VectorApproxMode::Normal) {
+        pb::VectorApproxMode::Fast => ApproxMode::Fast,
+        pb::VectorApproxMode::Normal => ApproxMode::Normal,
+        pb::VectorApproxMode::Accurate => ApproxMode::Accurate,
+    }
+}
+
 pub fn query_to_proto(query: &Query) -> Result<pb::VectorQueryProto> {
     let query_vector_arrow_ipc = query_vector_to_ipc_bytes(query.key.as_ref())?;
 
@@ -100,6 +117,7 @@ pub fn query_to_proto(query: &Query) -> Result<pb::VectorQueryProto> {
         use_index: query.use_index,
         dist_q_c: Some(query.dist_q_c),
         query_parallelism: Some(query.query_parallelism),
+        approx_mode: approx_mode_to_proto(query.approx_mode) as i32,
     })
 }
 
@@ -129,6 +147,7 @@ pub fn query_from_proto(proto: pb::VectorQueryProto) -> Result<Query> {
         use_index: proto.use_index,
         query_parallelism: proto.query_parallelism.unwrap_or(DEFAULT_QUERY_PARALLELISM),
         dist_q_c: proto.dist_q_c.unwrap_or(0.0),
+        approx_mode: approx_mode_from_proto(proto.approx_mode),
     })
 }
 
@@ -146,7 +165,7 @@ pub async fn ann_ivf_partition_exec_to_proto(
     Ok(pb::AnnIvfPartitionExecProto {
         query: Some(query),
         table: Some(table),
-        index_uuids: exec.index_uuids.clone(),
+        index_uuids: exec.index_uuids.iter().map(Uuid::to_string).collect(),
     })
 }
 
@@ -168,7 +187,17 @@ pub async fn ann_ivf_partition_exec_from_proto(
         ));
     }
 
-    ANNIvfPartitionExec::try_new(dataset, proto.index_uuids, query)
+    let index_uuids: Vec<Uuid> = proto
+        .index_uuids
+        .iter()
+        .map(|s| Uuid::parse_str(s))
+        .collect::<std::result::Result<_, _>>()
+        .map_err(|e| {
+            Error::invalid_input_source(
+                format!("Invalid UUID in AnnIvfPartitionExecProto: {e}").into(),
+            )
+        })?;
+    ANNIvfPartitionExec::try_new(dataset, index_uuids, query)
 }
 
 // =============================================================================
@@ -324,6 +353,7 @@ mod tests {
             use_index: true,
             query_parallelism: -1,
             dist_q_c: 0.42,
+            approx_mode: ApproxMode::Accurate,
         };
 
         let proto = query_to_proto(&query).unwrap();
@@ -341,6 +371,7 @@ mod tests {
         assert_eq!(query.use_index, back.use_index);
         assert_eq!(query.query_parallelism, back.query_parallelism);
         assert_eq!(query.dist_q_c, back.dist_q_c);
+        assert_eq!(query.approx_mode, back.approx_mode);
         assert_eq!(query.key.len(), back.key.len());
         assert_eq!(query.key.data_type(), back.key.data_type());
     }
@@ -362,12 +393,19 @@ mod tests {
             use_index: false,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: ApproxMode::Normal,
         };
 
         let proto = query_to_proto(&query).unwrap();
         let back = query_from_proto(proto).unwrap();
         assert!(back.metric_type.is_none());
         assert!(!back.use_index);
+        assert_eq!(back.approx_mode, ApproxMode::Normal);
+
+        let mut proto = query_to_proto(&query).unwrap();
+        proto.approx_mode = i32::MAX;
+        let back = query_from_proto(proto).unwrap();
+        assert_eq!(back.approx_mode, ApproxMode::Normal);
     }
 
     async fn make_vector_dataset() -> (Arc<Dataset>, tempfile::TempDir) {
@@ -433,11 +471,12 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: ApproxMode::Normal,
         };
 
         let exec = ANNIvfPartitionExec::try_new(
             dataset.clone(),
-            indices.iter().map(|idx| idx.uuid.to_string()).collect(),
+            indices.iter().map(|idx| idx.uuid).collect(),
             query,
         )
         .unwrap();
@@ -480,6 +519,7 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: ApproxMode::Normal,
         };
 
         // Use a TestMemoryExec as a mock input child (provides the KNN_PARTITION_SCHEMA)
@@ -543,6 +583,7 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: ApproxMode::Normal,
         };
         let input: Arc<dyn datafusion::physical_plan::ExecutionPlan> =
             TestMemoryExec::try_new_exec(
diff --git a/rust/lance/src/io/exec/count_from_mask.rs b/rust/lance/src/io/exec/count_from_mask.rs
new file mode 100644
index 00000000000..e2f7f00efb5
--- /dev/null
+++ b/rust/lance/src/io/exec/count_from_mask.rs
@@ -0,0 +1,707 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Execute-time half of the count-from-mask category of aggregate pushdown.
+//!
+//! [`CountFromMaskExec`] computes a `COUNT(*)`-style aggregate's partial
+//! state directly from index/manifest metadata, without scanning column
+//! data. Conceptually:
+//!
+//! ```text
+//! result = | fragments_allow ∩ optional_prefilter_mask − deletion_mask |
+//! ```
+//!
+//! Its output schema matches what `AggregateExec(AggregateMode::Partial)`
+//! would produce for the same `COUNT` aggregates, so a downstream
+//! `AggregateExec(Final)` can combine the result unchanged.
+//!
+//! This is one of four categories of aggregate acceleration we plan to
+//! support; the others (mask-to-answer, zone-aware, dimension-keyed) each
+//! need additional plumbing — see the corresponding design issue.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow_array::{Array, BinaryArray, Int64Array, RecordBatch};
+use arrow_schema::{Schema, SchemaRef};
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+    execution_plan::{Boundedness, EmissionType},
+    metrics::{ExecutionPlanMetricsSet, MetricsSet},
+};
+use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use futures::{StreamExt, TryStreamExt, stream};
+use lance_core::{Error, Result};
+use lance_select::{RowAddrMask, RowAddrSelection, RowAddrTreeMap};
+use lance_table::format::Fragment;
+use roaring::RoaringBitmap;
+use tracing::instrument;
+
+use super::utils::InstrumentedRecordBatchStreamAdapter;
+use crate::Dataset;
+use crate::dataset::rowids::load_row_id_sequences;
+use crate::index::prefilter::DatasetPreFilter;
+
+/// An execution node that computes a `COUNT(*)`-style aggregate from an
+/// optional row-address mask supplied by an upstream scalar-index search,
+/// combined with the dataset's deletion mask and an optional restriction to
+/// a fragment subset.
+///
+/// The node returns one record batch with one row whose columns are the
+/// partial-state representation of each `COUNT` in `aggregate_funcs` — i.e.
+/// the same shape an `AggregateExec(Partial)` would emit.
+#[derive(Debug)]
+pub struct CountFromMaskExec {
+    dataset: Arc<Dataset>,
+    /// One per output column. Used only for `state_fields()` to build the
+    /// output schema; the actual count is computed identically for all of
+    /// them since every entry is a non-distinct `COUNT(<literal>)`.
+    aggregate_funcs: Vec<Arc<AggregateFunctionExpr>>,
+    /// Optional [`super::scalar_index::ScalarIndexExec`] producing the row-
+    /// address mask to count.
+    prefilter_input: Option<Arc<dyn ExecutionPlan>>,
+    /// Restrict the count to this fragment subset. `None` means "every
+    /// fragment in the dataset." The optimizer rule uses this to scope the
+    /// pushdown branch of a partial-coverage split plan to the indexed
+    /// fragments only — the uncovered fragments are handled by a parallel
+    /// scan branch.
+    restrict_to_fragments: Option<RoaringBitmap>,
+    schema: SchemaRef,
+    properties: Arc<PlanProperties>,
+    metrics: ExecutionPlanMetricsSet,
+}
+
+impl DisplayAs for CountFromMaskExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                write!(f, "CountFromMask")
+            }
+            DisplayFormatType::TreeRender => write!(f, "CountFromMask"),
+        }
+    }
+}
+
+impl CountFromMaskExec {
+    /// Build a new node.
+    ///
+    /// `aggregate_funcs` must be a non-empty set of non-distinct `COUNT`
+    /// aggregates (the optimizer rule guarantees this). `prefilter_input`,
+    /// if present, must produce a single batch in the scalar-index result
+    /// schema; that mask is intersected with the dataset's covered
+    /// fragments and the active deletion mask.
+    pub fn try_new(
+        dataset: Arc<Dataset>,
+        aggregate_funcs: Vec<Arc<AggregateFunctionExpr>>,
+        prefilter_input: Option<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Self> {
+        Self::try_new_restricted(dataset, aggregate_funcs, prefilter_input, None)
+    }
+
+    /// Like [`Self::try_new`] but scopes the count to a fragment subset
+    /// rather than the whole dataset. The optimizer rule uses this for the
+    /// pushdown branch of a partial-coverage split plan, so the count only
+    /// covers the fragments the prefilter's index can answer for.
+    pub fn try_new_restricted(
+        dataset: Arc<Dataset>,
+        aggregate_funcs: Vec<Arc<AggregateFunctionExpr>>,
+        prefilter_input: Option<Arc<dyn ExecutionPlan>>,
+        restrict_to_fragments: Option<RoaringBitmap>,
+    ) -> Result<Self> {
+        if aggregate_funcs.is_empty() {
+            return Err(Error::invalid_input(
+                "CountFromMaskExec requires at least one aggregate".to_string(),
+            ));
+        }
+
+        let state_fields = aggregate_funcs
+            .iter()
+            .map(|agg| agg.state_fields())
+            .collect::<datafusion::error::Result<Vec<_>>>()
+            .map_err(|e| Error::invalid_input(e.to_string()))?
+            .into_iter()
+            .flatten()
+            .collect::<Vec<_>>();
+        let state_fields_owned: Vec<arrow_schema::Field> =
+            state_fields.iter().map(|f| f.as_ref().clone()).collect();
+        let schema: SchemaRef = Arc::new(Schema::new(state_fields_owned));
+
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(schema.clone()),
+            Partitioning::RoundRobinBatch(1),
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ));
+
+        Ok(Self {
+            dataset,
+            aggregate_funcs,
+            prefilter_input,
+            restrict_to_fragments,
+            schema,
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+        })
+    }
+
+    /// Drain `prefilter_input` (a [`super::scalar_index::ScalarIndexExec`])
+    /// to produce the row-address mask it serialized.
+    async fn load_prefilter(
+        prefilter_input: Arc<dyn ExecutionPlan>,
+        context: Arc<datafusion::execution::context::TaskContext>,
+    ) -> Result<RowAddrMask> {
+        let mut stream = prefilter_input.execute(0, context).map_err(Error::from)?;
+        let batch = stream
+            .try_next()
+            .await
+            .map_err(Error::from)?
+            .ok_or_else(|| {
+                Error::internal(
+                    "CountFromMaskExec: prefilter input produced no batches".to_string(),
+                )
+            })?;
+        // Drain any remaining batches so the upstream sees a clean shutdown.
+        while stream.try_next().await.map_err(Error::from)?.is_some() {}
+
+        let result_col = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .ok_or_else(|| {
+                Error::internal(format!(
+                    "CountFromMaskExec: prefilter result column has type {:?}, expected Binary",
+                    batch.column(0).data_type()
+                ))
+            })?;
+        RowAddrMask::from_arrow(result_col)
+    }
+
+    /// Fold the prefilter, fragment allow list, and deletion mask into a
+    /// single `AllowList`-shaped [`RowAddrMask`] suitable for counting.
+    fn combine_masks(
+        fragments_allow: RowAddrTreeMap,
+        prefilter: Option<RowAddrMask>,
+        deletion_mask: Option<Arc<RowAddrMask>>,
+    ) -> RowAddrMask {
+        let base = RowAddrMask::AllowList(fragments_allow);
+        let after_prefilter = match prefilter {
+            None => base,
+            Some(prefilter) => base & prefilter,
+        };
+        match deletion_mask {
+            None => after_prefilter,
+            Some(deletion_mask) => after_prefilter & (*deletion_mask).clone(),
+        }
+    }
+
+    /// Count the rows selected by `mask`, looking up `Full`-marker fragments
+    /// in the manifest so we never need to materialize a
+    /// `RoaringBitmap::full()`.
+    fn count_from_mask(mask: &RowAddrMask, dataset: &Dataset) -> Result<i64> {
+        let allow = mask.allow_list().ok_or_else(|| {
+            Error::internal("CountFromMaskExec: combined mask is not an AllowList".to_string())
+        })?;
+        let frag_map: HashMap<u32, &Fragment> = dataset
+            .fragments()
+            .iter()
+            .map(|f| (f.id as u32, f))
+            .collect();
+        let mut count = 0i64;
+        for (frag_id, sel) in allow.iter() {
+            match sel {
+                RowAddrSelection::Full => {
+                    // The fragment is in the allow list with no deletions
+                    // touching it — its row count is the physical row count.
+                    let frag = frag_map.get(frag_id).ok_or_else(|| {
+                        Error::internal(format!(
+                            "CountFromMaskExec: fragment {} not found in manifest",
+                            frag_id
+                        ))
+                    })?;
+                    let n = frag.physical_rows.ok_or_else(|| {
+                        Error::internal(format!(
+                            "CountFromMaskExec: physical_rows missing for fragment {}",
+                            frag_id
+                        ))
+                    })?;
+                    count += n as i64;
+                }
+                RowAddrSelection::Partial(bitmap) => {
+                    count += bitmap.len() as i64;
+                }
+            }
+        }
+        Ok(count)
+    }
+
+    /// Row-address-space fragments-allow list: concrete `[0..physical_rows)`
+    /// ranges per covered fragment.
+    ///
+    /// Concrete ranges, not `Full` markers: subtracting a `BlockList` from a
+    /// `Full` entry materializes a `RoaringBitmap::full()` (2^32) per fragment,
+    /// which is slow and throws off `len()`.
+    fn address_fragments_allow(
+        dataset: &Dataset,
+        fragments_covered: &RoaringBitmap,
+    ) -> Result<RowAddrTreeMap> {
+        let frag_map: HashMap<u32, &Fragment> = dataset
+            .fragments()
+            .iter()
+            .map(|f| (f.id as u32, f))
+            .collect();
+        let mut fragments_allow = RowAddrTreeMap::new();
+        for frag_id in fragments_covered.iter() {
+            let frag = frag_map.get(&frag_id).ok_or_else(|| {
+                Error::internal(format!(
+                    "CountFromMaskExec: fragment {} not in manifest",
+                    frag_id
+                ))
+            })?;
+            let physical = frag.physical_rows.ok_or_else(|| {
+                Error::internal(format!(
+                    "CountFromMaskExec: physical_rows missing for fragment {}",
+                    frag_id
+                ))
+            })?;
+            let mut bitmap = RoaringBitmap::new();
+            bitmap.insert_range(0u32..(physical as u32));
+            fragments_allow.insert_bitmap(frag_id, bitmap);
+        }
+        Ok(fragments_allow)
+    }
+
+    /// Live (non-deleted) row count of the covered fragments, from fragment
+    /// metadata. Used for an unfiltered count: no prefilter to intersect, so no
+    /// need to build the stable-id universe.
+    async fn count_live_rows(dataset: &Dataset, fragments_covered: &RoaringBitmap) -> Result<i64> {
+        let frags = dataset
+            .get_fragments()
+            .into_iter()
+            .filter(|f| fragments_covered.contains(f.id() as u32));
+        let counts = stream::iter(frags)
+            .map(|f| async move { f.count_rows(None).await })
+            .buffer_unordered(dataset.object_store.as_ref().io_parallelism())
+            .try_collect::<Vec<_>>()
+            .await?;
+        Ok(counts.iter().sum::<usize>() as i64)
+    }
+
+    /// Count universe in stable-id space: live stable row ids whose current home
+    /// is in `fragments_covered`. Staying in stable-id space lets it intersect
+    /// the index prefilter directly; deletions are already folded in, so the
+    /// caller passes no separate deletion mask.
+    async fn stable_id_universe(
+        dataset: &Arc<Dataset>,
+        fragments_covered: RoaringBitmap,
+    ) -> Result<RowAddrTreeMap> {
+        // create_restricted_deletion_mask gives a live-id allow list restricted
+        // to `fragments_covered`. It returns None only with no deletions and full
+        // coverage — then the universe is every stable id, loaded below.
+        if let Some(fut) = DatasetPreFilter::create_restricted_deletion_mask(
+            dataset.clone(),
+            fragments_covered.clone(),
+        ) {
+            let mask = fut.await?;
+            return mask.allow_list().cloned().ok_or_else(|| {
+                Error::internal(
+                    "CountFromMaskExec: stable-row-id deletion mask must be an AllowList"
+                        .to_string(),
+                )
+            });
+        }
+        Self::load_stable_id_universe(dataset, &fragments_covered).await
+    }
+
+    /// Every stable row id in the covered fragments, from their row-id sequences
+    /// (metadata, not column data). Only used with no deletions and full coverage.
+    async fn load_stable_id_universe(
+        dataset: &Dataset,
+        fragments_covered: &RoaringBitmap,
+    ) -> Result<RowAddrTreeMap> {
+        let frags: Vec<Fragment> = dataset
+            .fragments()
+            .iter()
+            .filter(|f| fragments_covered.contains(f.id as u32))
+            .cloned()
+            .collect();
+        let mut sequences = load_row_id_sequences(dataset, &frags);
+        let mut universe = RowAddrTreeMap::new();
+        while let Some((_frag_id, sequence)) = sequences.try_next().await? {
+            universe |= RowAddrTreeMap::from(sequence.as_ref());
+        }
+        Ok(universe)
+    }
+
+    #[instrument(name = "count_from_mask", skip_all, level = "debug")]
+    async fn do_execute(
+        dataset: Arc<Dataset>,
+        aggregate_funcs_len: usize,
+        prefilter_input: Option<Arc<dyn ExecutionPlan>>,
+        restrict_to_fragments: Option<RoaringBitmap>,
+        context: Arc<datafusion::execution::context::TaskContext>,
+        schema: SchemaRef,
+    ) -> Result<RecordBatch> {
+        let prefilter = match prefilter_input {
+            None => None,
+            Some(input) => Some(Self::load_prefilter(input, context.clone()).await?),
+        };
+
+        // Anchor the deletion mask against either every dataset fragment or
+        // the caller-supplied restricted subset.
+        let dataset_fragments: RoaringBitmap =
+            dataset.fragments().iter().map(|f| f.id as u32).collect();
+        let fragments_covered = match restrict_to_fragments {
+            Some(restrict) => dataset_fragments & restrict,
+            None => dataset_fragments,
+        };
+
+        // Under stable row ids the prefilter and deletion masks are in stable-id
+        // space, so the universe must be too (see `stable_id_universe`); the
+        // default path builds it in row-address space.
+        let count = if dataset.manifest.uses_stable_row_ids() {
+            match prefilter {
+                // No prefilter: just the live row count, from metadata.
+                None => Self::count_live_rows(&dataset, &fragments_covered).await?,
+                Some(prefilter) => {
+                    let universe = Self::stable_id_universe(&dataset, fragments_covered).await?;
+                    let combined = Self::combine_masks(universe, Some(prefilter), None);
+                    Self::count_from_mask(&combined, dataset.as_ref())?
+                }
+            }
+        } else {
+            let fragments_allow = Self::address_fragments_allow(&dataset, &fragments_covered)?;
+            // Load the deletion mask for the covered fragments.
+            let deletion_mask =
+                match DatasetPreFilter::create_deletion_mask(dataset.clone(), fragments_covered) {
+                    Some(fut) => Some(fut.await?),
+                    None => None,
+                };
+            let combined = Self::combine_masks(fragments_allow, prefilter, deletion_mask);
+            Self::count_from_mask(&combined, dataset.as_ref())?
+        };
+
+        // Every aggregate is the same non-distinct COUNT shape — emit the
+        // count once per output column.
+        let arrays: Vec<Arc<dyn Array>> = (0..aggregate_funcs_len)
+            .map(|_| Arc::new(Int64Array::from(vec![count])) as Arc<dyn Array>)
+            .collect();
+        Ok(RecordBatch::try_new(schema, arrays)?)
+    }
+}
+
+impl ExecutionPlan for CountFromMaskExec {
+    fn name(&self) -> &str {
+        "CountFromMaskExec"
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        match &self.prefilter_input {
+            Some(input) => vec![input],
+            None => vec![],
+        }
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> datafusion::error::Result<Arc<dyn ExecutionPlan>> {
+        let prefilter_input = match children.len() {
+            0 => None,
+            1 => Some(children.into_iter().next().unwrap()),
+            n => {
+                return Err(datafusion::error::DataFusionError::Internal(format!(
+                    "CountFromMaskExec accepts 0 or 1 children, got {}",
+                    n
+                )));
+            }
+        };
+        Ok(Arc::new(Self {
+            dataset: self.dataset.clone(),
+            aggregate_funcs: self.aggregate_funcs.clone(),
+            prefilter_input,
+            restrict_to_fragments: self.restrict_to_fragments.clone(),
+            schema: self.schema.clone(),
+            properties: self.properties.clone(),
+            metrics: self.metrics.clone(),
+        }))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<datafusion::execution::context::TaskContext>,
+    ) -> datafusion::error::Result<datafusion::physical_plan::SendableRecordBatchStream> {
+        let schema = self.schema.clone();
+        let batch_fut = Self::do_execute(
+            self.dataset.clone(),
+            self.aggregate_funcs.len(),
+            self.prefilter_input.clone(),
+            self.restrict_to_fragments.clone(),
+            context,
+            schema.clone(),
+        );
+        let stream = futures::stream::iter(vec![batch_fut])
+            .then(|fut| async move { fut.await.map_err(|err| err.into()) })
+            .boxed();
+        Ok(Box::pin(InstrumentedRecordBatchStreamAdapter::new(
+            schema,
+            stream,
+            partition,
+            &self.metrics,
+        )))
+    }
+
+    fn partition_statistics(
+        &self,
+        _partition: Option<usize>,
+    ) -> datafusion::error::Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(datafusion::physical_plan::Statistics {
+            num_rows: datafusion::common::stats::Precision::Exact(1),
+            ..datafusion::physical_plan::Statistics::new_unknown(&self.schema)
+        }))
+    }
+
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.properties
+    }
+
+    fn supports_limit_pushdown(&self) -> bool {
+        false
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{ops::Bound, sync::Arc};
+
+    use arrow::datatypes::{Int64Type, UInt64Type};
+    use datafusion::common::DFSchema;
+    use datafusion::execution::TaskContext;
+    use datafusion::functions_aggregate;
+    use datafusion::logical_expr::lit;
+    use datafusion::physical_expr::execution_props::ExecutionProps;
+    use datafusion::physical_plan::ExecutionPlan;
+    use datafusion::scalar::ScalarValue;
+    use futures::TryStreamExt;
+    use lance_core::utils::tempfile::TempStrDir;
+    use lance_datagen::gen_batch;
+    use lance_index::IndexType;
+    use lance_index::scalar::{
+        SargableQuery, ScalarIndexParams,
+        expression::{ScalarIndexExpr, ScalarIndexSearch},
+    };
+    use lance_select::result::IndexExprResultWireFormat;
+    use lance_select::{RowAddrMask, RowAddrTreeMap};
+
+    use super::*;
+    use crate::Dataset;
+    use crate::index::DatasetIndexExt;
+    use crate::io::exec::scalar_index::ScalarIndexExec;
+    use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
+    #[allow(deprecated)]
+    use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter;
+
+    /// Build an `AggregateFunctionExpr` matching `COUNT(*)`.
+    // TODO(datafusion-54): migrate off the deprecated
+    // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder.
+    #[allow(deprecated)]
+    fn count_star_expr(input_schema: &SchemaRef) -> Arc<AggregateFunctionExpr> {
+        let expr = functions_aggregate::count::count(lit(1));
+        let df_schema = DFSchema::try_from(input_schema.as_ref().clone()).unwrap();
+        let (agg_expr, _filter, _order_by) = create_aggregate_expr_and_maybe_filter(
+            &expr,
+            &df_schema,
+            input_schema.as_ref(),
+            &ExecutionProps::default(),
+        )
+        .unwrap();
+        agg_expr
+    }
+
+    struct Fixture {
+        dataset: Arc<Dataset>,
+        _tmp: TempStrDir,
+    }
+
+    /// 4 fragments × 10 rows, ascending `ordered` column with a BTree index.
+    async fn make_fixture() -> Fixture {
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(10),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        Fixture {
+            dataset: Arc::new(dataset),
+            _tmp: tmp,
+        }
+    }
+
+    fn input_schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![arrow_schema::Field::new(
+            "ordered",
+            arrow_schema::DataType::UInt64,
+            false,
+        )]))
+    }
+
+    async fn run(plan: CountFromMaskExec) -> i64 {
+        let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        assert_eq!(batches.len(), 1);
+        assert_eq!(batches[0].num_rows(), 1);
+        batches[0]
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow_array::PrimitiveArray<Int64Type>>()
+            .expect("count partial state should be Int64")
+            .value(0)
+    }
+
+    #[tokio::test]
+    async fn try_new_rejects_empty_aggregate_funcs() {
+        let fixture = make_fixture().await;
+        let err = CountFromMaskExec::try_new(fixture.dataset, vec![], None).unwrap_err();
+        assert!(err.to_string().contains("at least one aggregate"), "{err}");
+    }
+
+    #[tokio::test]
+    async fn count_from_mask_mixes_full_and_partial() {
+        // Synthesize an AllowList containing one Full-marker fragment and
+        // one Partial bitmap; verify the Full fragment falls back to
+        // physical_rows from the manifest and Partial falls back to
+        // bitmap.len().
+        let fixture = make_fixture().await;
+        let mut tm = RowAddrTreeMap::new();
+        // Fragment 0: full (10 physical rows).
+        tm.insert_fragment(0);
+        // Fragment 1: partial with explicit row addrs.
+        let row_addr_for = |frag_id: u32, offset: u32| ((frag_id as u64) << 32) | offset as u64;
+        tm.insert(row_addr_for(1, 0));
+        tm.insert(row_addr_for(1, 1));
+        tm.insert(row_addr_for(1, 2));
+
+        let mask = RowAddrMask::AllowList(tm);
+        let count = CountFromMaskExec::count_from_mask(&mask, fixture.dataset.as_ref()).unwrap();
+        assert_eq!(count, 10 + 3);
+    }
+
+    #[tokio::test]
+    async fn execute_count_no_prefilter() {
+        let fixture = make_fixture().await;
+        let schema = input_schema();
+        let plan = CountFromMaskExec::try_new(
+            fixture.dataset.clone(),
+            vec![count_star_expr(&schema)],
+            None,
+        )
+        .unwrap();
+        let count = run(plan).await;
+        assert_eq!(count, 40); // 4 fragments × 10 rows
+    }
+
+    #[tokio::test]
+    async fn execute_count_with_allow_list_prefilter() {
+        let fixture = make_fixture().await;
+        let schema = input_schema();
+
+        // `ordered < 25` matches 25 rows across the four fragments.
+        let prefilter_expr = ScalarIndexExpr::Query(ScalarIndexSearch {
+            column: "ordered".to_string(),
+            index_name: "ordered_idx".to_string(),
+            index_type: "BTree".to_string(),
+            query: Arc::new(SargableQuery::Range(
+                Bound::Unbounded,
+                Bound::Excluded(ScalarValue::UInt64(Some(25))),
+            )),
+            needs_recheck: false,
+            fragment_bitmap: None,
+        });
+        let prefilter: Arc<dyn ExecutionPlan> = Arc::new(ScalarIndexExec::new(
+            fixture.dataset.clone(),
+            prefilter_expr,
+            IndexExprResultWireFormat::default(),
+        ));
+
+        let plan = CountFromMaskExec::try_new(
+            fixture.dataset.clone(),
+            vec![count_star_expr(&schema)],
+            Some(prefilter),
+        )
+        .unwrap();
+        let count = run(plan).await;
+        assert_eq!(count, 25);
+    }
+
+    #[tokio::test]
+    async fn execute_count_with_block_list_prefilter() {
+        let fixture = make_fixture().await;
+        let schema = input_schema();
+
+        // NOT(ordered < 25) is a block list of those 25 rows — 40 − 25 = 15.
+        let prefilter_expr =
+            ScalarIndexExpr::Not(Box::new(ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: "ordered".to_string(),
+                index_name: "ordered_idx".to_string(),
+                index_type: "BTree".to_string(),
+                query: Arc::new(SargableQuery::Range(
+                    Bound::Unbounded,
+                    Bound::Excluded(ScalarValue::UInt64(Some(25))),
+                )),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            })));
+        let prefilter: Arc<dyn ExecutionPlan> = Arc::new(ScalarIndexExec::new(
+            fixture.dataset.clone(),
+            prefilter_expr,
+            IndexExprResultWireFormat::default(),
+        ));
+
+        let plan = CountFromMaskExec::try_new(
+            fixture.dataset.clone(),
+            vec![count_star_expr(&schema)],
+            Some(prefilter),
+        )
+        .unwrap();
+        let count = run(plan).await;
+        assert_eq!(count, 15);
+    }
+
+    #[tokio::test]
+    async fn execute_count_respects_deletions() {
+        let fixture = make_fixture().await;
+        let mut dataset = (*fixture.dataset).clone();
+        // Delete the first ten rows of the dataset (which live in fragment 0).
+        dataset.delete("ordered < 10").await.unwrap();
+        let dataset = Arc::new(dataset);
+
+        let schema = input_schema();
+        let plan =
+            CountFromMaskExec::try_new(dataset, vec![count_star_expr(&schema)], None).unwrap();
+        let count = run(plan).await;
+        assert_eq!(count, 30);
+    }
+}
diff --git a/rust/lance/src/io/exec/count_pushdown.rs b/rust/lance/src/io/exec/count_pushdown.rs
new file mode 100644
index 00000000000..a25724ac5f1
--- /dev/null
+++ b/rust/lance/src/io/exec/count_pushdown.rs
@@ -0,0 +1,827 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Physical optimizer rule that rewrites a `COUNT(*)`-style aggregate into
+//! [`CountFromMaskExec`], answering the count from index metadata and the
+//! deletion mask without scanning column data.
+//!
+//! This is the "count-from-mask" category of aggregate pushdown — one of
+//! four planned. The other categories (mask-to-answer, zone-aware,
+//! dimension-keyed) will each need their own rule and exec; the surrounding
+//! infrastructure (the `fragment_bitmap` plumbed on each `ScalarIndexSearch`,
+//! the `IndexInformationProvider::fragment_bitmap` lookup) is general
+//! enough to be reused.
+//!
+//! Two rewritten shapes are emitted depending on whether the scalar index
+//! backing the filter covers every dataset fragment.
+//!
+//! **Full coverage** (index ⊇ dataset, or no filter at all):
+//!
+//! ```text
+//! AggregateExec(Final, aggs=[count(...)], group_by=[])
+//!   └── CountFromMaskExec { prefilter_input = index_input }
+//! ```
+//!
+//! **Partial coverage** (index ⊊ dataset — typically appended fragments):
+//!
+//! ```text
+//! AggregateExec(Final, aggs=[count(...)], group_by=[])
+//!   └── UnionExec
+//!         ├── CountFromMaskExec(restrict_to_fragments = indexed)
+//!         └── AggregateExec(Partial)
+//!               └── FilteredReadExec(fragments = unindexed, full_filter = …)
+//! ```
+//!
+//! [`CountFromMaskExec`] emits partial-state, so the outer
+//! `AggregateExec(Final)` performs the final combine in either shape.
+//!
+//! If the prefilter's index coverage is unknown (any leaf is missing
+//! `fragment_bitmap`, e.g. constructed outside scanner planning), the rule
+//! refuses to fire and leaves the existing scan path in place.
+
+use std::sync::Arc;
+
+use datafusion::common::tree_node::{Transformed, TreeNode};
+use datafusion::config::ConfigOptions;
+use datafusion::error::Result as DFResult;
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+#[allow(deprecated)]
+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
+use datafusion::physical_plan::{
+    ExecutionPlan, ExecutionPlanProperties,
+    aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy},
+    coalesce_partitions::CoalescePartitionsExec,
+    projection::ProjectionExec,
+    repartition::RepartitionExec,
+    union::UnionExec,
+};
+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+use datafusion_physical_expr::expressions::{Column, Literal};
+use lance_index::scalar::expression::ScalarIndexExpr;
+use log::warn;
+use roaring::RoaringBitmap;
+
+use super::count_from_mask::CountFromMaskExec;
+use super::filtered_read::{FilteredReadExec, FilteredReadOptions};
+use super::scalar_index::ScalarIndexExec;
+
+/// Physical optimizer rule that rewrites a `COUNT(*)`-style aggregate into
+/// [`CountFromMaskExec`], optionally splitting into a parallel scan branch
+/// when the index has partial coverage of the dataset.
+///
+/// Only fires when the shape is verifiably safe; everything outside that
+/// envelope (GROUP BY, residual filters, scan ranges, etc.) is left alone
+/// for the normal scan path.
+#[derive(Debug)]
+pub struct CountPushdown;
+
+impl PhysicalOptimizerRule for CountPushdown {
+    fn optimize(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        _config: &ConfigOptions,
+    ) -> DFResult<Arc<dyn ExecutionPlan>> {
+        Ok(plan
+            .transform_down(|plan| {
+                let Some(agg) = plan.downcast_ref::<AggregateExec>() else {
+                    return Ok(Transformed::no(plan));
+                };
+                if let Some(rewritten) = try_rewrite(agg)? {
+                    return Ok(Transformed::yes(rewritten));
+                }
+                Ok(Transformed::no(plan))
+            })?
+            .data)
+    }
+
+    fn name(&self) -> &str {
+        "count_pushdown"
+    }
+
+    fn schema_check(&self) -> bool {
+        true
+    }
+}
+
+fn try_rewrite(agg: &AggregateExec) -> DFResult<Option<Arc<dyn ExecutionPlan>>> {
+    // We can accelerate Single (Lance scanner shape) and Partial (the shape
+    // DataFusion's SQL planner emits at the leaf of an aggregate pipeline);
+    // both produce results we know how to compute from the index. We will
+    // never accelerate Final or FinalPartitioned — those combine an existing
+    // partial stream, and the value of this rule is replacing the work that
+    // produces the partial stream.
+    let mode = match agg.mode() {
+        AggregateMode::Single => AggregateMode::Single,
+        AggregateMode::Partial => AggregateMode::Partial,
+        _ => return Ok(None),
+    };
+    if !agg.group_expr().is_empty() {
+        return Ok(None);
+    }
+    if agg.aggr_expr().is_empty() {
+        return Ok(None);
+    }
+
+    // Every aggregate must be a `COUNT(<literal>)` shape (i.e. COUNT(*) /
+    // COUNT(1) / etc.) with no per-aggregate `FILTER (WHERE ...)`. This rule
+    // is scoped to the count-from-mask category only; other aggregate
+    // categories (mask-to-answer, zone-aware, dimension-keyed) will need
+    // their own rules with their own gates.
+    for (af, filter) in agg.aggr_expr().iter().zip(agg.filter_expr().iter()) {
+        if !is_count_star(af) {
+            return Ok(None);
+        }
+        if filter.is_some() {
+            return Ok(None);
+        }
+    }
+
+    // The input must be a FilteredReadExec we can prove is safe to skip.
+    // DataFusion's SQL planner inserts a few row-preserving wrappers above
+    // the leaf — a `RepartitionExec` for parallelism, an empty
+    // `ProjectionExec` once the count expression has been resolved to need
+    // no columns, and `CoalesceBatchesExec` here and there. Walk through
+    // those to reach the FilteredReadExec.
+    let Some(filtered_read) = strip_row_preserving_wrappers(agg.children()[0]) else {
+        return Ok(None);
+    };
+
+    let options = filtered_read.options();
+    // A refine filter is a residual the index couldn't fully evaluate — it
+    // needs column data to apply, which we can't.
+    if options.refine_filter.is_some() {
+        return Ok(None);
+    }
+    // A full_filter without an index_input means the filter is evaluated by
+    // scanning every row; not pushdownable.
+    if options.full_filter.is_some() && filtered_read.index_input().is_none() {
+        return Ok(None);
+    }
+    // LIMIT/OFFSET would change the count.
+    if options.scan_range_before_filter.is_some() || options.scan_range_after_filter.is_some() {
+        return Ok(None);
+    }
+    // We rely on the deletion mask being applied; with_deleted_rows changes
+    // that contract. Surfacing as a warning because it shouldn't normally
+    // pair with an aggregate plan — if we see it, the planner produced a
+    // shape we could in principle accelerate but currently can't.
+    if options.with_deleted_rows {
+        warn!(
+            "count_pushdown: skipped because the FilteredReadExec was built \
+             with with_deleted_rows; the count will be computed via a full \
+             scan."
+        );
+        return Ok(None);
+    }
+    // Same story for an explicit fragment subset: legitimate, but unexpected
+    // alongside an aggregate, and we lose the pushdown opportunity.
+    if options.fragments.is_some() {
+        warn!(
+            "count_pushdown: skipped because the FilteredReadExec was scoped \
+             to an explicit fragment subset; the count will be computed via a \
+             full scan. Intersecting that subset into the coverage logic would \
+             let this query be answered from index metadata."
+        );
+        return Ok(None);
+    }
+
+    let dataset = filtered_read.dataset().clone();
+    let dataset_fragments: RoaringBitmap =
+        dataset.fragments().iter().map(|f| f.id as u32).collect();
+    let prefilter_input = filtered_read.index_input().cloned();
+
+    // If there is a prefilter, inspect its ScalarIndexExpr leaves:
+    //   - Refuse to fire if any leaf is inexact (`needs_recheck`). The
+    //     prefilter's serialized batch carries an Exact/AtMost/AtLeast
+    //     discriminant but `load_prefilter` only reads the row-address mask
+    //     and would treat AtMost as Exact, silently overcounting (and
+    //     symmetrically AtLeast would undercount).
+    //   - Compute the index's fragment coverage from leaf `fragment_bitmap`s.
+    //     `None` means at least one leaf has no bitmap and we can't reason
+    //     about coverage synchronously — refuse to fire.
+    let index_coverage = match &prefilter_input {
+        None => None,
+        Some(input) => {
+            let scalar_exec = input.downcast_ref::<ScalarIndexExec>().ok_or_else(|| {
+                datafusion::error::DataFusionError::Internal(
+                    "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec"
+                        .to_string(),
+                )
+            })?;
+            if scalar_exec.expr().needs_recheck() {
+                return Ok(None);
+            }
+            let Some(coverage) = collect_coverage(scalar_exec.expr()) else {
+                return Ok(None);
+            };
+            Some(coverage)
+        }
+    };
+
+    let aggr_exprs: Vec<Arc<AggregateFunctionExpr>> = agg.aggr_expr().to_vec();
+
+    // Decide on the plan shape. Three cases:
+    //
+    // 1. No prefilter (no filter at all): single pushdown branch over every
+    //    dataset fragment. Always safe.
+    // 2. Prefilter + index covers every dataset fragment: single pushdown
+    //    branch, prefilter feeds in directly.
+    // 3. Prefilter + index covers a strict subset: split into pushdown over
+    //    indexed fragments + parallel scan over unindexed fragments.
+    let (partial_stream, partial_state_schema): (Arc<dyn ExecutionPlan>, _) = match index_coverage {
+        None => {
+            // No prefilter at all (verified above): nothing to restrict.
+            let exec = CountFromMaskExec::try_new_restricted(
+                dataset,
+                aggr_exprs.clone(),
+                prefilter_input,
+                None,
+            )?;
+            let schema = exec.schema();
+            (Arc::new(exec), schema)
+        }
+        Some(coverage) if (&dataset_fragments - &coverage).is_empty() => {
+            // Prefilter exists and the index covers every dataset fragment —
+            // safe to push the whole count down.
+            let exec = CountFromMaskExec::try_new_restricted(
+                dataset,
+                aggr_exprs.clone(),
+                prefilter_input,
+                None,
+            )?;
+            let schema = exec.schema();
+            (Arc::new(exec), schema)
+        }
+        Some(coverage) => {
+            // Split plan: CountFromMaskExec for the indexed fragments, a
+            // normal scan + AggregateExec(Partial) for the rest.
+            let uncovered = &dataset_fragments - &coverage;
+            let pushdown_exec = CountFromMaskExec::try_new_restricted(
+                dataset,
+                aggr_exprs.clone(),
+                prefilter_input,
+                Some(&dataset_fragments & &coverage),
+            )?;
+            let partial_state_schema = pushdown_exec.schema();
+            let pushdown_branch: Arc<dyn ExecutionPlan> = Arc::new(pushdown_exec);
+            let scan_branch =
+                build_scan_branch(filtered_read, options, &uncovered, aggr_exprs.clone())?;
+            let union: Arc<dyn ExecutionPlan> =
+                UnionExec::try_new(vec![pushdown_branch, scan_branch])?;
+            (union, partial_state_schema)
+        }
+    };
+
+    match mode {
+        AggregateMode::Partial => {
+            // Caller's parent is already an AggregateExec(Final) that knows
+            // how to consume multi-partition partial state — substitute our
+            // partial stream and we're done.
+            Ok(Some(partial_stream))
+        }
+        AggregateMode::Single => {
+            // The original AggregateExec(Single) produced final output in one
+            // step; our exec emits partial state, so add a Final on top to
+            // recover the original output schema. Final expects a single
+            // partition of partial-state rows, so coalesce when we have a
+            // union producing multiple partitions.
+            let final_input: Arc<dyn ExecutionPlan> =
+                if partial_stream.output_partitioning().partition_count() > 1 {
+                    Arc::new(CoalescePartitionsExec::new(partial_stream))
+                } else {
+                    partial_stream
+                };
+            // `AggregateExec::try_new` requires one
+            // `Option<Arc<dyn PhysicalExpr>>` per aggregate expression for
+            // the optional per-aggregate `FILTER (WHERE ...)` clause. We
+            // rejected any aggregate carrying a filter back at the gate, so
+            // every slot is `None` here.
+            let filters: Vec<Option<Arc<dyn datafusion::physical_expr::PhysicalExpr>>> =
+                (0..aggr_exprs.len()).map(|_| None).collect();
+            let final_agg = AggregateExec::try_new(
+                AggregateMode::Final,
+                PhysicalGroupBy::default(),
+                aggr_exprs,
+                filters,
+                final_input,
+                partial_state_schema,
+            )?;
+            Ok(Some(Arc::new(final_agg)))
+        }
+        _ => unreachable!("mode was checked at the top of try_rewrite"),
+    }
+}
+
+/// Build the scan branch of a partial-coverage split: a `FilteredReadExec`
+/// restricted to the uncovered fragments (no `index_input`, the original
+/// `full_filter` applied per row) wrapped in `AggregateExec(Partial)` so its
+/// partial state can be unioned with the pushdown branch.
+fn build_scan_branch(
+    filtered_read: &FilteredReadExec,
+    options: &FilteredReadOptions,
+    uncovered: &RoaringBitmap,
+    aggr_exprs: Vec<Arc<AggregateFunctionExpr>>,
+) -> DFResult<Arc<dyn ExecutionPlan>> {
+    let dataset = filtered_read.dataset().clone();
+    let uncovered_fragments: Vec<_> = dataset
+        .manifest()
+        .fragments
+        .iter()
+        .filter(|f| uncovered.contains(f.id as u32))
+        .cloned()
+        .collect();
+    let mut scan_options = options.clone();
+    scan_options.fragments = Some(Arc::new(uncovered_fragments));
+    let scan = FilteredReadExec::try_new(dataset, scan_options, None)?;
+    let scan: Arc<dyn ExecutionPlan> = Arc::new(scan);
+    let scan_schema = scan.schema();
+    // Per-aggregate `FILTER (WHERE ...)` placeholders — see the matching
+    // comment in `try_rewrite`; we've already rejected any aggregate that
+    // carried a filter, so every slot is `None`.
+    let filters: Vec<Option<Arc<dyn datafusion::physical_expr::PhysicalExpr>>> =
+        (0..aggr_exprs.len()).map(|_| None).collect();
+    let partial = AggregateExec::try_new(
+        AggregateMode::Partial,
+        PhysicalGroupBy::default(),
+        aggr_exprs,
+        filters,
+        scan,
+        scan_schema,
+    )?;
+    Ok(Arc::new(partial))
+}
+
+/// Walk through row-preserving wrappers (`RepartitionExec`,
+/// `CoalesceBatchesExec`, and identity-or-empty `ProjectionExec`) that
+/// DataFusion's planner inserts between an `AggregateExec` and the leaf, and
+/// return the underlying `FilteredReadExec` if one is reached.
+///
+/// "Row-preserving" here means the wrapper changes neither the number of rows
+/// nor the predicate applied to them — it may reshape partitions, batches, or
+/// drop unused columns, but the row population at the bottom is what reaches
+/// the aggregate. That's all the rule needs from these layers, so it's safe to
+/// look past them.
+fn strip_row_preserving_wrappers(plan: &Arc<dyn ExecutionPlan>) -> Option<&FilteredReadExec> {
+    let mut current: &dyn ExecutionPlan = plan.as_ref();
+    loop {
+        if let Some(filtered_read) = current.downcast_ref::<FilteredReadExec>() {
+            return Some(filtered_read);
+        }
+        let next: &Arc<dyn ExecutionPlan> =
+            if let Some(inner) = current.downcast_ref::<RepartitionExec>() {
+                inner.input()
+            } else if let Some(inner) = {
+                #[allow(deprecated)]
+                current.downcast_ref::<CoalesceBatchesExec>()
+            } {
+                inner.input()
+            } else if let Some(inner) = current.downcast_ref::<CoalescePartitionsExec>() {
+                inner.input()
+            } else {
+                let proj = current.downcast_ref::<ProjectionExec>()?;
+                // Only walk through projections that are row-preserving: every
+                // output expression is a direct column reference back to the
+                // input. (Empty projections trivially qualify — DataFusion uses
+                // one when a `COUNT(*)`'s argument no longer needs any actual
+                // columns.)
+                let input_schema = proj.input().schema();
+                let identity = proj.expr().iter().all(|projection_expr| {
+                    projection_expr
+                        .expr
+                        .downcast_ref::<Column>()
+                        .is_some_and(|c| c.name() == input_schema.field(c.index()).name())
+                });
+                if !identity {
+                    return None;
+                }
+                proj.input()
+            };
+        current = next.as_ref();
+    }
+}
+
+/// Walk a `ScalarIndexExpr` and intersect the per-leaf `fragment_bitmap`.
+///
+/// Returns `None` if any leaf is missing a bitmap (coverage unknown). All
+/// three combinators (`And`, `Or`, `Not`) reduce to "every leaf must cover the
+/// fragment for us to give a definitive answer about it" — i.e. intersection.
+fn collect_coverage(expr: &ScalarIndexExpr) -> Option<RoaringBitmap> {
+    match expr {
+        ScalarIndexExpr::Not(inner) => collect_coverage(inner),
+        ScalarIndexExpr::And(lhs, rhs) | ScalarIndexExpr::Or(lhs, rhs) => {
+            let l = collect_coverage(lhs)?;
+            let r = collect_coverage(rhs)?;
+            Some(l & r)
+        }
+        ScalarIndexExpr::Query(search) => search.fragment_bitmap.clone(),
+    }
+}
+
+/// Returns `true` if `af` is `COUNT(<literal>)` with no DISTINCT.
+fn is_count_star(af: &Arc<AggregateFunctionExpr>) -> bool {
+    if af.fun().name() != "count" {
+        return false;
+    }
+    if af.is_distinct() {
+        return false;
+    }
+    let args = af.expressions();
+    if args.len() != 1 {
+        return false;
+    }
+    let Some(lit) = args[0].downcast_ref::<Literal>() else {
+        return false;
+    };
+    // `COUNT(NULL)` would always return 0; rule it out so we don't accidentally
+    // produce a wrong answer if the planner ever lets it through.
+    !lit.value().is_null()
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow::datatypes::{Int64Type, UInt64Type};
+    use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
+    use datafusion::physical_plan::{ExecutionPlan, displayable};
+    use futures::TryStreamExt;
+    use lance_core::utils::tempfile::TempStrDir;
+    use lance_datagen::gen_batch;
+    use lance_index::IndexType;
+    use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
+
+    use super::*;
+    use crate::Dataset;
+    use crate::dataset::scanner::AggregateExpr;
+    use crate::index::DatasetIndexExt;
+    use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
+
+    struct Fixture {
+        dataset: Arc<Dataset>,
+        _tmp: TempStrDir,
+    }
+
+    /// 4 fragments × 10 rows, ascending `ordered` column with a BTree index.
+    async fn make_fixture() -> Fixture {
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(10),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        Fixture {
+            dataset: Arc::new(dataset),
+            _tmp: tmp,
+        }
+    }
+
+    fn plan_contains_pushdown(plan: &Arc<dyn ExecutionPlan>) -> bool {
+        let mut found = false;
+        plan.apply(|node| {
+            if node.is::<CountFromMaskExec>() {
+                found = true;
+                Ok(TreeNodeRecursion::Stop)
+            } else {
+                Ok(TreeNodeRecursion::Continue)
+            }
+        })
+        .unwrap();
+        found
+    }
+
+    fn plan_contains_union(plan: &Arc<dyn ExecutionPlan>) -> bool {
+        let mut found = false;
+        plan.apply(|node| {
+            if node.is::<UnionExec>() {
+                found = true;
+                Ok(TreeNodeRecursion::Stop)
+            } else {
+                Ok(TreeNodeRecursion::Continue)
+            }
+        })
+        .unwrap();
+        found
+    }
+
+    async fn run_count(
+        scanner: &mut crate::dataset::scanner::Scanner,
+    ) -> (Arc<dyn ExecutionPlan>, i64) {
+        scanner
+            .aggregate(AggregateExpr::builder().count_star().build())
+            .unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        let stream = datafusion::physical_plan::execute_stream(
+            plan.clone(),
+            Arc::new(datafusion::execution::TaskContext::default()),
+        )
+        .unwrap();
+        let batches: Vec<_> = stream.try_collect().await.unwrap();
+        assert_eq!(
+            batches.len(),
+            1,
+            "count plan emitted {} batches",
+            batches.len()
+        );
+        let count = batches[0]
+            .column(0)
+            .as_any()
+            .downcast_ref::<arrow_array::PrimitiveArray<Int64Type>>()
+            .expect("count column should be Int64")
+            .value(0);
+        (plan, count)
+    }
+
+    #[tokio::test]
+    async fn rule_fires_on_unfiltered_count_star() {
+        let fixture = make_fixture().await;
+        let mut scanner = fixture.dataset.scan();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 40);
+        assert!(
+            plan_contains_pushdown(&plan),
+            "expected CountFromMaskExec in plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+        assert!(
+            !plan_contains_union(&plan),
+            "no union expected for unfiltered count, got: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_fires_when_filter_fully_indexed() {
+        let fixture = make_fixture().await;
+        let mut scanner = fixture.dataset.scan();
+        scanner.filter("ordered < 25").unwrap();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 25);
+        assert!(
+            plan_contains_pushdown(&plan),
+            "expected CountFromMaskExec in plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+        assert!(
+            !plan_contains_union(&plan),
+            "no union expected when index covers every fragment, got: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_emits_split_plan_for_partial_index_coverage() {
+        // Build index over 4 fragments, then append a 5th — the index now
+        // covers a strict subset of the dataset. The rule must split into a
+        // pushdown branch over the indexed fragments and a scan branch over
+        // the rest, then sum the partials.
+        use crate::dataset::WriteParams;
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(10),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        let extra = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_reader_rows(
+                lance_datagen::RowCount::from(10),
+                lance_datagen::BatchCount::from(1),
+            );
+        let dataset = Dataset::write(
+            extra,
+            tmp.as_str(),
+            Some(WriteParams {
+                mode: crate::dataset::WriteMode::Append,
+                max_rows_per_file: 10,
+                ..Default::default()
+            }),
+        )
+        .await
+        .unwrap();
+        let dataset = Arc::new(dataset);
+
+        let mut scanner = dataset.scan();
+        scanner.filter("ordered < 100").unwrap();
+        let (plan, count) = run_count(&mut scanner).await;
+        // 5 fragments × 10 rows, all match `< 100`.
+        assert_eq!(count, 50);
+        assert!(
+            plan_contains_pushdown(&plan),
+            "expected pushdown branch in split plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+        assert!(
+            plan_contains_union(&plan),
+            "expected UnionExec for partial-coverage split, got: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_fires_with_stable_row_ids() {
+        // Unfiltered count, stable row ids, with a deletion.
+        use crate::dataset::WriteParams;
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset_with_params(
+                tmp.as_str(),
+                FragmentCount::from(2),
+                FragmentRowCount::from(10),
+                Some(WriteParams {
+                    max_rows_per_file: 10,
+                    enable_stable_row_ids: true,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap();
+        dataset.delete("ordered = 0").await.unwrap();
+        let dataset = Arc::new(dataset);
+
+        let mut scanner = dataset.scan();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 19);
+        assert!(
+            plan_contains_pushdown(&plan),
+            "rule should fire under stable row IDs, got plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_fires_with_stable_row_ids_and_filter() {
+        // Indexed filter, stable row ids, deletions spread across fragments --
+        // the case the pre-fix code got wrong (dropped rows in fragments > 0).
+        use crate::dataset::WriteParams;
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset_with_params(
+                tmp.as_str(),
+                FragmentCount::from(3),
+                FragmentRowCount::from(10),
+                Some(WriteParams {
+                    max_rows_per_file: 10,
+                    enable_stable_row_ids: true,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        // Delete one row from fragment 1 and one from fragment 2.
+        dataset
+            .delete("ordered = 15 OR ordered = 25")
+            .await
+            .unwrap();
+        let dataset = Arc::new(dataset);
+
+        let mut scanner = dataset.scan();
+        // Matches every row across all three fragments; with the two deletions
+        // the live count is 28.
+        scanner.filter("ordered >= 0").unwrap();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 28);
+        assert!(
+            plan_contains_pushdown(&plan),
+            "rule should fire under stable row IDs with a filter, got plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_skips_when_filter_needs_refine() {
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .col("unindexed", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(10),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::BTree,
+                None,
+                &ScalarIndexParams::default(),
+                true,
+            )
+            .await
+            .unwrap();
+        let dataset = Arc::new(dataset);
+
+        let mut scanner = dataset.scan();
+        scanner.filter("unindexed > 5").unwrap();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 34);
+        assert!(
+            !plan_contains_pushdown(&plan),
+            "rule should not fire with non-indexed filter, got plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_skips_when_index_is_inexact() {
+        // Zonemap-style indices return AtMost (over-approximation) and set
+        // ScalarIndexSearch.needs_recheck = true. CountFromMaskExec
+        // ignores the discriminant on the prefilter batch, so firing the
+        // rule against an inexact index would silently overcount. The rule
+        // must refuse — and the scan path with its recheck still answers
+        // correctly.
+        let tmp = TempStrDir::default();
+        let mut dataset = gen_batch()
+            .col("ordered", lance_datagen::array::step::<UInt64Type>())
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(10),
+            )
+            .await
+            .unwrap();
+        dataset
+            .create_index(
+                &["ordered"],
+                IndexType::ZoneMap,
+                None,
+                &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap),
+                true,
+            )
+            .await
+            .unwrap();
+        let dataset = Arc::new(dataset);
+
+        let mut scanner = dataset.scan();
+        scanner.filter("ordered < 25").unwrap();
+        let (plan, count) = run_count(&mut scanner).await;
+        assert_eq!(count, 25);
+        assert!(
+            !plan_contains_pushdown(&plan),
+            "rule must not fire when the index produces inexact (needs_recheck) results, \
+             got plan: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+
+    #[tokio::test]
+    async fn rule_skips_count_with_group_by() {
+        let fixture = make_fixture().await;
+        let mut scanner = fixture.dataset.scan();
+        scanner
+            .aggregate(
+                AggregateExpr::builder()
+                    .group_by("ordered")
+                    .count_star()
+                    .build(),
+            )
+            .unwrap();
+        let plan = scanner.create_plan().await.unwrap();
+        assert!(
+            !plan_contains_pushdown(&plan),
+            "rule should not fire for GROUP BY: {}",
+            displayable(plan.as_ref()).indent(true)
+        );
+    }
+}
diff --git a/rust/lance/src/io/exec/filter.rs b/rust/lance/src/io/exec/filter.rs
index 3a36f8d6712..71f1a5b2b4a 100644
--- a/rust/lance/src/io/exec/filter.rs
+++ b/rust/lance/src/io/exec/filter.rs
@@ -48,10 +48,6 @@ impl ExecutionPlan for LanceFilterExec {
         "LanceFilterExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         self.filter.properties()
     }
@@ -71,7 +67,6 @@ impl ExecutionPlan for LanceFilterExec {
         // Rewrap the result in a LanceFilterExec to preserve the logical expression
         let new_filter_plan = self.filter.clone().with_new_children(children)?;
         let new_filter = new_filter_plan
-            .as_any()
             .downcast_ref::<FilterExec>()
             .expect("FilterExec::with_new_children should return FilterExec")
             .clone();
@@ -93,7 +88,7 @@ impl ExecutionPlan for LanceFilterExec {
         self.filter.metrics()
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         self.filter.partition_statistics(partition)
     }
 
diff --git a/rust/lance/src/io/exec/filtered_read.rs b/rust/lance/src/io/exec/filtered_read.rs
index c491e3f194b..414ce601763 100644
--- a/rust/lance/src/io/exec/filtered_read.rs
+++ b/rust/lance/src/io/exec/filtered_read.rs
@@ -1,20 +1,18 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
-use std::any::Any;
 use std::collections::{BTreeMap, HashMap};
-use std::pin::Pin;
 use std::sync::Mutex;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::{ops::Range, sync::Arc};
 
 use arrow_array::RecordBatch;
-use arrow_schema::SchemaRef;
+use arrow_schema::{Schema as ArrowSchema, SchemaRef};
 use datafusion::common::runtime::SpawnedTask;
 use datafusion::common::stats::Precision;
 use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::execution::{SendableRecordBatchStream, TaskContext};
 use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::stream::{RecordBatchReceiverStream, RecordBatchStreamAdapter};
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
     execution_plan::{Boundedness, EmissionType},
@@ -60,6 +58,13 @@ use crate::dataset::scanner::{
 
 use super::utils::IoMetrics;
 
+fn public_blob_v2_binary_projection_schema(projection: &Projection) -> SchemaRef {
+    let schema = projection.to_schema();
+    let schema = crate::dataset::blob::public_blob_v2_binary_output_schema(&schema);
+    let schema: ArrowSchema = (&schema).into();
+    Arc::new(schema)
+}
+
 #[derive(Debug)]
 pub struct EvaluatedIndex {
     index_result: IndexExprResult,
@@ -328,6 +333,116 @@ struct FilteredReadStream {
     threading_mode: FilteredReadThreadingMode,
     /// Range to apply to the result stream if not already pushed down in planning phase
     scan_range_after_filter: Option<Range<u64>>,
+    /// Fragments planned non-empty, and their total planned rows; the output
+    /// side uses these to detect take-shaped plans (batch size resolves at
+    /// execute time, so the detection lives there too)
+    touched_fragments: usize,
+    planned_rows: u64,
+}
+
+/// Below this many fragments there are too few handoffs to be worth
+/// consolidating
+const CONSOLIDATE_MIN_FRAGMENTS: usize = 8;
+
+/// Above this per-fragment average, batches are big enough to amortize
+/// their handoff
+const CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT: u64 = 1024;
+
+/// Pump a take-shaped read on a spawned task, handing the consumer
+/// consolidated batches. Inline polling would otherwise execute the
+/// per-batch pipeline work on the consumer, which serializes concurrent
+/// small reads.
+fn consolidated_stream(
+    inner: SendableRecordBatchStream,
+    target: usize,
+) -> SendableRecordBatchStream {
+    let mut builder = RecordBatchReceiverStream::builder(inner.schema(), 4);
+    let tx = builder.tx();
+    builder.spawn(async move {
+        let mut stream = coalesce_batches(inner, target).boxed();
+        while let Some(item) = stream.next().await {
+            if tx.send(item).await.is_err() {
+                // Receiver dropped: the query was cancelled
+                break;
+            }
+        }
+        Ok(())
+    });
+    builder.build()
+}
+
+/// Merge batches up to `target` rows; batches already at the target pass
+/// through whole (never split). Order is preserved.
+fn coalesce_batches(
+    input: SendableRecordBatchStream,
+    target: usize,
+) -> impl Stream<Item = DataFusionResult<RecordBatch>> {
+    struct Coalescer {
+        input: SendableRecordBatchStream,
+        schema: SchemaRef,
+        target: usize,
+        buffered: Vec<RecordBatch>,
+        buffered_rows: usize,
+        exhausted: bool,
+    }
+
+    impl Coalescer {
+        fn ready_to_emit(&self) -> bool {
+            self.buffered_rows >= self.target || (self.exhausted && !self.buffered.is_empty())
+        }
+
+        fn buffer(&mut self, batch: RecordBatch) {
+            self.buffered_rows += batch.num_rows();
+            self.buffered.push(batch);
+        }
+
+        fn emit(&mut self) -> DataFusionResult<RecordBatch> {
+            self.buffered_rows = 0;
+            if self.buffered.len() > 1 {
+                let batch = arrow::compute::concat_batches(&self.schema, self.buffered.iter())?;
+                self.buffered.clear();
+                Ok(batch)
+            } else {
+                self.buffered.pop().ok_or_else(|| {
+                    DataFusionError::Internal(
+                        "coalesce_batches emitted with an empty buffer".to_string(),
+                    )
+                })
+            }
+        }
+    }
+
+    let schema = input.schema();
+    let coalescer = Coalescer {
+        input,
+        schema,
+        target,
+        buffered: Vec::new(),
+        buffered_rows: 0,
+        exhausted: false,
+    };
+    futures::stream::try_unfold(coalescer, |mut this| async move {
+        loop {
+            if this.ready_to_emit() {
+                return Ok(Some((this.emit()?, this)));
+            }
+            if this.exhausted {
+                return Ok(None);
+            }
+            match this.input.try_next().await? {
+                Some(batch) if batch.num_rows() >= this.target && !this.buffered.is_empty() => {
+                    // Emit the partial buffer on its own; the large batch
+                    // then passes through whole on the next iteration
+                    let out = this.emit()?;
+                    this.buffer(batch);
+                    return Ok(Some((out, this)));
+                }
+                Some(batch) if batch.num_rows() > 0 => this.buffer(batch),
+                Some(_) => {}
+                None => this.exhausted = true,
+            }
+        }
+    })
 }
 
 impl std::fmt::Debug for FilteredReadStream {
@@ -385,7 +500,7 @@ impl FilteredReadStream {
             .try_collect::<Vec<_>>()
             .await?;
 
-        let output_schema = Arc::new(options.projection.to_arrow_schema());
+        let output_schema = public_blob_v2_binary_projection_schema(&options.projection);
 
         let obj_store = dataset.object_store.clone();
         // Explicit options take precedence; otherwise fall back to the
@@ -417,11 +532,14 @@ impl FilteredReadStream {
         let fragment_streams = futures::stream::iter(scoped_fragments)
             .map({
                 let scan_range_after_filter = scan_range_after_filter.clone();
+                let dataset = dataset.clone();
                 move |scoped_fragment| {
                     let metrics = global_metrics_clone.clone();
                     let limit = scan_range_after_filter.as_ref().map(|r| r.end);
+                    let dataset = dataset.clone();
                     SpawnedTask::spawn(
-                        Self::read_fragment(scoped_fragment, metrics, limit).in_current_span(),
+                        Self::read_fragment(dataset, scoped_fragment, metrics, limit)
+                            .in_current_span(),
                     )
                     .map(|thread_result| thread_result.unwrap())
                 }
@@ -429,6 +547,22 @@ impl FilteredReadStream {
             .buffered(fragment_readahead);
         let task_stream = fragment_streams.try_flatten().boxed();
 
+        // A batch never spans fragments, so a plan touching many fragments
+        // with few rows each emits one tiny batch per fragment. Fragments
+        // planned empty produce no batch and don't count. Filtered scans
+        // stay dense here: their planned rows are a pre-refine upper bound.
+        let (touched_fragments, planned_rows) =
+            plan.rows
+                .values()
+                .fold((0usize, 0u64), |(fragments, rows), ranges| {
+                    let fragment_rows: u64 =
+                        ranges.iter().map(|range| range.end - range.start).sum();
+                    if fragment_rows > 0 {
+                        (fragments + 1, rows + fragment_rows)
+                    } else {
+                        (fragments, rows)
+                    }
+                });
         Ok(Self {
             output_schema,
             task_stream: Arc::new(AsyncMutex::new(task_stream)),
@@ -437,6 +571,8 @@ impl FilteredReadStream {
             active_partitions_counter: Arc::new(AtomicUsize::new(0)),
             threading_mode,
             scan_range_after_filter,
+            touched_fragments,
+            planned_rows,
         })
     }
 
@@ -997,6 +1133,10 @@ impl FilteredReadStream {
                     base_batch_stream.boxed()
                 };
 
+                // Clone so the finally handler can record a final snapshot even when
+                // no output batches were produced (inspect_ok never fires in that case).
+                let global_metrics_final = global_metrics.clone();
+                let scan_scheduler_final = scan_scheduler.clone();
                 let batch_stream = batch_stream
                     .inspect_ok(move |batch| {
                         partition_metrics_clone
@@ -1005,6 +1145,9 @@ impl FilteredReadStream {
                         global_metrics.io_metrics.record(&scan_scheduler);
                     })
                     .finally(move || {
+                        global_metrics_final
+                            .io_metrics
+                            .record(&scan_scheduler_final);
                         partition_metrics.baseline_metrics.done();
                     })
                     .map_err(|e: lance_core::Error| DataFusionError::External(e.into()))
@@ -1067,13 +1210,15 @@ impl FilteredReadStream {
     }
 
     // Reads a single fragment into a stream of batch tasks
-    #[instrument(name = "read_fragment", skip_all)]
+    #[instrument(name = "read_fragment", level = "debug", skip_all)]
     async fn read_fragment(
+        dataset: Arc<Dataset>,
         mut fragment_read_task: ScopedFragmentRead,
         global_metrics: Arc<FilteredReadGlobalMetrics>,
         fragment_soft_limit: Option<u64>,
-    ) -> Result<impl Stream<Item = Result<ReadBatchFut>>> {
-        let output_schema = Arc::new(fragment_read_task.projection.to_arrow_schema());
+    ) -> Result<BoxStream<'static, Result<ReadBatchFut>>> {
+        let output_schema =
+            public_blob_v2_binary_projection_schema(fragment_read_task.projection.as_ref());
 
         if let Some(filter) = &fragment_read_task.filter {
             let filter_cols = Planner::column_names_in_expr(filter);
@@ -1088,10 +1233,22 @@ impl FilteredReadStream {
             }
         }
 
-        let read_schema = fragment_read_task.projection.to_bare_schema();
+        let output_read_schema = Arc::new(fragment_read_task.projection.to_schema());
+        let bare_read_schema = fragment_read_task.projection.to_bare_schema();
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(&bare_read_schema);
+        let read_schema = if materialize_blob_v2_binary {
+            crate::dataset::blob::blob_v2_descriptor_schema(&bare_read_schema)
+        } else {
+            bare_read_schema
+        };
+        let mut frag_read_config = fragment_read_task.frag_read_config();
+        if materialize_blob_v2_binary {
+            frag_read_config = frag_read_config.with_row_address(true);
+        }
         let mut fragment_reader = fragment_read_task
             .fragment
-            .open(&read_schema, fragment_read_task.frag_read_config())
+            .open(&read_schema, frag_read_config)
             .await?;
 
         if fragment_read_task.with_deleted_rows {
@@ -1105,8 +1262,9 @@ impl FilteredReadStream {
         let physical_filter = fragment_read_task
             .filter
             .map(|filter| {
-                let planner =
-                    Planner::new(Arc::new(fragment_read_task.projection.to_arrow_schema()));
+                let planner = Planner::new(public_blob_v2_binary_projection_schema(
+                    fragment_read_task.projection.as_ref(),
+                ));
                 planner.create_physical_expr(&filter)
             })
             .transpose()?;
@@ -1129,7 +1287,7 @@ impl FilteredReadStream {
                 let global_metrics = global_metrics.clone();
                 let fragment_counted = fragment_counted.clone();
                 let range_tracker = range_tracker.clone();
-                batch_fut
+                let batch_fut = batch_fut
                     .inspect_ok(move |batch| {
                         let num_rows = batch.num_rows();
                         global_metrics.rows_scanned.add(num_rows);
@@ -1144,7 +1302,23 @@ impl FilteredReadStream {
                             global_metrics.ranges_scanned.add(additional_ranges);
                         }
                     })
-                    .boxed()
+                    .boxed();
+                if materialize_blob_v2_binary {
+                    let dataset = dataset.clone();
+                    let output_read_schema = output_read_schema.clone();
+                    batch_fut
+                        .and_then(move |batch| async move {
+                            crate::dataset::blob::materialize_blob_v2_binary_batch(
+                                &dataset,
+                                output_read_schema.as_ref(),
+                                batch,
+                            )
+                            .await
+                        })
+                        .boxed()
+                } else {
+                    batch_fut
+                }
             })
             .zip(futures::stream::repeat((
                 physical_filter.clone(),
@@ -1152,12 +1326,11 @@ impl FilteredReadStream {
             )))
             .map(|(batch_fut, args)| Self::wrap_with_filter(batch_fut, args.0, args.1));
 
-        let result: Pin<Box<dyn Stream<Item = Result<ReadBatchFut>> + Send>> =
-            if let Some(limit) = fragment_soft_limit {
-                Box::pin(Self::apply_soft_limit(fragment_stream, limit))
-            } else {
-                Box::pin(fragment_stream)
-            };
+        let result = if let Some(limit) = fragment_soft_limit {
+            Self::apply_soft_limit(fragment_stream, limit).boxed()
+        } else {
+            fragment_stream.boxed()
+        };
         Ok(result)
     }
 
@@ -1475,6 +1648,19 @@ impl FilteredReadOptions {
         self.only_indexed_fragments = true;
         self
     }
+
+    /// Specify the threading mode to use for the scan.
+    ///
+    /// This controls how decode work is parallelized.  For the default single-partition
+    /// scan, the parameter of [`FilteredReadThreadingMode::OnePartitionMultipleThreads`]
+    /// bounds how many batch-decode tasks are buffered in flight (via `try_buffered`).
+    ///
+    /// The parallelism must be greater than 0.  A value of 0 is rejected by
+    /// [`FilteredReadExec::try_new`].
+    pub fn with_threading_mode(mut self, threading_mode: FilteredReadThreadingMode) -> Self {
+        self.threading_mode = threading_mode;
+        self
+    }
 }
 
 /// A plan node that reads a dataset, applying an optional filter and projection.
@@ -1566,6 +1752,23 @@ impl FilteredReadExec {
                 .into()));
         }
 
+        // A parallelism of 0 would cause `try_buffered(0)` to hang forever instead of erroring
+        match options.threading_mode {
+            FilteredReadThreadingMode::OnePartitionMultipleThreads(0) => {
+                return Err(Error::invalid_input_source(
+                    "FilteredReadThreadingMode::OnePartitionMultipleThreads must be greater than 0, got 0"
+                        .into(),
+                ));
+            }
+            FilteredReadThreadingMode::MultiplePartitions(0) => {
+                return Err(Error::invalid_input_source(
+                    "FilteredReadThreadingMode::MultiplePartitions must be greater than 0, got 0"
+                        .into(),
+                ));
+            }
+            _ => {}
+        }
+
         if options.scan_range_after_filter.is_some() {
             // Validate that there's a filter when using scan_range_after_filter
             if options.full_filter.is_none()
@@ -1588,7 +1791,7 @@ impl FilteredReadExec {
                 ));
             }
         }
-        let output_schema = Arc::new(options.projection.to_arrow_schema());
+        let output_schema = public_blob_v2_binary_projection_schema(&options.projection);
         let num_partitions = match options.threading_mode {
             FilteredReadThreadingMode::OnePartitionMultipleThreads(_) => 1,
             FilteredReadThreadingMode::MultiplePartitions(n) => n,
@@ -1732,7 +1935,14 @@ impl FilteredReadExec {
         // Second, multiple partitions all share the same underlying task stream (see get_stream)
         let running_stream_lock = self.running_stream.clone();
         let dataset = self.dataset.clone();
-        let options = self.options.clone();
+        let target_partitions = context.session_config().target_partitions();
+        let mut options = self.options.clone();
+        if let FilteredReadThreadingMode::OnePartitionMultipleThreads(n) = options.threading_mode {
+            options.threading_mode = FilteredReadThreadingMode::OnePartitionMultipleThreads(
+                n.min(target_partitions).max(1),
+            );
+        }
+        let batch_size_rows = options.batch_size;
         let batch_size_bytes = options
             .file_reader_options
             .as_ref()
@@ -1764,8 +1974,30 @@ impl FilteredReadExec {
                 *running_stream = Some(new_running_stream);
                 first_stream
             };
-            let stream: SendableRecordBatchStream = match batch_size_bytes {
-                Some(target) => {
+            // Only masked reads consolidate; plain scans keep their batch
+            // boundaries, and the byte-based rechunk merges on its own
+            let consolidate = if index_input.is_some() && batch_size_bytes.is_none() {
+                running_stream.as_ref().and_then(|running| {
+                    // Explicit option → lance env default → session batch size
+                    let batch_target_rows = batch_size_rows
+                        .map(|batch_size| batch_size as usize)
+                        .or_else(get_default_batch_size)
+                        .unwrap_or_else(|| context.session_config().batch_size());
+                    let is_sparse_plan = batch_target_rows > 0
+                        && running.touched_fragments >= CONSOLIDATE_MIN_FRAGMENTS
+                        && running.planned_rows
+                            < running.touched_fragments as u64
+                                * CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT;
+                    is_sparse_plan.then_some(batch_target_rows)
+                })
+            } else {
+                None
+            };
+            drop(running_stream);
+
+            let stream = match (consolidate, batch_size_bytes) {
+                (Some(target), _) => consolidated_stream(inner, target),
+                (None, Some(bytes)) => {
                     let schema = inner.schema();
                     Box::pin(RecordBatchStreamAdapter::new(
                         schema.clone(),
@@ -1773,11 +2005,11 @@ impl FilteredReadExec {
                             inner,
                             schema,
                             0,
-                            target as usize,
+                            bytes as usize,
                         ),
                     ))
                 }
-                None => inner,
+                (None, None) => inner,
             };
             DataFusionResult::<SendableRecordBatchStream>::Ok(stream)
         })
@@ -1879,10 +2111,6 @@ impl ExecutionPlan for FilteredReadExec {
         "FilteredReadExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -1902,7 +2130,7 @@ impl ExecutionPlan for FilteredReadExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> datafusion::error::Result<Statistics> {
+    ) -> datafusion::error::Result<Arc<Statistics>> {
         let fragments = self
             .options
             .fragments
@@ -1939,10 +2167,10 @@ impl ExecutionPlan for FilteredReadExec {
                 total_rows
             };
 
-            return Ok(Statistics {
+            return Ok(Arc::new(Statistics {
                 num_rows: Precision::Exact(total_rows as usize),
                 ..datafusion::physical_plan::Statistics::new_unknown(self.schema().as_ref())
-            });
+            }));
         };
 
         // We could evaluate the indexed filter here but this is still during the planning
@@ -1960,7 +2188,7 @@ impl ExecutionPlan for FilteredReadExec {
             .clone()
             .union_columns(filter_columns, OnMissing::Error)?;
 
-        let read_schema = Arc::new(read_projection.to_arrow_schema());
+        let read_schema = public_blob_v2_binary_projection_schema(&read_projection);
 
         let planner = Arc::new(Planner::new(read_schema.clone()));
         let physical_filter = planner.create_physical_expr(filter)?;
@@ -1977,7 +2205,7 @@ impl ExecutionPlan for FilteredReadExec {
             None,
         )?);
         let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?;
-        let mut df_stats = df_filter_exec.partition_statistics(partition)?;
+        let mut df_stats = Arc::unwrap_or_clone(df_filter_exec.partition_statistics(partition)?);
 
         // If we have an after-filter range, we should apply it to the stats (the before-filter range
         // is applied in the mock input)
@@ -2013,7 +2241,7 @@ impl ExecutionPlan for FilteredReadExec {
             }
         });
 
-        Ok(df_stats)
+        Ok(Arc::new(df_stats))
     }
 
     fn with_new_children(
@@ -2129,7 +2357,9 @@ mod tests {
     };
     use itertools::Itertools;
     use lance_core::datatypes::OnMissing;
+    use lance_core::utils::address::RowAddress;
     use lance_core::utils::tempfile::TempStrDir;
+    use lance_datafusion::exec::OneShotExec;
     use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch};
     use lance_index::{
         IndexType,
@@ -2137,6 +2367,7 @@ mod tests {
         scalar::{ScalarIndexParams, expression::PlannerIndexExt},
     };
     use lance_select::result::IndexExprResultWireFormat;
+    use lance_select::{RowAddrMask, RowAddrTreeMap};
 
     use crate::{
         dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams},
@@ -2363,6 +2594,77 @@ mod tests {
         ))
     }
 
+    /// Take-shaped masked reads consolidate their tiny per-fragment batches;
+    /// few-fragment and dense masked reads keep per-fragment boundaries.
+    #[test_log::test(tokio::test)]
+    async fn test_take_shaped_mask_consolidation() {
+        // 20 fragments x 2000 rows, value = global row number
+        let tmp_path = TempStrDir::default();
+        let data = gen_batch()
+            .col("value", array::step::<UInt32Type>())
+            .into_reader_rows(RowCount::from(2000), BatchCount::from(20));
+        let dataset = Arc::new(
+            Dataset::write(
+                data,
+                tmp_path.as_str(),
+                Some(WriteParams {
+                    max_rows_per_file: 2000,
+                    ..Default::default()
+                }),
+            )
+            .await
+            .unwrap(),
+        );
+
+        let mask_input = |addrs: Vec<u64>| -> Arc<dyn ExecutionPlan> {
+            let covered: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect();
+            let batch =
+                IndexExprResult::exact(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(addrs)))
+                    .serialize(&covered, IndexExprResultWireFormat::default())
+                    .unwrap();
+            let schema = batch.schema();
+            let stream = futures::stream::once(async move { Ok(batch) });
+            Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new(
+                schema, stream,
+            ))))
+        };
+        let run = |input: Arc<dyn ExecutionPlan>| {
+            let dataset = dataset.clone();
+            async move {
+                // Pin the batch size so batch-count assertions don't depend
+                // on LANCE_DEFAULT_BATCH_SIZE
+                let options = FilteredReadOptions::basic_full_read(&dataset).with_batch_size(2000);
+                let plan =
+                    FilteredReadExec::try_new(dataset.clone(), options, Some(input)).unwrap();
+                let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
+                stream.try_collect::<Vec<_>>().await.unwrap()
+            }
+        };
+        let addr = |frag: u32, offset: u32| u64::from(RowAddress::new_from_parts(frag, offset));
+
+        // Take shape: 20 fragments, 2 rows each -> one consolidated batch,
+        // rows in fragment order
+        let addrs: Vec<u64> = (0..20u32).flat_map(|f| [addr(f, 3), addr(f, 7)]).collect();
+        let batches = run(mask_input(addrs)).await;
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 40);
+        assert_eq!(batches.len(), 1);
+        let expected =
+            UInt32Array::from_iter_values((0..20u32).flat_map(|f| [f * 2000 + 3, f * 2000 + 7]));
+        assert_eq!(batches[0].column(0).as_ref(), &expected);
+
+        // Too few fragments -> inline path, one batch per fragment
+        let batches = run(mask_input(vec![addr(0, 3), addr(1, 7)])).await;
+        assert_eq!(batches.len(), 2);
+
+        // Dense (2000 planned rows per fragment) -> inline path
+        let addrs: Vec<u64> = (0..8u32)
+            .flat_map(|f| (0..2000u32).map(move |o| addr(f, o)))
+            .collect();
+        let batches = run(mask_input(addrs)).await;
+        assert_eq!(batches.len(), 8);
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 16000);
+    }
+
     /// Round-trip every interval shape through the arrow wire format and
     /// confirm the endpoints survive. Exercises both
     /// `IndexExprResult::serialize` and `EvaluatedIndex::try_from_arrow`
@@ -2954,10 +3256,7 @@ mod tests {
             assert_eq!(plan.options().scan_range_before_filter, None);
             assert_eq!(plan.fetch(), None);
             let new_plan = plan.with_fetch(Some(100)).unwrap();
-            let new_plan = new_plan
-                .as_any()
-                .downcast_ref::<FilteredReadExec>()
-                .unwrap();
+            let new_plan = new_plan.downcast_ref::<FilteredReadExec>().unwrap();
             assert_eq!(new_plan.options().scan_range_before_filter, Some(0..100));
             assert_eq!(new_plan.fetch(), Some(100));
         }
@@ -2985,10 +3284,7 @@ mod tests {
             assert_eq!(plan.options().scan_range_after_filter, None);
             assert_eq!(plan.fetch(), None);
             let new_plan = plan.with_fetch(Some(50)).unwrap();
-            let new_plan = new_plan
-                .as_any()
-                .downcast_ref::<FilteredReadExec>()
-                .unwrap();
+            let new_plan = new_plan.downcast_ref::<FilteredReadExec>().unwrap();
             assert_eq!(new_plan.options().scan_range_after_filter, Some(0..50));
             assert_eq!(new_plan.fetch(), Some(50));
         }
@@ -3042,10 +3338,7 @@ mod tests {
         assert!(plan.options().refine_filter.is_some());
 
         let limited_plan = plan.with_fetch(Some(10)).unwrap();
-        let limited_plan = limited_plan
-            .as_any()
-            .downcast_ref::<FilteredReadExec>()
-            .unwrap();
+        let limited_plan = limited_plan.downcast_ref::<FilteredReadExec>().unwrap();
         assert_eq!(limited_plan.options().scan_range_after_filter, Some(0..10));
 
         let stream = limited_plan
@@ -3716,6 +4009,55 @@ mod tests {
         assert!(iops > 0, "Should have recorded IO operations");
     }
 
+    // Reproduces a bug where bytes_read (and iops/requests) stay at 0 when a filter matches
+    // no rows. io_metrics.record is only called inside inspect_ok on the output batch stream,
+    // so when the filter produces zero output batches, the I/O that did occur is never counted.
+    #[tokio::test]
+    async fn test_io_metrics_recorded_when_filter_matches_no_rows() {
+        let fixture = TestFixture::new().await;
+        // not_indexed values in the fixture go up to ~400; this filter matches nothing
+        let filter_plan = fixture.filter_plan("not_indexed > 10000", false).await;
+        let options =
+            FilteredReadOptions::basic_full_read(&fixture.dataset).with_filter_plan(filter_plan);
+        let filtered_read =
+            Arc::new(FilteredReadExec::try_new(fixture.dataset.clone(), options, None).unwrap());
+
+        let batches = filtered_read
+            .execute(0, Arc::new(TaskContext::default()))
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(
+            batches.iter().map(|b| b.num_rows()).sum::<usize>(),
+            0,
+            "filter should match no rows"
+        );
+
+        let metrics = filtered_read.metrics().unwrap();
+
+        let rows_scanned = metrics
+            .sum_by_name("rows_scanned")
+            .map(|v| v.as_usize())
+            .unwrap_or(0);
+        assert!(
+            rows_scanned > 0,
+            "rows_scanned ({}) should be > 0: data was read even though filter matched nothing",
+            rows_scanned
+        );
+
+        let bytes_read = metrics
+            .sum_by_name("bytes_read")
+            .map(|v| v.as_usize())
+            .unwrap_or(0);
+        assert!(
+            bytes_read > 0,
+            "bytes_read ({}) should be > 0: io_metrics.record is only called when output batches \
+             are produced, so bytes_read stays 0 even though I/O occurred",
+            bytes_read
+        );
+    }
+
     /// Test that direct execution gives the same result as get_plan + execute_with_plan
     #[test_log::test(tokio::test)]
     async fn test_plan_round_trip() {
@@ -3798,4 +4140,37 @@ mod tests {
             assert_eq!(result1.column(i).as_ref(), result3.column(i).as_ref());
         }
     }
+
+    /// Verify that executing with target_partitions=1 produces the same results as the default
+    /// context and does not panic. This is a regression guard for the parallelism cap.
+    #[test_log::test(tokio::test)]
+    async fn test_target_partitions_cap_produces_correct_results() {
+        use datafusion::prelude::SessionConfig;
+
+        let fixture = TestFixture::new().await;
+
+        let options = FilteredReadOptions::basic_full_read(&fixture.dataset);
+        let plan =
+            FilteredReadExec::try_new(fixture.dataset.clone(), options.clone(), None).unwrap();
+
+        // Execute with default context (high thread count)
+        let default_ctx = Arc::new(TaskContext::default());
+        let stream = plan.execute(0, default_ctx).unwrap();
+        let schema = stream.schema();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let default_result = concat_batches(&schema, &batches).unwrap();
+
+        // Execute fresh plan with target_partitions=1
+        let plan2 = FilteredReadExec::try_new(fixture.dataset.clone(), options, None).unwrap();
+        let low_ctx = Arc::new(
+            TaskContext::default()
+                .with_session_config(SessionConfig::default().with_target_partitions(1)),
+        );
+        let stream2 = plan2.execute(0, low_ctx).unwrap();
+        let schema2 = stream2.schema();
+        let batches2 = stream2.try_collect::<Vec<_>>().await.unwrap();
+        let capped_result = concat_batches(&schema2, &batches2).unwrap();
+
+        assert_eq!(default_result.num_rows(), capped_result.num_rows());
+    }
 }
diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs
index b72bf0a2205..e827a5bfd63 100644
--- a/rust/lance/src/io/exec/fts.rs
+++ b/rust/lance/src/io/exec/fts.rs
@@ -13,7 +13,7 @@ use datafusion::error::{DataFusionError, Result as DataFusionResult};
 use datafusion::execution::SendableRecordBatchStream;
 use datafusion::physical_plan::empty::EmptyExec;
 use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
-use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, Gauge, MetricsSet};
 use datafusion::physical_plan::repartition::RepartitionExec;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::union::UnionExec;
@@ -34,10 +34,13 @@ use lance_datafusion::utils::{ExecutionPlanMetricsSetExt, MetricsExt, PARTITIONS
 use lance_table::format::IndexMetadata;
 
 use super::PreFilterSource;
-use super::utils::{IndexMetrics, InstrumentedChildInputStream, build_prefilter};
+use super::utils::{IndexMetrics, build_prefilter};
 use crate::index::scalar::inverted::{load_segment_details, load_segments};
 use crate::{Dataset, index::DatasetIndexInternalExt};
-use lance_index::metrics::MetricsCollector;
+use lance_index::metrics::{
+    AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, AND_CANDIDATES_SEEN_METRIC, AND_FULL_SCORES_METRIC,
+    FREQS_COLLECTED_METRIC, MetricsCollector,
+};
 use lance_index::scalar::inverted::builder::ScoredDoc;
 use lance_index::scalar::inverted::builder::document_input;
 use lance_index::scalar::inverted::document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer};
@@ -61,15 +64,16 @@ async fn open_fts_segment(
     segment: &IndexMetadata,
     metrics: &IndexMetrics,
 ) -> Result<Arc<InvertedIndex>> {
-    let uuid = segment.uuid.to_string();
-    let index = dataset.open_generic_index(column, &uuid, metrics).await?;
+    let index = dataset
+        .open_scalar_index(column, &segment.uuid, metrics)
+        .await?;
     let inverted = index
         .as_any()
         .downcast_ref::<InvertedIndex>()
         .ok_or_else(|| {
             Error::invalid_input(format!(
                 "Index for column {} and segment {} is not an inverted index",
-                column, uuid
+                column, segment.uuid
             ))
         })?;
     Ok(Arc::new(inverted.clone()))
@@ -131,7 +135,7 @@ async fn search_segments(
     let mut searches = searches;
 
     while let Some((doc_ids, scores)) = searches.try_next().await? {
-        for (row_id, score) in doc_ids.into_iter().zip(scores.into_iter()) {
+        for (row_id, score) in doc_ids.into_iter().zip(scores) {
             if candidates.len() < limit {
                 candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score)));
             } else if candidates.peek().unwrap().0.score.0 < score {
@@ -158,6 +162,13 @@ fn default_text_tokenizer() -> Box<dyn LanceTokenizer> {
 pub struct FtsIndexMetrics {
     index_metrics: IndexMetrics,
     partitions_searched: Count,
+    and_candidates_seen: Count,
+    and_candidates_pruned_before_return: Count,
+    and_full_scores: Count,
+    freqs_collected: Count,
+    /// Wall time (ms) of the exec-local `build_global_bm25_scorer`
+    /// fallback; zero when a preset base scorer was injected.
+    scorer_build_ms: Gauge,
     baseline_metrics: BaselineMetrics,
 }
 
@@ -166,6 +177,12 @@ impl FtsIndexMetrics {
         Self {
             index_metrics: IndexMetrics::new(metrics, partition),
             partitions_searched: metrics.new_count(PARTITIONS_SEARCHED_METRIC, partition),
+            and_candidates_seen: metrics.new_count(AND_CANDIDATES_SEEN_METRIC, partition),
+            and_candidates_pruned_before_return: metrics
+                .new_count(AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, partition),
+            and_full_scores: metrics.new_count(AND_FULL_SCORES_METRIC, partition),
+            freqs_collected: metrics.new_count(FREQS_COLLECTED_METRIC, partition),
+            scorer_build_ms: metrics.new_gauge("scorer_build_ms", partition),
             baseline_metrics: BaselineMetrics::new(metrics, partition),
         }
     }
@@ -173,6 +190,10 @@ impl FtsIndexMetrics {
     pub fn record_parts_searched(&self, num_parts: usize) {
         self.partitions_searched.add(num_parts);
     }
+
+    pub fn record_scorer_build(&self, elapsed: std::time::Duration) {
+        self.scorer_build_ms.set(elapsed.as_millis() as usize);
+    }
 }
 
 impl MetricsCollector for FtsIndexMetrics {
@@ -187,6 +208,22 @@ impl MetricsCollector for FtsIndexMetrics {
     fn record_comparisons(&self, num_comparisons: usize) {
         self.index_metrics.record_comparisons(num_comparisons);
     }
+
+    fn record_and_candidates_seen(&self, num_candidates: usize) {
+        self.and_candidates_seen.add(num_candidates);
+    }
+
+    fn record_and_candidates_pruned_before_return(&self, num_candidates: usize) {
+        self.and_candidates_pruned_before_return.add(num_candidates);
+    }
+
+    fn record_and_full_scores(&self, num_scores: usize) {
+        self.and_full_scores.add(num_scores);
+    }
+
+    fn record_freqs_collected(&self, num_collections: usize) {
+        self.freqs_collected.add(num_collections);
+    }
 }
 
 #[derive(Debug)]
@@ -212,7 +249,7 @@ impl DisplayAs for MatchQueryExec {
             DisplayFormatType::Default | DisplayFormatType::Verbose => {
                 write!(
                     f,
-                    "MatchQuery: column={}, query={}",
+                    "MatchQuery: column={}, query=[{}]",
                     self.query.column.as_deref().unwrap_or_default(),
                     self.query.terms
                 )
@@ -346,10 +383,6 @@ impl ExecutionPlan for MatchQueryExec {
         "MatchQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.prefilter_source {
             PreFilterSource::None => vec![],
@@ -496,11 +529,16 @@ impl ExecutionPlan for MatchQueryExec {
             let tokens = collect_query_tokens(&query.terms, &mut tokenizer);
             let base_scorer = match preset_base_scorer {
                 Some(scorer) => scorer,
-                None => Arc::new(
-                    build_global_bm25_scorer(&indices, &tokens, &params)
-                        .boxed()
-                        .await?,
-                ),
+                None => {
+                    let scorer_start = std::time::Instant::now();
+                    let scorer = Arc::new(
+                        build_global_bm25_scorer(&indices, &tokens, &params)
+                            .boxed()
+                            .await?,
+                    );
+                    metrics.record_scorer_build(scorer_start.elapsed());
+                    scorer
+                }
             };
 
             pre_filter.wait_for_ready().await?;
@@ -699,11 +737,6 @@ impl FlatMatchFilterExec {
         metrics_set: ExecutionPlanMetricsSet,
     ) -> DataFusionResult<SendableRecordBatchStream> {
         let metrics = Arc::new(FtsIndexMetrics::new(&metrics_set, partition));
-        let elapsed_compute = metrics.baseline_metrics.elapsed_compute().clone();
-        // Time the one-shot setup (tokenizer load + query tokenization) so it's
-        // attributed to this node's elapsed_compute. The helper itself only
-        // times per-batch work.
-        let setup_start = std::time::Instant::now();
         let column = query
             .column
             .clone()
@@ -724,44 +757,45 @@ impl FlatMatchFilterExec {
             None => Self::load_tokenizer(&dataset, &column, &metrics.index_metrics).await?,
         };
         let query_tokens = Arc::new(collect_query_tokens(&query.terms, &mut tokenizer));
-        elapsed_compute.add_duration(setup_start.elapsed());
 
-        let helper = InstrumentedChildInputStream::new(
-            input,
-            schema,
-            move |batch| {
-                // Clone per-batch so the work runs *inside* the async block
-                // (i.e., during the helper's timed in_flight poll, not during
-                // its untimed input-pulling phase).
-                let column = column.clone();
-                let query_tokens = query_tokens.clone();
-                let mut tokenizer = tokenizer.box_clone();
-                async move {
-                    let text_column = batch.column_by_name(&column).ok_or_else(|| {
-                        DataFusionError::Execution(format!("Column {} not found in batch", column,))
-                    })?;
-                    let predicate = match text_column.data_type() {
-                        DataType::Utf8 => {
-                            Self::find_matches::<i32>(text_column, &mut tokenizer, &query_tokens)
-                        }
-                        DataType::LargeUtf8 => {
-                            Self::find_matches::<i64>(text_column, &mut tokenizer, &query_tokens)
-                        }
-                        _ => {
-                            return Err(DataFusionError::Execution(format!(
-                                "Column {} is not a string",
-                                column,
-                            )));
-                        }
-                    };
-                    Ok(arrow::compute::filter_record_batch(&batch, &predicate)?)
-                }
-            },
-            1,
-            partition,
-            &metrics_set,
-        );
-        Ok(Box::pin(helper))
+        let baseline = BaselineMetrics::new(&metrics_set, partition);
+        let elapsed_compute = baseline.elapsed_compute().clone();
+        let stream = input.then(move |batch_result| {
+            let column = column.clone();
+            let query_tokens = query_tokens.clone();
+            let mut tokenizer = tokenizer.box_clone();
+            let elapsed_compute = elapsed_compute.clone();
+            async move {
+                let batch = batch_result?;
+                let _t = elapsed_compute.timer();
+                let text_column = batch.column_by_name(&column).ok_or_else(|| {
+                    DataFusionError::Execution(format!("Column {} not found in batch", column,))
+                })?;
+                let predicate = match text_column.data_type() {
+                    DataType::Utf8 => {
+                        Self::find_matches::<i32>(text_column, &mut tokenizer, &query_tokens)
+                    }
+                    DataType::LargeUtf8 => {
+                        Self::find_matches::<i64>(text_column, &mut tokenizer, &query_tokens)
+                    }
+                    _ => {
+                        return Err(DataFusionError::Execution(format!(
+                            "Column {} is not a string",
+                            column,
+                        )));
+                    }
+                };
+                Ok(arrow::compute::filter_record_batch(&batch, &predicate)?)
+            }
+        });
+        let stream = stream.map(move |batch| {
+            let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch)));
+            match poll {
+                std::task::Poll::Ready(Some(b)) => b,
+                _ => unreachable!("record_poll preserves Ready(Some) input"),
+            }
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
     }
 }
 
@@ -770,10 +804,6 @@ impl ExecutionPlan for FlatMatchFilterExec {
         "FlatMatchFilterExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.input]
     }
@@ -825,7 +855,7 @@ impl ExecutionPlan for FlatMatchFilterExec {
         Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
@@ -966,14 +996,18 @@ impl ExecutionPlan for FlatMatchQueryExec {
         "FlatMatchQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.unindexed_input]
     }
 
+    fn required_input_distribution(&self) -> Vec<Distribution> {
+        // `execute()` only reads `unindexed_input.execute(partition)` for the single
+        // output partition, so the input must be coalesced to one partition. Without
+        // this, EnforceDistribution may round-robin the scan across `target_partitions`
+        // and only partition 0 is consumed, silently dropping the other fragments.
+        vec![Distribution::SinglePartition]
+    }
+
     fn with_new_children(
         self: Arc<Self>,
         mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -1014,11 +1048,8 @@ impl ExecutionPlan for FlatMatchQueryExec {
         // so it can attribute the spawn_cpu tokenize work and synchronous
         // scoring back onto this node's `elapsed_compute`. Sharing the same
         // `Time` handle that's already inside the FtsIndexMetrics avoids
-        // registering a duplicate metric. Cloned once for use during setup
-        // timing (below) and again moved into the async block for the
-        // streaming-phase call.
+        // registering a duplicate metric.
         let elapsed_compute = metrics.baseline_metrics.elapsed_compute().clone();
-        let elapsed_compute_for_stream = elapsed_compute.clone();
 
         let column = query.column.ok_or(DataFusionError::Execution(format!(
             "column not set for MatchQuery {}",
@@ -1028,9 +1059,6 @@ impl ExecutionPlan for FlatMatchQueryExec {
             document_input(self.unindexed_input.execute(partition, context)?, &column)?;
 
         let stream = stream::once(async move {
-            // Time the one-shot setup (load segments / open indices / build
-            // scorer / acquire tokenizer) and attribute it to elapsed_compute.
-            let setup_start = std::time::Instant::now();
             let segments = match preset_segments {
                 Some(segments) => Some(segments),
                 None => load_segments(&ds, &column).await?,
@@ -1051,13 +1079,16 @@ impl ExecutionPlan for FlatMatchQueryExec {
                         Some(scorer) => (*scorer).clone(),
                         None => {
                             let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer);
-                            build_global_bm25_scorer(
+                            let scorer_start = std::time::Instant::now();
+                            let scorer = build_global_bm25_scorer(
                                 &indices,
                                 &query_tokens,
                                 &FtsSearchParams::new(),
                             )
                             .boxed()
-                            .await?
+                            .await?;
+                            metrics.record_scorer_build(scorer_start.elapsed());
+                            scorer
                         }
                     };
                     (tokenizer, Some(base_scorer))
@@ -1067,7 +1098,6 @@ impl ExecutionPlan for FlatMatchQueryExec {
                     preset_base_scorer.map(|s| (*s).clone()),
                 ),
             };
-            elapsed_compute.add_duration(setup_start.elapsed());
 
             flat_bm25_search_stream_with_metrics(
                 unindexed_input,
@@ -1076,7 +1106,7 @@ impl ExecutionPlan for FlatMatchQueryExec {
                 tokenizer,
                 base_scorer,
                 target_batch_size,
-                Some(elapsed_compute_for_stream),
+                Some(elapsed_compute),
             )
             .await
         })
@@ -1242,10 +1272,6 @@ impl ExecutionPlan for PhraseQueryExec {
         "PhraseQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.prefilter_source {
             PreFilterSource::None => vec![],
@@ -1369,11 +1395,16 @@ impl ExecutionPlan for PhraseQueryExec {
             let tokens = collect_query_tokens(&query.terms, &mut tokenizer);
             let base_scorer = match preset_base_scorer {
                 Some(scorer) => scorer,
-                None => Arc::new(
-                    build_global_bm25_scorer(&indices, &tokens, &params)
-                        .boxed()
-                        .await?,
-                ),
+                None => {
+                    let scorer_start = std::time::Instant::now();
+                    let scorer = Arc::new(
+                        build_global_bm25_scorer(&indices, &tokens, &params)
+                            .boxed()
+                            .await?,
+                    );
+                    metrics.record_scorer_build(scorer_start.elapsed());
+                    scorer
+                }
             };
 
             pre_filter.wait_for_ready().await?;
@@ -1495,10 +1526,6 @@ impl ExecutionPlan for BoostQueryExec {
         "BoostQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         vec![&self.positive, &self.negative]
     }
@@ -1763,10 +1790,6 @@ impl ExecutionPlan for BooleanQueryExec {
         "BooleanQueryExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
         match &self.must {
             Some(must) => vec![&self.should, &self.must_not, must],
@@ -2199,7 +2222,7 @@ mod tests {
             .unwrap()
             .unwrap();
         let index = dataset
-            .open_generic_index("text", &index_meta.uuid.to_string(), &NoOpMetricsCollector)
+            .open_generic_index("text", &index_meta.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let inverted_index = index.as_any().downcast_ref::<InvertedIndex>().unwrap();
@@ -2262,7 +2285,7 @@ mod tests {
             .unwrap()
             .unwrap();
         let index = dataset
-            .open_generic_index("text", &index_meta.uuid.to_string(), &NoOpMetricsCollector)
+            .open_generic_index("text", &index_meta.uuid, &NoOpMetricsCollector)
             .await
             .unwrap();
         let inverted_index = index.as_any().downcast_ref::<InvertedIndex>().unwrap();
@@ -2516,7 +2539,7 @@ mod tests {
             .unwrap()
             .expect("Should slot always returns Some");
         assert!(
-            plan.as_any().downcast_ref::<EmptyExec>().is_some(),
+            plan.downcast_ref::<EmptyExec>().is_some(),
             "expected EmptyExec for empty Should slot, got {plan:?}"
         );
     }
@@ -2544,12 +2567,10 @@ mod tests {
         .unwrap()
         .expect("Should slot always returns Some");
         let repartition = plan
-            .as_any()
             .downcast_ref::<RepartitionExec>()
             .expect("multi-child Should should be wrapped in RepartitionExec");
         let inner = repartition
             .input()
-            .as_any()
             .downcast_ref::<UnionExec>()
             .expect("RepartitionExec should wrap a UnionExec");
         assert_eq!(inner.children().len(), 2);
@@ -2588,7 +2609,7 @@ mod tests {
         // there are N-1 joins.
         let mut joins = 0usize;
         let mut current: Arc<dyn ExecutionPlan> = plan;
-        while let Some(join) = current.clone().as_any().downcast_ref::<HashJoinExec>() {
+        while let Some(join) = current.clone().downcast_ref::<HashJoinExec>() {
             joins += 1;
             current = join.children()[0].clone();
         }
@@ -2604,12 +2625,10 @@ mod tests {
         .unwrap()
         .expect("MustNot slot always returns Some");
         let repartition = plan
-            .as_any()
             .downcast_ref::<RepartitionExec>()
             .expect("multi-child MustNot should be wrapped in RepartitionExec");
         let inner = repartition
             .input()
-            .as_any()
             .downcast_ref::<UnionExec>()
             .expect("RepartitionExec should wrap a UnionExec");
         assert_eq!(inner.children().len(), 2);
diff --git a/rust/lance/src/io/exec/knn.rs b/rust/lance/src/io/exec/knn.rs
index 71239b4e34b..fde0289621d 100644
--- a/rust/lance/src/io/exec/knn.rs
+++ b/rust/lance/src/io/exec/knn.rs
@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
+#[cfg(test)]
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::cmp::Ordering as CmpOrdering;
 use std::collections::{BinaryHeap, HashMap, HashSet};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -17,7 +18,6 @@ use arrow_array::{
     cast::AsArray,
 };
 use arrow_schema::{DataType, Field, Schema, SchemaRef};
-use arrow_select::concat::concat_batches;
 use datafusion::physical_plan::PlanProperties;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
@@ -58,6 +58,7 @@ use lance_linalg::distance::DistanceType;
 use lance_linalg::kernels::normalize_arrow;
 use lance_table::format::IndexMetadata;
 use tokio::sync::Notify;
+use uuid::Uuid;
 
 use crate::dataset::Dataset;
 use crate::index::DatasetIndexInternalExt;
@@ -67,8 +68,8 @@ use crate::{Error, Result};
 use lance_arrow::*;
 
 use super::utils::{
-    FilteredRowIdsToPrefilter, IndexMetrics, InstrumentedChildInputStream,
-    InstrumentedRecordBatchStreamAdapter, PreFilterSource, SelectionVectorToPrefilter,
+    FilteredRowIdsToPrefilter, IndexMetrics, InstrumentedRecordBatchStreamAdapter, PreFilterSource,
+    SelectionVectorToPrefilter,
 };
 
 pub const QUERY_INDEX_COL: &str = "query_index";
@@ -156,6 +157,7 @@ pub struct KNNVectorDistanceExec {
     pub upper_bound: Option<f32>,
     pub column: String,
     pub distance_type: DistanceType,
+    retain_vector: bool,
 
     input_schema: SchemaRef,
     output_schema: SchemaRef,
@@ -171,10 +173,11 @@ pub struct KnnBatchParams {
     pub lower_bound: Option<f32>,
     pub upper_bound: Option<f32>,
     pub distance_type: DistanceType,
+    pub retain_vector: bool,
 }
 
 struct BatchKnnConfig {
-    input_schema: SchemaRef,
+    stored_schema: SchemaRef,
     output_schema: SchemaRef,
     column: String,
     query: ArrayRef,
@@ -183,6 +186,7 @@ struct BatchKnnConfig {
     lower_bound: Option<f32>,
     upper_bound: Option<f32>,
     distance_type: DistanceType,
+    retain_vector: bool,
 }
 
 impl DisplayAs for KNNVectorDistanceExec {
@@ -215,6 +219,120 @@ impl DisplayAs for KNNVectorDistanceExec {
 }
 
 impl KNNVectorDistanceExec {
+    fn remove_field_path_from_fields(
+        fields: &[Arc<Field>],
+        path: &[String],
+    ) -> DataFusionResult<Vec<Arc<Field>>> {
+        if path.is_empty() {
+            return Ok(fields.to_vec());
+        }
+        let mut removed = false;
+        let mut new_fields = Vec::with_capacity(fields.len());
+        for field in fields {
+            if field.name() != &path[0] {
+                new_fields.push(field.clone());
+                continue;
+            }
+            removed = true;
+            if path.len() == 1 {
+                continue;
+            }
+            match field.data_type() {
+                DataType::Struct(children) => {
+                    let child_fields = children.iter().cloned().collect::<Vec<_>>();
+                    let projected_children =
+                        Self::remove_field_path_from_fields(&child_fields, &path[1..])?;
+                    if projected_children.is_empty() {
+                        continue;
+                    }
+                    let updated = Field::new(
+                        field.name(),
+                        DataType::Struct(projected_children.into()),
+                        field.is_nullable(),
+                    )
+                    .with_metadata(field.metadata().clone());
+                    new_fields.push(Arc::new(updated));
+                }
+                _ => {
+                    return Err(DataFusionError::Internal(format!(
+                        "batch KNN cannot remove nested path '{}': '{}' is not a struct",
+                        path.join("."),
+                        field.name()
+                    )));
+                }
+            }
+        }
+        if !removed {
+            return Err(DataFusionError::Internal(format!(
+                "batch KNN expected vector column '{}' in scan batch schema",
+                path.join(".")
+            )));
+        }
+        Ok(new_fields)
+    }
+
+    fn remove_vector_from_schema(schema: &Schema, column: &str) -> DataFusionResult<Schema> {
+        let path = lance_core::datatypes::parse_field_path(column).map_err(|err| {
+            DataFusionError::Internal(format!(
+                "batch KNN failed to parse vector column path '{column}': {err}"
+            ))
+        })?;
+        let fields = schema.fields().iter().cloned().collect::<Vec<_>>();
+        let updated_fields = Self::remove_field_path_from_fields(&fields, &path)?;
+        Ok(Schema::new_with_metadata(
+            updated_fields,
+            schema.metadata().clone(),
+        ))
+    }
+
+    fn remove_vector_from_batch(
+        batch: &RecordBatch,
+        column: &str,
+    ) -> DataFusionResult<RecordBatch> {
+        let slim_schema = Self::remove_vector_from_schema(batch.schema().as_ref(), column)?;
+        batch
+            .project_by_schema(&slim_schema)
+            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+    }
+
+    fn resolve_vector_column(batch: &RecordBatch, column: &str) -> DataFusionResult<ArrayRef> {
+        if let Some(col) = batch.column_by_name(column) {
+            return Ok(col.clone());
+        }
+        let parts = lance_core::datatypes::parse_field_path(column).map_err(|e| {
+            DataFusionError::Internal(format!(
+                "batch KNN failed to parse vector column path '{column}': {e}"
+            ))
+        })?;
+        if parts.is_empty() {
+            return Err(DataFusionError::Internal(format!(
+                "batch KNN has invalid empty vector column path '{column}'"
+            )));
+        }
+        let mut current = batch.column_by_name(&parts[0]).cloned().ok_or_else(|| {
+            DataFusionError::Internal(format!(
+                "batch KNN expected vector column '{column}' in scan batch (missing root field '{}')",
+                parts[0]
+            ))
+        })?;
+        for part in &parts[1..] {
+            let struct_array = current
+                .as_any()
+                .downcast_ref::<arrow_array::StructArray>()
+                .ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "batch KNN expected struct while resolving '{column}', but parent of '{part}' was not a struct"
+                    ))
+                })?;
+            current = struct_array.column_by_name(part).cloned().ok_or_else(|| {
+                DataFusionError::Internal(format!(
+                    "batch KNN expected vector column '{column}' in scan batch (missing nested field '{part}')"
+                ))
+            })?;
+        }
+        Ok(current)
+    }
+
     /// Create a new [`KNNVectorDistanceExec`] node.
     ///
     /// Returns an error if the preconditions are not met.
@@ -235,6 +353,7 @@ impl KNNVectorDistanceExec {
                 lower_bound: None,
                 upper_bound: None,
                 distance_type,
+                retain_vector: false,
             },
         )
     }
@@ -252,6 +371,7 @@ impl KNNVectorDistanceExec {
             lower_bound,
             upper_bound,
             distance_type,
+            retain_vector,
         } = params;
         if query_count == 0 {
             return Err(Error::invalid_input(
@@ -286,13 +406,19 @@ impl KNNVectorDistanceExec {
                 "batch KNN cannot run when the input already contains reserved column '{QUERY_INDEX_COL}'"
             )));
         }
-        let input_schema = Arc::new(input_schema);
+
+        let stored_schema = if is_batch && !retain_vector {
+            Arc::new(Self::remove_vector_from_schema(&input_schema, column)?)
+        } else {
+            Arc::new(input_schema)
+        };
+
         let output_schema = if is_batch {
-            input_schema
+            stored_schema
                 .as_ref()
                 .try_with_column_at(0, query_index_field())?
         } else {
-            input_schema.as_ref().clone()
+            stored_schema.as_ref().clone()
         };
         let output_schema = Arc::new(output_schema.try_with_column(Field::new(
             DIST_COL,
@@ -329,19 +455,230 @@ impl KNNVectorDistanceExec {
             upper_bound,
             column: column.to_string(),
             distance_type,
-            input_schema,
+            retain_vector,
+            input_schema: stored_schema,
             output_schema,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
         })
     }
 
+    fn take_vector_row(vectors: &dyn Array, row_index: u32) -> DataFusionResult<ArrayRef> {
+        let indices = UInt32Array::from_iter([Some(row_index)]);
+        arrow_select::take::take(vectors, &indices, None)
+            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+    }
+
+    fn take_slim_batch_field(
+        results: &[BatchKnnCandidate],
+        field_name: &str,
+    ) -> DataFusionResult<ArrayRef> {
+        Self::take_slim_batch_field_if_present(results, field_name)?.ok_or_else(|| {
+            DataFusionError::Internal(format!("column '{field_name}' missing from slim batch"))
+        })
+    }
+
+    fn take_slim_batch_field_if_present(
+        results: &[BatchKnnCandidate],
+        field_name: &str,
+    ) -> DataFusionResult<Option<ArrayRef>> {
+        use std::collections::HashMap;
+
+        type SlimBatchGroup = (Arc<RecordBatch>, Vec<(usize, u32)>);
+        let mut groups: HashMap<*const RecordBatch, SlimBatchGroup> = HashMap::new();
+        for (result_index, candidate) in results.iter().enumerate() {
+            let BatchKnnExtra::WithSlimBatch {
+                slim_batch,
+                row_index,
+                ..
+            } = &candidate.extra
+            else {
+                return Err(DataFusionError::Internal(
+                    "batch KNN expected slim batch in candidate heap".to_string(),
+                ));
+            };
+            groups
+                .entry(Arc::as_ptr(slim_batch))
+                .or_insert_with(|| (Arc::clone(slim_batch), Vec::new()))
+                .1
+                .push((result_index, *row_index));
+        }
+
+        let mut ordered: Vec<Option<ArrayRef>> = vec![None; results.len()];
+        for (_, (slim_batch, entries)) in groups {
+            let indices =
+                UInt32Array::from_iter(entries.iter().map(|(_, row_index)| Some(*row_index)));
+            let taken = arrow_select::take::take_record_batch(slim_batch.as_ref(), &indices)
+                .map_err(|e| {
+                    DataFusionError::ArrowError(Box::new(e), Some("take top-k rows".to_string()))
+                })?;
+            let Some(column) = taken.column_by_name(field_name) else {
+                continue;
+            };
+            for (offset, (result_index, _)) in entries.iter().enumerate() {
+                ordered[*result_index] = Some(column.slice(offset, 1));
+            }
+        }
+        if ordered.iter().all(Option::is_none) {
+            return Ok(None);
+        }
+        if ordered.iter().any(Option::is_none) {
+            return Err(DataFusionError::Internal(format!(
+                "column '{field_name}' inconsistently present in slim batches"
+            )));
+        }
+
+        let row_arrays: Vec<&dyn Array> = ordered
+            .iter()
+            .map(|array| {
+                array
+                    .as_ref()
+                    .expect("every result mapped from slim batch")
+                    .as_ref()
+            })
+            .collect();
+        arrow::compute::concat(&row_arrays)
+            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+            .map(Some)
+    }
+
+    fn build_struct_column_for_path(
+        field: &Field,
+        path: &[String],
+        leaf_column: ArrayRef,
+        slim_column: Option<&dyn Array>,
+    ) -> DataFusionResult<ArrayRef> {
+        if path.is_empty() {
+            return Ok(leaf_column);
+        }
+        let DataType::Struct(children) = field.data_type() else {
+            return Err(DataFusionError::Internal(format!(
+                "batch KNN expected struct field '{}' while rebuilding nested vector path '{}'",
+                field.name(),
+                path.join(".")
+            )));
+        };
+        let slim_struct = slim_column
+            .map(|column| {
+                column
+                    .as_any()
+                    .downcast_ref::<arrow_array::StructArray>()
+                    .ok_or_else(|| {
+                        DataFusionError::Internal(format!(
+                            "batch KNN expected slim column '{}' to be a struct while rebuilding nested vector path '{}'",
+                            field.name(),
+                            path.join(".")
+                        ))
+                    })
+            })
+            .transpose()?;
+        let mut columns = Vec::with_capacity(children.len());
+        for child in children.iter() {
+            if child.name() == &path[0] {
+                if path.len() == 1 {
+                    columns.push(leaf_column.clone());
+                } else {
+                    let child_slim_column = slim_struct
+                        .and_then(|struct_array| struct_array.column_by_name(child.name()));
+                    columns.push(Self::build_struct_column_for_path(
+                        child,
+                        &path[1..],
+                        leaf_column.clone(),
+                        child_slim_column.map(|column| column.as_ref()),
+                    )?);
+                }
+            } else if let Some(column) =
+                slim_struct.and_then(|struct_array| struct_array.column_by_name(child.name()))
+            {
+                columns.push(column.clone());
+            } else {
+                columns.push(arrow_array::new_null_array(
+                    child.data_type(),
+                    leaf_column.len(),
+                ));
+            }
+        }
+        let struct_array = arrow_array::StructArray::try_new(
+            children.clone(),
+            columns,
+            slim_struct.and_then(|struct_array| struct_array.nulls().cloned()),
+        )
+        .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
+        Ok(Arc::new(struct_array))
+    }
+
+    fn take_retained_vector_column(
+        results: &[BatchKnnCandidate],
+        field: &Field,
+        field_path: &[String],
+    ) -> DataFusionResult<ArrayRef> {
+        let vector_rows: Vec<&dyn Array> = results
+            .iter()
+            .map(|candidate| {
+                let BatchKnnExtra::WithSlimBatch {
+                    vector_row: Some(vector_row),
+                    ..
+                } = &candidate.extra
+                else {
+                    return Err(DataFusionError::Internal(
+                        "batch KNN expected vector rows in candidate heap".to_string(),
+                    ));
+                };
+                Ok(vector_row.as_ref())
+            })
+            .collect::<DataFusionResult<Vec<_>>>()?;
+        let leaf_column = arrow::compute::concat(&vector_rows)
+            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
+        if field_path.len() <= 1 {
+            Ok(leaf_column)
+        } else {
+            let slim_column = Self::take_slim_batch_field_if_present(results, field.name())?;
+            Self::build_struct_column_for_path(
+                field,
+                &field_path[1..],
+                leaf_column,
+                slim_column.as_deref(),
+            )
+        }
+    }
+
+    fn assemble_batch_output(
+        results: &[BatchKnnCandidate],
+        stored_schema: &Schema,
+        column: &str,
+        retain_vector: bool,
+    ) -> DataFusionResult<RecordBatch> {
+        let field_path = lance_core::datatypes::parse_field_path(column).map_err(|e| {
+            DataFusionError::Internal(format!(
+                "batch KNN failed to parse vector column path '{column}': {e}"
+            ))
+        })?;
+        let mut columns: Vec<ArrayRef> = Vec::with_capacity(stored_schema.fields().len());
+        for field in stored_schema.fields() {
+            if field.name() == ROW_ID {
+                let row_ids =
+                    UInt64Array::from_iter(results.iter().map(|candidate| Some(candidate.row_id)));
+                columns.push(Arc::new(row_ids));
+            } else if retain_vector && !field_path.is_empty() && field.name() == &field_path[0] {
+                columns.push(Self::take_retained_vector_column(
+                    results,
+                    field,
+                    &field_path,
+                )?);
+            } else {
+                columns.push(Self::take_slim_batch_field(results, field.name())?);
+            }
+        }
+        RecordBatch::try_new(Arc::new(stored_schema.clone()), columns)
+            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
+    }
+
     async fn execute_batch(
         input: SendableRecordBatchStream,
         config: BatchKnnConfig,
     ) -> DataFusionResult<RecordBatch> {
         let BatchKnnConfig {
-            input_schema,
+            stored_schema,
             output_schema,
             column,
             query,
@@ -350,8 +687,10 @@ impl KNNVectorDistanceExec {
             lower_bound,
             upper_bound,
             distance_type,
+            retain_vector,
         } = config;
         let query_dim = query.len() / query_count;
+        let needs_slim_batch = stored_schema.fields().iter().any(|f| f.name() != ROW_ID);
         let mut heaps = (0..query_count)
             .map(|_| BinaryHeap::<BatchKnnCandidate>::with_capacity(k))
             .collect::<Vec<_>>();
@@ -373,6 +712,13 @@ impl KNNVectorDistanceExec {
                 .as_primitive::<UInt64Type>()
                 .clone();
 
+            let mut slim_batch: Option<Arc<RecordBatch>> = None;
+            let vectors = if retain_vector {
+                Some(Self::resolve_vector_column(&batch, &column)?)
+            } else {
+                None
+            };
+
             for (query_index, heap) in heaps.iter_mut().enumerate().take(query_count) {
                 let key = query.slice(query_index * query_dim, query_dim);
                 let with_distances = compute_distance(key, distance_type, &column, batch.clone())
@@ -397,20 +743,42 @@ impl KNNVectorDistanceExec {
                     }
                     let query_index = query_index as i32;
                     let row_id = row_ids.value(row_index);
-                    let row_index = row_index as u32;
+                    if !would_enter_heap(heap, k, distance, row_id, query_index) {
+                        continue;
+                    }
+
+                    let extra = if retain_vector || needs_slim_batch {
+                        let row_index = row_index as u32;
+                        if slim_batch.is_none() {
+                            let slim = Self::remove_vector_from_batch(&batch, &column)?;
+                            slim_batch = Some(Arc::new(slim));
+                        }
+                        let slim_batch = slim_batch.as_ref().expect("slim batch");
+                        let vector_row = if retain_vector {
+                            Some(Self::take_vector_row(
+                                vectors.as_ref().expect("vectors"),
+                                row_index,
+                            )?)
+                        } else {
+                            None
+                        };
+                        BatchKnnExtra::WithSlimBatch {
+                            slim_batch: Arc::clone(slim_batch),
+                            row_index,
+                            vector_row,
+                        }
+                    } else {
+                        BatchKnnExtra::RowIdOnly
+                    };
                     let candidate = BatchKnnCandidate {
                         query_index,
                         distance,
                         row_id,
-                        batch: batch.clone(),
-                        row_index,
+                        extra,
                     };
                     if heap.len() < k {
                         heap.push(candidate);
-                    } else if heap
-                        .peek()
-                        .is_some_and(|worst| candidate.cmp(worst).is_lt())
-                    {
+                    } else {
                         heap.pop();
                         heap.push(candidate);
                     }
@@ -435,20 +803,14 @@ impl KNNVectorDistanceExec {
 
         let mut query_indices = Int32Builder::with_capacity(results.len());
         let mut distances = Float32Builder::with_capacity(results.len());
-        let mut row_batches = Vec::with_capacity(results.len());
-        for result in results {
+        for result in &results {
             query_indices.append_value(result.query_index);
             distances.append_value(result.distance);
-            let indices = UInt32Array::from(vec![result.row_index]);
-            row_batches.push(
-                arrow_select::take::take_record_batch(&result.batch, &indices).map_err(|e| {
-                    DataFusionError::ArrowError(Box::new(e), Some("take top-k row".to_string()))
-                })?,
-            );
         }
 
-        let output = concat_batches(&input_schema, &row_batches)
-            .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))?;
+        let output =
+            Self::assemble_batch_output(&results, stored_schema.as_ref(), &column, retain_vector)?;
+
         output
             .try_with_column_at(0, query_index_field(), Arc::new(query_indices.finish()))
             .and_then(|batch| {
@@ -466,10 +828,6 @@ impl ExecutionPlan for KNNVectorDistanceExec {
         "KNNVectorDistanceExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     /// Flat KNN inherits the schema from input node, and add one distance column.
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.output_schema.clone()
@@ -500,6 +858,7 @@ impl ExecutionPlan for KNNVectorDistanceExec {
                 lower_bound: self.lower_bound,
                 upper_bound: self.upper_bound,
                 distance_type: self.distance_type,
+                retain_vector: self.retain_vector,
             },
         )?))
     }
@@ -514,7 +873,7 @@ impl ExecutionPlan for KNNVectorDistanceExec {
             let stream = stream::once(Self::execute_batch(
                 input_stream,
                 BatchKnnConfig {
-                    input_schema: self.input_schema.clone(),
+                    stored_schema: self.input_schema.clone(),
                     output_schema: self.output_schema.clone(),
                     column: self.column.clone(),
                     query: self.query.clone(),
@@ -523,6 +882,7 @@ impl ExecutionPlan for KNNVectorDistanceExec {
                     lower_bound: self.lower_bound,
                     upper_bound: self.upper_bound,
                     distance_type: self.distance_type,
+                    retain_vector: self.retain_vector,
                 },
             ));
             let schema = self.schema();
@@ -533,43 +893,35 @@ impl ExecutionPlan for KNNVectorDistanceExec {
                 &self.metrics,
             )) as SendableRecordBatchStream);
         }
-        let input_schema = self.input.schema();
         let key = self.query.clone();
         let column = self.column.clone();
         let dt = self.distance_type;
         let schema = self.schema();
 
         // Empty batches don't have a vector column to score; filter them out
-        // before reaching the helper so the transform always sees real work.
-        let filtered_input = Box::pin(RecordBatchStreamAdapter::new(
-            input_schema,
-            input_stream.try_filter(|batch| future::ready(batch.num_rows() > 0)),
-        )) as SendableRecordBatchStream;
-
-        // Mirror of the helper's elapsed_compute counter; used to attribute
-        // wall-clock from the spawn_blocking distance kernel back onto the
-        // node's `elapsed_compute` metric.
-        let elapsed_compute = BaselineMetrics::new(&self.metrics, partition)
-            .elapsed_compute()
-            .clone();
-
-        let stream = InstrumentedChildInputStream::new(
-            filtered_input,
-            schema,
-            move |batch| {
+        // before the transform so it always sees real work.
+        let filtered_input = input_stream.try_filter(|batch| future::ready(batch.num_rows() > 0));
+
+        let baseline = BaselineMetrics::new(&self.metrics, partition);
+        let elapsed_compute = baseline.elapsed_compute().clone();
+
+        let stream = filtered_input
+            .map(move |batch_result| {
                 let key = key.clone();
                 let column = column.clone();
                 let elapsed_compute = elapsed_compute.clone();
                 async move {
+                    let batch = batch_result?;
                     // Time around the .await to capture the spawn_blocking
                     // distance work, which otherwise runs while this future is
-                    // Pending and is missed by the helper's own poll timer.
+                    // Pending and is missed by a poll-time timer.
                     let start = Instant::now();
                     let batch = compute_distance(key, dt, &column, batch)
                         .await
                         .map_err(|e| DataFusionError::External(Box::new(e)))?;
                     elapsed_compute.add_duration(start.elapsed());
 
+                    let _t = elapsed_compute.timer();
                     let distances = batch[DIST_COL].as_primitive::<Float32Type>();
                     let distance_values = distances.values();
                     let mask = BooleanArray::from_iter((0..distances.len()).map(|row_index| {
@@ -578,50 +930,61 @@ impl ExecutionPlan for KNNVectorDistanceExec {
                     arrow::compute::filter_record_batch(&batch, &mask)
                         .map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
                 }
-            },
-            get_num_compute_intensive_cpus(),
-            partition,
-            &self.metrics,
-        );
-        Ok(Box::pin(stream) as SendableRecordBatchStream)
+            })
+            .buffer_unordered(get_num_compute_intensive_cpus());
+
+        let stream = stream.map(move |batch| {
+            let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch)));
+            match poll {
+                std::task::Poll::Ready(Some(b)) => b,
+                _ => unreachable!("record_poll preserves Ready(Some) input"),
+            }
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
         let inner_stats = self.input.partition_statistics(partition)?;
-        let schema = self.input.schema();
-        let dist_stats = inner_stats
+        let input_schema = self.input.schema();
+        let input_stats_by_name = inner_stats
             .column_statistics
             .iter()
-            .zip(schema.fields())
-            .find(|(_, field)| field.name() == &self.column)
-            .map(|(stats, _)| ColumnStatistics {
+            .zip(input_schema.fields())
+            .map(|(stats, field)| (field.name().as_str(), stats.clone()))
+            .collect::<HashMap<_, _>>();
+        let vector_root = lance_core::datatypes::parse_field_path(&self.column)
+            .ok()
+            .and_then(|parts| parts.first().cloned())
+            .unwrap_or_else(|| self.column.clone());
+        let dist_stats = input_stats_by_name
+            .get(vector_root.as_str())
+            .map(|stats| ColumnStatistics {
                 null_count: stats.null_count,
                 ..Default::default()
             })
             .unwrap_or_default();
-        let column_statistics = inner_stats
-            .column_statistics
-            .into_iter()
-            .zip(schema.fields())
-            .filter(|(_, field)| field.name() != DIST_COL)
-            .map(|(stats, _)| stats)
+        let column_statistics = self
+            .output_schema
+            .fields()
+            .iter()
+            .map(|field| {
+                if field.name() == QUERY_INDEX_COL {
+                    ColumnStatistics::default()
+                } else if field.name() == DIST_COL {
+                    dist_stats.clone()
+                } else {
+                    input_stats_by_name
+                        .get(field.name().as_str())
+                        .cloned()
+                        .unwrap_or_default()
+                }
+            })
             .collect::<Vec<_>>();
-        let column_statistics = if self.is_batch {
-            std::iter::once(ColumnStatistics::default())
-                .chain(column_statistics)
-                .chain(std::iter::once(dist_stats))
-                .collect::<Vec<_>>()
-        } else {
-            column_statistics
-                .into_iter()
-                .chain(std::iter::once(dist_stats))
-                .collect::<Vec<_>>()
-        };
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows: inner_stats.num_rows,
             column_statistics,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -637,11 +1000,9 @@ impl ExecutionPlan for KNNVectorDistanceExec {
     }
 
     fn required_input_distribution(&self) -> Vec<Distribution> {
-        if self.is_batch {
-            vec![Distribution::SinglePartition]
-        } else {
-            vec![Distribution::UnspecifiedDistribution]
-        }
+        // Both batch and non-batch modes execute a single input partition at a time,
+        // so all input must be coalesced to one partition before distance computation.
+        vec![Distribution::SinglePartition]
     }
 }
 
@@ -650,8 +1011,37 @@ struct BatchKnnCandidate {
     query_index: i32,
     distance: f32,
     row_id: u64,
-    batch: RecordBatch,
-    row_index: u32,
+    extra: BatchKnnExtra,
+}
+
+#[derive(Clone)]
+enum BatchKnnExtra {
+    RowIdOnly,
+    WithSlimBatch {
+        slim_batch: Arc<RecordBatch>,
+        row_index: u32,
+        vector_row: Option<ArrayRef>,
+    },
+}
+
+fn would_enter_heap(
+    heap: &BinaryHeap<BatchKnnCandidate>,
+    k: usize,
+    distance: f32,
+    row_id: u64,
+    query_index: i32,
+) -> bool {
+    if heap.len() < k {
+        return true;
+    }
+    let worst = heap.peek().expect("heap non-empty when len >= k");
+    let probe = BatchKnnCandidate {
+        query_index,
+        distance,
+        row_id,
+        extra: BatchKnnExtra::RowIdOnly,
+    };
+    probe.cmp(worst).is_lt()
 }
 
 impl PartialEq for BatchKnnCandidate {
@@ -659,7 +1049,6 @@ impl PartialEq for BatchKnnCandidate {
         self.query_index == other.query_index
             && self.distance == other.distance
             && self.row_id == other.row_id
-            && self.row_index == other.row_index
     }
 }
 
@@ -677,7 +1066,6 @@ impl Ord for BatchKnnCandidate {
             .total_cmp(&other.distance)
             .then_with(|| self.row_id.cmp(&other.row_id))
             .then_with(|| self.query_index.cmp(&other.query_index))
-            .then_with(|| self.row_index.cmp(&other.row_index))
     }
 }
 
@@ -719,7 +1107,7 @@ pub fn new_knn_exec(
 ) -> Result<Arc<dyn ExecutionPlan>> {
     let ivf_node = ANNIvfPartitionExec::try_new(
         dataset.clone(),
-        indices.iter().map(|idx| idx.uuid.to_string()).collect_vec(),
+        indices.iter().map(|idx| idx.uuid).collect_vec(),
         query.clone(),
     )?;
 
@@ -765,7 +1153,7 @@ pub struct ANNIvfPartitionExec {
     pub query: Query,
 
     /// The UUIDs of the indices to search.
-    pub index_uuids: Vec<String>,
+    pub index_uuids: Vec<Uuid>,
 
     pub properties: Arc<PlanProperties>,
 
@@ -773,7 +1161,7 @@ pub struct ANNIvfPartitionExec {
 }
 
 impl ANNIvfPartitionExec {
-    pub fn try_new(dataset: Arc<Dataset>, index_uuids: Vec<String>, query: Query) -> Result<Self> {
+    pub fn try_new(dataset: Arc<Dataset>, index_uuids: Vec<Uuid>, query: Query) -> Result<Self> {
         let dataset_schema = dataset.schema();
         get_vector_type(dataset_schema, &query.column)?;
         if index_uuids.is_empty() {
@@ -832,10 +1220,6 @@ impl ExecutionPlan for ANNIvfPartitionExec {
         "ANNIVFPartitionExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         KNN_PARTITION_SCHEMA.clone()
     }
@@ -844,11 +1228,11 @@ impl ExecutionPlan for ANNIvfPartitionExec {
         &self.properties
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Statistics> {
-        Ok(Statistics {
+    fn partition_statistics(&self, _partition: Option<usize>) -> DataFusionResult<Arc<Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(self.query.minimum_nprobes),
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -875,10 +1259,11 @@ impl ExecutionPlan for ANNIvfPartitionExec {
     fn execute(
         &self,
         partition: usize,
-        _context: Arc<datafusion::execution::TaskContext>,
+        context: Arc<datafusion::execution::TaskContext>,
     ) -> DataFusionResult<SendableRecordBatchStream> {
         let timer = Instant::now();
 
+        let target_partitions = context.session_config().target_partitions();
         let query = self.query.clone();
         let ds = self.dataset.clone();
         let metrics = Arc::new(AnnPartitionMetrics::new(&self.metrics, partition));
@@ -914,7 +1299,7 @@ impl ExecutionPlan for ANNIvfPartitionExec {
                     dist_q_c_list_builder.append_value(dist_q_c.iter());
                     let dist_q_c_col = dist_q_c_list_builder.finish();
 
-                    let uuid_col = StringArray::from(vec![uuid.as_str()]);
+                    let uuid_col = StringArray::from(vec![uuid.to_string()]);
                     let batch = RecordBatch::try_new(
                         KNN_PARTITION_SCHEMA.clone(),
                         vec![
@@ -927,8 +1312,11 @@ impl ExecutionPlan for ANNIvfPartitionExec {
                     Ok::<_, DataFusionError>(batch)
                 }
             })
-            .buffered(self.index_uuids.len())
+            .buffered(self.index_uuids.len().min(target_partitions).max(1))
             .finally(move || {
+                // Partition ranking reads centroids from memory, so this is
+                // typically zero; flushed for symmetry with ANNSubIndex.
+                metrics_clone.index_metrics.flush_io();
                 metrics_clone.baseline_metrics.done();
                 metrics_clone
                     .baseline_metrics
@@ -1148,8 +1536,14 @@ impl PartitionSearchControl for LatePartitionSearchControl {
     }
 }
 
-fn effective_query_parallelism(query: &Query, index: &dyn VectorIndex) -> usize {
-    let cpu_pool_size = get_num_compute_intensive_cpus();
+fn effective_query_parallelism(
+    query: &Query,
+    index: &dyn VectorIndex,
+    target_partitions: usize,
+) -> usize {
+    let cpu_pool_size = get_num_compute_intensive_cpus()
+        .min(target_partitions)
+        .max(1);
     effective_query_parallelism_for(
         query,
         cpu_pool_size,
@@ -1211,6 +1605,7 @@ impl ANNIvfSubIndexExec {
             .boxed()
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn late_search(
         index: Arc<dyn VectorIndex>,
         query: Query,
@@ -1219,6 +1614,7 @@ impl ANNIvfSubIndexExec {
         prefilter: Arc<DatasetPreFilter>,
         metrics: Arc<AnnIndexMetrics>,
         state: Arc<ANNIvfEarlySearchResults>,
+        target_partitions: usize,
     ) -> impl Stream<Item = DataFusionResult<RecordBatch>> {
         let stream = futures::stream::once(async move {
             let max_nprobes = query
@@ -1288,7 +1684,8 @@ impl ANNIvfSubIndexExec {
 
             let state_clone = state.clone();
 
-            let query_parallelism = effective_query_parallelism(&query, index.as_ref());
+            let query_parallelism =
+                effective_query_parallelism(&query, index.as_ref(), target_partitions);
             if query_parallelism <= 1 {
                 return stream::once(async move {
                     let prefilter: Arc<dyn PreFilter> = prefilter;
@@ -1359,6 +1756,7 @@ impl ANNIvfSubIndexExec {
         stream.flatten()
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn initial_search(
         index: Arc<dyn VectorIndex>,
         query: Query,
@@ -1367,10 +1765,12 @@ impl ANNIvfSubIndexExec {
         prefilter: Arc<DatasetPreFilter>,
         metrics: Arc<AnnIndexMetrics>,
         state: Arc<ANNIvfEarlySearchResults>,
+        target_partitions: usize,
     ) -> impl Stream<Item = DataFusionResult<RecordBatch>> {
         let minimum_nprobes = query.minimum_nprobes.min(partitions.len());
 
-        let query_parallelism = effective_query_parallelism(&query, index.as_ref());
+        let query_parallelism =
+            effective_query_parallelism(&query, index.as_ref(), target_partitions);
         if query_parallelism <= 1 {
             metrics.partitions_searched.add(minimum_nprobes);
             return stream::once(async move {
@@ -1434,10 +1834,6 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
         "ANNSubIndexExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         KNN_INDEX_SCHEMA.clone()
     }
@@ -1502,6 +1898,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
     ) -> DataFusionResult<datafusion::physical_plan::SendableRecordBatchStream> {
         let input_stream = self.input.execute(partition, context.clone())?;
         let schema = self.schema();
+        let target_partitions = context.session_config().target_partitions();
         let query = self.query.clone();
         let ds = self.dataset.clone();
         let column = self.query.column.clone();
@@ -1540,7 +1937,11 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                             Arc::new(part_id.unwrap().as_primitive::<UInt32Type>().clone());
                         let dist_q_c =
                             Arc::new(dist_q_c.unwrap().as_primitive::<Float32Type>().clone());
-                        let uuid = uuid.unwrap().to_string();
+                        let uuid = Uuid::parse_str(uuid.unwrap()).map_err(|e| {
+                            DataFusionError::Execution(format!(
+                                "Invalid UUID in __index_uuid column: {e}"
+                            ))
+                        })?;
                         Ok((partitions, dist_q_c, uuid))
                     })
                     .collect_vec();
@@ -1593,6 +1994,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                             pre_filter.clone(),
                             metrics.clone(),
                             state.clone(),
+                            target_partitions,
                         );
                         let late_search = Self::late_search(
                             raw_index.clone(),
@@ -1602,6 +2004,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                             pre_filter,
                             metrics,
                             state,
+                            target_partitions,
                         );
                         DataFusionResult::Ok(early_search.chain(late_search).boxed())
                     }
@@ -1611,6 +2014,9 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                 // will not start until the early search is complete across all deltas.
                 .try_flatten_unordered(None)
                 .finally(move || {
+                    // Publish the exact index-file I/O measured for this query
+                    // (cache misses only) to the iops/requests/bytes_read gauges.
+                    metrics_clone.index_metrics.flush_io();
                     metrics_clone
                         .baseline_metrics
                         .elapsed_compute()
@@ -1624,8 +2030,8 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> DataFusionResult<datafusion::physical_plan::Statistics> {
-        Ok(Statistics {
+    ) -> DataFusionResult<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: Precision::Exact(
                 self.query.k
                     * self.query.refine_factor.unwrap_or(1) as usize
@@ -1637,7 +2043,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec {
                         .unwrap_or(&1),
             ),
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -1720,10 +2126,6 @@ impl ExecutionPlan for MultivectorScoringExec {
         "MultivectorScoringExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         KNN_INDEX_SCHEMA.clone()
     }
@@ -1871,17 +2273,20 @@ impl ExecutionPlan for MultivectorScoringExec {
 mod tests {
     use super::*;
 
+    use std::any::Any;
+
     use crate::index::DatasetIndexExt;
     use arrow::compute::{concat_batches, sort_to_indices, take_record_batch};
     use arrow::datatypes::Float32Type;
     use arrow_array::{
         ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatchIterator, StringArray,
+        StructArray,
     };
     use arrow_schema::{Field as ArrowField, Schema as ArrowSchema};
     use async_trait::async_trait;
     use datafusion::error::Result as DataFusionResult;
     use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-    use deepsize::DeepSizeOf;
+    use lance_core::deepsize::DeepSizeOf;
     use lance_core::utils::tempfile::TempStrDir;
     use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts};
     use lance_datafusion::utils::FIND_PARTITIONS_ELAPSED_METRIC;
@@ -1901,6 +2306,7 @@ mod tests {
 
     use crate::dataset::{WriteMode, WriteParams};
     use crate::index::vector::VectorIndexParams;
+    use crate::index::vector::ivf::v2::STREAMING_SEARCH_BATCH_SIZE;
     use crate::io::exec::testing::TestingExec;
 
     fn base_query() -> Query {
@@ -1918,6 +2324,7 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: Default::default(),
         }
     }
 
@@ -1943,6 +2350,36 @@ mod tests {
         assert_eq!(effective_query_parallelism_for(&query, 16, 1), 16);
     }
 
+    #[test]
+    fn test_effective_query_parallelism_respects_target_partitions() {
+        // effective_query_parallelism caps cpu_pool_size at target_partitions before
+        // passing it to effective_query_parallelism_for, so the ceiling is
+        // min(cpu_pool_size, target_partitions).
+        let mut query = base_query();
+        let cpu_pool_size = 16;
+
+        // use-all-cpus mode: capped at target_partitions
+        query.query_parallelism = -1;
+        assert_eq!(
+            effective_query_parallelism_for(&query, cpu_pool_size.min(4), 1),
+            4
+        );
+
+        // auto mode: auto_parallelism also clamped to the reduced cpu_pool_size
+        query.query_parallelism = 0;
+        assert_eq!(
+            effective_query_parallelism_for(&query, cpu_pool_size.min(4), 8),
+            4
+        );
+
+        // explicit parallelism > target_partitions: clamped down
+        query.query_parallelism = 16;
+        assert_eq!(
+            effective_query_parallelism_for(&query, cpu_pool_size.min(4), 1),
+            4
+        );
+    }
+
     #[derive(Debug, DeepSizeOf)]
     struct ThreadCapturingIndex {
         thread_name: Arc<Mutex<Option<String>>>,
@@ -1967,10 +2404,6 @@ mod tests {
             self
         }
 
-        fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-            Ok(self)
-        }
-
         fn statistics(&self) -> Result<serde_json::Value> {
             Ok(serde_json::json!({}))
         }
@@ -2047,7 +2480,7 @@ mod tests {
             Box::new(self.row_ids.iter())
         }
 
-        async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+        async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
             Ok(())
         }
 
@@ -2091,10 +2524,6 @@ mod tests {
             self
         }
 
-        fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-            Ok(self)
-        }
-
         fn statistics(&self) -> Result<serde_json::Value> {
             Ok(serde_json::json!({}))
         }
@@ -2191,7 +2620,6 @@ mod tests {
             _metrics: Arc<dyn lance_index::metrics::MetricsCollector>,
         ) -> Result<SendableRecordBatchStream> {
             let (batch_tx, batch_rx) = mpsc::channel(1);
-            let batch_tx_for_search = batch_tx.clone();
             let prepared_partition_ids = (start_idx..end_idx)
                 .map(|idx| partitions.value(idx) as usize)
                 .collect::<Vec<_>>();
@@ -2199,42 +2627,74 @@ mod tests {
                 .lock()
                 .unwrap()
                 .extend(prepared_partition_ids.iter().copied());
+            // Mirror the production streaming path (v2.rs): search prepared partitions
+            // in batches of STREAMING_SEARCH_BATCH_SIZE, one `spawn_cpu` per batch, with
+            // the channel send in async code so no CPU-pool thread parks (#7642).
             tokio::spawn(async move {
-                let search_result = spawn_cpu(move || -> DataFusionResult<()> {
-                    for partition_id in prepared_partition_ids {
-                        if control
-                            .as_ref()
-                            .is_some_and(|control| control.should_stop())
-                        {
-                            return Ok(());
-                        }
-                        let batch = self
-                            .search_prepared_partition(
-                                Box::new(partition_id),
-                                &lance_index::metrics::NoOpMetricsCollector,
-                            )
-                            .map_err(datafusion::error::DataFusionError::from);
-                        match batch {
-                            Ok(batch) => {
-                                if let Some(control) = control.as_ref() {
-                                    control.record_batch(&batch);
+                for chunk in prepared_partition_ids.chunks(*STREAMING_SEARCH_BATCH_SIZE) {
+                    if control
+                        .as_ref()
+                        .is_some_and(|control| control.should_stop())
+                        || batch_tx.is_closed()
+                    {
+                        return;
+                    }
+                    let chunk = chunk.to_vec();
+                    let index = self.clone();
+                    let control_for_search = control.clone();
+                    let cancel_probe = batch_tx.clone();
+                    let search_output = spawn_cpu(move || {
+                        let mut outputs: Vec<DataFusionResult<RecordBatch>> =
+                            Vec::with_capacity(chunk.len());
+                        let mut stopped = false;
+                        for partition_id in chunk {
+                            if control_for_search
+                                .as_ref()
+                                .is_some_and(|control| control.should_stop())
+                                || cancel_probe.is_closed()
+                            {
+                                stopped = true;
+                                break;
+                            }
+                            match index
+                                .search_prepared_partition(
+                                    Box::new(partition_id),
+                                    &lance_index::metrics::NoOpMetricsCollector,
+                                )
+                                .map_err(datafusion::error::DataFusionError::from)
+                            {
+                                Ok(batch) => {
+                                    if let Some(control) = control_for_search.as_ref() {
+                                        control.record_batch(&batch);
+                                    }
+                                    outputs.push(Ok(batch));
                                 }
-                                if batch_tx_for_search.blocking_send(Ok(batch)).is_err() {
-                                    return Ok(());
+                                Err(err) => {
+                                    outputs.push(Err(err));
+                                    stopped = true;
+                                    break;
                                 }
                             }
-                            Err(err) => {
-                                let _ = batch_tx_for_search.blocking_send(Err(err));
-                                return Ok(());
-                            }
                         }
-                    }
-                    Ok(())
-                })
-                .await;
+                        Ok::<_, datafusion::error::DataFusionError>((outputs, stopped))
+                    })
+                    .await;
 
-                if let Err(err) = search_result {
-                    let _ = batch_tx.send(Err(err)).await;
+                    let (outputs, stopped) = match search_output {
+                        Ok(output) => output,
+                        Err(err) => {
+                            let _ = batch_tx.send(Err(err)).await;
+                            return;
+                        }
+                    };
+                    for output in outputs {
+                        if batch_tx.send(output).await.is_err() {
+                            return;
+                        }
+                    }
+                    if stopped {
+                        return;
+                    }
                 }
             });
 
@@ -2269,7 +2729,7 @@ mod tests {
             Box::new(self.row_ids.iter())
         }
 
-        async fn remap(&mut self, _mapping: &HashMap<u64, Option<u64>>) -> Result<()> {
+        async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> {
             Ok(())
         }
 
@@ -2428,32 +2888,44 @@ mod tests {
         );
     }
 
+    // All partitions fit in a single search batch, so they are searched in one
+    // `spawn_cpu` dispatch and therefore share one cpu thread. The partition count
+    // adapts to the configured batch size so the single-batch property holds under
+    // any valid `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE`, including 1.
     #[tokio::test]
     async fn test_sequential_initial_search_prepares_all_then_searches_on_one_cpu_thread() {
+        let num_partitions = 3.min(*STREAMING_SEARCH_BATCH_SIZE);
+        let row_ids = (0..num_partitions).map(|i| 10 + i as u64).collect();
         let (index, prepared_partitions, searched_partitions, search_threads) =
-            prepared_index(vec![10, 11, 12]);
+            prepared_index(row_ids);
         let mut query = base_query();
-        query.minimum_nprobes = 3;
+        query.minimum_nprobes = num_partitions;
         let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k));
 
+        let partition_idx = (0..num_partitions as u32).collect::<Vec<_>>();
+        let q_c_dists = (0..num_partitions)
+            .map(|i| i as f32 * 0.1)
+            .collect::<Vec<_>>();
         let batches = ANNIvfSubIndexExec::initial_search(
             index,
             query,
-            Arc::new(UInt32Array::from(vec![0, 1, 2])),
-            Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3])),
+            Arc::new(UInt32Array::from(partition_idx)),
+            Arc::new(Float32Array::from(q_c_dists)),
             empty_prefilter().await,
             prepared_metrics(),
             state,
+            usize::MAX,
         )
         .try_collect::<Vec<_>>()
         .await
         .unwrap();
 
-        assert_eq!(batches.len(), 3);
-        assert_eq!(*prepared_partitions.lock().unwrap(), vec![0, 1, 2]);
-        assert_eq!(*searched_partitions.lock().unwrap(), vec![0, 1, 2]);
+        let expected: Vec<usize> = (0..num_partitions).collect();
+        assert_eq!(batches.len(), num_partitions);
+        assert_eq!(*prepared_partitions.lock().unwrap(), expected);
+        assert_eq!(*searched_partitions.lock().unwrap(), expected);
         let search_threads = search_threads.lock().unwrap().clone();
-        assert_eq!(search_threads.len(), 3);
+        assert_eq!(search_threads.len(), num_partitions);
         assert!(
             search_threads.iter().all(|name| name.contains("lance-cpu")),
             "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}",
@@ -2464,6 +2936,52 @@ mod tests {
         );
     }
 
+    // Regression guard for the batched streaming search (#7642): with more partitions
+    // than a single batch, the search spans multiple `spawn_cpu` dispatches. Verify that
+    // every partition is still prepared and searched in order across the batch boundary,
+    // and that all search work stays on the cpu runtime.
+    //
+    // Note: this does not reproduce the single-thread-pool deadlock the async recv/send
+    // fixes -- that requires a 1-thread CPU pool, which is a process-global singleton and
+    // impractical to force in a unit test (same limitation noted for the #7423 fix).
+    #[tokio::test]
+    async fn test_sequential_search_spans_multiple_cpu_batches() {
+        let num_partitions = *STREAMING_SEARCH_BATCH_SIZE + 3;
+        let row_ids = (0..num_partitions).map(|i| i as u64 * 10).collect();
+        let (index, prepared_partitions, searched_partitions, search_threads) =
+            prepared_index(row_ids);
+        let mut query = base_query();
+        query.minimum_nprobes = num_partitions;
+        let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k));
+
+        let partition_idx = (0..num_partitions as u32).collect::<Vec<_>>();
+        let q_c_dists = (0..num_partitions).map(|i| i as f32).collect::<Vec<_>>();
+        let batches = ANNIvfSubIndexExec::initial_search(
+            index,
+            query,
+            Arc::new(UInt32Array::from(partition_idx.clone())),
+            Arc::new(Float32Array::from(q_c_dists)),
+            empty_prefilter().await,
+            prepared_metrics(),
+            state,
+            usize::MAX,
+        )
+        .try_collect::<Vec<_>>()
+        .await
+        .unwrap();
+
+        let expected: Vec<usize> = (0..num_partitions).collect();
+        assert_eq!(batches.len(), num_partitions);
+        assert_eq!(*prepared_partitions.lock().unwrap(), expected);
+        assert_eq!(*searched_partitions.lock().unwrap(), expected);
+        let search_threads = search_threads.lock().unwrap().clone();
+        assert_eq!(search_threads.len(), num_partitions);
+        assert!(
+            search_threads.iter().all(|name| name.contains("lance-cpu")),
+            "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}",
+        );
+    }
+
     #[tokio::test]
     async fn test_sequential_late_search_prepares_all_then_stops_search_early() {
         let (index, prepared_partitions, searched_partitions, _search_threads) =
@@ -2492,6 +3010,7 @@ mod tests {
             empty_prefilter().await,
             prepared_metrics(),
             state.clone(),
+            usize::MAX,
         )
         .try_collect::<Vec<_>>()
         .await
@@ -2629,6 +3148,254 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_batch_partition_statistics_aligns_with_output_schema() {
+        let schema = Arc::new(ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            ArrowField::new(
+                "vec",
+                DataType::FixedSizeList(
+                    Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                    4,
+                ),
+                true,
+            ),
+            ROW_ID_FIELD.clone(),
+        ]));
+        let batch = RecordBatch::new_empty(schema);
+        let input: Arc<dyn ExecutionPlan> = Arc::new(TestingExec::new(vec![batch]));
+        let query = Arc::new(Float32Array::from(vec![0.0, 1.0, 2.0, 3.0])) as ArrayRef;
+        let plan = KNNVectorDistanceExec::try_new_batch(
+            input,
+            "vec",
+            query,
+            KnnBatchParams {
+                is_batch: true,
+                query_count: 1,
+                k: 2,
+                lower_bound: None,
+                upper_bound: None,
+                distance_type: DistanceType::L2,
+                retain_vector: false,
+            },
+        )
+        .unwrap();
+        let stats = plan.partition_statistics(None).unwrap();
+        assert_eq!(
+            stats.column_statistics.len(),
+            plan.schema().fields().len(),
+            "partition stats must align with output schema"
+        );
+        let schema = plan.schema();
+        let query_index_pos = schema
+            .column_with_name(QUERY_INDEX_COL)
+            .expect("query_index must exist")
+            .0;
+        let dist_pos = schema
+            .column_with_name(DIST_COL)
+            .expect("distance must exist")
+            .0;
+        assert_eq!(
+            stats.column_statistics[query_index_pos],
+            ColumnStatistics::default(),
+        );
+        assert_eq!(
+            stats.column_statistics[dist_pos].null_count,
+            stats.column_statistics[schema.column_with_name("i").unwrap().0].null_count,
+            "distance null-count should be derived from vector/input nullability and remain aligned"
+        );
+    }
+
+    #[test]
+    fn test_remove_vector_from_schema_nested_path() {
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(
+                vec![
+                    ArrowField::new(
+                        "vec",
+                        DataType::FixedSizeList(
+                            Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                            4,
+                        ),
+                        true,
+                    ),
+                    ArrowField::new("tag", DataType::Utf8, true),
+                ]
+                .into(),
+            ),
+            true,
+        );
+        let schema = ArrowSchema::new(vec![
+            ArrowField::new("i", DataType::Int32, true),
+            payload_field,
+            ROW_ID_FIELD.clone(),
+        ]);
+        let without_vec =
+            KNNVectorDistanceExec::remove_vector_from_schema(&schema, "payload.vec").unwrap();
+        let payload = without_vec.field_with_name("payload").unwrap();
+        let DataType::Struct(children) = payload.data_type() else {
+            panic!("payload should remain struct");
+        };
+        assert!(children.iter().all(|f| f.name() != "vec"));
+        assert!(children.iter().any(|f| f.name() == "tag"));
+    }
+
+    #[test]
+    fn test_take_vector_row_copies_single_row() {
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from((0..12).map(|v| v as f32).collect::<Vec<_>>()),
+            4,
+        )
+        .unwrap();
+        let row = KNNVectorDistanceExec::take_vector_row(&vectors, 2).unwrap();
+        assert_eq!(row.len(), 1);
+        assert_eq!(
+            row.to_data().offset(),
+            0,
+            "take/copy should not retain row offset into the full input buffer"
+        );
+    }
+
+    #[test]
+    fn test_resolve_vector_column_supports_escaped_nested_path() {
+        let vec_field = ArrowField::new(
+            "vec.with.dot",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                4,
+            ),
+            true,
+        );
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(vec![vec_field.clone()].into()),
+            true,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![payload_field]));
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from((0..8).map(|v| v as f32).collect::<Vec<_>>()),
+            4,
+        )
+        .unwrap();
+        let payload = StructArray::from(vec![(Arc::new(vec_field), Arc::new(vectors) as ArrayRef)]);
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(payload)]).unwrap();
+        let vector =
+            KNNVectorDistanceExec::resolve_vector_column(&batch, "payload.`vec.with.dot`").unwrap();
+        assert_eq!(vector.len(), 2);
+    }
+
+    #[test]
+    fn test_remove_vector_from_batch_nested_keeps_siblings() {
+        let vec_field = ArrowField::new(
+            "vec.with.dot",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                4,
+            ),
+            true,
+        );
+        let tag_field = ArrowField::new("tag", DataType::Utf8, true);
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(vec![vec_field.clone(), tag_field.clone()].into()),
+            true,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![payload_field]));
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from((0..8).map(|v| v as f32).collect::<Vec<_>>()),
+            4,
+        )
+        .unwrap();
+        let tags = StringArray::from(vec!["a", "b"]);
+        let payload = StructArray::from(vec![
+            (Arc::new(vec_field), Arc::new(vectors) as ArrayRef),
+            (Arc::new(tag_field), Arc::new(tags) as ArrayRef),
+        ]);
+        let batch = RecordBatch::try_new(schema, vec![Arc::new(payload)]).unwrap();
+
+        let slim =
+            KNNVectorDistanceExec::remove_vector_from_batch(&batch, "payload.`vec.with.dot`")
+                .unwrap();
+        let payload = slim.column_by_name("payload").unwrap().as_struct();
+        assert!(payload.column_by_name("vec.with.dot").is_none());
+        assert!(payload.column_by_name("tag").is_some());
+    }
+
+    #[test]
+    fn test_assemble_batch_output_retained_nested_vector_keeps_sibling_values() {
+        let vec_field = ArrowField::new(
+            "vec",
+            DataType::FixedSizeList(
+                Arc::new(ArrowField::new("item", DataType::Float32, true)),
+                4,
+            ),
+            true,
+        );
+        let tag_field = ArrowField::new("tag", DataType::Utf8, true);
+        let payload_field = ArrowField::new(
+            "payload",
+            DataType::Struct(vec![vec_field.clone(), tag_field.clone()].into()),
+            true,
+        );
+        let schema = Arc::new(ArrowSchema::new(vec![payload_field, ROW_ID_FIELD.clone()]));
+        let vectors = FixedSizeListArray::try_new_from_values(
+            Float32Array::from((0..12).map(|v| v as f32).collect::<Vec<_>>()),
+            4,
+        )
+        .unwrap();
+        let tags = StringArray::from(vec!["a", "b", "c"]);
+        let payload = StructArray::from(vec![
+            (Arc::new(vec_field), Arc::new(vectors) as ArrayRef),
+            (Arc::new(tag_field), Arc::new(tags) as ArrayRef),
+        ]);
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(payload) as ArrayRef,
+                Arc::new(UInt64Array::from(vec![10, 11, 12])) as ArrayRef,
+            ],
+        )
+        .unwrap();
+        let slim_batch = Arc::new(
+            KNNVectorDistanceExec::remove_vector_from_batch(&batch, "payload.vec").unwrap(),
+        );
+        let vectors = KNNVectorDistanceExec::resolve_vector_column(&batch, "payload.vec").unwrap();
+        let results = [2, 0]
+            .into_iter()
+            .map(|row_index| BatchKnnCandidate {
+                query_index: 0,
+                distance: row_index as f32,
+                row_id: 10 + row_index as u64,
+                extra: BatchKnnExtra::WithSlimBatch {
+                    slim_batch: Arc::clone(&slim_batch),
+                    row_index,
+                    vector_row: Some(
+                        KNNVectorDistanceExec::take_vector_row(vectors.as_ref(), row_index)
+                            .unwrap(),
+                    ),
+                },
+            })
+            .collect::<Vec<_>>();
+
+        let output = KNNVectorDistanceExec::assemble_batch_output(
+            &results,
+            schema.as_ref(),
+            "payload.vec",
+            true,
+        )
+        .unwrap();
+
+        let payload = output.column_by_name("payload").unwrap().as_struct();
+        let tags = payload.column_by_name("tag").unwrap().as_string::<i32>();
+        assert!(tags.is_valid(0));
+        assert!(tags.is_valid(1));
+        assert_eq!(tags.value(0), "c");
+        assert_eq!(tags.value(1), "a");
+        let vectors = payload.column_by_name("vec").unwrap();
+        assert_eq!(vectors.len(), 2);
+    }
+
     #[tokio::test]
     async fn test_multivector_score() {
         let query = Query {
@@ -2645,6 +3412,7 @@ mod tests {
             use_index: true,
             query_parallelism: DEFAULT_QUERY_PARALLELISM,
             dist_q_c: 0.0,
+            approx_mode: Default::default(),
         };
 
         async fn multivector_scoring(
@@ -2869,6 +3637,128 @@ mod tests {
         assert_find_partitions_elapsed_recorded(&stats);
     }
 
+    /// The ANN operators report the exact index-file I/O performed for a query
+    /// (bytes_read / iops), measured only on cache misses.  A cold search loads
+    /// partitions from storage and reports non-zero I/O; an immediately
+    /// following warm search serves every partition from the index cache and
+    /// reports zero -- which is the cache-effectiveness signal the metric adds.
+    #[tokio::test]
+    async fn test_io_metrics_cold_vs_warm() {
+        let fixture = NprobesTestFixture::new(100, 1).await;
+        let q = fixture.get_centroid(0);
+
+        let run = |holder: &StatsHolder| {
+            let setter = holder.get_setter();
+            async {
+                fixture
+                    .dataset
+                    .scan()
+                    .nearest("vector", q.as_ref(), 10)
+                    .unwrap()
+                    .minimum_nprobes(10)
+                    .scan_stats_callback(setter)
+                    .project(&Vec::<String>::new())
+                    .unwrap()
+                    .with_row_id()
+                    .try_into_batch()
+                    .await
+                    .unwrap()
+            }
+        };
+
+        // Cold: a freshly opened dataset has an empty index cache, so the
+        // sub-index search must read partitions (and their quantization storage)
+        // from disk.  Those reads flow through the per-query I/O sink.
+        let cold_holder = StatsHolder::default();
+        run(&cold_holder).await;
+        let cold = cold_holder.consume();
+        assert!(
+            cold.parts_loaded > 0,
+            "cold search should load partitions, got parts_loaded={}",
+            cold.parts_loaded
+        );
+        assert!(
+            cold.bytes_read > 0,
+            "cold search should report index-file I/O, got bytes_read={}",
+            cold.bytes_read
+        );
+        assert!(
+            cold.iops > 0,
+            "cold search should report index-file IOPS, got iops={}",
+            cold.iops
+        );
+
+        // Warm: the same query on the same dataset finds every partition it
+        // needs already cached, so no index-file I/O is performed.
+        let warm_holder = StatsHolder::default();
+        run(&warm_holder).await;
+        let warm = warm_holder.consume();
+        assert_eq!(
+            warm.parts_loaded, 0,
+            "warm search should not reload partitions, got parts_loaded={}",
+            warm.parts_loaded
+        );
+        assert_eq!(
+            warm.bytes_read, 0,
+            "warm search should report no index-file I/O, got bytes_read={}",
+            warm.bytes_read
+        );
+    }
+
+    /// The new I/O metrics must actually surface in `EXPLAIN ANALYZE` text on
+    /// the ANN operators: non-zero on a cold query (partition reads on
+    /// `ANNSubIndex`, index-open reads on `ANNIvfPartition`) and zero on a warm
+    /// query (everything served from the index cache).
+    #[tokio::test]
+    async fn test_io_metrics_visible_in_explain_analyze() {
+        // Returns the value of `metric=` from the analyzed-plan line for `node`.
+        fn node_metric<'a>(plan: &'a str, node: &str, metric: &str) -> &'a str {
+            let line = plan
+                .lines()
+                .find(|l| l.trim_start().starts_with(node))
+                .unwrap_or_else(|| panic!("plan missing node {node}:\n{plan}"));
+            let after = line
+                .split_once(&format!("{metric}="))
+                .unwrap_or_else(|| panic!("node {node} line missing {metric}=:\n{line}"))
+                .1;
+            after.split([',', ']']).next().unwrap().trim()
+        }
+
+        let fixture = NprobesTestFixture::new(100, 1).await;
+        let q = fixture.get_centroid(0);
+
+        // Cold: a freshly opened dataset must show real index-file I/O.
+        let cold = fixture
+            .dataset
+            .scan()
+            .nearest("vector", q.as_ref(), 10)
+            .unwrap()
+            .minimum_nprobes(10)
+            .analyze_plan()
+            .await
+            .unwrap();
+        // Sub-index partition reads.
+        assert_ne!(node_metric(&cold, "ANNSubIndex", "bytes_read"), "0");
+        assert_ne!(node_metric(&cold, "ANNSubIndex", "iops"), "0");
+        // Index-open reads (centroids/metadata) now attributed to the partition
+        // operator -- the value this part of the change adds.
+        assert_ne!(node_metric(&cold, "ANNIvfPartition", "bytes_read"), "0");
+        assert_ne!(node_metric(&cold, "ANNIvfPartition", "iops"), "0");
+
+        // Warm: same query, everything cache-resident -> zero index-file I/O.
+        let warm = fixture
+            .dataset
+            .scan()
+            .nearest("vector", q.as_ref(), 10)
+            .unwrap()
+            .minimum_nprobes(10)
+            .analyze_plan()
+            .await
+            .unwrap();
+        assert_eq!(node_metric(&warm, "ANNSubIndex", "bytes_read"), "0");
+        assert_eq!(node_metric(&warm, "ANNIvfPartition", "bytes_read"), "0");
+    }
+
     #[rstest]
     #[tokio::test]
     async fn test_no_prefilter_results(#[values(1, 20)] num_deltas: usize) {
diff --git a/rust/lance/src/io/exec/optimizer.rs b/rust/lance/src/io/exec/optimizer.rs
index f031e10ce19..bc1df37f0bb 100644
--- a/rust/lance/src/io/exec/optimizer.rs
+++ b/rust/lance/src/io/exec/optimizer.rs
@@ -78,21 +78,19 @@ impl PhysicalOptimizerRule for CoalesceTake {
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
         Ok(plan
             .transform_down(|plan| {
-                if let Some(outer_take) = plan.as_any().downcast_ref::<TakeExec>() {
+                if let Some(outer_take) = plan.downcast_ref::<TakeExec>() {
                     let child = outer_take.children()[0];
                     // Case 1: TakeExec -> TakeExec
-                    if let Some(inner_take) = child.as_any().downcast_ref::<TakeExec>() {
+                    if let Some(inner_take) = child.downcast_ref::<TakeExec>() {
                         return Ok(Transformed::yes(Self::collapse_takes(
                             inner_take,
                             outer_take,
                             plan.clone(),
                         )));
                     // Case 2: TakeExec -> CoalesceBatchesExec -> TakeExec
-                    } else if let Some(exec_child) =
-                        child.as_any().downcast_ref::<CoalesceBatchesExec>()
-                    {
+                    } else if let Some(exec_child) = child.downcast_ref::<CoalesceBatchesExec>() {
                         let inner_child = exec_child.children()[0].clone();
-                        if let Some(inner_take) = inner_child.as_any().downcast_ref::<TakeExec>() {
+                        if let Some(inner_take) = inner_child.downcast_ref::<TakeExec>() {
                             return Ok(Transformed::yes(Self::collapse_takes(
                                 inner_take,
                                 outer_take,
@@ -128,7 +126,7 @@ impl PhysicalOptimizerRule for SimplifyProjection {
     ) -> DFResult<Arc<dyn ExecutionPlan>> {
         Ok(plan
             .transform_down(|plan| {
-                if let Some(proj) = plan.as_any().downcast_ref::<ProjectionExec>() {
+                if let Some(proj) = plan.downcast_ref::<ProjectionExec>() {
                     let children = proj.children();
                     if children.len() != 1 {
                         return Ok(Transformed::no(plan));
@@ -145,7 +143,7 @@ impl PhysicalOptimizerRule for SimplifyProjection {
                     }
 
                     if proj.expr().iter().enumerate().all(|(index, proj_expr)| {
-                        if let Some(expr) = proj_expr.expr.as_any().downcast_ref::<Column>() {
+                        if let Some(expr) = proj_expr.expr.downcast_ref::<Column>() {
                             // no renaming, no reordering
                             expr.index() == index && expr.name() == proj_expr.alias
                         } else {
@@ -171,9 +169,16 @@ impl PhysicalOptimizerRule for SimplifyProjection {
 
 pub fn get_physical_optimizer() -> PhysicalOptimizer {
     PhysicalOptimizer::with_rules(vec![
+        // Rewrite `COUNT(*)`-style aggregates into CountFromMaskExec so they
+        // can be answered without scanning column data. Runs before the
+        // generic rules so they don't see the rewritten subtree.
+        Arc::new(crate::io::exec::count_pushdown::CountPushdown),
         Arc::new(crate::io::exec::optimizer::CoalesceTake),
         Arc::new(crate::io::exec::optimizer::SimplifyProjection),
         // Push down limit into FilteredReadExec and other Execs via with_fetch()
         Arc::new(datafusion::physical_optimizer::limit_pushdown::LimitPushdown::new()),
+        // Insert exchange nodes (RepartitionExec, CoalescePartitionsExec) where needed
+        // to satisfy distribution requirements as exec nodes migrate to multi-partition output.
+        Arc::new(datafusion::physical_optimizer::enforce_distribution::EnforceDistribution::new()),
     ])
 }
diff --git a/rust/lance/src/io/exec/projection.rs b/rust/lance/src/io/exec/projection.rs
index 3106fcfac61..06bc0b8de67 100644
--- a/rust/lance/src/io/exec/projection.rs
+++ b/rust/lance/src/io/exec/projection.rs
@@ -44,7 +44,7 @@ pub fn project(input: Arc<dyn ExecutionPlan>, projection: &ArrowSchema) -> Resul
 
     let field_names = projection.fields().iter().map(|f| f.name()).cloned();
 
-    for (name, selection) in field_names.zip(selections.into_iter()) {
+    for (name, selection) in field_names.zip(selections) {
         let expr = selection_as_expr(&selection, input_schema.fields(), None);
         exprs.push((expr, name));
     }
diff --git a/rust/lance/src/io/exec/pushdown_scan.rs b/rust/lance/src/io/exec/pushdown_scan.rs
index 0d49232c094..b82434116b8 100644
--- a/rust/lance/src/io/exec/pushdown_scan.rs
+++ b/rust/lance/src/io/exec/pushdown_scan.rs
@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
 use std::collections::HashMap;
-use std::{any::Any, sync::Arc};
+use std::sync::Arc;
 
 use arrow_array::cast::AsArray;
 use arrow_array::types::{Int64Type, UInt64Type};
@@ -25,7 +25,8 @@ use datafusion::{
 };
 use datafusion_functions::core::expr_ext::FieldAccessor;
 use datafusion_physical_expr::EquivalenceProperties;
-use futures::{FutureExt, Stream, StreamExt, TryStreamExt};
+use futures::stream::BoxStream;
+use futures::{FutureExt, StreamExt, TryStreamExt};
 use lance_arrow::{RecordBatchExt, SchemaExt};
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::{ROW_ADDR, ROW_ADDR_FIELD, ROW_ID_FIELD};
@@ -158,10 +159,6 @@ impl ExecutionPlan for LancePushdownScanExec {
         "LancePushdownScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -218,7 +215,7 @@ impl ExecutionPlan for LancePushdownScanExec {
             )
             .await?;
 
-            frag_scanner.scan().await
+            frag_scanner.scan()
         });
 
         let batch_stream = batch_stream
@@ -325,7 +322,7 @@ impl FragmentScanner {
         })
     }
 
-    pub async fn scan(self) -> Result<impl Stream<Item = Result<RecordBatch>> + 'static + Send> {
+    pub fn scan(self) -> Result<BoxStream<'static, Result<RecordBatch>>> {
         let batch_readahead = self.config.batch_readahead;
         let simplified_predicates = self.simplified_predicates()?;
         let ordered_output = self.config.ordered_output;
@@ -671,7 +668,7 @@ impl FragmentScanner {
                 .collect();
             let schema =
                 Arc::new(ArrowSchema::from(self.predicate_projection.as_ref()).try_into()?);
-            let context = SimplifyContext::default().with_schema(schema);
+            let context = SimplifyContext::builder().with_schema(schema).build();
             let mut simplifier = ExprSimplifier::new(context);
 
             let mut predicates = Vec::with_capacity(num_batches);
diff --git a/rust/lance/src/io/exec/rowids.rs b/rust/lance/src/io/exec/rowids.rs
index cb8280ea874..dc9a54cc182 100644
--- a/rust/lance/src/io/exec/rowids.rs
+++ b/rust/lance/src/io/exec/rowids.rs
@@ -10,7 +10,7 @@ use datafusion::common::ColumnStatistics;
 use datafusion::common::stats::Precision;
 use datafusion::error::{DataFusionError, Result};
 use datafusion::execution::SendableRecordBatchStream;
-use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
 use datafusion_physical_expr::EquivalenceProperties;
 use datafusion_physical_plan::Statistics;
@@ -29,8 +29,6 @@ use crate::Dataset;
 use crate::dataset::rowids::get_row_id_index;
 use crate::utils::future::SharedPrerequisite;
 
-use super::utils::InstrumentedChildInputStream;
-
 /// Add a `_rowaddr` column to a stream of record batches that have a `_rowid`.
 ///
 /// It's generally more efficient to scan the `_rowaddr` column, but this can be
@@ -191,33 +189,41 @@ impl AddRowAddrExec {
         let rowid_pos = self.rowid_pos;
         let rowaddr_pos = self.rowaddr_pos;
         let output_schema = self.output_schema.clone();
-        let stream = InstrumentedChildInputStream::new(
-            input_stream,
+        let baseline = BaselineMetrics::new(&self.metrics, partition);
+        let elapsed_compute = baseline.elapsed_compute().clone();
+        let stream = input_stream.then(move |batch_result| {
+            let output_schema = output_schema.clone();
+            let index_prereq = index_prereq.clone();
+            let elapsed_compute = elapsed_compute.clone();
+            async move {
+                let batch = batch_result?;
+                index_prereq.wait_ready().await?;
+                let row_id_index = index_prereq.get_ready();
+                let index_ref = row_id_index.as_deref();
+
+                let _t = elapsed_compute.timer();
+                let row_addr = Self::compute_row_addrs(batch.column(rowid_pos), index_ref)?;
+
+                let mut columns = Vec::with_capacity(batch.num_columns() + 1);
+                let existing_columns = batch.columns();
+                columns.extend_from_slice(&existing_columns[..rowaddr_pos]);
+                columns.push(row_addr);
+                columns.extend_from_slice(&existing_columns[rowaddr_pos..]);
+
+                Ok(RecordBatch::try_new(output_schema.clone(), columns)?)
+            }
+        });
+        let stream = stream.map(move |batch| {
+            let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch)));
+            match poll {
+                std::task::Poll::Ready(Some(b)) => b,
+                _ => unreachable!("record_poll preserves Ready(Some) input"),
+            }
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
             self.output_schema.clone(),
-            move |batch| {
-                let output_schema = output_schema.clone();
-                let index_prereq = index_prereq.clone();
-                async move {
-                    index_prereq.wait_ready().await?;
-                    let row_id_index = index_prereq.get_ready();
-                    let index_ref = row_id_index.as_deref();
-
-                    let row_addr = Self::compute_row_addrs(batch.column(rowid_pos), index_ref)?;
-
-                    let mut columns = Vec::with_capacity(batch.num_columns() + 1);
-                    let existing_columns = batch.columns();
-                    columns.extend_from_slice(&existing_columns[..rowaddr_pos]);
-                    columns.push(row_addr);
-                    columns.extend_from_slice(&existing_columns[rowaddr_pos..]);
-
-                    Ok(RecordBatch::try_new(output_schema.clone(), columns)?)
-                }
-            },
-            1,
-            partition,
-            &self.metrics,
-        );
-        Ok(Box::pin(stream))
+            stream,
+        )))
     }
 }
 
@@ -236,10 +242,6 @@ impl ExecutionPlan for AddRowAddrExec {
         "AddRowAddrExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> Arc<Schema> {
         self.output_schema.clone()
     }
@@ -285,8 +287,8 @@ impl ExecutionPlan for AddRowAddrExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> Result<datafusion::physical_plan::Statistics> {
-        let mut stats = self.input.partition_statistics(partition)?;
+    ) -> Result<Arc<datafusion::physical_plan::Statistics>> {
+        let mut stats = Arc::unwrap_or_clone(self.input.partition_statistics(partition)?);
 
         let row_id_col_stats = stats.column_statistics.get(self.rowid_pos).ok_or_else(|| {
             DataFusionError::Internal("RowAddrExec: rowid column stats not found".into())
@@ -321,7 +323,7 @@ impl ExecutionPlan for AddRowAddrExec {
             .column_statistics
             .insert(self.rowaddr_pos, row_addr_col_stats);
 
-        Ok(stats)
+        Ok(Arc::new(stats))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -500,10 +502,6 @@ impl ExecutionPlan for AddRowOffsetExec {
         "AddRowOffsetExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn properties(&self) -> &Arc<PlanProperties> {
         &self.properties
     }
@@ -520,7 +518,7 @@ impl ExecutionPlan for AddRowOffsetExec {
         vec![false]
     }
 
-    fn partition_statistics(&self, partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {
         self.input.partition_statistics(partition)
     }
 
diff --git a/rust/lance/src/io/exec/scalar_index.rs b/rust/lance/src/io/exec/scalar_index.rs
index ade4995fb4b..e67ba3b7b34 100644
--- a/rust/lance/src/io/exec/scalar_index.rs
+++ b/rust/lance/src/io/exec/scalar_index.rs
@@ -3,12 +3,10 @@
 
 use std::sync::{Arc, LazyLock};
 
-use super::utils::{
-    IndexMetrics, InstrumentedChildInputStream, InstrumentedRecordBatchStreamAdapter,
-};
+use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter};
 use crate::{
     Dataset,
-    dataset::rowids::load_row_id_sequences,
+    dataset::rowids::{load_row_id_sequence, load_row_id_sequences},
     index::{
         prefilter::DatasetPreFilter,
         scalar_logical::{open_named_scalar_index, scalar_index_fragment_bitmap},
@@ -22,7 +20,7 @@ use datafusion::{
     physical_plan::{
         DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
         execution_plan::{Boundedness, EmissionType},
-        metrics::{ExecutionPlanMetricsSet, MetricsSet},
+        metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet},
         stream::RecordBatchStreamAdapter,
     },
     scalar::ScalarValue,
@@ -44,7 +42,8 @@ use lance_index::{
     },
 };
 use lance_select::{
-    IndexExprResult, RowAddrMask, RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat,
+    IndexExprResult, NullableIndexExprResult, NullableRowAddrMask, NullableRowAddrSet, RowAddrMask,
+    RowAddrSelection, RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat,
 };
 use lance_table::format::Fragment;
 use roaring::RoaringBitmap;
@@ -60,6 +59,112 @@ impl ScalarIndexLoader for Dataset {
     ) -> Result<Arc<dyn ScalarIndex>> {
         open_named_scalar_index(self, column, index_name, metrics).await
     }
+
+    async fn row_addr_result_to_row_ids(
+        &self,
+        result: NullableIndexExprResult,
+    ) -> Result<NullableIndexExprResult> {
+        // Addresses and row ids only diverge under stable row ids; otherwise the
+        // address is the row id and there is nothing to translate.
+        if !self.manifest.uses_stable_row_ids() {
+            return Ok(result);
+        }
+
+        let NullableIndexExprResult { lower, upper, .. } = result;
+        let lower = translate_addr_mask_to_row_ids(self, lower).await?;
+        let upper = translate_addr_mask_to_row_ids(self, upper).await?;
+        Ok(NullableIndexExprResult::new(lower, upper))
+    }
+}
+
+/// Translate an address-domain [`NullableRowAddrMask`] into the row-id domain
+///
+/// Address-domain index results are always positive allow-lists (`AtMost`), so
+/// a block-list here would mean a boolean op was applied before translation,
+/// which is unsupported.
+async fn translate_addr_mask_to_row_ids(
+    dataset: &Dataset,
+    mask: NullableRowAddrMask,
+) -> Result<NullableRowAddrMask> {
+    match mask {
+        NullableRowAddrMask::AllowList(set) => Ok(NullableRowAddrMask::AllowList(
+            translate_addr_set_to_row_ids(dataset, set).await?,
+        )),
+        NullableRowAddrMask::BlockList(_) => Err(Error::internal(
+            "cannot translate a block-list address mask to the row-id domain",
+        )),
+    }
+}
+
+async fn translate_addr_set_to_row_ids(
+    dataset: &Dataset,
+    set: NullableRowAddrSet,
+) -> Result<NullableRowAddrSet> {
+    let selected = translate_addr_treemap_to_row_ids(dataset, set.selected_rows()).await?;
+    let nulls = translate_addr_treemap_to_row_ids(dataset, set.null_rows()).await?;
+    Ok(NullableRowAddrSet::new(selected, nulls))
+}
+
+/// Map a set of physical row addresses to their stable row ids
+///
+/// For each fragment present in `addrs`, the live rows in physical order carry
+/// the stable ids yielded by the fragment's [`RowIdSequence`] in the same
+/// order. Zipping the two (skipping deleted physical offsets) gives the
+/// `physical offset -> stable id` mapping. Addresses that point at deleted rows
+/// have no live counterpart and are dropped, which is correct: those rows are
+/// not part of the answer.
+async fn translate_addr_treemap_to_row_ids(
+    dataset: &Dataset,
+    addrs: &RowAddrTreeMap,
+) -> Result<RowAddrTreeMap> {
+    let mut row_ids = RowAddrTreeMap::new();
+    for (fragment_id, selection) in addrs.iter() {
+        let file_fragment = dataset.get_fragment(*fragment_id as usize).ok_or_else(|| {
+            Error::internal(format!(
+                "fragment {fragment_id} referenced by an address-domain index result \
+                 was not found in the dataset"
+            ))
+        })?;
+        let sequence = load_row_id_sequence(dataset, file_fragment.metadata()).await?;
+
+        match selection {
+            RowAddrSelection::Full => {
+                // The whole fragment is selected: every live row's id qualifies.
+                row_ids |= RowAddrTreeMap::from(sequence.as_ref());
+            }
+            RowAddrSelection::Partial(offsets) => {
+                let Some(max_offset) = offsets.max() else {
+                    continue;
+                };
+                let (deletion_vector, num_physical_rows) = futures::try_join!(
+                    file_fragment.get_deletion_vector(),
+                    file_fragment.physical_rows()
+                )?;
+                let num_physical_rows = num_physical_rows as u32;
+                let mut ids = sequence.iter();
+                for physical_offset in 0..num_physical_rows {
+                    if physical_offset > max_offset {
+                        break;
+                    }
+                    let deleted = deletion_vector
+                        .as_ref()
+                        .is_some_and(|dv| dv.contains(physical_offset));
+                    if deleted {
+                        continue;
+                    }
+                    match ids.next() {
+                        Some(id) => {
+                            if offsets.contains(physical_offset) {
+                                row_ids.insert(id);
+                            }
+                        }
+                        None => break,
+                    }
+                }
+            }
+        }
+    }
+    Ok(row_ids)
 }
 
 /// An execution node that performs a scalar index search
@@ -105,7 +210,7 @@ impl ScalarIndexExec {
         ));
         Self {
             dataset,
-            expr,
+            expr: expr.optimize(),
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
             result_format,
@@ -116,6 +221,7 @@ impl ScalarIndexExec {
         &self.dataset
     }
 
+    /// The parsed scalar-index expression this node will evaluate.
     pub fn expr(&self) -> &ScalarIndexExpr {
         &self.expr
     }
@@ -183,10 +289,6 @@ impl ExecutionPlan for ScalarIndexExec {
         "ScalarIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.result_format.schema().clone()
     }
@@ -234,11 +336,11 @@ impl ExecutionPlan for ScalarIndexExec {
     fn partition_statistics(
         &self,
         _partition: Option<usize>,
-    ) -> datafusion::error::Result<datafusion::physical_plan::Statistics> {
-        Ok(datafusion::physical_plan::Statistics {
+    ) -> datafusion::error::Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(datafusion::physical_plan::Statistics {
             num_rows: datafusion::common::stats::Precision::Exact(2),
             ..datafusion::physical_plan::Statistics::new_unknown(self.result_format.schema())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -257,14 +359,40 @@ impl ExecutionPlan for ScalarIndexExec {
 pub static INDEX_LOOKUP_SCHEMA: LazyLock<SchemaRef> =
     LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone()])));
 
+/// A single scalar-index lookup used by [`MapIndexExec`].
+///
+/// `column` identifies a column whose values will be probed against the
+/// index named `index_name`. Multiple lookups are intersected with logical
+/// AND semantics inside `MapIndexExec`.
+#[derive(Debug, Clone)]
+pub struct IndexLookup {
+    pub column: String,
+    pub index_name: String,
+}
+
+impl IndexLookup {
+    pub fn new(column: impl Into<String>, index_name: impl Into<String>) -> Self {
+        Self {
+            column: column.into(),
+            index_name: index_name.into(),
+        }
+    }
+}
+
 /// An execution node that translates index values into row addresses
 ///
-/// This can be combined with TakeExec to perform an "indexed take"
+/// This can be combined with TakeExec to perform an "indexed take".
+///
+/// Multiple `(column, index_name)` lookups can be supplied: the operator
+/// expects one input column per lookup (in matching order) and emits the
+/// row addresses where every column's value is present in its respective
+/// index — that is, the AND of the per-column index probes. This lets a
+/// composite-key join trim the candidate row set with every available
+/// scalar index before the downstream take.
 #[derive(Debug)]
 pub struct MapIndexExec {
     dataset: Arc<Dataset>,
-    column_name: String,
-    index_name: String,
+    lookups: Vec<IndexLookup>,
     input: Arc<dyn ExecutionPlan>,
     properties: Arc<PlanProperties>,
     metrics: ExecutionPlanMetricsSet,
@@ -276,19 +404,49 @@ impl DisplayAs for MapIndexExec {
             DisplayFormatType::Default
             | DisplayFormatType::Verbose
             | DisplayFormatType::TreeRender => {
-                write!(f, "IndexedLookup")
+                write!(f, "IndexedLookup")?;
+                if self.lookups.len() > 1 {
+                    let cols = self
+                        .lookups
+                        .iter()
+                        .map(|l| l.column.as_str())
+                        .collect::<Vec<_>>()
+                        .join(", ");
+                    write!(f, " [{cols}]")?;
+                }
+                Ok(())
             }
         }
     }
 }
 
 impl MapIndexExec {
+    /// Convenience constructor for the common single-column case.
     pub fn new(
         dataset: Arc<Dataset>,
         column_name: String,
         index_name: String,
         input: Arc<dyn ExecutionPlan>,
     ) -> Self {
+        Self::new_multi(
+            dataset,
+            vec![IndexLookup::new(column_name, index_name)],
+            input,
+        )
+    }
+
+    /// Build a `MapIndexExec` that probes one or more scalar indices and
+    /// emits the AND of their results. `lookups` must be non-empty and
+    /// `input` must produce one column per lookup, in the same order.
+    pub fn new_multi(
+        dataset: Arc<Dataset>,
+        lookups: Vec<IndexLookup>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Self {
+        debug_assert!(
+            !lookups.is_empty(),
+            "MapIndexExec requires at least one index lookup"
+        );
         let properties = Arc::new(PlanProperties::new(
             EquivalenceProperties::new(INDEX_LOOKUP_SCHEMA.clone()),
             Partitioning::RoundRobinBatch(1),
@@ -297,8 +455,7 @@ impl MapIndexExec {
         ));
         Self {
             dataset,
-            column_name,
-            index_name,
+            lookups,
             input,
             properties,
             metrics: ExecutionPlanMetricsSet::new(),
@@ -309,24 +466,30 @@ impl MapIndexExec {
         input: datafusion::physical_plan::SendableRecordBatchStream,
         partition: usize,
         dataset: Arc<Dataset>,
-        column_name: String,
-        index_name: String,
+        lookups: Vec<IndexLookup>,
         index_metrics: Arc<IndexMetrics>,
         metrics_set: ExecutionPlanMetricsSet,
     ) -> datafusion::error::Result<datafusion::physical_plan::SendableRecordBatchStream> {
-        // Time the one-shot setup (fragment bitmap + deletion mask) so it's
-        // attributed to this node's elapsed_compute. The helper itself only
-        // times per-batch work.
-        let elapsed_compute = datafusion::physical_plan::metrics::MetricBuilder::new(&metrics_set)
-            .elapsed_compute(partition);
-        let setup_start = std::time::Instant::now();
-        let fragment_bitmap = scalar_index_fragment_bitmap(&dataset, &column_name, &index_name)
-            .await?
-            .ok_or_else(|| {
-                datafusion::error::DataFusionError::Internal(format!(
-                    "IndexedLookupExec: index '{index_name}' on column '{column_name}' disappeared after planning"
-                ))
-            })?;
+        // A row can be found by the composite probe only if it lives in a
+        // fragment covered by *every* index in `lookups`; restrict the
+        // deletion mask to that intersection so we only filter deletes we
+        // could actually see.
+        let mut fragment_bitmap: Option<RoaringBitmap> = None;
+        for lookup in &lookups {
+            let bm = scalar_index_fragment_bitmap(&dataset, &lookup.column, &lookup.index_name)
+                .await?
+                .ok_or_else(|| {
+                    datafusion::error::DataFusionError::Internal(format!(
+                        "IndexedLookupExec: index '{}' on column '{}' disappeared after planning",
+                        lookup.index_name, lookup.column,
+                    ))
+                })?;
+            fragment_bitmap = Some(match fragment_bitmap {
+                None => bm,
+                Some(acc) => acc & bm,
+            });
+        }
+        let fragment_bitmap = fragment_bitmap.expect("MapIndexExec built with no lookups");
         let deletion_mask_fut =
             DatasetPreFilter::create_restricted_deletion_mask(dataset.clone(), fragment_bitmap);
         let deletion_mask = if let Some(fut) = deletion_mask_fut {
@@ -334,53 +497,73 @@ impl MapIndexExec {
         } else {
             None
         };
-        elapsed_compute.add_duration(setup_start.elapsed());
 
-        let helper = InstrumentedChildInputStream::new(
-            input,
+        let baseline = BaselineMetrics::new(&metrics_set, partition);
+        let elapsed_compute = baseline.elapsed_compute().clone();
+        let stream = input.then(move |batch_result| {
+            let lookups = lookups.clone();
+            let dataset = dataset.clone();
+            let deletion_mask = deletion_mask.clone();
+            let metrics = index_metrics.clone();
+            let elapsed_compute = elapsed_compute.clone();
+            async move {
+                let batch = batch_result?;
+                // Timer spans `map_batch`'s `.await` on purpose: that await is
+                // the per-batch sargable index evaluation, which is the work
+                // we want attributed here.
+                let _t = elapsed_compute.timer();
+                Self::map_batch(lookups, dataset, deletion_mask, batch, metrics).await
+            }
+        });
+        let stream = stream.map(move |batch| {
+            let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch)));
+            match poll {
+                std::task::Poll::Ready(Some(b)) => b,
+                _ => unreachable!("record_poll preserves Ready(Some) input"),
+            }
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
             INDEX_LOOKUP_SCHEMA.clone(),
-            move |batch| {
-                let column_name = column_name.clone();
-                let index_name = index_name.clone();
-                let dataset = dataset.clone();
-                let deletion_mask = deletion_mask.clone();
-                let metrics = index_metrics.clone();
-                Self::map_batch(
-                    column_name,
-                    index_name,
-                    dataset,
-                    deletion_mask,
-                    batch,
-                    metrics,
-                )
-            },
-            1,
-            partition,
-            &metrics_set,
-        );
-        Ok(Box::pin(helper))
+            stream,
+        )))
+    }
+
+    /// Build the AND-of-IsIn `ScalarIndexExpr` describing this batch's
+    /// composite lookup: each input column contributes one `IsIn` query
+    /// against its matching index.
+    fn build_query(
+        lookups: &[IndexLookup],
+        batch: &RecordBatch,
+    ) -> datafusion::error::Result<ScalarIndexExpr> {
+        let per_column = lookups.iter().enumerate().map(|(idx, lookup)| {
+            let column = batch.column(idx);
+            let values = (0..column.len())
+                .map(|row| ScalarValue::try_from_array(column, row))
+                .collect::<datafusion::error::Result<Vec<_>>>()?;
+            Ok::<_, datafusion::error::DataFusionError>(ScalarIndexExpr::Query(ScalarIndexSearch {
+                column: lookup.column.clone(),
+                index_name: lookup.index_name.clone(),
+                // Internal IndexedLookup-style query — type is unknown at this layer
+                index_type: String::new(),
+                query: Arc::new(SargableQuery::IsIn(values)),
+                needs_recheck: false,
+                fragment_bitmap: None,
+            }))
+        });
+
+        per_column
+            .reduce(|lhs, rhs| Ok(ScalarIndexExpr::And(Box::new(lhs?), Box::new(rhs?))))
+            .expect("MapIndexExec built with no lookups")
     }
 
     async fn map_batch(
-        column_name: String,
-        index_name: String,
+        lookups: Vec<IndexLookup>,
         dataset: Arc<Dataset>,
         deletion_mask: Option<Arc<RowAddrMask>>,
         batch: RecordBatch,
         metrics: Arc<IndexMetrics>,
     ) -> datafusion::error::Result<RecordBatch> {
-        let index_vals = batch.column(0);
-        let index_vals = (0..index_vals.len())
-            .map(|idx| ScalarValue::try_from_array(index_vals, idx))
-            .collect::<datafusion::error::Result<Vec<_>>>()?;
-        let query = ScalarIndexExpr::Query(ScalarIndexSearch {
-            column: column_name,
-            index_name,
-            // Internal IndexedLookup-style query — type is unknown at this layer
-            index_type: String::new(),
-            query: Arc::new(SargableQuery::IsIn(index_vals)),
-            needs_recheck: false,
-        });
+        let query = Self::build_query(&lookups, &batch)?;
         let query_result = query.evaluate(dataset.as_ref(), metrics.as_ref()).await?;
         if !query_result.is_exact() {
             todo!("Support for non-exact query results as input for merge_insert")
@@ -409,10 +592,6 @@ impl ExecutionPlan for MapIndexExec {
         "MapIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         INDEX_LOOKUP_SCHEMA.clone()
     }
@@ -430,10 +609,9 @@ impl ExecutionPlan for MapIndexExec {
                 "MapIndexExec requires exactly one child".to_string(),
             ))
         } else {
-            Ok(Arc::new(Self::new(
+            Ok(Arc::new(Self::new_multi(
                 self.dataset.clone(),
-                self.column_name.clone(),
-                self.index_name.clone(),
+                self.lookups.clone(),
                 children.into_iter().next().unwrap(),
             )))
         }
@@ -449,8 +627,7 @@ impl ExecutionPlan for MapIndexExec {
             input,
             partition,
             self.dataset.clone(),
-            self.column_name.clone(),
-            self.index_name.clone(),
+            self.lookups.clone(),
             Arc::new(IndexMetrics::new(&self.metrics, partition)),
             self.metrics.clone(),
         );
@@ -698,10 +875,6 @@ impl ExecutionPlan for MaterializeIndexExec {
         "MaterializeIndexExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         MATERIALIZE_INDEX_SCHEMA.clone()
     }
@@ -771,13 +944,15 @@ mod tests {
 
     use crate::index::DatasetIndexExt;
     use arrow::datatypes::UInt64Type;
+    use arrow::record_batch::RecordBatchIterator;
+    use arrow_array::{ArrayRef, Int32Array, RecordBatch};
     use arrow_schema::Schema;
     use datafusion::{
         execution::TaskContext, physical_plan::ExecutionPlan, prelude::SessionConfig,
         scalar::ScalarValue,
     };
     use futures::TryStreamExt;
-    use lance_core::utils::tempfile::TempStrDir;
+    use lance_core::utils::{address::RowAddress, tempfile::TempStrDir};
     use lance_datagen::gen_batch;
     use lance_index::{
         IndexType,
@@ -786,10 +961,11 @@ mod tests {
             expression::{ScalarIndexExpr, ScalarIndexSearch},
         },
     };
-    use lance_select::result::IndexExprResultWireFormat;
+    use lance_select::{RowAddrTreeMap, result::IndexExprResultWireFormat};
 
     use crate::{
         Dataset,
+        dataset::WriteParams,
         io::exec::scalar_index::MaterializeIndexExec,
         utils::test::{DatagenExt, FragmentCount, FragmentRowCount, NoContextTestFixture},
     };
@@ -848,8 +1024,8 @@ mod tests {
                 Bound::Excluded(ScalarValue::UInt64(Some(47))),
             )),
             needs_recheck: false,
+            fragment_bitmap: None,
         });
-
         let fragments = dataset.fragments().clone();
 
         let plan = MaterializeIndexExec::new(dataset, query, fragments);
@@ -870,6 +1046,51 @@ mod tests {
         assert_eq!(batches[0].num_rows(), 5);
     }
 
+    #[tokio::test]
+    async fn test_translate_addr_treemap_to_stable_row_ids() {
+        let test_dir = TempStrDir::default();
+        let batch = RecordBatch::try_from_iter(vec![(
+            "id",
+            Arc::new(Int32Array::from((0..10).collect::<Vec<_>>())) as ArrayRef,
+        )])
+        .unwrap();
+        let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
+        let write_params = WriteParams {
+            enable_stable_row_ids: true,
+            max_rows_per_file: 5,
+            ..Default::default()
+        };
+        let dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params))
+            .await
+            .unwrap();
+        let fragment_id = dataset.get_fragments()[1].id() as u32;
+
+        let mut full_fragment = RowAddrTreeMap::new();
+        full_fragment.insert_fragment(fragment_id);
+        let translated = super::translate_addr_treemap_to_row_ids(&dataset, &full_fragment)
+            .await
+            .unwrap();
+        let row_ids = translated
+            .get_fragment_bitmap(0)
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>();
+        assert_eq!(row_ids, vec![5, 6, 7, 8, 9]);
+
+        let mut partial_fragment = RowAddrTreeMap::new();
+        partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 1).into());
+        partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 3).into());
+        let translated = super::translate_addr_treemap_to_row_ids(&dataset, &partial_fragment)
+            .await
+            .unwrap();
+        let row_ids = translated
+            .get_fragment_bitmap(0)
+            .unwrap()
+            .iter()
+            .collect::<Vec<_>>();
+        assert_eq!(row_ids, vec![6, 8]);
+    }
+
     /// `ScalarIndexExec::schema()` (and the stream it emits) must advertise
     /// the same schema the batch actually carries — otherwise downstream
     /// consumers that trust `ExecutionPlan::schema()` will see a different
@@ -892,6 +1113,7 @@ mod tests {
                 Bound::Excluded(ScalarValue::UInt64(Some(47))),
             )),
             needs_recheck: false,
+            fragment_bitmap: None,
         });
 
         let verify = async |plan: ScalarIndexExec, schema: Arc<Schema>| {
@@ -943,6 +1165,7 @@ mod tests {
                 Bound::Excluded(ScalarValue::UInt64(Some(47))),
             )),
             needs_recheck: false,
+            fragment_bitmap: None,
         });
 
         // These plans aren't even valid but it appears we defer all work (even validation) until
diff --git a/rust/lance/src/io/exec/scan.rs b/rust/lance/src/io/exec/scan.rs
index 15d8d181eab..a8c3b2e3dc3 100644
--- a/rust/lance/src/io/exec/scan.rs
+++ b/rust/lance/src/io/exec/scan.rs
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
-use std::any::Any;
 use std::ops::Range;
 use std::pin::Pin;
 use std::sync::Arc;
@@ -26,7 +25,10 @@ use futures::{StreamExt, TryStreamExt};
 use lance_arrow::SchemaExt;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::utils::tracing::StreamTracingExt;
-use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD};
+use lance_core::{
+    Error, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID_FIELD,
+    ROW_LAST_UPDATED_AT_VERSION_FIELD,
+};
 use lance_file::reader::FileReaderOptions;
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
 use lance_table::format::Fragment;
@@ -192,7 +194,36 @@ impl LanceStream {
     ) -> Result<Self> {
         let scan_metrics = ScanMetrics::new(metrics, partition);
         let timer = scan_metrics.baseline_metrics.elapsed_compute().timer();
-        let project_schema = projection.clone();
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(projection.as_ref());
+        let read_projection = if materialize_blob_v2_binary {
+            Arc::new(crate::dataset::blob::blob_v2_descriptor_schema(
+                projection.as_ref(),
+            ))
+        } else {
+            projection.clone()
+        };
+        let project_schema = read_projection;
+        let output_projection = if materialize_blob_v2_binary {
+            let mut output_projection = projection.as_ref().clone();
+            let mut system_fields = Vec::with_capacity(4);
+            if config.with_row_id {
+                system_fields.push(ROW_ID_FIELD.clone());
+            }
+            if config.with_row_address {
+                system_fields.push(ROW_ADDR_FIELD.clone());
+            }
+            if config.with_row_last_updated_at_version {
+                system_fields.push(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone());
+            }
+            if config.with_row_created_at_version {
+                system_fields.push(ROW_CREATED_AT_VERSION_FIELD.clone());
+            }
+            output_projection.extend(&system_fields)?;
+            Arc::new(output_projection)
+        } else {
+            projection.clone()
+        };
         let io_parallelism = dataset.object_store.io_parallelism();
         // First, use the value specified by the user in the call
         // Second, use the default from the environment variable, if specified
@@ -275,12 +306,14 @@ impl LanceStream {
 
         let scan_scheduler_clone = scan_scheduler.clone();
 
+        let materialize_dataset = dataset;
         let config_for_stream = config.clone();
         let batches = stream::iter(file_fragments.into_iter().enumerate())
             .map(move |(priority, file_fragment)| {
                 let project_schema = project_schema.clone();
                 let scan_scheduler = scan_scheduler.clone();
                 let config = config_for_stream.clone();
+                let force_row_address = materialize_blob_v2_binary;
                 #[allow(clippy::type_complexity)]
                 let frag_task: BoxFuture<
                     Result<BoxStream<Result<BoxFuture<Result<RecordBatch>>>>>,
@@ -288,7 +321,7 @@ impl LanceStream {
                     (async move {
                         let mut frag_config = FragReadConfig::default()
                             .with_row_id(config.with_row_id)
-                            .with_row_address(config.with_row_address)
+                            .with_row_address(config.with_row_address || force_row_address)
                             .with_row_last_updated_at_version(
                                 config.with_row_last_updated_at_version,
                             )
@@ -342,9 +375,32 @@ impl LanceStream {
             // TODO: Ideally this will eventually get tied into datafusion as a # of partitions.  This will let
             // us fully fuse decode into the first half of the plan.  Currently there is likely to be a thread
             // transfer between the two steps.
-            .try_buffered(get_num_compute_intensive_cpus())
+            .try_buffered(
+                get_num_compute_intensive_cpus()
+                    .min(config.parallelism_cap.unwrap_or(usize::MAX))
+                    .max(1),
+            )
             .stream_in_current_span()
             .boxed();
+        let inner_stream = if materialize_blob_v2_binary {
+            inner_stream
+                .and_then(move |batch| {
+                    let dataset = materialize_dataset.clone();
+                    let output_projection = output_projection.clone();
+                    async move {
+                        crate::dataset::blob::materialize_blob_v2_binary_batch(
+                            &dataset,
+                            output_projection.as_ref(),
+                            batch,
+                        )
+                        .await
+                        .map_err(DataFusionError::from)
+                    }
+                })
+                .boxed()
+        } else {
+            inner_stream
+        };
 
         timer.done();
         Ok(Self {
@@ -371,9 +427,13 @@ impl LanceStream {
         let fragment_readahead = config
             .fragment_readahead
             .unwrap_or(LEGACY_DEFAULT_FRAGMENT_READAHEAD);
+        let batch_readahead = config
+            .batch_readahead
+            .min(config.parallelism_cap.unwrap_or(usize::MAX))
+            .max(1);
         debug!(
             "Scanning v1 dataset with frag_readahead={} and batch_readahead={}",
-            fragment_readahead, config.batch_readahead
+            fragment_readahead, batch_readahead
         );
 
         let file_fragments = fragments
@@ -410,7 +470,7 @@ impl LanceStream {
                 // We must be waiting to finish a file before moving onto thenext. That's an issue.
                 .try_flatten()
                 // We buffer up to `batch_readahead` batches across all streams.
-                .try_buffered(config.batch_readahead)
+                .try_buffered(batch_readahead)
                 .stream_in_current_span()
                 .boxed()
         } else {
@@ -443,7 +503,7 @@ impl LanceStream {
             tasks
                 .try_flatten_unordered(config.fragment_readahead)
                 // We buffer up to `batch_readahead` batches across all streams.
-                .try_buffer_unordered(config.batch_readahead)
+                .try_buffer_unordered(batch_readahead)
                 .stream_in_current_span()
                 .boxed()
         };
@@ -474,7 +534,9 @@ impl core::fmt::Debug for LanceStream {
 
 impl RecordBatchStream for LanceStream {
     fn schema(&self) -> SchemaRef {
-        let mut schema: ArrowSchema = self.projection.as_ref().into();
+        let output_projection =
+            crate::dataset::blob::public_blob_v2_binary_output_schema(self.projection.as_ref());
+        let mut schema: ArrowSchema = (&output_projection).into();
         if self.config.with_row_id {
             schema = schema.try_with_column(ROW_ID_FIELD.clone()).unwrap();
         }
@@ -508,6 +570,9 @@ pub struct LanceScanConfig {
     pub with_make_deletions_null: bool,
     pub ordered_output: bool,
     pub file_reader_options: Option<FileReaderOptions>,
+    /// Upper bound on frag_parallelism and CPU decode concurrency. Set from
+    /// DataFusion's `target_partitions` session config in `LanceScanExec::execute`.
+    pub parallelism_cap: Option<usize>,
 }
 
 // This is mostly for testing purposes, end users are unlikely to create this
@@ -526,6 +591,7 @@ impl Default for LanceScanConfig {
             with_make_deletions_null: false,
             ordered_output: false,
             file_reader_options: None,
+            parallelism_cap: None,
         }
     }
 }
@@ -590,7 +656,9 @@ impl LanceScanExec {
         projection: Arc<Schema>,
         config: LanceScanConfig,
     ) -> Self {
-        let mut output_schema: ArrowSchema = projection.as_ref().into();
+        let output_projection =
+            crate::dataset::blob::public_blob_v2_binary_output_schema(projection.as_ref());
+        let mut output_schema: ArrowSchema = (&output_projection).into();
 
         if config.with_row_id {
             output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap();
@@ -661,10 +729,6 @@ impl ExecutionPlan for LanceScanExec {
         "LanceScanExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -690,13 +754,17 @@ impl ExecutionPlan for LanceScanExec {
     fn execute(
         &self,
         partition: usize,
-        _context: Arc<datafusion::execution::context::TaskContext>,
+        context: Arc<datafusion::execution::context::TaskContext>,
     ) -> Result<SendableRecordBatchStream> {
         let dataset = self.dataset.clone();
         let fragments = self.fragments.clone();
         let range = self.range.clone();
         let projection = self.projection.clone();
-        let config = self.config.clone();
+        let target_partitions = context.session_config().target_partitions();
+        let config = LanceScanConfig {
+            parallelism_cap: Some(target_partitions),
+            ..self.config.clone()
+        };
         let metrics = self.metrics.clone();
 
         let lance_fut_stream = stream::once(async move {
@@ -711,7 +779,7 @@ impl ExecutionPlan for LanceScanExec {
         )))
     }
 
-    fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {
+    fn partition_statistics(&self, _partition: Option<usize>) -> Result<Arc<Statistics>> {
         // Some fragments from older datasets might have the row count stats missing.
         let (row_count, is_exact) =
             self.fragments
@@ -728,10 +796,10 @@ impl ExecutionPlan for LanceScanExec {
             false => Precision::Absent,
         };
 
-        Ok(Statistics {
+        Ok(Arc::new(Statistics {
             num_rows,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn metrics(&self) -> Option<MetricsSet> {
@@ -750,6 +818,9 @@ impl ExecutionPlan for LanceScanExec {
 #[cfg(test)]
 mod tests {
     use datafusion::execution::TaskContext;
+    use datafusion::prelude::SessionConfig;
+    use futures::TryStreamExt;
+    use lance_datagen::gen_batch;
 
     use crate::utils::test::NoContextTestFixture;
 
@@ -772,4 +843,47 @@ mod tests {
 
         scan.execute(0, Arc::new(TaskContext::default())).unwrap();
     }
+
+    /// Verify that executing with target_partitions=1 produces the same row count as the
+    /// default context. Regression guard for the parallelism cap.
+    #[tokio::test]
+    async fn test_target_partitions_cap_produces_correct_results() {
+        use lance_core::utils::tempfile::TempStrDir;
+        use lance_datagen::{Dimension, array};
+
+        use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount};
+
+        let tmp = TempStrDir::default();
+        let dataset = gen_batch()
+            .col("x", array::step::<arrow_array::types::Int32Type>())
+            .col(
+                "v",
+                array::rand_vec::<arrow_array::types::Float32Type>(Dimension::from(4)),
+            )
+            .into_dataset(
+                tmp.as_str(),
+                FragmentCount::from(4),
+                FragmentRowCount::from(100),
+            )
+            .await
+            .unwrap();
+        let dataset = Arc::new(dataset);
+
+        let scan = LanceScanExec::new(
+            dataset.clone(),
+            dataset.fragments().clone(),
+            None,
+            Arc::new(dataset.schema().clone()),
+            LanceScanConfig::default(),
+        );
+
+        let low_ctx = Arc::new(
+            TaskContext::default()
+                .with_session_config(SessionConfig::default().with_target_partitions(1)),
+        );
+        let stream = scan.execute(0, low_ctx).unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
+        assert_eq!(total_rows, 400);
+    }
 }
diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs
index 977a9c88dce..bd95812984d 100644
--- a/rust/lance/src/io/exec/take.rs
+++ b/rust/lance/src/io/exec/take.rs
@@ -4,6 +4,7 @@
 use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, Mutex};
+use std::task::Poll;
 
 use arrow::array::AsArray;
 use arrow::compute::{TakeOptions, concat_batches};
@@ -27,6 +28,7 @@ use lance_arrow::RecordBatchExt;
 use lance_core::datatypes::{Field, OnMissing, Projection};
 use lance_core::error::{DataFusionResult, LanceOptionExt};
 use lance_core::utils::address::RowAddress;
+use lance_core::utils::futures::FinallyStreamExt;
 use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use lance_core::{ROW_ADDR, ROW_ID};
 use lance_io::scheduler::{ScanScheduler, SchedulerConfig};
@@ -68,6 +70,10 @@ struct TakeStream {
     dataset: Arc<Dataset>,
     /// The fields to take from the input stream
     fields_to_take: Arc<Schema>,
+    /// The descriptor-view schema used for storage reads when blob payloads
+    /// must be materialized after take.
+    read_fields: Arc<Schema>,
+    materialize_blob_v2_binary: bool,
     /// The output schema, needed for us to merge the new columns
     /// into the input data in the correct order
     output_schema: SchemaRef,
@@ -90,9 +96,20 @@ impl TakeStream {
         metrics: &ExecutionPlanMetricsSet,
         partition: usize,
     ) -> Self {
+        let materialize_blob_v2_binary =
+            crate::dataset::blob::schema_has_blob_v2_binary_view(fields_to_take.as_ref());
+        let read_fields = if materialize_blob_v2_binary {
+            Arc::new(crate::dataset::blob::blob_v2_descriptor_schema(
+                fields_to_take.as_ref(),
+            ))
+        } else {
+            fields_to_take.clone()
+        };
         Self {
             dataset,
             fields_to_take,
+            read_fields,
+            materialize_blob_v2_binary,
             output_schema,
             readers_cache: Arc::new(Mutex::new(HashMap::new())),
             scan_scheduler,
@@ -129,14 +146,12 @@ impl TakeStream {
                 ))
             })?;
 
-        let reader = Arc::new(
-            fragment
-                .open(
-                    &self.fields_to_take,
-                    FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()),
-                )
-                .await?,
-        );
+        let mut read_config =
+            FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone());
+        if self.materialize_blob_v2_binary {
+            read_config = read_config.with_row_address(true);
+        }
+        let reader = Arc::new(fragment.open(&self.read_fields, read_config).await?);
 
         let mut readers = self.readers_cache.lock().unwrap();
         readers.insert(fragment_id, reader.clone());
@@ -353,10 +368,15 @@ impl TakeStream {
             (None, None) => {}
         }
 
-        self.metrics
-            .baseline_metrics
-            .record_output(new_data.num_rows());
-        self.metrics.batches_processed.add(1);
+        if self.materialize_blob_v2_binary {
+            new_data = crate::dataset::blob::materialize_blob_v2_binary_batch(
+                &self.dataset,
+                self.fields_to_take.as_ref(),
+                new_data,
+            )
+            .await?;
+        }
+
         Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?)
     }
 
@@ -364,8 +384,10 @@ impl TakeStream {
         self: Arc<Self>,
         input: S,
     ) -> impl Stream<Item = Result<RecordBatch>> {
-        let scan_scheduler = self.scan_scheduler.clone();
-        let metrics = self.metrics.clone();
+        let result_scan_scheduler = self.scan_scheduler.clone();
+        let final_scan_scheduler = self.scan_scheduler.clone();
+        let result_metrics = self.metrics.clone();
+        let final_metrics = self.metrics.clone();
         let batches = input
             .enumerate()
             .map(move |(batch_index, batch)| {
@@ -378,8 +400,24 @@ impl TakeStream {
             })
             .boxed();
         batches
-            .inspect_ok(move |_| metrics.io_metrics.record(&scan_scheduler))
             .try_buffered(get_num_compute_intensive_cpus())
+            .map(move |result| {
+                if result.is_ok() {
+                    result_metrics.batches_processed.add(1);
+                }
+                result_metrics.io_metrics.record(&result_scan_scheduler);
+                match result_metrics
+                    .baseline_metrics
+                    .record_poll(Poll::Ready(Some(result)))
+                {
+                    Poll::Ready(Some(result)) => result,
+                    _ => unreachable!("record_poll returned a different poll state"),
+                }
+            })
+            .finally(move || {
+                final_metrics.baseline_metrics.done();
+                final_metrics.io_metrics.record(&final_scan_scheduler);
+            })
     }
 }
 
@@ -471,10 +509,10 @@ impl TakeExec {
             projection
         );
 
-        let output_schema = Arc::new(Self::calculate_output_schema(
-            dataset.schema(),
-            &input.schema(),
-            &projection,
+        let output_schema =
+            Self::calculate_output_schema(dataset.schema(), &input.schema(), &projection);
+        let output_schema = Arc::new(crate::dataset::blob::public_blob_v2_binary_output_schema(
+            &output_schema,
         ));
         let output_arrow = Arc::new(ArrowSchema::from(output_schema.as_ref()));
         let properties = Arc::new(
@@ -560,10 +598,6 @@ impl ExecutionPlan for TakeExec {
         "TakeExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> SchemaRef {
         self.output_schema.clone()
     }
@@ -647,11 +681,11 @@ impl ExecutionPlan for TakeExec {
     fn partition_statistics(
         &self,
         partition: Option<usize>,
-    ) -> Result<datafusion::physical_plan::Statistics> {
-        Ok(Statistics {
+    ) -> Result<Arc<datafusion::physical_plan::Statistics>> {
+        Ok(Arc::new(Statistics {
             num_rows: self.input.partition_statistics(partition)?.num_rows,
             ..Statistics::new_unknown(self.schema().as_ref())
-        })
+        }))
     }
 
     fn properties(&self) -> &Arc<PlanProperties> {
@@ -839,6 +873,80 @@ mod tests {
         }
     }
 
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_take_records_output_and_io_metrics() {
+        use datafusion::physical_plan::metrics::MetricValue;
+        use lance_datafusion::utils::{BYTES_READ_METRIC, IOPS_METRIC, REQUESTS_METRIC};
+        let TestFixture {
+            dataset,
+            _tmp_dir_guard,
+        } = test_fixture().await;
+
+        let row_addrs = UInt64Array::from(vec![0_u64, 1, 2, 3, 4]);
+        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
+            ROW_ADDR,
+            DataType::UInt64,
+            true,
+        )]));
+        let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(row_addrs)]).unwrap();
+        let stream = futures::stream::iter(vec![Ok(batch)]);
+        let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+        let input = Arc::new(OneShotExec::new(stream));
+
+        let projection = dataset
+            .empty_projection()
+            .union_column("s", OnMissing::Error)
+            .unwrap();
+
+        let take_exec = TakeExec::try_new(dataset, input, projection)
+            .unwrap()
+            .unwrap();
+
+        let stream = take_exec
+            .execute(0, Arc::new(TaskContext::default()))
+            .unwrap();
+        let batches: Vec<RecordBatch> = stream.try_collect().await.unwrap();
+        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 5);
+
+        let metrics = take_exec.metrics().unwrap();
+
+        let output_batches: usize = metrics
+            .iter()
+            .filter_map(|m| match m.value() {
+                MetricValue::OutputBatches(count) => Some(count.value()),
+                _ => None,
+            })
+            .sum();
+
+        let output_bytes: usize = metrics
+            .iter()
+            .filter_map(|m| match m.value() {
+                MetricValue::OutputBytes(count) => Some(count.value()),
+                _ => None,
+            })
+            .sum();
+
+        let gauge = |name: &str| -> usize {
+            metrics
+                .iter_gauges()
+                .find_map(|(metric_name, gauge)| {
+                    (metric_name.as_ref() == name).then(|| gauge.value())
+                })
+                .unwrap_or(0)
+        };
+
+        let bytes_read = gauge(BYTES_READ_METRIC);
+        let iops = gauge(IOPS_METRIC);
+        let requests = gauge(REQUESTS_METRIC);
+
+        assert_eq!(metrics.output_rows(), Some(5));
+        assert_eq!(metrics.find_count("batches_processed").unwrap().value(), 1);
+        assert!(
+            output_batches > 0 && output_bytes > 0 && bytes_read > 0 && iops > 0 && requests > 0,
+            "expected positive TakeExec metrics, got output_batches={output_batches}, output_bytes={output_bytes}, bytes_read={bytes_read}, iops={iops}, requests={requests}"
+        );
+    }
+
     #[tokio::test]
     async fn test_take_order() {
         let TestFixture {
diff --git a/rust/lance/src/io/exec/testing.rs b/rust/lance/src/io/exec/testing.rs
index 2d5911a4e46..f979c403431 100644
--- a/rust/lance/src/io/exec/testing.rs
+++ b/rust/lance/src/io/exec/testing.rs
@@ -4,7 +4,6 @@
 //! Testing Node
 //!
 
-use std::any::Any;
 use std::sync::Arc;
 
 use arrow_array::RecordBatch;
@@ -51,10 +50,6 @@ impl ExecutionPlan for TestingExec {
         "TestingExec"
     }
 
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.batches[0].schema()
     }
diff --git a/rust/lance/src/io/exec/utils.rs b/rust/lance/src/io/exec/utils.rs
index 5def0fb254d..1af0bc3f4ef 100644
--- a/rust/lance/src/io/exec/utils.rs
+++ b/rust/lance/src/io/exec/utils.rs
@@ -6,9 +6,10 @@ use lance_datafusion::utils::{
     IOPS_METRIC, PARTS_LOADED_METRIC, REQUESTS_METRIC,
 };
 use lance_index::metrics::MetricsCollector;
-use lance_io::scheduler::ScanScheduler;
+use lance_io::scheduler::{IoStats, ScanScheduler, ScanStats};
 use lance_table::format::IndexMetadata;
 use pin_project::pin_project;
+use std::future::Future;
 use std::pin::Pin;
 use std::sync::{Arc, Mutex};
 use std::task::{Context, Poll};
@@ -30,7 +31,6 @@ use lance_core::utils::futures::{Capacity, SharedStreamExt};
 use lance_core::{ROW_ID, Result};
 use lance_index::prefilter::FilterLoader;
 use lance_select::{RowAddrMask, RowAddrTreeMap, result::IndexExprResult};
-use std::future::Future;
 
 use crate::Dataset;
 use crate::index::prefilter::DatasetPreFilter;
@@ -300,7 +300,7 @@ where
 /// applies a per-batch async transform.
 ///
 /// `elapsed_compute` measures only the time spent driving the transform
-/// futures — never the time spent polling the child input — so wrapping a
+/// futures -- never the time spent polling the child input -- so wrapping a
 /// chain of nodes does not double-count child CPU. `output_rows` and
 /// `output_batches` are recorded as the transform produces batches.
 ///
@@ -363,7 +363,7 @@ where
         let this = self.get_mut();
 
         // Fill in-flight transforms up to `concurrency` from the input.
-        // Polling the input does NOT count toward `elapsed_compute`.
+        // Polling the input does not count toward `elapsed_compute`.
         while !this.input_done && this.in_flight.len() < this.concurrency {
             match this.input.poll_next_unpin(cx) {
                 Poll::Ready(Some(Ok(batch))) => {
@@ -379,7 +379,7 @@ where
             }
         }
 
-        // Drive in-flight transforms; their poll time IS counted.
+        // Drive in-flight transforms; their poll time is counted.
         if !this.in_flight.is_empty() {
             let timer = this.baseline_metrics.elapsed_compute().timer();
             let poll = this.in_flight.poll_next_unpin(cx);
@@ -391,13 +391,9 @@ where
                     }
                     return this.baseline_metrics.record_poll(Poll::Ready(Some(result)));
                 }
-                // Unreachable: `FuturesUnordered::poll_next` returns
-                // `Ready(None)` only when empty, and we just checked
-                // `!is_empty` above. A panic here is preferable to a
-                // silent infinite loop if the invariant ever breaks.
-                Poll::Ready(None) => {
-                    unreachable!("FuturesUnordered yielded None while non-empty")
-                }
+                // `FuturesUnordered::poll_next` returns `Ready(None)` only
+                // when empty, and we just checked `!is_empty` above.
+                Poll::Ready(None) => unreachable!("non-empty transform queue yielded None"),
                 Poll::Pending => return Poll::Pending,
             }
         }
@@ -425,10 +421,6 @@ impl ExecutionPlan for ReplayExec {
         "ReplayExec"
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn schema(&self) -> arrow_schema::SchemaRef {
         self.input.schema()
     }
@@ -506,12 +498,17 @@ impl IoMetrics {
     }
 
     pub fn record(&self, scan_scheduler: &ScanScheduler) {
-        let current_stats = scan_scheduler.stats();
+        self.record_stats(scan_scheduler.stats());
+    }
 
-        // Use set_max to ensure gauge always shows the highest value seen
-        self.iops.set_max(current_stats.iops as usize);
-        self.requests.set_max(current_stats.requests as usize);
-        self.bytes_read.set_max(current_stats.bytes_read as usize);
+    /// Record a snapshot of cumulative I/O statistics.
+    ///
+    /// Uses `set_max` because the underlying counters are cumulative; the gauge
+    /// always reflects the highest (i.e. final) value seen.
+    pub fn record_stats(&self, stats: ScanStats) {
+        self.iops.set_max(stats.iops as usize);
+        self.requests.set_max(stats.requests as usize);
+        self.bytes_read.set_max(stats.bytes_read as usize);
     }
 }
 
@@ -520,6 +517,12 @@ pub struct IndexMetrics {
     indices_loaded: Count,
     parts_loaded: Count,
     index_comparisons: Count,
+    /// Per-query sink that accumulates exact index-file I/O as partitions are
+    /// loaded from storage.  Shared by all clones of this `IndexMetrics`, so
+    /// concurrent partition loads all funnel into the same counters.  Published
+    /// to `io_metrics` for display via [`IndexMetrics::flush_io`].
+    io_stats: IoStats,
+    io_metrics: IoMetrics,
 }
 
 impl IndexMetrics {
@@ -528,8 +531,18 @@ impl IndexMetrics {
             indices_loaded: metrics.new_count(INDICES_LOADED_METRIC, partition),
             parts_loaded: metrics.new_count(PARTS_LOADED_METRIC, partition),
             index_comparisons: metrics.new_count(INDEX_COMPARISONS_METRIC, partition),
+            io_stats: IoStats::new(),
+            io_metrics: IoMetrics::new(metrics, partition),
         }
     }
+
+    /// Publish the I/O accumulated in the per-query sink to the displayed
+    /// `iops`/`requests`/`bytes_read` metrics.  Call once when the operator's
+    /// stream finishes; the sink only accumulates on cache misses, so a fully
+    /// cache-resident query publishes zeros.
+    pub fn flush_io(&self) {
+        self.io_metrics.record_stats(self.io_stats.snapshot());
+    }
 }
 
 impl MetricsCollector for IndexMetrics {
@@ -542,6 +555,9 @@ impl MetricsCollector for IndexMetrics {
     fn record_comparisons(&self, num_comparisons: usize) {
         self.index_comparisons.add(num_comparisons);
     }
+    fn io_stats(&self) -> Option<IoStats> {
+        Some(self.io_stats.clone())
+    }
 }
 
 #[cfg(test)]
@@ -579,11 +595,7 @@ mod tests {
 
         let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
         let n_batches: usize = 3;
-        // child_delay is intentionally several times larger than transform_delay
-        // so the assertion tolerates significant `std::thread::sleep` overshoot
-        // on busy CI runners (we've seen ~2-3x overshoot on macOS Actions).
         let child_delay = Duration::from_millis(150);
-        let transform_delay = Duration::from_millis(30);
 
         let counter = Arc::new(AtomicUsize::new(0));
         let s = schema.clone();
@@ -607,10 +619,7 @@ mod tests {
         let stream = InstrumentedChildInputStream::new(
             child,
             schema,
-            move |batch| async move {
-                std::thread::sleep(transform_delay);
-                Ok(batch)
-            },
+            move |batch| async move { Ok(batch) },
             1,
             0,
             &metrics,
@@ -625,16 +634,10 @@ mod tests {
             .expect("elapsed_compute should be recorded");
         let elapsed = Duration::from_nanos(elapsed_ns as u64);
 
-        // Expect ~ transform_delay * n. The upper bound is set generously to
-        // absorb sleep overshoot on slow CI (~4-5x per call) while still
-        // cleanly rejecting any version that double-counts child poll time,
-        // which would yield ~ (transform_delay + child_delay) * n.
-        let upper = Duration::from_millis(400);
-        assert!(
-            elapsed >= transform_delay * (n_batches as u32 - 1),
-            "elapsed_compute={:?} too low; transform time was not measured",
-            elapsed,
-        );
+        // The transform is immediate, so `elapsed_compute` should stay well
+        // below even one child poll delay. A version that double-counts child
+        // input time would include roughly `child_delay * n_batches`.
+        let upper = child_delay;
         assert!(
             elapsed < upper,
             "elapsed_compute={:?} >= {:?}; child input time was double-counted",
diff --git a/rust/lance/src/lib.rs b/rust/lance/src/lib.rs
index 284e10a9b6f..66bc876bd88 100644
--- a/rust/lance/src/lib.rs
+++ b/rust/lance/src/lib.rs
@@ -81,6 +81,8 @@ pub mod datafusion;
 pub mod dataset;
 pub mod index;
 pub mod io;
+#[cfg(feature = "metrics")]
+pub mod metrics;
 pub mod session;
 pub mod table;
 pub mod utils;
@@ -90,7 +92,11 @@ pub mod pb {
     include!(concat!(env!("OUT_DIR"), "/lance.pb.rs"));
 }
 
-pub use blob::{BlobArrayBuilder, blob_field};
+pub use blob::{
+    BlobArrayBuilder, BlobDescriptor, BlobDescriptorArrayBuilder, BlobDescriptorColumn,
+    BlobFieldOptions, BlobRange, DedicatedBlobWriter, PackedBlobWriter, blob_field,
+    blob_field_with_options,
+};
 pub use dataset::Dataset;
 use lance_index::vector::DIST_COL;
 
diff --git a/rust/lance/src/metrics.md b/rust/lance/src/metrics.md
new file mode 100644
index 00000000000..05d9df0c0e3
--- /dev/null
+++ b/rust/lance/src/metrics.md
@@ -0,0 +1,40 @@
+Lance publishes metrics through the [`metrics`](https://docs.rs/metrics) crate
+facade. Install any recorder (Prometheus, OpenTelemetry, etc.) in your
+application and Lance will emit into it; when no recorder is installed, emission
+is a cheap no-op. Metrics are only emitted when Lance is built with the
+`metrics` feature.
+
+## Object store metrics
+
+These track I/O against the underlying object store. The `base` label
+identifies the store; its cardinality is controlled by the
+`LANCE_OBJECT_STORE_METRICS_LABEL` environment variable:
+
+- `scheme` (default) — the scheme only (`s3`, `gs`, `az`, `file`, `memory`);
+  low, bounded cardinality.
+- `full` — the store's unique prefix (`s3$my-bucket`, `az$container@account`
+  where Azure's account also matters), so multiple buckets on the same cloud
+  can be told apart. Cardinality grows with the number of stores accessed.
+- `off` — omit the `base` label entirely.
+
+`operation` is one of `get`, `put`, `put_part`, `head`, `list`, `delete`,
+`copy`, `rename`, `complete_multipart`, or `abort_multipart`.
+
+Request counts are per logical operation: a `list` or `delete` that spans many
+objects is one request, matching how backends batch them.
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `lance_object_store_requests_total` | counter | `operation`, `base` | Object store requests issued. |
+| `lance_object_store_request_bytes_total` | counter | `operation`, `base` | Bytes transferred by `get`/`put` requests. A `get` is counted once its response body has been fully read. |
+| `lance_object_store_request_duration_seconds` | histogram | `operation`, `base` | Per-request latency, in seconds. For `get` this covers the full body transfer, not just time-to-first-byte. |
+| `lance_object_store_errors_total` | counter | `operation`, `base` | Requests that returned an error. |
+| `lance_object_store_in_flight_requests` | gauge | `operation`, `base` | Requests currently in flight. |
+| `lance_object_store_throttle_total` | counter | `status`, `base` | Throttle responses (HTTP 429 / 503) seen at the HTTP layer, counted per attempt including retries. The `status` label is the numeric HTTP status. |
+| `lance_object_store_retryable_responses_total` | counter | `status`, `base` | Retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, counted per attempt including retries. A superset of `throttle_total`; 409 (conflict) is excluded so commit conflicts are not counted. |
+
+`lance_object_store_throttle_total` and
+`lance_object_store_retryable_responses_total` are recorded only for the native
+cloud stores (S3, GCS, Azure); Opendal-backed stores bypass the HTTP client
+where the counters are installed, so they report the other object store metrics
+but not throttle/retryable counts.
diff --git a/rust/lance/src/metrics.rs b/rust/lance/src/metrics.rs
new file mode 100644
index 00000000000..1d48865e859
--- /dev/null
+++ b/rust/lance/src/metrics.rs
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Metrics published by Lance.
+#![doc = include_str!("metrics.md")]
+//!
+//! The metrics themselves are emitted from the relevant subsystems (for
+//! example object store I/O is instrumented in
+//! [`lance_io::object_store::metrics`]); this module exists to document the
+//! full catalogue of metric names, types, and labels in one place.
diff --git a/rust/lance/src/session.rs b/rust/lance/src/session.rs
index b032cbaa15e..c6214298341 100644
--- a/rust/lance/src/session.rs
+++ b/rust/lance/src/session.rs
@@ -4,11 +4,12 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
-use deepsize::DeepSizeOf;
-use lance_core::cache::{CacheBackend, LanceCache};
+use lance_core::cache::{CacheBackend, CacheKeyIterator, LanceCache};
+use lance_core::deepsize::DeepSizeOf;
 use lance_core::{Error, Result};
 use lance_index::IndexType;
 use lance_io::object_store::ObjectStoreRegistry;
+use lance_io::spill::{LocalSpillStore, SpillStore};
 
 use crate::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE};
 use crate::session::caches::GlobalMetadataCache;
@@ -53,10 +54,12 @@ pub struct Session {
     pub(crate) index_extensions: HashMap<(IndexType, String), Arc<dyn IndexExtension>>,
 
     store_registry: Arc<ObjectStoreRegistry>,
+
+    spill_store: Arc<dyn SpillStore>,
 }
 
 impl DeepSizeOf for Session {
-    fn deep_size_of_children(&self, context: &mut deepsize::Context) -> usize {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
         let mut size = 0;
         // Measure the actual cache contents through the wrapper types
         size += self.index_cache.deep_size_of_children(context);
@@ -107,6 +110,7 @@ impl Session {
             metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)),
             index_extensions: HashMap::new(),
             store_registry,
+            spill_store: Arc::new(LocalSpillStore::default()),
         }
     }
 
@@ -124,9 +128,35 @@ impl Session {
             metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)),
             index_extensions: HashMap::new(),
             store_registry,
+            spill_store: Arc::new(LocalSpillStore::default()),
         }
     }
 
+    /// Replace the spill store used by this session.
+    ///
+    /// This is a builder-style method that consumes and returns `self`, making
+    /// it easy to chain during session construction:
+    ///
+    /// ```rust,no_run
+    /// # use lance::session::Session;
+    /// # use lance_io::spill::LocalSpillStore;
+    /// # use std::sync::Arc;
+    /// let session = Session::default()
+    ///     .with_spill_store(Arc::new(LocalSpillStore::with_cap(1 << 30).unwrap()));
+    /// ```
+    pub fn with_spill_store(mut self, store: Arc<dyn SpillStore>) -> Self {
+        self.spill_store = store;
+        self
+    }
+
+    /// Return a reference to the session's spill store.
+    ///
+    /// Callers use this to obtain reclaimable scratch space for intermediate
+    /// state that overflows memory (e.g. index builders).
+    pub fn spill_store(&self) -> &dyn SpillStore {
+        &*self.spill_store
+    }
+
     /// Register a new index extension.
     ///
     /// A name can only be registered once per type of index extension.
@@ -209,6 +239,44 @@ impl Session {
     pub async fn index_cache_stats(&self) -> lance_core::cache::CacheStats {
         self.index_cache.0.stats().await
     }
+
+    /// Return an iterator over keys currently held by the index cache.
+    ///
+    /// Returns `None` when the index cache backend does not support key
+    /// inventory.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use lance::session::Session;
+    /// # async fn example() {
+    /// let session = Session::default();
+    /// let keys = session.index_cache_keys().await;
+    /// assert!(keys.is_some());
+    /// # }
+    /// ```
+    pub async fn index_cache_keys(&self) -> Option<CacheKeyIterator<'_>> {
+        self.index_cache.0.keys().await
+    }
+
+    /// Return an iterator over keys currently held by the metadata cache.
+    ///
+    /// Returns `None` when the metadata cache backend does not support key
+    /// inventory.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use lance::session::Session;
+    /// # async fn example() {
+    /// let session = Session::default();
+    /// let keys = session.metadata_cache_keys().await;
+    /// assert!(keys.is_some());
+    /// # }
+    /// ```
+    pub async fn metadata_cache_keys(&self) -> Option<CacheKeyIterator<'_>> {
+        self.metadata_cache.0.keys().await
+    }
 }
 
 impl Default for Session {
@@ -224,9 +292,23 @@ impl Default for Session {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use lance_core::cache::UnsizedCacheKey;
+    use lance_core::cache::{CacheKey, UnsizedCacheKey};
     use lance_index::vector::VectorIndex;
     use std::borrow::Cow;
+    use tokio::io::AsyncWriteExt;
+
+    struct TestKey(&'static str);
+    impl CacheKey for TestKey {
+        type ValueType = Vec<i32>;
+
+        fn key(&self) -> Cow<'_, str> {
+            Cow::Borrowed(self.0)
+        }
+
+        fn type_name() -> &'static str {
+            "TestVec"
+        }
+    }
 
     struct TestUnsizedKey(&'static str);
     impl UnsizedCacheKey for TestUnsizedKey {
@@ -251,4 +333,71 @@ mod tests {
                 .is_none()
         );
     }
+
+    #[tokio::test]
+    async fn test_session_cache_keys() {
+        let session = Session::new(10_000, 10_000, Default::default());
+
+        session
+            .index_cache
+            .insert_with_key(&TestKey("index-key"), Arc::new(vec![1]))
+            .await;
+        session
+            .metadata_cache
+            .0
+            .insert_with_key(&TestKey("metadata-key"), Arc::new(vec![2]))
+            .await;
+
+        let index_keys = session
+            .index_cache_keys()
+            .await
+            .unwrap()
+            .collect::<Vec<_>>();
+        assert_eq!(index_keys.len(), 1);
+        assert_eq!(index_keys[0].prefix(), "");
+        assert_eq!(index_keys[0].key(), "index-key");
+        assert_eq!(index_keys[0].type_name(), "TestVec");
+
+        let metadata_keys = session
+            .metadata_cache_keys()
+            .await
+            .unwrap()
+            .collect::<Vec<_>>();
+        assert_eq!(metadata_keys.len(), 1);
+        assert_eq!(metadata_keys[0].prefix(), "");
+        assert_eq!(metadata_keys[0].key(), "metadata-key");
+        assert_eq!(metadata_keys[0].type_name(), "TestVec");
+
+        assert_ne!(index_keys, metadata_keys);
+    }
+
+    #[tokio::test]
+    async fn test_default_session_has_spill_store() {
+        let session = Session::default();
+        // Should be able to allocate a spill and write to it without error.
+        let (mut writer, _spill) = session.spill_store().new_spill().await.unwrap();
+        writer.write_all(b"scratch").await.unwrap();
+        lance_io::traits::Writer::shutdown(writer.as_mut())
+            .await
+            .unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_custom_spill_store_injected() {
+        let capped = Arc::new(LocalSpillStore::with_cap(50).unwrap());
+        let session = Session::default().with_spill_store(capped);
+
+        let (mut writer, _spill) = session.spill_store().new_spill().await.unwrap();
+        // Writing 51 bytes exceeds the 50-byte cap; the typed error is wrapped
+        // in an io::Error by the writer and recovered on conversion.
+        let io_err = writer.write_all(&[0u8; 51]).await.unwrap_err();
+        let err: lance_core::Error = io_err.into();
+        assert!(
+            matches!(
+                err,
+                lance_core::Error::DiskCapExceeded { cap_bytes: 50, .. }
+            ),
+            "expected DiskCapExceeded, got {err}"
+        );
+    }
 }
diff --git a/rust/lance/src/session/caches.rs b/rust/lance/src/session/caches.rs
index 82dc755f6c0..a2dda6069ab 100644
--- a/rust/lance/src/session/caches.rs
+++ b/rust/lance/src/session/caches.rs
@@ -12,7 +12,7 @@
 
 use std::{borrow::Cow, ops::Deref};
 
-use deepsize::{Context, DeepSizeOf};
+use lance_core::deepsize::{Context, DeepSizeOf};
 use lance_core::{
     cache::{CacheKey, LanceCache},
     utils::deletion::DeletionVector,
diff --git a/rust/lance/src/session/index_caches.rs b/rust/lance/src/session/index_caches.rs
index 3ae777880aa..e93b7208b09 100644
--- a/rust/lance/src/session/index_caches.rs
+++ b/rust/lance/src/session/index_caches.rs
@@ -12,8 +12,8 @@
 
 use std::{borrow::Cow, ops::Deref, sync::Arc};
 
-use deepsize::{Context, DeepSizeOf};
 use lance_core::cache::{CacheKey, LanceCache};
+use lance_core::deepsize::{Context, DeepSizeOf};
 use lance_index::frag_reuse::FragReuseIndex;
 use lance_table::format::IndexMetadata;
 use uuid::Uuid;
@@ -63,14 +63,14 @@ impl Deref for DSIndexCache {
 
 impl DSIndexCache {
     /// Create an index-specific cache with the given UUID prefix.
-    pub fn for_index(&self, uuid: &str, fri_uuid: Option<&Uuid>) -> LanceCache {
+    pub fn for_index(&self, uuid: &Uuid, fri_uuid: Option<&Uuid>) -> LanceCache {
         if let Some(fri_uuid) = fri_uuid {
             // If a FRI UUID is provided, use it to create a more specific cache key.
             let cache_key = format!("{}-{}", uuid, fri_uuid);
             self.0.with_key_prefix(&cache_key)
         } else {
             // Otherwise, just use the index UUID as the key prefix.
-            self.0.with_key_prefix(uuid)
+            self.0.with_key_prefix(&uuid.to_string())
         }
     }
 }
@@ -79,7 +79,7 @@ impl DSIndexCache {
 
 #[derive(Debug)]
 pub struct FragReuseIndexKey<'a> {
-    pub uuid: &'a str,
+    pub uuid: &'a Uuid,
 }
 
 impl CacheKey for FragReuseIndexKey<'_> {
@@ -131,7 +131,7 @@ impl DeepSizeOf for ProstAny {
 /// what they are.  These we cache.
 #[derive(Debug)]
 pub struct ScalarIndexDetailsKey<'a> {
-    pub uuid: &'a str,
+    pub uuid: &'a Uuid,
 }
 
 impl CacheKey for ScalarIndexDetailsKey<'_> {
diff --git a/rust/lance/src/session/index_extension.rs b/rust/lance/src/session/index_extension.rs
index f5e7741441f..0f4d2ecf310 100644
--- a/rust/lance/src/session/index_extension.rs
+++ b/rust/lance/src/session/index_extension.rs
@@ -1,12 +1,15 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
 
+#[cfg(test)]
+use lance_core::utils::row_addr_remap::RowAddrRemap;
 use std::sync::Arc;
 
-use deepsize::DeepSizeOf;
 use lance_core::Result;
+use lance_core::deepsize::DeepSizeOf;
 use lance_file::previous::reader::FileReader as PreviousFileReader;
 use lance_index::{IndexParams, IndexType, vector::VectorIndex};
+use uuid::Uuid;
 
 use crate::Dataset;
 
@@ -35,7 +38,7 @@ pub trait VectorIndexExtension: IndexExtension {
         // if we wrap into an Arc, the mutable reference is lost
         dataset: &Dataset,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         params: &dyn IndexParams,
     ) -> Result<()>;
 
@@ -44,7 +47,7 @@ pub trait VectorIndexExtension: IndexExtension {
         &self,
         dataset: Arc<Dataset>,
         column: &str,
-        uuid: &str,
+        uuid: &Uuid,
         reader: PreviousFileReader,
     ) -> Result<Arc<dyn VectorIndex>>;
 }
@@ -61,7 +64,6 @@ mod test {
 
     use std::{
         any::Any,
-        collections::HashMap,
         sync::{Arc, atomic::AtomicBool},
     };
 
@@ -69,7 +71,7 @@ mod test {
     use arrow_array::{Float32Array, RecordBatch, UInt32Array};
     use arrow_schema::Schema;
     use datafusion::execution::SendableRecordBatchStream;
-    use deepsize::DeepSizeOf;
+    use lance_core::deepsize::DeepSizeOf;
     use lance_file::previous::writer::{
         FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions,
     };
@@ -95,7 +97,7 @@ mod test {
     struct MockIndex;
 
     impl DeepSizeOf for MockIndex {
-        fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
             0
         }
     }
@@ -110,10 +112,6 @@ mod test {
             self
         }
 
-        fn as_vector_index(self: Arc<Self>) -> Result<Arc<dyn VectorIndex>> {
-            Ok(self)
-        }
-
         async fn prewarm(&self) -> Result<()> {
             Ok(())
         }
@@ -185,7 +183,7 @@ mod test {
             unimplemented!()
         }
 
-        async fn remap(&mut self, _: &HashMap<u64, Option<u64>>) -> Result<()> {
+        async fn remap(&mut self, _: &RowAddrRemap) -> Result<()> {
             Ok(())
         }
 
@@ -230,7 +228,7 @@ mod test {
     }
 
     impl DeepSizeOf for MockIndexExtension {
-        fn deep_size_of_children(&self, _context: &mut deepsize::Context) -> usize {
+        fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize {
             todo!()
         }
     }
@@ -259,7 +257,7 @@ mod test {
             &self,
             dataset: &Dataset,
             _column: &str,
-            uuid: &str,
+            uuid: &Uuid,
             _params: &dyn IndexParams,
         ) -> Result<()> {
             let store = dataset.object_store.clone();
@@ -305,7 +303,7 @@ mod test {
             &self,
             _dataset: Arc<Dataset>,
             _column: &str,
-            _uuid: &str,
+            _uuid: &Uuid,
             _reader: PreviousFileReader,
         ) -> Result<Arc<dyn VectorIndex>> {
             self.load_index_called
@@ -391,7 +389,7 @@ mod test {
         let idx = ds_without_extension.load_indices().await.unwrap();
         assert_eq!(idx.len(), 1);
         // get the index uuid
-        let index_uuid = idx.first().unwrap().uuid.to_string();
+        let index_uuid = idx.first().unwrap().uuid;
 
         // trying to open the index should fail as there is no extension loader
         assert!(
diff --git a/rust/lance/src/utils/test.rs b/rust/lance/src/utils/test.rs
index 3338eee07a8..f804a7cc38a 100644
--- a/rust/lance/src/utils/test.rs
+++ b/rust/lance/src/utils/test.rs
@@ -243,6 +243,7 @@ impl TestDatasetGenerator {
         Fragment {
             id: 0,
             files,
+            overlays: vec![],
             deletion_file: None,
             row_id_meta: None,
             physical_rows: Some(batch.num_rows()),
diff --git a/rust/lance/tests/count_pushdown/mod.rs b/rust/lance/tests/count_pushdown/mod.rs
new file mode 100644
index 00000000000..d8afa051bca
--- /dev/null
+++ b/rust/lance/tests/count_pushdown/mod.rs
@@ -0,0 +1,183 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! End-to-end integration tests for [`CountPushdown`] when DataFusion
+//! does the planning (i.e. an aggregate built from SQL through the public
+//! [`LanceTableProvider`] surface), as opposed to going through
+//! `Scanner::create_plan`.
+//!
+//! The plan shape DataFusion produces for a SQL aggregate differs from the
+//! scanner's: it emits `AggregateExec(Final) → CoalescePartitionsExec →
+//! AggregateExec(Partial) → LanceTableScan` rather than a single
+//! `AggregateExec(Single)`. This file pins that down so future aggregate-
+//! pushdown categories can be added here with the test scaffolding already
+//! in place.
+
+use std::sync::Arc;
+
+use arrow::datatypes::UInt64Type;
+use arrow_array::types::Int64Type;
+use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion};
+use datafusion::execution::SessionStateBuilder;
+use datafusion::physical_plan::{ExecutionPlan, displayable};
+use datafusion::prelude::SessionContext;
+use futures::TryStreamExt;
+use lance::Dataset;
+use lance::datafusion::LanceTableProvider;
+use lance::dataset::WriteParams;
+use lance::index::DatasetIndexExt;
+use lance::io::exec::count_from_mask::CountFromMaskExec;
+use lance::io::exec::count_pushdown::CountPushdown;
+use lance_core::utils::tempfile::TempStrDir;
+use lance_datagen::{BatchCount, RowCount, array, gen_batch};
+use lance_index::IndexType;
+use lance_index::scalar::ScalarIndexParams;
+
+/// Build a 4-fragment, 10-row-per-fragment dataset with a BTREE index on `x`.
+async fn make_indexed_dataset() -> (Arc<Dataset>, TempStrDir) {
+    let tmp = TempStrDir::default();
+    let reader = gen_batch()
+        .col("x", array::step::<UInt64Type>())
+        .into_reader_rows(RowCount::from(10), BatchCount::from(4));
+    let mut dataset = Dataset::write(
+        reader,
+        tmp.as_str(),
+        Some(WriteParams {
+            max_rows_per_file: 10,
+            ..Default::default()
+        }),
+    )
+    .await
+    .unwrap();
+    dataset
+        .create_index(
+            &["x"],
+            IndexType::BTree,
+            None,
+            &ScalarIndexParams::default(),
+            true,
+        )
+        .await
+        .unwrap();
+    (Arc::new(dataset), tmp)
+}
+
+/// Build a `SessionContext` configured with the Lance physical optimizer rule
+/// for aggregate pushdown, then register `dataset` under the name `t`.
+fn lance_aware_context(dataset: Arc<Dataset>) -> SessionContext {
+    let state = SessionStateBuilder::new()
+        .with_default_features()
+        .with_physical_optimizer_rule(Arc::new(CountPushdown))
+        .build();
+    let ctx = SessionContext::new_with_state(state);
+    ctx.register_table(
+        "t",
+        Arc::new(LanceTableProvider::new(dataset, false, false)),
+    )
+    .unwrap();
+    ctx
+}
+
+fn plan_contains_pushdown(plan: &Arc<dyn ExecutionPlan>) -> bool {
+    let mut found = false;
+    plan.apply(|node| {
+        if node.is::<CountFromMaskExec>() {
+            found = true;
+            Ok(TreeNodeRecursion::Stop)
+        } else {
+            Ok(TreeNodeRecursion::Continue)
+        }
+    })
+    .unwrap();
+    found
+}
+
+async fn execute_count(plan: Arc<dyn ExecutionPlan>) -> i64 {
+    let stream = datafusion::physical_plan::execute_stream(
+        plan,
+        Arc::new(datafusion::execution::TaskContext::default()),
+    )
+    .unwrap();
+    let batches: Vec<_> = stream.try_collect().await.unwrap();
+    let total: i64 = batches
+        .iter()
+        .map(|b| {
+            b.column(0)
+                .as_any()
+                .downcast_ref::<arrow_array::PrimitiveArray<Int64Type>>()
+                .expect("count column should be Int64")
+                .iter()
+                .map(|v| v.unwrap_or(0))
+                .sum::<i64>()
+        })
+        .sum();
+    total
+}
+
+#[tokio::test]
+async fn sql_count_star_with_indexed_filter() {
+    // SELECT COUNT(*) FROM t WHERE x < 25
+    //
+    // The rule should fire on DataFusion's `AggregateExec(Partial)` node at
+    // the leaf of the aggregate pipeline, replacing the column scan with
+    // `CountFromMaskExec` while the outer `AggregateExec(Final)` keeps
+    // doing the cross-partition combine.
+    let (dataset, _tmp) = make_indexed_dataset().await;
+    let ctx = lance_aware_context(dataset);
+
+    let df = ctx
+        .sql("SELECT COUNT(*) FROM t WHERE x < 25")
+        .await
+        .unwrap();
+    let plan = df.create_physical_plan().await.unwrap();
+    assert!(
+        plan_contains_pushdown(&plan),
+        "expected CountFromMaskExec in SQL plan, got:\n{}",
+        displayable(plan.as_ref()).indent(true)
+    );
+    assert_eq!(execute_count(plan).await, 25);
+}
+
+#[tokio::test]
+async fn sql_unfiltered_count_star_uses_statistics() {
+    // SELECT COUNT(*) FROM t with no filter is answered by DataFusion
+    // statically from LanceTableProvider's row-count statistic — never
+    // reaches an `AggregateExec` for our rule to look at. Pin that
+    // behaviour: the rule should not fire, and the answer is correct.
+    let (dataset, _tmp) = make_indexed_dataset().await;
+    let ctx = lance_aware_context(dataset);
+
+    let df = ctx.sql("SELECT COUNT(*) FROM t").await.unwrap();
+    let plan = df.create_physical_plan().await.unwrap();
+    assert!(
+        !plan_contains_pushdown(&plan),
+        "unfiltered COUNT(*) should be resolved from statistics, got:\n{}",
+        displayable(plan.as_ref()).indent(true)
+    );
+    assert_eq!(execute_count(plan).await, 40);
+}
+
+#[tokio::test]
+async fn sql_count_distinct_does_not_fire_yet() {
+    // SELECT COUNT(DISTINCT x) FROM t WHERE x < 25
+    //
+    // `is_count_star` rejects distinct, so this rule never fires for
+    // distinct counts — they belong to the mask-to-answer category and will
+    // need their own rule (e.g. over a bitmap-index dictionary). This test
+    // pins the not-firing behaviour and the scaffold for the future test.
+    let (dataset, _tmp) = make_indexed_dataset().await;
+    let ctx = lance_aware_context(dataset);
+
+    let df = ctx
+        .sql("SELECT COUNT(DISTINCT x) FROM t WHERE x < 25")
+        .await
+        .unwrap();
+    let plan = df.create_physical_plan().await.unwrap();
+    assert!(
+        !plan_contains_pushdown(&plan),
+        "CountFromMaskExec must not fire for COUNT(DISTINCT) yet: \n{}",
+        displayable(plan.as_ref()).indent(true)
+    );
+    // Correctness via the scan path: values 0..25 are all distinct.
+    assert_eq!(execute_count(plan).await, 25);
+}
diff --git a/rust/lance/tests/integration_tests.rs b/rust/lance/tests/integration_tests.rs
index 81c2535dd9c..7a6d3e71ca4 100644
--- a/rust/lance/tests/integration_tests.rs
+++ b/rust/lance/tests/integration_tests.rs
@@ -3,6 +3,7 @@
 
 // NOTE: we only create one integration test binary, to keep compilation overhead down.
 
+mod count_pushdown;
 #[cfg(feature = "slow_tests")]
 mod query;
 #[cfg(feature = "slow_tests")]
diff --git a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
index 6fcff711a06..5edd16c2fed 100644
--- a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
+++ b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock
@@ -1124,9 +1124,9 @@ dependencies = [
 
 [[package]]
 name = "cmov"
-version = "0.5.3"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746"
+checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
 
 [[package]]
 name = "colorchoice"